From 303f894a70265a38d56b085382e4ec48835407ad Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 16:59:42 -0600 Subject: [PATCH 1/9] js helpers for migrating --- source/quickjs.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/quickjs.h b/source/quickjs.h index d100dcd8..98353893 100644 --- a/source/quickjs.h +++ b/source/quickjs.h @@ -587,6 +587,11 @@ JSValue __js_printf_like (2, 3) /* Log to "memory" channel + disrupt. Skips JS callback (can't allocate). */ JSValue JS_RaiseOOM (JSContext *ctx); +#define JS_ThrowOutOfMemory JS_RaiseOOM +#define JS_ThrowReferenceError JS_RaiseDisrupt +#define JS_ThrowTypeError JS_RaiseDisrupt +#define JS_ThrowInternalError JS_RaiseDisrupt +#define JS_ThrowRangeError JS_RaiseDisrupt /* ============================================================ 8. Function Creation and Invocation From 81561d426bb150728be3c58a029c6f19785ddb57 Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 17:39:22 -0600 Subject: [PATCH 2/9] use unstone jstext for string creation --- archive/miniz.c | 6 +- playdate/json_playdate.c | 2 +- qop.c | 37 ++-- source/quickjs-internal.h | 10 - source/quickjs.h | 3 - source/runtime.c | 400 ++++++++++++++++++-------------------- 6 files changed, 207 insertions(+), 251 deletions(-) diff --git a/archive/miniz.c b/archive/miniz.c index d9404247..7bf9a919 100644 --- a/archive/miniz.c +++ b/archive/miniz.c @@ -115,7 +115,7 @@ static JSValue js_miniz_compress(JSContext *js, JSValue this_val, /* ─── 2. Allocate an output buffer big enough ────────────── */ mz_ulong out_len_est = mz_compressBound(in_len); - void *out_buf = js_malloc(js, out_len_est); + void *out_buf = js_malloc_rt(out_len_est); if (!out_buf) { if (cstring) JS_FreeCString(js, cstring); return JS_EXCEPTION; @@ -130,14 +130,14 @@ static JSValue js_miniz_compress(JSContext *js, JSValue this_val, if (cstring) JS_FreeCString(js, cstring); if (st != MZ_OK) { - js_free(js, out_buf); + js_free_rt(out_buf); return JS_ThrowInternalError(js, "miniz: compression failed (%d)", st); } /* ─── 4. Hand JavaScript a copy of the compressed data ────── */ JSValue abuf = js_new_blob_stoned_copy(js, out_buf, out_len); - js_free(js, out_buf); + js_free_rt(out_buf); return abuf; } diff --git a/playdate/json_playdate.c b/playdate/json_playdate.c index 6aa2b432..30198fd8 100644 --- a/playdate/json_playdate.c +++ b/playdate/json_playdate.c @@ -116,7 +116,7 @@ static void encode_js_object(json_encoder *enc, JSContext *js, JSValue obj) { JS_FreeCString(js, key); JS_FreeAtom(js, props[i].atom); } - js_free(js, props); + js_free_rt(props); } enc->endTable(enc); } diff --git a/qop.c b/qop.c index af8db7cd..1fe0459a 100644 --- a/qop.c +++ b/qop.c @@ -84,11 +84,11 @@ static qop_desc *js2qop(JSContext *js, JSValue v) { static int js_qop_ensure_index(JSContext *js, qop_desc *qop) { if (qop->hashmap != NULL) return 1; - void *buffer = js_malloc(js, qop->hashmap_size); + void *buffer = js_malloc_rt(qop->hashmap_size); if (!buffer) return 0; int num = qop_read_index(qop, buffer); if (num == 0) { - js_free(js, buffer); + js_free_rt(buffer); return 0; } return 1; @@ -102,13 +102,13 @@ JSC_CCALL(qop_open, else if (!data) ret = JS_ThrowReferenceError(js, "Empty blob"); else { - qop_desc *qop = js_malloc(js, sizeof(qop_desc)); + qop_desc *qop = js_malloc_rt(sizeof(qop_desc)); if (!qop) ret = JS_ThrowOutOfMemory(js); else { int size = qop_open_data((const unsigned char *)data, len, qop); if (size == 0) { - js_free(js, qop); + js_free_rt(qop); ret = JS_ThrowReferenceError(js, "Failed to open QOP archive from blob"); } else { JSValue obj = JS_NewObjectClass(js, js_qop_archive_class_id); @@ -127,7 +127,7 @@ JSC_CCALL(qop_write, JS_FreeCString(js, path); if (!fh) return JS_ThrowInternalError(js, "Could not open file for writing"); - qop_writer *w = js_malloc(js, sizeof(qop_writer)); + qop_writer *w = js_malloc_rt(sizeof(qop_writer)); if (!w) { fclose(fh); return JS_ThrowOutOfMemory(js); @@ -137,10 +137,10 @@ JSC_CCALL(qop_write, w->capacity = 1024; w->len = 0; w->size = 0; - w->files = js_malloc(js, sizeof(qop_file) * w->capacity); + w->files = js_malloc_rt(sizeof(qop_file) * w->capacity); if (!w->files) { fclose(fh); - js_free(js, w); + js_free_rt(w); return JS_ThrowOutOfMemory(js); } @@ -183,18 +183,18 @@ static JSValue js_qop_read(JSContext *js, JSValue self, int argc, JSValue *argv) return JS_NULL; } - unsigned char *dest = js_malloc(js, file->size); + unsigned char *dest = js_malloc_rt(file->size); if (!dest) return JS_ThrowOutOfMemory(js); int bytes = qop_read(qop, file, dest); if (bytes == 0) { - js_free(js, dest); + js_free_rt(dest); return JS_ThrowReferenceError(js, "Failed to read file"); } JSValue blob = js_new_blob_stoned_copy(js, dest, bytes); - js_free(js, dest); + js_free_rt(dest); return blob; } @@ -223,18 +223,18 @@ static JSValue js_qop_read_ex(JSContext *js, JSValue self, int argc, JSValue *ar if (JS_ToUint32(js, &start, argv[1]) < 0 || JS_ToUint32(js, &len, argv[2]) < 0) return JS_ThrowTypeError(js, "Invalid start or len"); - unsigned char *dest = js_malloc(js, len); + unsigned char *dest = js_malloc_rt(len); if (!dest) return JS_ThrowOutOfMemory(js); int bytes = qop_read_ex(qop, file, dest, start, len); if (bytes == 0) { - js_free(js, dest); + js_free_rt(dest); return JS_ThrowReferenceError(js, "Failed to read file part"); } JSValue blob = js_new_blob_stoned_copy(js, dest, bytes); - js_free(js, dest); + js_free_rt(dest); return blob; } @@ -254,19 +254,19 @@ static JSValue js_qop_list(JSContext *js, JSValue self, int argc, JSValue *argv) qop_file *file = &qop->hashmap[i]; if (file->size == 0) continue; // empty slot - char *path = js_malloc(js, file->path_len); + char *path = js_malloc_rt(file->path_len); if (!path) { return JS_ThrowOutOfMemory(js); } int len = qop_read_path(qop, file, path); if (len == 0) { - js_free(js, path); + js_free_rt(path); continue; // skip on error } JSValue str = JS_NewStringLen(js, path, len - 1); // -1 for null terminator - js_free(js, path); + js_free_rt(path); JS_SetPropertyNumber(js, arr, count++, str); } @@ -311,7 +311,7 @@ static JSValue js_qop_is_directory(JSContext *js, JSValue self, int argc, JSValu // Check if any file starts with path + "/" size_t path_len = strlen(path); - char *prefix = js_malloc(js, path_len + 2); + char *prefix = alloca(path_len + 2); memcpy(prefix, path, path_len); prefix[path_len] = '/'; prefix[path_len + 1] = '\0'; @@ -339,7 +339,6 @@ static JSValue js_qop_is_directory(JSContext *js, JSValue self, int argc, JSValu } } - js_free(js, prefix); JS_FreeCString(js, path); return JS_NewBool(js, found); } @@ -366,7 +365,7 @@ static JSValue js_writer_add_file(JSContext *js, JSValue self, int argc, JSValue if (w->len >= w->capacity) { w->capacity *= 2; - qop_file *new_files = js_realloc(js, w->files, sizeof(qop_file) * w->capacity); + qop_file *new_files = realloc(w->files, sizeof(qop_file) * w->capacity); if (!new_files) { JS_FreeCString(js, path); return JS_ThrowOutOfMemory(js); diff --git a/source/quickjs-internal.h b/source/quickjs-internal.h index 0f2a051e..3c2068ad 100644 --- a/source/quickjs-internal.h +++ b/source/quickjs-internal.h @@ -230,8 +230,6 @@ static inline JS_BOOL JS_VALUE_IS_NUMBER (JSValue v) { /* JS_IsPretext, JS_KeyGetStr, JS_PushGCRef, JS_PopGCRef, JS_AddGCRef, JS_DeleteGCRef are defined after JSContext (they need its fields) */ -/* Forward declarations for memory functions (now declared in quickjs.h) */ -void *js_realloc (JSContext *ctx, void *ptr, size_t size); /* Forward declaration for string_get */ static inline int string_get (const JSText *p, int idx); @@ -1575,7 +1573,6 @@ JSText *pretext_putc (JSContext *ctx, JSText *s, uint32_t c); JSText *pretext_concat_value (JSContext *ctx, JSText *s, JSValue v); JSValue js_new_blob (JSContext *ctx, blob *b); /* Functions from header region (defined in runtime.c) */ -void *js_realloc (JSContext *ctx, void *ptr, size_t size); void *ct_alloc (JSContext *ctx, size_t bytes, size_t align); void ct_free_all (JSContext *ctx); int ct_resize (JSContext *ctx); @@ -1629,13 +1626,6 @@ static inline int to_digit (int c) { else return 36; } -no_inline int js_realloc_array (JSContext *ctx, void **parray, int elem_size, int *psize, int req_size); -static inline int js_resize_array (JSContext *ctx, void **parray, int elem_size, int *psize, int req_size) { - if (unlikely (req_size > *psize)) - return js_realloc_array (ctx, parray, elem_size, psize, req_size); - else - return 0; -} JSText *js_alloc_string (JSContext *ctx, int max_len); JSValue js_key_from_string (JSContext *ctx, JSValue val); diff --git a/source/quickjs.h b/source/quickjs.h index 1c02f2eb..93aeb164 100644 --- a/source/quickjs.h +++ b/source/quickjs.h @@ -1007,9 +1007,6 @@ void *js_debugger_val_address (JSContext *js, JSValue val); ============================================================ */ void *js_malloc (JSContext *ctx, size_t size); void *js_mallocz (JSContext *ctx, size_t size); -void *js_realloc (JSContext *ctx, void *ptr, size_t size); -void js_free (JSContext *ctx, void *ptr); -char *js_strdup (JSContext *ctx, const char *str); /* Runtime-level memory functions */ void *js_malloc_rt (size_t size); diff --git a/source/runtime.c b/source/runtime.c index f99557e8..45fcb328 100644 --- a/source/runtime.c +++ b/source/runtime.c @@ -291,24 +291,6 @@ int ct_resize (JSContext *ctx) { return 0; } -void *js_realloc (JSContext *ctx, void *ptr, size_t size) { - void *new_ptr; - - /* Align size to 8 bytes */ - size = (size + 7) & ~7; - - if (!ptr) { - /* New allocation */ - new_ptr = js_malloc (ctx, size); - if (!new_ptr) return NULL; - return new_ptr; - } - - /* Bump allocator: just allocate new space. - Caller is responsible for protecting ptr and copying data. */ - new_ptr = js_malloc (ctx, size); - return new_ptr; -} JSValue intern_text_to_value (JSContext *ctx, const uint32_t *utf32, uint32_t len) { /* Pack UTF-32 for hashing and comparison */ @@ -583,6 +565,8 @@ JSValue rec_get (JSContext *ctx, JSRecord *rec, JSValue k) { return JS_NULL; } +size_t gc_object_size (void *ptr); /* forward declaration for growth-forward size storage */ + int rec_resize (JSContext *ctx, JSValue *pobj, uint64_t new_mask) { /* Protect the source object with a GC ref in case js_malloc triggers GC */ JSGCRef obj_ref; @@ -646,7 +630,9 @@ int rec_resize (JSContext *ctx, JSValue *pobj, uint64_t new_mask) { } /* Install forward header at old location so stale references can find the new record */ + size_t old_size = gc_object_size (rec); rec->mist_hdr = objhdr_make_fwd (new_rec); + *((size_t *)((uint8_t *)rec + sizeof (objhdr_t))) = old_size; /* Update caller's JSValue to point to new record */ *pobj = JS_MKPTR (new_rec); @@ -865,11 +851,6 @@ void *js_mallocz (JSContext *ctx, size_t size) { return memset (ptr, 0, size); } -void js_free (JSContext *ctx, void *ptr) { - /* Bump allocator doesn't free individual allocations - GC handles it */ - (void)ctx; - (void)ptr; -} /* Parser memory functions - use system allocator to avoid GC issues */ @@ -937,41 +918,6 @@ JSValue ppretext_end (JSContext *ctx, PPretext *p) { return JS_MKPTR (str); } -no_inline int js_realloc_array (JSContext *ctx, void **parray, int elem_size, int *psize, int req_size) { - int new_size; - void *new_array; - void *old_array = *parray; - int old_size = *psize; - - /* XXX: potential arithmetic overflow */ - new_size = max_int (req_size, old_size * 3 / 2); - - /* Protect source object with a GC ref before allocating (GC may move it) */ - JSGCRef src_ref; - JS_PushGCRef (ctx, &src_ref); - if (old_array) { - src_ref.val = JS_MKPTR (old_array); - } - - new_array = js_malloc (ctx, new_size * elem_size); - if (!new_array) { - JS_PopGCRef (ctx, &src_ref); - return -1; - } - - /* Get possibly-moved source pointer after GC */ - if (old_array) { - old_array = (void *)chase (src_ref.val); - memcpy (new_array, old_array, old_size * elem_size); - } - JS_PopGCRef (ctx, &src_ref); - - *psize = new_size; - *parray = new_array; - return 0; -} - - /* Append a JSValue string to a PPretext (parser pretext) */ PPretext *ppretext_append_jsvalue (PPretext *p, JSValue str) { @@ -1386,6 +1332,7 @@ JSValue gc_copy_value (JSContext *ctx, JSValue v, uint8_t *from_base, uint8_t *f *to_free += size; *hdr_ptr = objhdr_make_fwd (new_ptr); + *((size_t *)((uint8_t *)hdr_ptr + sizeof (objhdr_t))) = size; return JS_MKPTR (new_ptr); } @@ -1732,6 +1679,70 @@ int ctx_gc (JSContext *ctx, int allow_grow, size_t alloc_size) { } #endif + /* Finalize garbage records that have class finalizers */ + { + uint8_t *p = from_base; + uint8_t *prev_p = NULL; + size_t prev_size = 0; + uint8_t prev_type = 0; + while (p < from_end) { + objhdr_t hdr = *(objhdr_t *)p; + uint8_t type = objhdr_type (hdr); + size_t size; + if (type == OBJ_FORWARD) { + size = *((size_t *)(p + sizeof (objhdr_t))); + if (size == 0 || size > (size_t)(from_end - from_base) || (size & 7) != 0) { + uint64_t *w = (uint64_t *)p; + fprintf (stderr, "gc_finalize_walk: bad fwd size=%zu at p=%p prev_p=%p prev_type=%d prev_size=%zu\n" + " words: [0]=0x%llx [1]=0x%llx [2]=0x%llx [3]=0x%llx\n" + " prev words: [0]=0x%llx [1]=0x%llx [2]=0x%llx [3]=0x%llx\n", + size, (void *)p, (void *)prev_p, prev_type, prev_size, + (unsigned long long)w[0], (unsigned long long)w[1], + (unsigned long long)w[2], (unsigned long long)w[3], + prev_p ? ((unsigned long long *)prev_p)[0] : 0, + prev_p ? ((unsigned long long *)prev_p)[1] : 0, + prev_p ? ((unsigned long long *)prev_p)[2] : 0, + prev_p ? ((unsigned long long *)prev_p)[3] : 0); + break; + } + } else if (type != OBJ_ARRAY && type != OBJ_BLOB && type != OBJ_TEXT && + type != OBJ_RECORD && type != OBJ_FUNCTION && type != OBJ_FRAME) { + uint64_t *w = (uint64_t *)p; + fprintf (stderr, "gc_finalize_walk: bad type=%d at p=%p hdr=0x%llx prev_p=%p prev_type=%d prev_size=%zu\n" + " words: [0]=0x%llx [1]=0x%llx [2]=0x%llx [3]=0x%llx\n" + " prev words: [0]=0x%llx [1]=0x%llx [2]=0x%llx [3]=0x%llx\n", + type, (void *)p, (unsigned long long)hdr, (void *)prev_p, prev_type, prev_size, + (unsigned long long)w[0], (unsigned long long)w[1], + (unsigned long long)w[2], (unsigned long long)w[3], + prev_p ? ((unsigned long long *)prev_p)[0] : 0, + prev_p ? ((unsigned long long *)prev_p)[1] : 0, + prev_p ? ((unsigned long long *)prev_p)[2] : 0, + prev_p ? ((unsigned long long *)prev_p)[3] : 0); + break; + } else { + size = gc_object_size (p); + if (type == OBJ_RECORD) { + JSRecord *rec = (JSRecord *)p; + uint32_t class_id = REC_GET_CLASS_ID (rec); + if (class_id != 0 && (int)class_id < ctx->class_count) { + JSClassFinalizer *fn = ctx->class_array[class_id].finalizer; + if (fn) { +#ifdef DUMP_GC_FINALIZER + fprintf (stderr, "gc_finalize: class_id=%u name=%s rec=%p\n", + class_id, ctx->class_array[class_id].class_name, (void *)rec); +#endif + fn (rt, JS_MKPTR (rec)); + } + } + } + } + prev_p = p; + prev_type = type; + prev_size = size; + p += size; + } + } + /* Return old block (in poison mode, just poison it and leak) */ heap_block_free (rt, from_base, old_heap_size); @@ -2083,6 +2094,37 @@ void JS_FreeContext (JSContext *ctx) { } for (i = 0; i < ctx->class_count; i++) { } + + /* Finalize all remaining records with class finalizers before teardown */ + if (ctx->heap_base) { + uint8_t *p = ctx->heap_base; + while (p < ctx->heap_free) { + objhdr_t hdr = *(objhdr_t *)p; + uint8_t type = objhdr_type (hdr); + size_t size; + if (type == OBJ_FORWARD) { + size = *((size_t *)(p + sizeof (objhdr_t))); + } else { + size = gc_object_size (p); + if (type == OBJ_RECORD) { + JSRecord *rec = (JSRecord *)p; + uint32_t class_id = REC_GET_CLASS_ID (rec); + if (class_id != 0 && (int)class_id < ctx->class_count) { + JSClassFinalizer *fn = ctx->class_array[class_id].finalizer; + if (fn) { +#ifdef DUMP_GC_FINALIZER + fprintf (stderr, "teardown_finalize: class_id=%u name=%s rec=%p\n", + class_id, ctx->class_array[class_id].class_name, (void *)rec); +#endif + fn (rt, JS_MKPTR (rec)); + } + } + } + } + p += size; + } + } + js_free_rt (ctx->class_array); js_free_rt (ctx->class_proto); @@ -2454,10 +2496,7 @@ JSText *pretext_concat_value (JSContext *ctx, JSText *s, JSValue v) { JSValue pretext_end (JSContext *ctx, JSText *s) { if (!s) return JS_EXCEPTION; int len = (int)s->length; - if (len == 0) { - js_free (ctx, s); - return JS_KEY_empty; - } + if (len == 0) return JS_KEY_empty; /* Promote short ASCII strings to immediate values */ if (len <= MIST_ASCII_MAX_LEN) { char buf[MIST_ASCII_MAX_LEN]; @@ -2957,7 +2996,9 @@ static int js_array_grow (JSContext *ctx, JSValue *arr_ptr, word_t min_cap) { new_arr->values[i] = JS_NULL; /* Install forward header at old location */ + size_t old_arr_size = gc_object_size (arr); arr->mist_hdr = objhdr_make_fwd (new_arr); + *((size_t *)((uint8_t *)arr + sizeof (objhdr_t))) = old_arr_size; /* Update the tracked JSValue to point to new array */ *arr_ptr = JS_MKPTR (new_arr); @@ -4213,25 +4254,10 @@ static __exception int JS_ToLength (JSContext *ctx, int64_t *plen, JSValue val) } static JSValue js_dtoa2 (JSContext *ctx, double d, int radix, int n_digits, int flags) { - char static_buf[128], *buf, *tmp_buf; - int len, len_max; - JSValue res; + char buf[1088]; JSDTOATempMem dtoa_mem; - len_max = js_dtoa_max_len (d, radix, n_digits, flags); - - /* longer buffer may be used if radix != 10 */ - if (len_max > sizeof (static_buf) - 1) { - tmp_buf = js_malloc (ctx, len_max + 1); - if (!tmp_buf) return JS_EXCEPTION; - buf = tmp_buf; - } else { - tmp_buf = NULL; - buf = static_buf; - } - len = js_dtoa (buf, d, radix, n_digits, flags, &dtoa_mem); - res = js_new_string8_len (ctx, buf, len); - js_free (ctx, tmp_buf); - return res; + int len = js_dtoa (buf, d, radix, n_digits, flags, &dtoa_mem); + return js_new_string8_len (ctx, buf, len); } JSValue JS_ToString (JSContext *ctx, JSValue val) { @@ -5074,7 +5100,7 @@ JSValue js_compile_regexp (JSContext *ctx, JSValue pattern, JSValue flags) { ret = js_new_string8_len (ctx, (const char *)re_bytecode_buf, re_bytecode_len); - js_free (ctx, re_bytecode_buf); + js_free_rt (re_bytecode_buf); return ret; } @@ -6182,12 +6208,7 @@ static JSValue js_cell_number (JSContext *ctx, JSValue this_val, int argc, JSVal return JS_EXCEPTION; } - char *clean = js_malloc (ctx, strlen (str) + 1); - if (!clean) { - JS_FreeCString (ctx, format); - JS_FreeCString (ctx, str); - return JS_EXCEPTION; - } + char *clean = alloca (strlen (str) + 1); const char *p = str; char *q = clean; @@ -6228,7 +6249,6 @@ static JSValue js_cell_number (JSContext *ctx, JSValue this_val, int argc, JSVal *q = '\0'; char *endptr; long long n = strtoll (str, &endptr, 2); - js_free (ctx, clean); JS_FreeCString (ctx, format); JS_FreeCString (ctx, str); if (endptr == str) return JS_NULL; @@ -6237,7 +6257,6 @@ static JSValue js_cell_number (JSContext *ctx, JSValue this_val, int argc, JSVal *q = '\0'; char *endptr; long long n = strtoll (str, &endptr, 8); - js_free (ctx, clean); JS_FreeCString (ctx, format); JS_FreeCString (ctx, str); if (endptr == str) return JS_NULL; @@ -6246,7 +6265,6 @@ static JSValue js_cell_number (JSContext *ctx, JSValue this_val, int argc, JSVal *q = '\0'; char *endptr; long long n = strtoll (str, &endptr, 16); - js_free (ctx, clean); JS_FreeCString (ctx, format); JS_FreeCString (ctx, str); if (endptr == str) return JS_NULL; @@ -6255,14 +6273,12 @@ static JSValue js_cell_number (JSContext *ctx, JSValue this_val, int argc, JSVal *q = '\0'; char *endptr; long long n = strtoll (str, &endptr, 32); - js_free (ctx, clean); JS_FreeCString (ctx, format); JS_FreeCString (ctx, str); if (endptr == str) return JS_NULL; return JS_NewInt64 (ctx, n); } else if (strcmp (format, "j") == 0) { /* JavaScript style prefix */ - js_free (ctx, clean); JS_FreeCString (ctx, format); int radix = 10; const char *start = str; @@ -6294,7 +6310,6 @@ static JSValue js_cell_number (JSContext *ctx, JSValue this_val, int argc, JSVal *q = '\0'; double d = strtod (clean, NULL); - js_free (ctx, clean); JS_FreeCString (ctx, format); JS_FreeCString (ctx, str); if (isnan (d)) return JS_NULL; @@ -6497,13 +6512,9 @@ static JSValue js_cell_number_to_radix_string (JSContext *ctx, double num, int r return JS_NewString (ctx, result); } -/* Helper: add separator every n digits from right */ -static char *add_separator (JSContext *ctx, const char *str, char sep, int n) { - if (n <= 0) { - char *result = js_malloc (ctx, strlen (str) + 1); - if (result) strcpy (result, str); - return result; - } +/* Helper: add separator every n digits from right, returns JSValue string */ +static JSValue add_separator (JSContext *ctx, const char *str, char sep, int n, int prepend_neg) { + if (n <= 0) return JS_NewString (ctx, str); int negative = (str[0] == '-'); const char *start = negative ? str + 1 : str; @@ -6511,34 +6522,35 @@ static char *add_separator (JSContext *ctx, const char *str, char sep, int n) { /* Find decimal point */ const char *decimal = strchr (start, '.'); int int_len = decimal ? (int)(decimal - start) : (int)strlen (start); + int dec_len = decimal ? (int)strlen (decimal) : 0; int num_seps = (int_len - 1) / n; - int result_len = strlen (str) + num_seps + 1; - char *result = js_malloc (ctx, result_len); - if (!result) return NULL; + int total = (negative ? 1 : 0) + prepend_neg + int_len + num_seps + dec_len; - char *q = result; - if (negative) *q++ = '-'; + JSText *pt = pretext_init (ctx, total); + if (!pt) return JS_EXCEPTION; + + int pos = 0; + if (prepend_neg) string_put (pt, pos++, '-'); + if (negative) string_put (pt, pos++, '-'); int count = int_len % n; if (count == 0) count = n; for (int i = 0; i < int_len; i++) { if (i > 0 && count == 0) { - *q++ = sep; + string_put (pt, pos++, (uint32_t)sep); count = n; } - *q++ = start[i]; + string_put (pt, pos++, (uint32_t)(unsigned char)start[i]); count--; } - if (decimal) { - strcpy (q, decimal); - } else { - *q = '\0'; - } + for (int i = 0; i < dec_len; i++) + string_put (pt, pos++, (uint32_t)(unsigned char)decimal[i]); - return result; + pt->length = pos; + return pretext_end (ctx, pt); } /* Helper: format number with format string */ @@ -6576,7 +6588,6 @@ static JSValue js_cell_format_number (JSContext *ctx, double num, const char *fo if (format[i] != '\0') return JS_NULL; char buf[128]; - char *result_str = NULL; switch (style) { case 'e': { @@ -6602,22 +6613,13 @@ static JSValue js_cell_format_number (JSContext *ctx, double num, const char *fo /* Space separated */ if (separation == 0) separation = 3; snprintf (buf, sizeof (buf), "%.*f", places, num); - result_str = add_separator (ctx, buf, ' ', separation); - if (!result_str) return JS_EXCEPTION; - JSValue ret = JS_NewString (ctx, result_str); - js_free (ctx, result_str); - return ret; + return add_separator (ctx, buf, ' ', separation, 0); } case 'u': { /* Underbar separated */ snprintf (buf, sizeof (buf), "%.*f", places, num); - if (separation > 0) { - result_str = add_separator (ctx, buf, '_', separation); - if (!result_str) return JS_EXCEPTION; - JSValue ret = JS_NewString (ctx, result_str); - js_free (ctx, result_str); - return ret; - } + if (separation > 0) + return add_separator (ctx, buf, '_', separation, 0); return JS_NewString (ctx, buf); } case 'd': @@ -6626,11 +6628,7 @@ static JSValue js_cell_format_number (JSContext *ctx, double num, const char *fo if (separation == 0) separation = 3; if (places == 0 && style == 'd') places = 2; snprintf (buf, sizeof (buf), "%.*f", places, num); - result_str = add_separator (ctx, buf, ',', separation); - if (!result_str) return JS_EXCEPTION; - JSValue ret = JS_NewString (ctx, result_str); - js_free (ctx, result_str); - return ret; + return add_separator (ctx, buf, ',', separation, 0); } case 'v': { /* European style: comma decimal, period separator */ @@ -6639,13 +6637,8 @@ static JSValue js_cell_format_number (JSContext *ctx, double num, const char *fo for (char *p = buf; *p; p++) { if (*p == '.') *p = ','; } - if (separation > 0) { - result_str = add_separator (ctx, buf, '.', separation); - if (!result_str) return JS_EXCEPTION; - JSValue ret = JS_NewString (ctx, result_str); - js_free (ctx, result_str); - return ret; - } + if (separation > 0) + return add_separator (ctx, buf, '.', separation, 0); return JS_NewString (ctx, buf); } case 'i': { @@ -6661,26 +6654,8 @@ static JSValue js_cell_format_number (JSContext *ctx, double num, const char *fo memmove (buf + (places - len), buf, len + 1); memset (buf, '0', places - len); } - if (separation > 0) { - result_str = add_separator (ctx, buf, '_', separation); - if (!result_str) return JS_EXCEPTION; - if (neg) { - char *final = js_malloc (ctx, strlen (result_str) + 2); - if (!final) { - js_free (ctx, result_str); - return JS_EXCEPTION; - } - final[0] = '-'; - strcpy (final + 1, result_str); - js_free (ctx, result_str); - JSValue ret = JS_NewString (ctx, final); - js_free (ctx, final); - return ret; - } - JSValue ret = JS_NewString (ctx, result_str); - js_free (ctx, result_str); - return ret; - } + if (separation > 0) + return add_separator (ctx, buf, '_', separation, neg); if (neg) { memmove (buf + 1, buf, strlen (buf) + 1); buf[0] = '-'; @@ -6902,69 +6877,72 @@ static JSValue js_cell_text (JSContext *ctx, JSValue this_val, int argc, JSValue if (format == 'h') { static const char hex[] = "0123456789abcdef"; - char *result = js_malloc (ctx, byte_len * 2 + 1); - if (!result) return JS_EXCEPTION; + int exact = (int)(byte_len * 2); + JSText *pt = pretext_init (ctx, exact); + if (!pt) return JS_EXCEPTION; + bd = (JSBlob *)chase (argv[0]); + data = (const uint8_t *)bd->bits; for (size_t i = 0; i < byte_len; i++) { - result[i * 2] = hex[(data[i] >> 4) & 0xF]; - result[i * 2 + 1] = hex[data[i] & 0xF]; + string_put (pt, (int)(i * 2), (uint32_t)hex[(data[i] >> 4) & 0xF]); + string_put (pt, (int)(i * 2 + 1), (uint32_t)hex[data[i] & 0xF]); } - result[byte_len * 2] = '\0'; - JSValue ret = JS_NewString (ctx, result); - js_free (ctx, result); - return ret; + pt->length = exact; + return pretext_end (ctx, pt); } else if (format == 'b') { - char *result = js_malloc (ctx, bd->length + 1); - if (!result) return JS_EXCEPTION; + int exact = (int)bd->length; + JSText *pt = pretext_init (ctx, exact); + if (!pt) return JS_EXCEPTION; + bd = (JSBlob *)chase (argv[0]); + data = (const uint8_t *)bd->bits; for (size_t i = 0; i < (size_t)bd->length; i++) { size_t byte_idx = i / 8; size_t bit_idx = i % 8; - result[i] = (data[byte_idx] & (1u << bit_idx)) ? '1' : '0'; + string_put (pt, (int)i, + (data[byte_idx] & (1u << bit_idx)) ? '1' : '0'); } - result[bd->length] = '\0'; - JSValue ret = JS_NewString (ctx, result); - js_free (ctx, result); - return ret; + pt->length = exact; + return pretext_end (ctx, pt); } else if (format == 'o') { - size_t octal_len = ((size_t)bd->length + 2) / 3; - char *result = js_malloc (ctx, octal_len + 1); - if (!result) return JS_EXCEPTION; - for (size_t i = 0; i < octal_len; i++) { + int exact = (int)(((size_t)bd->length + 2) / 3); + JSText *pt = pretext_init (ctx, exact); + if (!pt) return JS_EXCEPTION; + bd = (JSBlob *)chase (argv[0]); + data = (const uint8_t *)bd->bits; + for (int i = 0; i < exact; i++) { int val = 0; for (int j = 0; j < 3; j++) { - size_t bit_pos = i * 3 + (size_t)j; + size_t bit_pos = (size_t)i * 3 + (size_t)j; if (bit_pos < (size_t)bd->length) { size_t byte_idx = bit_pos / 8; size_t bit_idx = bit_pos % 8; if (data[byte_idx] & (1u << bit_idx)) val |= (1 << j); } } - result[i] = (char)('0' + val); + string_put (pt, i, (uint32_t)('0' + val)); } - result[octal_len] = '\0'; - JSValue ret = JS_NewString (ctx, result); - js_free (ctx, result); - return ret; + pt->length = exact; + return pretext_end (ctx, pt); } else if (format == 't') { static const char b32[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; - size_t b32_len = ((size_t)bd->length + 4) / 5; - char *result = js_malloc (ctx, b32_len + 1); - if (!result) return JS_EXCEPTION; - for (size_t i = 0; i < b32_len; i++) { + int exact = (int)(((size_t)bd->length + 4) / 5); + JSText *pt = pretext_init (ctx, exact); + if (!pt) return JS_EXCEPTION; + bd = (JSBlob *)chase (argv[0]); + data = (const uint8_t *)bd->bits; + for (int i = 0; i < exact; i++) { int val = 0; for (int j = 0; j < 5; j++) { - size_t bit_pos = i * 5 + (size_t)j; + size_t bit_pos = (size_t)i * 5 + (size_t)j; if (bit_pos < (size_t)bd->length) { size_t byte_idx = bit_pos / 8; size_t bit_idx = bit_pos % 8; if (data[byte_idx] & (1u << bit_idx)) val |= (1 << j); } } - result[i] = b32[val & 31]; + string_put (pt, i, (uint32_t)b32[val & 31]); } - result[b32_len] = '\0'; - JSValue ret = JS_NewString (ctx, result); - js_free (ctx, result); - return ret; + pt->length = exact; + return pretext_end (ctx, pt); } else { if (bd->length % 8 != 0) return JS_ThrowTypeError (ctx, @@ -7074,32 +7052,28 @@ static JSValue js_cell_text (JSContext *ctx, JSValue this_val, int argc, JSValue const char *pref = "function "; const char *suff = "() {\n [native code]\n}"; - const char *name = ""; const char *name_cstr = NULL; + int nlen = 0; if (!JS_IsNull (fn->name)) { name_cstr = JS_ToCString (ctx, fn->name); - if (name_cstr) name = name_cstr; + if (name_cstr) nlen = (int)strlen (name_cstr); } - size_t plen = strlen (pref); - size_t nlen = strlen (name); - size_t slen = strlen (suff); + int plen = (int)strlen (pref); + int slen = (int)strlen (suff); - char *result = js_malloc (ctx, plen + nlen + slen + 1); - if (!result) { + JSText *pt = pretext_init (ctx, plen + nlen + slen); + if (!pt) { if (name_cstr) JS_FreeCString (ctx, name_cstr); return JS_EXCEPTION; } - memcpy (result, pref, plen); - memcpy (result + plen, name, nlen); - memcpy (result + plen + nlen, suff, slen + 1); - - JSValue ret = JS_NewString (ctx, result); - js_free (ctx, result); + pt = pretext_puts8 (ctx, pt, pref); + if (pt && name_cstr) pt = pretext_write8 (ctx, pt, (const uint8_t *)name_cstr, nlen); if (name_cstr) JS_FreeCString (ctx, name_cstr); - return ret; + if (pt) pt = pretext_puts8 (ctx, pt, suff); + return pretext_end (ctx, pt); } return JS_ToString (ctx, arg); @@ -9059,7 +9033,6 @@ static JSValue js_cell_array_sort (JSContext *ctx, JSValue this_val, int argc, J if (str_keys) { for (word_t j = 0; j < i; j++) JS_FreeCString (ctx, str_keys[j]); - js_free (ctx, str_keys); } JS_PopGCRef (ctx, &result_ref); JS_PopGCRef (ctx, &arr_ref); @@ -9375,18 +9348,13 @@ static JSValue js_cell_fn_apply (JSContext *ctx, JSValue this_val, int argc, JSV if (len == 0) return JS_CallInternal (ctx, argv[0], JS_NULL, 0, NULL, 0); - JSValue *args = js_malloc (ctx, sizeof (JSValue) * len); - if (!args) return JS_EXCEPTION; - arr = JS_VALUE_GET_ARRAY (argv[1]); /* re-chase after malloc */ + JSValue *args = alloca (sizeof (JSValue) * len); for (int i = 0; i < len; i++) { args[i] = arr->values[i]; } - JSValue result = JS_CallInternal (ctx, argv[0], JS_NULL, len, args, 0); - - js_free (ctx, args); - return result; + return JS_CallInternal (ctx, argv[0], JS_NULL, len, args, 0); } /* ============================================================================ @@ -9444,7 +9412,9 @@ static int blob_grow (JSContext *ctx, JSValue *pblob, size_t need_bits) { memcpy (nb->bits, old->bits, old_words * sizeof (word_t)); /* Install forward pointer at old location */ + size_t old_blob_size = gc_object_size (old); old->mist_hdr = objhdr_make_fwd (nb); + *((size_t *)((uint8_t *)old + sizeof (objhdr_t))) = old_blob_size; /* Update caller's JSValue */ *pblob = JS_MKPTR (nb); From 34521e44f1ced9f27d5db6019e2d483ddf4844f2 Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 17:58:36 -0600 Subject: [PATCH 3/9] jstext properly used for oncat --- source/quickjs-internal.h | 6 +++--- source/runtime.c | 32 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/source/quickjs-internal.h b/source/quickjs-internal.h index 3c2068ad..79a600cb 100644 --- a/source/quickjs-internal.h +++ b/source/quickjs-internal.h @@ -906,8 +906,9 @@ typedef struct JSBlob { } JSBlob; typedef struct JSText { - objhdr_t hdr; /* mist header */ - word_t length; /* length (or hash for stoned text) */ + objhdr_t hdr; /* mist header — cap56 = allocated capacity */ + word_t length; /* character count (always) */ + word_t hash; /* cached hash (stoned text only) */ word_t packed[]; /* two chars per packed word */ } JSText; @@ -1029,7 +1030,6 @@ static inline uint64_t fash64_hash_one (uint64_t word) { } static inline word_t JSText_len (const JSText *text) { - if (objhdr_s (text->hdr)) return objhdr_cap56 (text->hdr); return text->length; } diff --git a/source/runtime.c b/source/runtime.c index 45fcb328..22a08791 100644 --- a/source/runtime.c +++ b/source/runtime.c @@ -110,16 +110,16 @@ JS_BOOL JS_IsFrame(JSValue v) { } uint64_t get_text_hash (JSText *text) { - uint64_t len = objhdr_cap56 (text->hdr); + uint64_t len = text->length; size_t word_count = (len + 1) / 2; if (objhdr_s (text->hdr)) { /* Stoned text: check for cached hash */ - if (text->length != 0) return text->length; - /* Compute and cache hash using content length from header */ - text->length = fash64_hash_words (text->packed, word_count, len); - if (!text->length) text->length = 1; - return text->length; + if (text->hash != 0) return text->hash; + /* Compute and cache hash */ + text->hash = fash64_hash_words (text->packed, word_count, len); + if (!text->hash) text->hash = 1; + return text->hash; } else { /* Pre-text: compute hash on the fly */ return fash64_hash_words (text->packed, word_count, len); @@ -137,7 +137,7 @@ void pack_utf32_to_words (const uint32_t *utf32, uint32_t len, uint64_t *packed) /* Compare two packed UTF-32 texts for equality */ int text_equal (JSText *a, const uint64_t *packed_b, uint32_t len_b) { - uint32_t len_a = (uint32_t)objhdr_cap56 (a->hdr); + uint32_t len_a = (uint32_t)a->length; if (len_a != len_b) return 0; size_t word_count = (len_a + 1) / 2; return memcmp (a->packed, packed_b, word_count * sizeof (uint64_t)) == 0; @@ -331,7 +331,8 @@ JSValue intern_text_to_value (JSContext *ctx, const uint32_t *utf32, uint32_t le /* Initialize the text */ text->hdr = objhdr_make (len, OBJ_TEXT, false, false, false, true); /* s=1 for stoned */ - text->length = hash; /* Store hash in length field for stoned text */ + text->length = len; + text->hash = hash; memcpy (text->packed, packed, word_count * sizeof (uint64_t)); /* Add to intern table */ @@ -911,7 +912,8 @@ JSValue ppretext_end (JSContext *ctx, PPretext *p) { for (int i = 0; i < len; i++) { string_put (str, i, p->data[i]); } - str->hdr = objhdr_set_cap56 (str->hdr, len); + str->length = len; + str->hash = 0; str->hdr = objhdr_set_s (str->hdr, true); ppretext_free (p); @@ -1917,7 +1919,8 @@ JSText *js_alloc_string (JSContext *ctx, int max_len) { } /* Initialize objhdr_t with OBJ_TEXT type and capacity in cap56 */ str->hdr = objhdr_make (max_len, OBJ_TEXT, false, false, false, false); - str->length = 0; /* length starts at 0, capacity is in hdr */ + str->length = 0; + str->hash = 0; /* Zero packed data so odd-length strings have deterministic padding. js_malloc is a bump allocator and does not zero memory; without this, the last word's unused low 32 bits contain garbage, causing @@ -2511,9 +2514,8 @@ JSValue pretext_end (JSContext *ctx, JSText *s) { if (!JS_IsNull (imm)) return imm; } } - /* Set final length in capacity field and clear length for hash storage */ - s->hdr = objhdr_set_cap56 (s->hdr, len); - s->length = 0; + /* length is already set by caller; cap56 stays as allocated capacity */ + s->hash = 0; s->hdr = objhdr_set_s (s->hdr, true); /* mark as stone */ return JS_MKPTR (s); } @@ -5455,8 +5457,8 @@ static JSValue js_regexp_exec (JSContext *ctx, JSValue this_val, int argc, JSVal } for (int ci = 0; ci < imm_len; ci++) string_put (hs, ci, MIST_GetImmediateASCIIChar (str_ref.val, ci)); - hs->hdr = objhdr_set_cap56 (hs->hdr, imm_len); - hs->length = 0; + hs->length = imm_len; + hs->hash = 0; hs->hdr = objhdr_set_s (hs->hdr, true); str_ref.val = JS_MKPTR (hs); } From dc70a15981e95601301947e81607f1846f2b512b Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 18:27:12 -0600 Subject: [PATCH 4/9] add guards for root cycles --- docs/c-modules.md | 7 +++++-- source/runtime.c | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/c-modules.md b/docs/c-modules.md index 98b4b052..adce79fb 100644 --- a/docs/c-modules.md +++ b/docs/c-modules.md @@ -410,13 +410,14 @@ JS_SetPropertyStr(js, result.val, "pixels", js_new_blob_stoned_copy(js, data, le JS_RETURN(result.val); ``` -**Array with loop** — root both the array and each element created in the loop: +**Array with loop** — root the element variable *before* the loop, then reassign `.val` each iteration: ```c JS_FRAME(js); JS_ROOT(arr, JS_NewArray(js)); +JS_ROOT(item, JS_NULL); for (int i = 0; i < count; i++) { - JS_ROOT(item, JS_NewObject(js)); + item.val = JS_NewObject(js); JS_SetPropertyStr(js, item.val, "index", JS_NewInt32(js, i)); JS_SetPropertyStr(js, item.val, "data", js_new_blob_stoned_copy(js, ptr, sz)); JS_SetPropertyNumber(js, arr.val, i, item.val); @@ -424,6 +425,8 @@ for (int i = 0; i < count; i++) { JS_RETURN(arr.val); ``` +**WARNING — NEVER put `JS_ROOT` inside a loop.** `JS_ROOT` declares a `JSGCRef` local and calls `JS_PushGCRef(&name)`, which pushes its address onto a linked list. Inside a loop the compiler reuses the same stack address, so on iteration 2+ the list becomes self-referential (`ref->prev == ref`). When GC triggers it walks the chain and **hangs forever**. This bug is intermittent — it only manifests when GC happens to run during the loop — making it very hard to reproduce. + **Nested objects** — root every object that persists across an allocating call: ```c diff --git a/source/runtime.c b/source/runtime.c index f257e24c..eb7813b7 100644 --- a/source/runtime.c +++ b/source/runtime.c @@ -150,6 +150,7 @@ int JS_IsPretext (JSValue v) { } JSValue *JS_PushGCRef (JSContext *ctx, JSGCRef *ref) { + assert(ref != ctx->top_gc_ref && "JS_ROOT used in a loop — same address pushed twice"); ref->prev = ctx->top_gc_ref; ctx->top_gc_ref = ref; ref->val = JS_NULL; @@ -157,11 +158,13 @@ JSValue *JS_PushGCRef (JSContext *ctx, JSGCRef *ref) { } JSValue JS_PopGCRef (JSContext *ctx, JSGCRef *ref) { + assert(ctx->top_gc_ref == ref && "JS_PopGCRef: not popping top of stack — mismatched push/pop"); ctx->top_gc_ref = ref->prev; return ref->val; } JSValue *JS_AddGCRef (JSContext *ctx, JSGCRef *ref) { + assert(ref != ctx->last_gc_ref && "JS_AddGCRef: same address added twice — cycle in GC ref list"); ref->prev = ctx->last_gc_ref; ctx->last_gc_ref = ref; ref->val = JS_NULL; @@ -193,6 +196,7 @@ JSLocalRef *JS_GetLocalFrame (JSContext *ctx) { } void JS_PushLocalRef (JSContext *ctx, JSLocalRef *ref) { + assert(ref != ctx->top_local_ref && "JS_LOCAL used in a loop — same address pushed twice"); ref->prev = ctx->top_local_ref; ctx->top_local_ref = ref; } From fa5c0416fbfca39d13b2e17dcbceb5f99a55817b Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 18:47:46 -0600 Subject: [PATCH 5/9] correct log line blames --- internal/engine.cm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/engine.cm b/internal/engine.cm index 53ffcae6..bb8c7057 100644 --- a/internal/engine.cm +++ b/internal/engine.cm @@ -572,7 +572,7 @@ log = function(name, args) { caller = {file: c_stack[0].file, line: c_stack[0].line} if (stack_channels[name]) stack = c_stack } else { - caller = caller_info(2) + caller = caller_info(0) if (stack_channels[name]) stack = os.stack(1) } From f7499c4f6052d1b91167996cd85ca296cbd7a9e7 Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 19:26:06 -0600 Subject: [PATCH 6/9] log reentrancy guard --- Makefile | 21 +++++++++++++++++++-- source/qjs_actor.c | 9 +++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 4dc24953..772eaaa9 100755 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ BUILD = build +BUILD_DBG = build_debug INSTALL_BIN = /opt/homebrew/bin INSTALL_LIB = /opt/homebrew/lib INSTALL_INC = /opt/homebrew/include @@ -12,6 +13,14 @@ all: $(BUILD)/build.ninja $(BUILD)/build.ninja: meson setup $(BUILD) -Dbuildtype=debugoptimized +debug: $(BUILD_DBG)/build.ninja + meson compile -C $(BUILD_DBG) + cp $(BUILD_DBG)/libcell_runtime.dylib . + cp $(BUILD_DBG)/cell . + +$(BUILD_DBG)/build.ninja: + meson setup $(BUILD_DBG) -Dbuildtype=debug -Db_sanitize=address + install: all $(CELL_SHOP) cp cell $(INSTALL_BIN)/cell cp libcell_runtime.dylib $(INSTALL_LIB)/ @@ -20,11 +29,19 @@ install: all $(CELL_SHOP) ln -s $(CURDIR) $(CELL_SHOP)/packages/core @echo "Installed cell to $(INSTALL_BIN) and $(INSTALL_LIB)" +install_debug: debug $(CELL_SHOP) + cp cell $(INSTALL_BIN)/cell + cp libcell_runtime.dylib $(INSTALL_LIB)/ + cp source/cell.h source/quickjs.h source/wota.h $(INSTALL_INC)/ + rm -rf $(CELL_SHOP)/packages/core + ln -s $(CURDIR) $(CELL_SHOP)/packages/core + @echo "Installed cell (debug+asan) to $(INSTALL_BIN) and $(INSTALL_LIB)" + $(CELL_SHOP): mkdir -p $(CELL_SHOP)/packages $(CELL_SHOP)/cache $(CELL_SHOP)/build clean: - rm -rf $(BUILD) + rm -rf $(BUILD) $(BUILD_DBG) rm -f cell libcell_runtime.dylib -.PHONY: all install clean +.PHONY: all install debug install_debug clean diff --git a/source/qjs_actor.c b/source/qjs_actor.c index 8dcf43da..2f40e554 100644 --- a/source/qjs_actor.c +++ b/source/qjs_actor.c @@ -131,12 +131,16 @@ JSC_CCALL(actor_removetimer, ) /* Log callback bridge: called from JS_Log, calls ƿit log(channel, [msg, stack]) - Captures the register VM stack trace so the log system can show the real error site. */ + Captures the register VM stack trace so the log system can show the real error site. + Uses a re-entrancy guard to prevent infinite recursion when JS_Call triggers + js_poll_interrupts -> JS_RaiseDisrupt -> JS_Log -> js_log_callback -> ... */ +static int js_log_reentrancy = 0; static void js_log_callback(JSContext *ctx, const char *channel, const char *msg) { - if (JS_IsNull(ctx->log_callback_js)) { + if (JS_IsNull(ctx->log_callback_js) || js_log_reentrancy) { fprintf(stderr, "%s\n", msg); return; } + js_log_reentrancy = 1; JS_FRAME(ctx); JS_ROOT(stack, JS_GetStack(ctx)); JS_ROOT(args_array, JS_NewArray(ctx)); @@ -147,6 +151,7 @@ static void js_log_callback(JSContext *ctx, const char *channel, const char *msg argv[1] = args_array.val; JS_Call(ctx, ctx->log_callback_js, JS_NULL, 2, argv); JS_RestoreFrame(ctx, _js_gc_frame, _js_local_frame); + js_log_reentrancy = 0; } JSC_CCALL(actor_set_log, From 6bc9dd53a7374d73113874b67a46087af9d4aa71 Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 19:27:28 -0600 Subject: [PATCH 7/9] better cache handling --- build.cm | 220 +++++++++++++++++++++++++++---------------- clean.ce | 38 ++------ internal/shop.cm | 193 ++++++++++++++++--------------------- resolve.ce | 21 ++--- tests/build_audit.cm | 20 ---- verify.ce | 17 +--- 6 files changed, 238 insertions(+), 271 deletions(-) diff --git a/build.cm b/build.cm index 8f343470..ab9fe5fd 100644 --- a/build.cm +++ b/build.cm @@ -80,6 +80,22 @@ function content_hash(str) { return text(crypto.blake2(bb, 32), 'h') } +// ============================================================================ +// Cache key salts — canonical registry +// Every artifact type has a unique salt so hash collisions between types +// are impossible, and no file extensions are needed in build/. +// ============================================================================ +var SALT_OBJ = 'obj' // compiled C object file +var SALT_DYLIB = 'dylib' // linked dynamic library +var SALT_NATIVE = 'native' // native-compiled .cm dylib +var SALT_MACH = 'mach' // mach bytecode blob +var SALT_MCODE = 'mcode' // mcode IR (JSON) +var SALT_DEPS = 'deps' // cached cc -MM dependency list + +function cache_path(content, salt) { + return get_build_dir() + '/' + content_hash(content + '\n' + salt) +} + function get_build_dir() { return shop.get_build_dir() } @@ -98,6 +114,52 @@ function ensure_dir(path) { Build.ensure_dir = ensure_dir +// ============================================================================ +// Dependency scanning helpers +// ============================================================================ + +// Parse make-style dependency output: +// foo.o: foo.c header1.h \ +// header2.h +// Returns array of dependency file paths (skips the target) +function parse_makefile_deps(dep_text) { + var joined = replace(dep_text, /\\\n\s*/, ' ') + var colon_pos = search(joined, ':') + if (colon_pos == null) return [] + var rest = trim(text(joined, colon_pos + 1)) + var parts = filter(array(rest, /\s+/), function(p) { + return length(p) > 0 + }) + return parts +} + +// Run cc -MM to get the preprocessor dependency list. +// Returns array of dependency file paths. +function get_c_deps(cc, flags, src_path) { + var dep_file = '/tmp/cell_deps_' + content_hash(src_path) + '.d' + var dep_cmd = [cc, '-MM', '-MG', '-MF', '"' + dep_file + '"'] + dep_cmd = array(dep_cmd, flags) + push(dep_cmd, '"' + src_path + '"') + var ret = os.system(text(dep_cmd, ' ') + ' 2>/dev/null') + if (ret != 0) return [src_path] + if (!fd.is_file(dep_file)) return [src_path] + var dep_text = text(fd.slurp(dep_file)) + return parse_makefile_deps(dep_text) +} + +// Build a full hash string from the compilation command and all dependency +// file contents. This is the content key for the object file. +function hash_all_deps(cmd_str, deps) { + var parts = [cmd_str] + arrfor(deps, function(dep_path) { + if (fd.is_file(dep_path)) + push(parts, dep_path + '\n' + text(fd.slurp(dep_path))) + else + push(parts, dep_path + '\n') + }) + return text(parts, '\n') +} + // ============================================================================ // Compilation // ============================================================================ @@ -124,30 +186,30 @@ Build.compile_file = function(pkg, file, target, opts) { // Symbol name for this file var sym_name = shop.c_symbol_for_file(pkg, file) - // Build command - var cmd_parts = [cc, '-c', '-fPIC'] + // Build common flags (shared between dep scan and compilation) + var common_flags = [] // Add buildtype-specific flags if (_buildtype == 'release') { - cmd_parts = array(cmd_parts, ['-O3', '-DNDEBUG']) + common_flags = array(common_flags, ['-O3', '-DNDEBUG']) } else if (_buildtype == 'debug') { - cmd_parts = array(cmd_parts, ['-O2', '-g']) + common_flags = array(common_flags, ['-O2', '-g']) } else if (_buildtype == 'minsize') { - cmd_parts = array(cmd_parts, ['-Os', '-DNDEBUG']) + common_flags = array(common_flags, ['-Os', '-DNDEBUG']) } - push(cmd_parts, '-DCELL_USE_NAME=' + sym_name) - push(cmd_parts, '-I"' + pkg_dir + '"') + push(common_flags, '-DCELL_USE_NAME=' + sym_name) + push(common_flags, '-I"' + pkg_dir + '"') // Auto-discover include/ directory if (fd.is_dir(pkg_dir + '/include')) { - push(cmd_parts, '-I"' + pkg_dir + '/include"') + push(common_flags, '-I"' + pkg_dir + '/include"') } // External packages need core's source dir for cell.h, quickjs.h, blob.h if (pkg != 'core') { core_dir = shop.get_package_dir('core') - push(cmd_parts, '-I"' + core_dir + '/source"') + push(common_flags, '-I"' + core_dir + '/source"') } // Add package CFLAGS (resolve relative -I paths) @@ -160,16 +222,19 @@ Build.compile_file = function(pkg, file, target, opts) { f = '-I"' + pkg_dir + '/' + ipath + '"' } } - push(cmd_parts, f) + push(common_flags, f) }) - + // Add target CFLAGS arrfor(target_cflags, function(flag) { - push(cmd_parts, flag) + push(common_flags, flag) }) - + + // Build full compilation command + var cmd_parts = [cc, '-c', '-fPIC'] + cmd_parts = array(cmd_parts, common_flags) push(cmd_parts, '"' + src_path + '"') - + var cmd_str = text(cmd_parts, ' ') if (_opts.verbose) { @@ -177,30 +242,56 @@ Build.compile_file = function(pkg, file, target, opts) { print('[verbose] compile: ' + cmd_str) } - // Content hash: command + file content + // Two-level cache: quick hash for deps file, full hash for object var file_content = fd.slurp(src_path) - var hash_input = cmd_str + '\n' + text(file_content) - var hash = content_hash(hash_input) - + var quick_content = cmd_str + '\n' + text(file_content) + var deps_path = cache_path(quick_content, SALT_DEPS) + var build_dir = get_build_dir() ensure_dir(build_dir) - var obj_path = build_dir + '/' + hash - - // Check if already compiled + + var deps = null + var full_content = null + var obj_path = null + + // Warm path: read cached dep list, verify by hashing all deps + if (fd.is_file(deps_path)) { + deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) { + return length(p) > 0 + }) + full_content = hash_all_deps(cmd_str, deps) + obj_path = cache_path(full_content, SALT_OBJ) + if (fd.is_file(obj_path)) { + if (_opts.verbose) print('[verbose] cache hit: ' + file) + log.shop('cache hit ' + file) + return obj_path + } + log.shop('cache stale ' + file + ' (header changed)') + } + + // Cold path: run cc -MM to discover deps + log.shop('dep scan ' + file) + deps = get_c_deps(cc, common_flags, src_path) + full_content = hash_all_deps(cmd_str, deps) + obj_path = cache_path(full_content, SALT_OBJ) + + // Check if object exists (might exist from previous build with same deps) if (fd.is_file(obj_path)) { - if (_opts.verbose) print('[verbose] cache hit: ' + file) + fd.slurpwrite(deps_path, stone(blob(text(deps, '\n')))) + if (_opts.verbose) print('[verbose] cache hit: ' + file + ' (after dep scan)') + log.shop('cache hit ' + file + ' (after dep scan)') return obj_path } - if (_opts.verbose) print('[verbose] cache miss: ' + file) - - // Compile — capture stderr to detect missing-header vs real errors - var err_path = '/tmp/cell_build_err_' + hash + '.log' + + // Compile + log.shop('compiling ' + file) + log.console('Compiling ' + file) + var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log' var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"' var err_text = null var missing = null var err_lines = null var first_err = null - log.console('Compiling ' + file) var ret = os.system(full_cmd) if (ret != 0) { if (fd.is_file(err_path)) { @@ -222,6 +313,8 @@ Build.compile_file = function(pkg, file, target, opts) { return null } + // Save deps for future warm-path lookups + fd.slurpwrite(deps_path, stone(blob(text(deps, '\n')))) return obj_path } @@ -249,8 +342,8 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) { // Dynamic library building // ============================================================================ -// Compute link key from all inputs that affect the dylib output -function compute_link_key(objects, ldflags, target_ldflags, opts) { +// Compute link content string from all inputs that affect the dylib output +function compute_link_content(objects, ldflags, target_ldflags, opts) { // Sort objects for deterministic hash var sorted_objects = sort(objects) @@ -269,7 +362,7 @@ function compute_link_key(objects, ldflags, target_ldflags, opts) { push(parts, 'target_ldflag:' + flag) }) - return content_hash(text(parts, '\n')) + return text(parts, '\n') } // Build a per-module dynamic library for a single C file @@ -283,7 +376,6 @@ Build.build_module_dylib = function(pkg, file, target, opts) { if (!obj) return null var tc = toolchains[_target] - var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so') var cc = tc.cpp || tc.c var local_dir = get_local_dir() var pkg_dir = shop.get_package_dir(pkg) @@ -307,10 +399,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) { // Content-addressed output: hash of (all objects + link flags + target) var all_objects = [obj] all_objects = array(all_objects, _extra) - var link_key = compute_link_key(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc}) + var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc}) var build_dir = get_build_dir() ensure_dir(build_dir) - var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext + var dylib_path = cache_path(link_content, SALT_DYLIB) var cmd_parts = null var cmd_str = null var ret = null @@ -352,29 +444,17 @@ Build.build_module_dylib = function(pkg, file, target, opts) { cmd_str = text(cmd_parts, ' ') if (_opts.verbose) print('[verbose] link: ' + cmd_str) + log.shop('linking ' + file) log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path)) ret = os.system(cmd_str) if (ret != 0) { print('Linking failed: ' + file) return null } + } else { + log.shop('link cache hit ' + file) } - // Always install to deterministic lib//.dylib - // Strip .c/.cpp extension so the loader can find it by module name - var file_stem = file - if (ends_with(file_stem, '.cpp')) file_stem = text(file_stem, 0, -4) - else if (ends_with(file_stem, '.c')) file_stem = text(file_stem, 0, -2) - var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) - var stem_dir = fd.dirname(file_stem) - if (stem_dir && stem_dir != '.') { - install_dir = install_dir + '/' + stem_dir - } - ensure_dir(install_dir) - var install_path = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) + '/' + file_stem + dylib_ext - fd.slurpwrite(install_path, fd.slurp(dylib_path)) - if (_opts.verbose) print('[verbose] install: ' + install_path) - return dylib_path } @@ -602,16 +682,12 @@ Build.compile_native = function(src_path, target, buildtype, pkg) { var _target = target || Build.detect_host_target() var _buildtype = buildtype || 'release' var qbe_rt_path = null - var native_stem = null - var native_install_dir = null - var native_install_path = null if (!fd.is_file(src_path)) { print('Source file not found: ' + src_path); disrupt } var tc = toolchains[_target] - var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so') var cc = tc.c // Step 1: Compile through pipeline @@ -627,16 +703,15 @@ Build.compile_native = function(src_path, target, buildtype, pkg) { var il_parts = qbe_emit(optimized, qbe_macros, sym_name) // Content hash for cache key - var hash = content_hash(text(fd.slurp(src_path)) + '\n' + _target + '\nnative') var build_dir = get_build_dir() ensure_dir(build_dir) - var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext + var dylib_path = cache_path(text(fd.slurp(src_path)) + '\n' + _target, SALT_NATIVE) if (fd.is_file(dylib_path)) return dylib_path // Compile and assemble via batched parallel pipeline - var tmp = '/tmp/cell_native_' + hash + var tmp = '/tmp/cell_native_' + content_hash(src_path) var rt_o_path = '/tmp/cell_qbe_rt.o' var o_paths = compile_native_batched(il_parts, cc, tmp) @@ -672,15 +747,6 @@ Build.compile_native = function(src_path, target, buildtype, pkg) { log.console('Built native: ' + fd.basename(dylib_path)) - // Install to deterministic lib//.dylib - if (pkg) { - native_stem = fd.basename(src_path) - native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) - ensure_dir(native_install_dir) - native_install_path = native_install_dir + '/' + native_stem + dylib_ext - fd.slurpwrite(native_install_path, fd.slurp(dylib_path)) - } - return dylib_path } @@ -692,12 +758,8 @@ Build.compile_native_ir = function(optimized, src_path, opts) { var _buildtype = (opts && opts.buildtype) || 'release' var pkg = opts && opts.pkg var qbe_rt_path = null - var native_stem = null - var native_install_dir = null - var native_install_path = null var tc = toolchains[_target] - var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so') var cc = tc.c var qbe_macros = use('qbe') @@ -710,16 +772,15 @@ Build.compile_native_ir = function(optimized, src_path, opts) { var il_parts = qbe_emit(optimized, qbe_macros, sym_name) var src = text(fd.slurp(src_path)) - var hash = content_hash(src + '\n' + _target + '\nnative') var build_dir = get_build_dir() ensure_dir(build_dir) - var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext + var dylib_path = cache_path(src + '\n' + _target, SALT_NATIVE) if (fd.is_file(dylib_path)) return dylib_path // Compile and assemble via batched parallel pipeline - var tmp = '/tmp/cell_native_' + hash + var tmp = '/tmp/cell_native_' + content_hash(src_path) var rt_o_path = '/tmp/cell_qbe_rt.o' var o_paths = compile_native_batched(il_parts, cc, tmp) @@ -755,14 +816,6 @@ Build.compile_native_ir = function(optimized, src_path, opts) { log.console('Built native: ' + fd.basename(dylib_path)) - if (pkg) { - native_stem = fd.basename(src_path) - native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) - ensure_dir(native_install_dir) - native_install_path = native_install_dir + '/' + native_stem + dylib_ext - fd.slurpwrite(native_install_path, fd.slurp(dylib_path)) - } - return dylib_path } @@ -884,4 +937,13 @@ Build.build_all_dynamic = function(target, buildtype, opts) { return results } +// Export salt constants and cache_path for shop.cm and others +Build.SALT_OBJ = SALT_OBJ +Build.SALT_DYLIB = SALT_DYLIB +Build.SALT_NATIVE = SALT_NATIVE +Build.SALT_MACH = SALT_MACH +Build.SALT_MCODE = SALT_MCODE +Build.SALT_DEPS = SALT_DEPS +Build.cache_path = cache_path + return Build diff --git a/clean.ce b/clean.ce index f3ab6b10..89c80bc7 100644 --- a/clean.ce +++ b/clean.ce @@ -119,37 +119,13 @@ var build_dir = shop.get_build_dir() var packages_dir = replace(shop.get_package_dir(''), /\/$/, '') // Get base packages dir if (clean_build) { - if (is_shop_scope) { - // Clean entire build and lib directories - if (fd.is_dir(build_dir)) { - push(dirs_to_delete, build_dir) - } - if (fd.is_dir(lib_dir)) { - push(dirs_to_delete, lib_dir) - } - } else { - // Clean specific package libraries - arrfor(packages_to_clean, function(p) { - if (p == 'core') return - - var lib_name = shop.lib_name_for_package(p) - var dylib_ext = '.dylib' - var lib_path = lib_dir + '/' + lib_name + dylib_ext - - if (fd.is_file(lib_path)) { - push(files_to_delete, lib_path) - } - - // Also check for .so and .dll - var so_path = lib_dir + '/' + lib_name + '.so' - var dll_path = lib_dir + '/' + lib_name + '.dll' - if (fd.is_file(so_path)) { - push(files_to_delete, so_path) - } - if (fd.is_file(dll_path)) { - push(files_to_delete, dll_path) - } - }) + // Nuke entire build cache (content-addressed, per-package clean impractical) + if (fd.is_dir(build_dir)) { + push(dirs_to_delete, build_dir) + } + // Clean orphaned lib/ directory if it exists (legacy) + if (fd.is_dir(lib_dir)) { + push(dirs_to_delete, lib_dir) } } diff --git a/internal/shop.cm b/internal/shop.cm index 9d6285c8..09796818 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -54,9 +54,10 @@ function ensure_dir(path) { } } -function hash_path(content) +function hash_path(content, salt) { - return global_shop_path + '/build' + '/' + content_hash(content) + var s = salt || 'mach' + return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s))) } var Shop = {} @@ -418,6 +419,7 @@ Shop.extract_commit_hash = function(pkg, response) { } var open_dls = {} +var package_dylibs = {} // pkg -> [{file, symbol, dylib}, ...] // Host target detection for native dylib resolution function detect_host_target() { @@ -434,37 +436,26 @@ function detect_host_target() { var host_target = detect_host_target() -// Check for a native .cm dylib at the deterministic lib path +// Check for a native .cm dylib in the build cache // Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists -// Also checks staleness: if source has changed, the content-addressed build artifact -// won't exist for the new hash, so the installed dylib is treated as stale. function try_native_mod_dylib(pkg, stem) { - var dylib_path = get_dylib_path(pkg, stem) - var src_path = null - var src = null - var host = null - var hash = null - var tc_ext = null - var build_path = null - var handle = null - var sym = null + var build_mod = use_cache['core/build'] + if (!build_mod) return null - if (!fd.is_file(dylib_path)) return null + var src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem + if (!fd.is_file(src_path)) return null - // Staleness check: verify the content-addressed build artifact exists - src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem - if (fd.is_file(src_path)) { - src = text(fd.slurp(src_path)) - host = detect_host_target() - hash = content_hash(src + '\n' + host + '\nnative') - tc_ext = dylib_ext - build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext - if (!fd.is_file(build_path)) return null - } + var src = text(fd.slurp(src_path)) + var host = detect_host_target() + if (!host) return null - handle = os.dylib_open(dylib_path) + var build_path = build_mod.cache_path(src + '\n' + host, build_mod.SALT_NATIVE) + if (!fd.is_file(build_path)) return null + + log.shop('native dylib cache hit: ' + stem) + var handle = os.dylib_open(build_path) if (!handle) return null - sym = Shop.c_symbol_for_file(pkg, stem) + var sym = Shop.c_symbol_for_file(pkg, stem) return {_native: true, _handle: handle, _sym: sym} } @@ -675,9 +666,6 @@ function resolve_mod_fn(path, pkg) { var cached = null var ast = null var compiled = null - var mach_path = null - var mach_blob = null - var mcode_path = null var ir = null var optimized = null var mcode_json = null @@ -729,9 +717,9 @@ function resolve_mod_fn(path, pkg) { } } - // Check for cached mcode in content-addressed store (salted hash to distinguish from mach) + // Check for cached mcode in content-addressed store if (policy.allow_compile) { - cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + cached_mcode_path = hash_path(content_key, 'mcode') if (fd.is_file(cached_mcode_path)) { mcode_json = text(fd.slurp(cached_mcode_path)) compiled = mach_compile_mcode_bin(path, mcode_json) @@ -855,52 +843,67 @@ function make_c_symbol(pkg, file) { return 'js_' + pkg_safe + '_' + file_safe + '_use' } -// Get the deterministic dylib path for a module in lib//.dylib -function get_dylib_path(pkg, stem) { - return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext -} +// Ensure all C modules for a package are built and loaded. +// Returns the array of {file, symbol, dylib} results, cached per package. +function ensure_package_dylibs(pkg) { + if (package_dylibs[pkg]) return package_dylibs[pkg] -// Get the deterministic mach path for a module in lib//.mach -function get_mach_path(pkg, stem) { - return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach' -} + var build_mod = use_cache['core/build'] + if (!build_mod) return null -// Open a per-module dylib and return the dlopen handle -// Pre-loads sibling dylibs with RTLD_LAZY|RTLD_GLOBAL so cross-dylib symbols resolve -function open_module_dylib(dylib_path) { - if (open_dls[dylib_path]) return open_dls[dylib_path] - if (!fd.is_file(dylib_path)) return null - var dir = fd.dirname(dylib_path) - var entries = fd.readdir(dir) - var i = 0 - var sibling = null - var handle = null - while (i < length(entries)) { - if (ends_with(entries[i], dylib_ext) && entries[i] != fd.basename(dylib_path)) { - sibling = dir + '/' + entries[i] - if (!open_dls[sibling]) { - handle = os.dylib_preload(sibling) - if (handle) open_dls[sibling] = handle - } - } - i = i + 1 + var target = detect_host_target() + if (!target) return null + + var c_files = pkg_tools.get_c_files(pkg, target, true) + if (!c_files || length(c_files) == 0) { + package_dylibs[pkg] = [] + return [] } - open_dls[dylib_path] = os.dylib_open(dylib_path) - return open_dls[dylib_path] + + log.shop('ensuring C modules for ' + pkg) + var results = build_mod.build_dynamic(pkg, target, 'release', {}) + package_dylibs[pkg] = results + + // Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL + arrfor(results, function(r) { + var handle = null + if (r.dylib && !open_dls[r.dylib]) { + handle = os.dylib_preload(r.dylib) + if (handle) open_dls[r.dylib] = handle + } + }) + + log.shop('built ' + text(length(results)) + ' C module(s) for ' + pkg) + return results } -// Try to resolve a C symbol from the deterministic dylib path +// Try to resolve a C symbol by building the package on demand // Returns a loader function or null function try_dylib_symbol(sym, pkg, file_stem) { - var dylib_path = get_dylib_path(pkg, file_stem) - var handle = open_module_dylib(dylib_path) + var dylibs = ensure_package_dylibs(pkg) + if (!dylibs || length(dylibs) == 0) return null + + var c_file = file_stem + '.c' + var cpp_file = file_stem + '.cpp' + var entry = find(dylibs, function(r) { + return r.file == c_file || r.file == cpp_file + }) + if (!entry || !entry.dylib) return null + + var handle = open_dls[entry.dylib] + if (!handle) { + handle = os.dylib_open(entry.dylib) + if (handle) open_dls[entry.dylib] = handle + } if (!handle) return null if (!os.dylib_has_symbol(handle, sym)) return null + + log.shop('resolved ' + sym + ' from build cache') return function() { return os.dylib_symbol(handle, sym) } } // Resolve a C symbol by searching: -// At each scope: check lib/ dylib first, then internal (static) +// At each scope: check build-cache dylib first, then internal (static) function resolve_c_symbol(path, package_context) { var explicit = split_explicit_package_import(path) var sym = null @@ -922,7 +925,7 @@ function resolve_c_symbol(path, package_context) { sym = make_c_symbol(explicit.package, explicit.path) file_stem = replace(explicit.path, '.c', '') - // Check lib/ dylib first + // Check build-cache dylib first if (policy.allow_dylib) { loader = try_dylib_symbol(sym, explicit.package, file_stem) if (loader) { @@ -950,7 +953,7 @@ function resolve_c_symbol(path, package_context) { if (!package_context || package_context == 'core') { core_sym = make_c_symbol('core', path) - // Check lib/ dylib first for core + // Check build-cache dylib first for core if (policy.allow_dylib) { loader = try_dylib_symbol(core_sym, 'core', path) if (loader) { @@ -972,7 +975,7 @@ function resolve_c_symbol(path, package_context) { return null } - // 1. Check own package (dylib first, then internal) + // 1. Check own package (build-cache dylib first, then internal) sym = make_c_symbol(package_context, path) if (policy.allow_dylib) { @@ -1028,7 +1031,7 @@ function resolve_c_symbol(path, package_context) { } } - // 3. Check core (dylib first, then internal) + // 3. Check core (build-cache dylib first, then internal) core_sym = make_c_symbol('core', path) if (policy.allow_dylib) { @@ -1541,11 +1544,8 @@ Shop.remove = function(pkg) { fd.rmdir(pkg_dir, 1) } - // Remove built dylibs - var lib_dir = global_shop_path + '/lib/' + safe_package_path(pkg) - if (fd.is_dir(lib_dir)) { - fd.rmdir(lib_dir, 1) - } + // Invalidate package dylib cache + package_dylibs[pkg] = null log.console("Removed " + pkg) return true @@ -1599,14 +1599,9 @@ Shop.module_reload = function(path, package) { var lookup_key = package ? package + ':' + path : ':' + path module_info_cache[lookup_key] = null - // Close old dylib handle if any - var old_dylib_path = null + // Invalidate package dylib cache so next resolve triggers rebuild if (package) { - old_dylib_path = get_dylib_path(package, path) - if (open_dls[old_dylib_path]) { - os.dylib_close(open_dls[old_dylib_path]) - open_dls[old_dylib_path] = null - } + package_dylibs[package] = null } var info = resolve_module_info(path, package) @@ -1717,17 +1712,6 @@ Shop.lib_name_for_package = function(pkg) { return safe_package_path(pkg) } -// Returns { ok: bool, results: [{pkg, ok, error}] } -// Get the deterministic dylib path for a module (public API) -Shop.get_dylib_path = function(pkg, stem) { - return get_dylib_path(pkg, stem) -} - -// Get the deterministic mach path for a module (public API) -Shop.get_mach_path = function(pkg, stem) { - return get_mach_path(pkg, stem) -} - // Load a module explicitly as mach bytecode, bypassing dylib resolution. // Returns the loaded module value. Disrupts if the module cannot be found. Shop.load_as_mach = function(path, pkg) { @@ -1742,9 +1726,6 @@ Shop.load_as_mach = function(path, pkg) { var ast = null var ir = null var optimized = null - var pkg_dir = null - var stem = null - var mach_path = null var file_info = null var inject = null var env = null @@ -1755,27 +1736,13 @@ Shop.load_as_mach = function(path, pkg) { content = text(fd.slurp(file_path)) content_key = stone(blob(content)) - // Try installed .mach in lib/ - if (pkg) { - pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg) - if (starts_with(file_path, pkg_dir + '/')) { - stem = text(file_path, length(pkg_dir) + 1) - mach_path = get_mach_path(pkg, stem) - if (fd.is_file(mach_path)) { - compiled = fd.slurp(mach_path) - } - } - } - // Try cached mach blob - if (!compiled) { - cached = pull_from_cache(content_key) - if (cached) compiled = cached - } + cached = pull_from_cache(content_key) + if (cached) compiled = cached // Try cached mcode -> compile to mach if (!compiled) { - cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + cached_mcode_path = hash_path(content_key, 'mcode') if (fd.is_file(cached_mcode_path)) { mcode_json = text(fd.slurp(cached_mcode_path)) compiled = mach_compile_mcode_bin(file_path, mcode_json) @@ -1795,7 +1762,7 @@ Shop.load_as_mach = function(path, pkg) { ir = _mcode_mod(ast) optimized = _streamline_mod(ir) mcode_json = shop_json.encode(optimized) - cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + cached_mcode_path = hash_path(content_key, 'mcode') ensure_dir(global_shop_path + '/build') fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json))) compiled = mach_compile_mcode_bin(file_path, mcode_json) diff --git a/resolve.ce b/resolve.ce index 8cee4696..e67dba7f 100644 --- a/resolve.ce +++ b/resolve.ce @@ -130,11 +130,7 @@ var is_linked = false var is_in_lock = false var is_local = false var is_fetched = false -var lib_dir = null -var lib_name = null -var dylib_ext = null -var lib_path = null -var is_built = false +var has_c_files = false var status_parts = null var commit_str = null var line = null @@ -164,12 +160,13 @@ for (i = 0; i < length(sorted); i++) { pkg_dir = shop.get_package_dir(locator) is_fetched = fd.is_dir(pkg_dir) || fd.is_link(pkg_dir) - // Check if built (library exists) - lib_dir = shop.get_lib_dir() - lib_name = shop.lib_name_for_package(locator) - dylib_ext = '.dylib' // TODO: detect from target - lib_path = lib_dir + '/' + lib_name + dylib_ext - is_built = fd.is_file(lib_path) + // Check if package has C modules (built on demand) + has_c_files = false + var _check_c = function() { + var c_files = pkg.get_c_files(locator, target_triple, true) + if (c_files && length(c_files) > 0) has_c_files = true + } disruption {} + _check_c() // Format output status_parts = [] @@ -177,7 +174,7 @@ for (i = 0; i < length(sorted); i++) { if (is_local) push(status_parts, "local") if (!is_in_lock) push(status_parts, "not in lock") if (!is_fetched) push(status_parts, "not fetched") - if (is_built) push(status_parts, "built") + if (has_c_files) push(status_parts, "has C modules") commit_str = "" if (lock_entry && lock_entry.commit) { diff --git a/tests/build_audit.cm b/tests/build_audit.cm index 375aff23..947342e0 100644 --- a/tests/build_audit.cm +++ b/tests/build_audit.cm @@ -5,26 +5,6 @@ var shop = use('internal/shop') var runtime = use('runtime') return { - // ======================================================================== - // DETERMINISTIC DYLIB PATHS - // ======================================================================== - - test_dylib_path_deterministic: function() { - var path = shop.get_dylib_path('core', 'time') - if (!ends_with(path, '/lib/core/time.dylib')) return "dylib path should end with /lib/core/time.dylib, got: " + path - }, - - test_dylib_path_internal: function() { - var path = shop.get_dylib_path('core', 'internal/os') - if (!ends_with(path, '/lib/core/internal/os.dylib')) return "dylib path should end with /lib/core/internal/os.dylib, got: " + path - }, - - test_dylib_path_external_package: function() { - var path = shop.get_dylib_path('gitea.pockle.world/john/prosperon', 'sprite') - if (!ends_with(path, '/lib/gitea.pockle.world/john/prosperon/sprite.dylib')) - return "dylib path should mirror package layout, got: " + path - }, - // ======================================================================== // SYMBOL NAMING // ======================================================================== diff --git a/verify.ce b/verify.ce index ee0ed4cf..cda42bba 100644 --- a/verify.ce +++ b/verify.ce @@ -85,10 +85,6 @@ function verify_package(locator) { var current_target = null var expected_target = null var target_dir = null - var lib_dir = null - var lib_name = null - var dylib_ext = null - var lib_path = null var c_files = null checked++ @@ -147,20 +143,9 @@ function verify_package(locator) { } } - // Check build output exists - lib_dir = shop.get_lib_dir() - lib_name = shop.lib_name_for_package(locator) - dylib_ext = '.dylib' // TODO: detect from target - lib_path = lib_dir + '/' + lib_name + dylib_ext - - // Only check for builds if package has C files + // Check if package has C files (builds happen lazily on first use) var _check_build = function() { c_files = pkg.get_c_files(locator, target_triple, true) - if (c_files && length(c_files) > 0) { - if (!fd.is_file(lib_path)) { - add_warning(locator + ": library not built at " + lib_path) - } - } } disruption { // Skip build check if can't determine C files } From 02eb58772c2db5684d1b5bfbcf1144c5950b811f Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 19:41:59 -0600 Subject: [PATCH 8/9] fix build hangs --- build.cm | 12 ++++++++++++ internal/os.c | 14 +++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/build.cm b/build.cm index ab9fe5fd..529f4def 100644 --- a/build.cm +++ b/build.cm @@ -91,6 +91,7 @@ var SALT_NATIVE = 'native' // native-compiled .cm dylib var SALT_MACH = 'mach' // mach bytecode blob var SALT_MCODE = 'mcode' // mcode IR (JSON) var SALT_DEPS = 'deps' // cached cc -MM dependency list +var SALT_FAIL = 'fail' // cached compilation failure function cache_path(content, salt) { return get_build_dir() + '/' + content_hash(content + '\n' + salt) @@ -246,10 +247,18 @@ Build.compile_file = function(pkg, file, target, opts) { var file_content = fd.slurp(src_path) var quick_content = cmd_str + '\n' + text(file_content) var deps_path = cache_path(quick_content, SALT_DEPS) + var fail_path = cache_path(quick_content, SALT_FAIL) var build_dir = get_build_dir() ensure_dir(build_dir) + // Check for cached failure (skip files that previously failed to compile) + if (fd.is_file(fail_path)) { + if (_opts.verbose) print('[verbose] skipping ' + file + ' (cached failure)') + log.shop('skip ' + file + ' (cached failure)') + return null + } + var deps = null var full_content = null var obj_path = null @@ -310,6 +319,8 @@ Build.compile_file = function(pkg, file, target, opts) { if (err_text) print(err_text) else print('Command: ' + full_cmd) } + // Cache the failure so we don't retry on every build + fd.slurpwrite(fail_path, stone(blob(err_text || 'compilation failed'))) return null } @@ -944,6 +955,7 @@ Build.SALT_NATIVE = SALT_NATIVE Build.SALT_MACH = SALT_MACH Build.SALT_MCODE = SALT_MCODE Build.SALT_DEPS = SALT_DEPS +Build.SALT_FAIL = SALT_FAIL Build.cache_path = cache_path return Build diff --git a/internal/os.c b/internal/os.c index ffe13694..d08083e0 100644 --- a/internal/os.c +++ b/internal/os.c @@ -1,4 +1,5 @@ #include "cell.h" +#include "cell_internal.h" #include #include @@ -306,7 +307,18 @@ static JSValue js_os_rusage(JSContext *js, JSValue self, int argc, JSValue *argv JSC_SCALL(os_system, int err = system(str); - ret = number2js(js,err); + /* Reset actor turn timer after blocking system() call. + The scheduler's kill timer may have fired while system() blocked, + setting pause_flag = 2. Bump turn_gen so stale timer events are + ignored, and clear the pause flag so the VM doesn't raise + "interrupted" on the next backward branch. */ + cell_rt *crt = JS_GetContextOpaque(js); + if (crt) { + atomic_fetch_add_explicit(&crt->turn_gen, 1, memory_order_relaxed); + JS_SetPauseFlag(js, 0); + crt->turn_start_ns = cell_ns(); + } + ret = number2js(js, err); ) JSC_CCALL(os_exit, From 777474ab4f2df4d3ca7e1db1a1d8ad709f4c0d3b Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Wed, 18 Feb 2026 20:30:54 -0600 Subject: [PATCH 9/9] updated docs for dylib paths --- CLAUDE.md | 2 +- build.cm | 11 +++++++ docs/_index.md | 2 +- docs/c-modules.md | 2 +- docs/cli.md | 15 ++++----- docs/packages.md | 20 +++++------- docs/shop.md | 80 ++++++++++++++++++++++++++--------------------- internal/shop.cm | 45 ++++++++++++++++++++------ source/runtime.c | 10 +++++- 9 files changed, 115 insertions(+), 72 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 793f415e..72abf10f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -177,7 +177,7 @@ When running locally with `./cell --dev`, these commands manage packages: ./cell --dev list # list installed packages ``` -Local paths are symlinked into `.cell/packages/`. The build step compiles C files to `.cell/lib//.dylib`. C files in `src/` are support files linked into module dylibs, not standalone modules. +Local paths are symlinked into `.cell/packages/`. The build step compiles C files to content-addressed dylibs in `~/.cell/build/` and writes a per-package manifest so the runtime can find them. C files in `src/` are support files linked into module dylibs, not standalone modules. ## Debugging Compiler Issues diff --git a/build.cm b/build.cm index 529f4def..6deba3d5 100644 --- a/build.cm +++ b/build.cm @@ -97,6 +97,11 @@ function cache_path(content, salt) { return get_build_dir() + '/' + content_hash(content + '\n' + salt) } +// Deterministic manifest path for a package's built dylibs +function manifest_path(pkg) { + return get_build_dir() + '/' + content_hash(pkg + '\n' + 'manifest') +} + function get_build_dir() { return shop.get_build_dir() } @@ -499,6 +504,11 @@ Build.build_dynamic = function(pkg, target, buildtype, opts) { } }) + // Write manifest so runtime can find dylibs without the build module + var json = use('json') + var mpath = manifest_path(pkg) + fd.slurpwrite(mpath, stone(blob(json.encode(results)))) + return results } @@ -957,5 +967,6 @@ Build.SALT_MCODE = SALT_MCODE Build.SALT_DEPS = SALT_DEPS Build.SALT_FAIL = SALT_FAIL Build.cache_path = cache_path +Build.manifest_path = manifest_path return Build diff --git a/docs/_index.md b/docs/_index.md index 3efc5a65..5517821d 100644 --- a/docs/_index.md +++ b/docs/_index.md @@ -81,7 +81,7 @@ cd cell make bootstrap ``` -The ƿit shop is stored at `~/.pit/`. +The ƿit shop is stored at `~/.cell/`. ## Development diff --git a/docs/c-modules.md b/docs/c-modules.md index adce79fb..e2aa7027 100644 --- a/docs/c-modules.md +++ b/docs/c-modules.md @@ -231,7 +231,7 @@ C files are automatically compiled when you run: cell --dev build ``` -Each C file is compiled into a per-file dynamic library at `.cell/lib//.dylib`. +Each C file is compiled into a per-file dynamic library at a content-addressed path in `~/.cell/build/`. A manifest is written for each package so the runtime can find dylibs without rerunning the build pipeline — see [Dylib Manifests](/docs/shop/#dylib-manifests). ## Compilation Flags (cell.toml) diff --git a/docs/cli.md b/docs/cli.md index 1adad63d..c07c207a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -53,7 +53,7 @@ For local paths, the package is symlinked into the shop rather than copied. Chan ### pit build -Build C modules for a package. Compiles each C file into a per-file dynamic library and installs them to `~/.pit/lib//.dylib`. C files in `src/` directories are compiled as support objects and linked into the module dylibs. +Build C modules for a package. Compiles each C file into a per-file dynamic library stored in the content-addressed build cache at `~/.cell/build/`. A per-package manifest is written so the runtime can find dylibs by package name. C files in `src/` directories are compiled as support objects and linked into the module dylibs. Files that previously failed to compile are skipped automatically (cached failure markers); they are retried when the source or compiler flags change. ```bash pit build # build all packages @@ -174,7 +174,7 @@ Output includes median, mean, standard deviation, and percentiles for each bench ## Shop Commands -These commands operate on the global shop (`~/.pit/`) or system-level state. +These commands operate on the global shop (`~/.cell/`) or system-level state. ### pit install @@ -430,7 +430,7 @@ pit qbe Compile a source file to a native dynamic library. ```bash -pit compile # outputs .dylib to .cell/lib/ +pit compile # outputs .dylib to ~/.cell/build/ pit compile ``` @@ -555,15 +555,12 @@ pit install /Users/john/work/mylib ## Configuration -ƿit stores its data in `~/.pit/`: +ƿit stores its data in `~/.cell/`: ``` -~/.pit/ +~/.cell/ ├── packages/ # installed package sources -├── lib/ # installed per-file dylibs and mach (persistent) -│ ├── core/ # core package: .dylib and .mach files -│ └── / # per-package subdirectories -├── build/ # ephemeral build cache (safe to delete) +├── build/ # content-addressed cache (bytecode, dylibs, manifests) ├── cache/ # downloaded archives ├── lock.toml # installed package versions └── link.toml # local development links diff --git a/docs/packages.md b/docs/packages.md index 6e44b203..8c898b3f 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -91,10 +91,10 @@ Local packages are symlinked into the shop, making development seamless. ## The Shop -ƿit stores all packages in the **shop** at `~/.pit/`: +ƿit stores all packages in the **shop** at `~/.cell/`: ``` -~/.pit/ +~/.cell/ ├── packages/ │ ├── core -> gitea.pockle.world/john/cell │ ├── gitea.pockle.world/ @@ -105,14 +105,8 @@ Local packages are symlinked into the shop, making development seamless. │ └── john/ │ └── work/ │ └── mylib -> /Users/john/work/mylib -├── lib/ -│ ├── core/ -│ │ ├── fd.dylib -│ │ └── time.mach -│ └── gitea_pockle_world_john_prosperon/ -│ └── sprite.dylib ├── build/ -│ └── +│ └── ├── cache/ │ └── ├── lock.toml @@ -175,16 +169,16 @@ pit link delete gitea.pockle.world/john/prosperon ## C Extensions -C files in a package are compiled into per-file dynamic libraries: +C files in a package are compiled into per-file dynamic libraries stored in the content-addressed build cache: ``` mypackage/ ├── cell.toml -├── render.c # compiled to lib/mypackage/render.dylib -└── physics.c # compiled to lib/mypackage/physics.dylib +├── render.c # compiled to ~/.cell/build/ +└── physics.c # compiled to ~/.cell/build/ ``` -Each `.c` file gets its own `.dylib` in `~/.pit/lib//`. A `.c` file and `.cm` file with the same stem at the same scope is a build error — use distinct names. +Each `.c` file gets its own `.dylib` at a content-addressed path in `~/.cell/build/`. A per-package manifest maps module names to their dylib paths so the runtime can find them — see [Dylib Manifests](/docs/shop/#dylib-manifests). A `.c` file and `.cm` file with the same stem at the same scope is a build error — use distinct names. See [Writing C Modules](/docs/c-modules/) for details. diff --git a/docs/shop.md b/docs/shop.md index 0a94cb16..180cd144 100644 --- a/docs/shop.md +++ b/docs/shop.md @@ -17,7 +17,7 @@ When `pit` runs a program, startup takes one of two paths: C runtime → engine.cm (from cache) → shop.cm → user program ``` -The C runtime hashes the source of `internal/engine.cm` with BLAKE2 and looks up the hash in the content-addressed cache (`~/.pit/build/`). On a cache hit, engine.cm loads directly — no bootstrap involved. +The C runtime hashes the source of `internal/engine.cm` with BLAKE2 and looks up the hash in the content-addressed cache (`~/.cell/build/`). On a cache hit, engine.cm loads directly — no bootstrap involved. ### Cold path (first run or cache cleared) @@ -37,7 +37,7 @@ On a cache miss, the C runtime loads `boot/bootstrap.cm.mcode` (a pre-compiled s ### Cache invalidation -All caching is content-addressed by BLAKE2 hash of the source. When any source file changes, its hash changes and the old cache entry is simply never looked up again. No manual invalidation is needed. To force a full rebuild, delete `~/.pit/build/`. +All caching is content-addressed by BLAKE2 hash of the source. When any source file changes, its hash changes and the old cache entry is simply never looked up again. No manual invalidation is needed. To force a full rebuild, delete `~/.cell/build/`. ## Module Resolution @@ -47,7 +47,7 @@ When `use('path')` is called from a package context, the shop resolves the modul For a call like `use('sprite')` from package `myapp`: -1. **Own package** — `~/.pit/packages/myapp/sprite.cm` and C symbol `js_myapp_sprite_use` +1. **Own package** — `~/.cell/packages/myapp/sprite.cm` and C symbol `js_myapp_sprite_use` 2. **Aliased dependencies** — if `myapp/cell.toml` has `renderer = "gitea.pockle.world/john/renderer"`, checks `renderer/sprite.cm` and its C symbols 3. **Core** — built-in core modules and internal C symbols @@ -80,31 +80,49 @@ Every module goes through a content-addressed caching pipeline. The cache key is When loading a module, the shop checks (in order): 1. **In-memory cache** — `use_cache[key]`, checked first on every `use()` call -2. **Installed dylib** — per-file `.dylib` in `~/.pit/lib//.dylib` -3. **Installed mach** — pre-compiled bytecode in `~/.pit/lib//.mach` -4. **Cached bytecode** — content-addressed in `~/.pit/build/` (no extension) -5. **Cached .mcode IR** — JSON IR in `~/.pit/build/.mcode` -6. **Internal symbols** — statically linked into the `pit` binary (fat builds) -7. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize +2. **Build-cache dylib** — content-addressed `.dylib` in `~/.cell/build/`, found via manifest (see [Dylib Manifests](#dylib-manifests)) +3. **Cached bytecode** — content-addressed in `~/.cell/build/` (no extension) +4. **Internal symbols** — statically linked into the `cell` binary (fat builds) +5. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize -When both a `.dylib` and `.mach` exist for the same module in `lib/`, the dylib is selected. Dylib resolution also wins over internal symbols, so a dylib in `lib/` can hot-patch a fat binary. Delete the dylib to fall back to mach or static. - -Results from steps 5-7 are cached back to the content-addressed store for future loads. +Dylib resolution wins over internal symbols, so a built dylib can hot-patch a fat binary. Results from compilation are cached back to the content-addressed store for future loads. Each loading method (except the in-memory cache) can be individually enabled or disabled via `shop.toml` policy flags — see [Shop Configuration](#shop-configuration) below. ### Content-Addressed Store -The build cache at `~/.pit/build/` stores ephemeral artifacts named by the BLAKE2 hash of their inputs: +The build cache at `~/.cell/build/` stores all compiled artifacts named by the BLAKE2 hash of their inputs: ``` -~/.pit/build/ -├── a1b2c3d4... # cached bytecode blob (no extension) -├── c9d0e1f2...mcode # cached JSON IR -└── f3a4b5c6... # compiled dylib (checked before copying to lib/) +~/.cell/build/ +├── a1b2c3d4... # cached bytecode blob, object file, dylib, or manifest (no extension) +└── ... ``` -This scheme provides automatic cache invalidation: when source changes, its hash changes, and the old cache entry is simply never looked up again. When building a dylib, the build cache is checked first — if a matching hash exists, it is copied to `lib/` without recompiling. +Every artifact type uses a unique salt appended to the content before hashing, so collisions between different artifact types are impossible: + +| Salt | Artifact | +|------|----------| +| `obj` | compiled C object file | +| `dylib` | linked dynamic library | +| `native` | native-compiled .cm dylib | +| `mach` | mach bytecode blob | +| `mcode` | mcode IR (JSON) | +| `deps` | cached `cc -MM` dependency list | +| `fail` | cached compilation failure marker | +| `manifest` | package dylib manifest (JSON) | + +This scheme provides automatic cache invalidation: when source changes, its hash changes, and the old cache entry is simply never looked up again. + +### Failure Caching + +When a C file fails to compile (missing SDK headers, syntax errors, etc.), the build system writes a failure marker to the cache using the `fail` salt. On subsequent builds, the failure marker is found and the file is skipped immediately — no time wasted retrying files that can't compile. The failure marker is keyed on the same content as the compilation (command string + source content), so if the source changes or compiler flags change, the failure is automatically invalidated and compilation is retried. + +### Dylib Manifests + +Dylibs live at content-addressed paths (`~/.cell/build/`) that can only be computed by running the full build pipeline. To allow the runtime to find pre-built dylibs without invoking the build module, `cell build` writes a **manifest** for each package. The manifest is a JSON file mapping each C module to its `{file, symbol, dylib}` entry. The manifest path is itself content-addressed (BLAKE2 hash of the package name + `manifest` salt), so the runtime can compute it from the package name alone. + +At runtime, when `use()` needs a C module from another package, the shop reads the manifest to find the dylib path. This means `cell build` must be run before C modules from packages can be loaded. ### Core Module Caching @@ -124,10 +142,10 @@ symbol: js_gitea_pockle_world_john_prosperon_sprite_use ### C Resolution Sources -1. **Installed dylibs** — per-file dylibs in `~/.pit/lib//.dylib` (deterministic paths, no manifests) -2. **Internal symbols** — statically linked into the `pit` binary (fat builds) +1. **Build-cache dylibs** — content-addressed dylibs in `~/.cell/build/`, found via per-package manifests written by `cell build` +2. **Internal symbols** — statically linked into the `cell` binary (fat builds) -Dylibs are checked first at each resolution scope, so an installed dylib always wins over a statically linked symbol. This enables hot-patching fat binaries by placing a dylib in `lib/`. +Dylibs are checked first at each resolution scope, so a built dylib always wins over a statically linked symbol. This enables hot-patching fat binaries. ### Name Collisions @@ -145,7 +163,7 @@ The set of injected capabilities is controlled by `script_inject_for()`, which c ## Shop Configuration -The shop reads an optional `shop.toml` file from the shop root (`~/.pit/shop.toml`). This file controls which loading methods are permitted through policy flags. +The shop reads an optional `shop.toml` file from the shop root (`~/.cell/shop.toml`). This file controls which loading methods are permitted through policy flags. ### Policy Flags @@ -188,22 +206,12 @@ If `shop.toml` is missing or has no `[policy]` section, all methods are enabled ## Shop Directory Layout ``` -~/.pit/ +~/.cell/ ├── packages/ # installed packages (directories and symlinks) │ └── core -> ... # symlink to the ƿit core -├── lib/ # INSTALLED per-file artifacts (persistent, human-readable) -│ ├── core/ -│ │ ├── fd.dylib -│ │ ├── time.mach -│ │ ├── time.dylib -│ │ └── internal/ -│ │ └── os.dylib -│ └── gitea_pockle_world_john_prosperon/ -│ ├── sprite.dylib -│ └── render.dylib -├── build/ # EPHEMERAL cache (safe to delete anytime) -│ ├── # cached bytecode or dylib blobs (no extension) -│ └── .mcode # cached JSON IR +├── build/ # content-addressed cache (safe to delete anytime) +│ ├── # cached bytecode, object file, dylib, or manifest +│ └── ... ├── cache/ # downloaded package zip archives ├── lock.toml # installed package versions and commit hashes ├── link.toml # local development link overrides diff --git a/internal/shop.cm b/internal/shop.cm index 09796818..4e5aba70 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -843,25 +843,51 @@ function make_c_symbol(pkg, file) { return 'js_' + pkg_safe + '_' + file_safe + '_use' } +// Compute the manifest path for a package (must match build.cm's manifest_path) +function dylib_manifest_path(pkg) { + var hash = content_hash(stone(blob(pkg + '\n' + 'manifest'))) + return global_shop_path + '/build/' + hash +} + +// Read a pre-built dylib manifest for a package. +// Returns the array of {file, symbol, dylib} or null. +function read_dylib_manifest(pkg) { + var mpath = dylib_manifest_path(pkg) + if (!fd.is_file(mpath)) return null + var content = text(fd.slurp(mpath)) + if (!content || length(content) == 0) return null + return json.decode(content) +} + // Ensure all C modules for a package are built and loaded. // Returns the array of {file, symbol, dylib} results, cached per package. function ensure_package_dylibs(pkg) { if (package_dylibs[pkg]) return package_dylibs[pkg] + var results = null var build_mod = use_cache['core/build'] - if (!build_mod) return null + var target = null + var c_files = null - var target = detect_host_target() - if (!target) return null + if (build_mod) { + target = detect_host_target() + if (!target) return null - var c_files = pkg_tools.get_c_files(pkg, target, true) - if (!c_files || length(c_files) == 0) { - package_dylibs[pkg] = [] - return [] + c_files = pkg_tools.get_c_files(pkg, target, true) + if (!c_files || length(c_files) == 0) { + package_dylibs[pkg] = [] + return [] + } + + log.shop('ensuring C modules for ' + pkg) + results = build_mod.build_dynamic(pkg, target, 'release', {}) + } else { + // No build module at runtime — read manifest from cell build + results = read_dylib_manifest(pkg) + if (!results) return null + log.shop('loaded manifest for ' + pkg + ' (' + text(length(results)) + ' modules)') } - log.shop('ensuring C modules for ' + pkg) - var results = build_mod.build_dynamic(pkg, target, 'release', {}) package_dylibs[pkg] = results // Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL @@ -873,7 +899,6 @@ function ensure_package_dylibs(pkg) { } }) - log.shop('built ' + text(length(results)) + ' C module(s) for ' + pkg) return results } diff --git a/source/runtime.c b/source/runtime.c index eb7813b7..06235eec 100644 --- a/source/runtime.c +++ b/source/runtime.c @@ -6889,7 +6889,15 @@ static JSValue js_cell_text (JSContext *ctx, JSValue this_val, int argc, JSValue if (bd->length % 8 != 0) return JS_RaiseDisrupt (ctx, "text: blob not byte-aligned for UTF-8"); - return JS_NewStringLen (ctx, (const char *)data, byte_len); + /* Copy blob data to a temp buffer before JS_NewStringLen, because + JS_NewStringLen allocates internally (js_alloc_string) which can + trigger GC, moving the blob and invalidating data. */ + char *tmp = pjs_malloc (byte_len); + if (!tmp) return JS_ThrowMemoryError (ctx); + memcpy (tmp, data, byte_len); + JSValue result = JS_NewStringLen (ctx, tmp, byte_len); + pjs_free (tmp); + return result; } }