john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

rm tokenizer/parser/mcode generators from C

Browse Source
This commit is contained in:
John Alanbrook
2026-02-09 20:05:50 -06:00
parent 27e852af5b
commit 3e42c57479
16 changed files with 198175 additions and 5998 deletions
Download Patch File Download Diff File Expand all files Collapse all files

423
source/cell.c
View File

@@ -11,7 +11,7 @@
#include "cell_internal.h"
#include "cJSON.h"
#define ENGINE "internal/engine.cm"
#define BOOTSTRAP_AST "internal/bootstrap.ast.json"
#define CELL_SHOP_DIR ".cell"
#define CELL_CORE_DIR "packages/core"
@@ -71,81 +71,41 @@ int find_cell_shop(void)
// Load a file from the core directory
static char* load_core_file(const char *filename, size_t *out_size) {
if (!core_path) return NULL;
size_t path_len = strlen(core_path) + 1 + strlen(filename) + 1;
char *full_path = malloc(path_len);
if (!full_path) return NULL;
snprintf(full_path, path_len, "%s/%s", core_path, filename);
FILE *fh = fopen(full_path, "rb");
free(full_path);
if (!fh) return NULL;
fseek(fh, 0, SEEK_END);
long file_size = ftell(fh);
fseek(fh, 0, SEEK_SET);
char *data = malloc(file_size + 1);
if (!data) {
fclose(fh);
return NULL;
}
if (fread(data, 1, file_size, fh) != (size_t)file_size) {
free(data);
fclose(fh);
return NULL;
}
fclose(fh);
data[file_size] = 0;
if (out_size) *out_size = file_size;
return data;
}
static int print_tree_errors(cJSON *root) {
if (!root) return 0;
cJSON *errors = cJSON_GetObjectItemCaseSensitive(root, "errors");
if (!cJSON_IsArray(errors) || cJSON_GetArraySize(errors) == 0)
return 0;
const char *filename = "<unknown>";
cJSON *fname = cJSON_GetObjectItemCaseSensitive(root, "filename");
if (cJSON_IsString(fname))
filename = fname->valuestring;
int prev_line = -1;
const char *prev_msg = NULL;
cJSON *e;
cJSON_ArrayForEach(e, errors) {
const char *msg = cJSON_GetStringValue(
cJSON_GetObjectItemCaseSensitive(e, "message"));
cJSON *line = cJSON_GetObjectItemCaseSensitive(e, "line");
cJSON *col = cJSON_GetObjectItemCaseSensitive(e, "column");
int cur_line = cJSON_IsNumber(line) ? (int)line->valuedouble : -1;
if (prev_msg && msg && cur_line == prev_line && strcmp(msg, prev_msg) == 0)
continue;
prev_line = cur_line;
prev_msg = msg;
if (msg && cJSON_IsNumber(line) && cJSON_IsNumber(col))
fprintf(stderr, "%s:%d:%d: error: %s\n",
filename, (int)line->valuedouble, (int)col->valuedouble, msg);
else if (msg)
fprintf(stderr, "%s: error: %s\n", filename, msg);
}
return 1;
}
static int print_json_errors(const char *json) {
if (!json) return 0;
cJSON *root = cJSON_Parse(json);
if (!root) return 0;
int result = print_tree_errors(root);
cJSON_Delete(root);
return result;
}
// Get the core path for use by scripts
const char* cell_get_core_path(void) {
return core_path;
@@ -190,23 +150,18 @@ void script_startup(cell_rt *prt)
cell_rt *crt = JS_GetContextOpaque(js);
JS_FreeValue(js, js_blob_use(js));
// Load and parse engine.cm to AST
size_t engine_size;
char *data = load_core_file(ENGINE, &engine_size);
if (!data) {
printf("ERROR: Could not load %s from %s!\n", ENGINE, core_path);
// Load pre-compiled bootstrap AST
size_t boot_size;
char *boot_json = load_core_file(BOOTSTRAP_AST, &boot_size);
if (!boot_json) {
printf("ERROR: Could not load %s from %s!\n", BOOTSTRAP_AST, core_path);
return;
}
cJSON *ast = JS_ASTTree(data, engine_size, ENGINE);
free(data);
cJSON *ast = cJSON_Parse(boot_json);
free(boot_json);
if (!ast) {
printf("ERROR: Failed to parse %s\n", ENGINE);
return;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
printf("ERROR: Failed to parse %s\n", BOOTSTRAP_AST);
return;
}
@@ -287,18 +242,12 @@ static int run_test_suite(size_t heap_size)
return result;
}
/* Run an immediate script string */
static void print_usage(const char *prog)
{
printf("Usage: %s [options] <script> [args...]\n\n", prog);
printf("Run a cell script (.ce actor or .cm module).\n\n");
printf("Options:\n");
printf(" --ast <code|file> Output AST as JSON\n");
printf(" --tokenize <code|file> Output token array as JSON\n");
printf(" --mcode <script> [args] Run through mcode compilation pipeline\n");
printf(" --mcode-print <code|file> Compile to MCODE JSON and print it\n");
printf(" --mach <code|file> Output MACH bytecode\n");
printf(" --mach-run <code|file> Compile and run through MACH VM\n");
printf(" --test [heap_size] Run C test suite\n");
printf(" -h, --help Show this help message\n");
printf("\nRecompile after changes: make\n");
@@ -326,322 +275,7 @@ int cell_init(int argc, char **argv)
return run_test_suite(heap_size);
}
/* Check for --ast flag to output AST JSON */
if (argc >= 3 && strcmp(argv[1], "--ast") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
cJSON *ast = JS_ASTTree(script, strlen(script), filename);
if (ast) {
int has_errors = print_tree_errors(ast);
char *pretty = cJSON_Print(ast);
cJSON_Delete(ast);
printf("%s\n", pretty);
free(pretty);
free(allocated_script);
return has_errors ? 1 : 0;
} else {
printf("Failed to parse AST\n");
free(allocated_script);
return 1;
}
}
/* Check for --tokenize flag to output token array JSON */
if (argc >= 3 && strcmp(argv[1], "--tokenize") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
char *json = JS_Tokenize(script, strlen(script), filename);
if (json) {
int has_errors = print_json_errors(json);
cJSON *root = cJSON_Parse(json);
free(json);
if (root) {
char *pretty = cJSON_Print(root);
cJSON_Delete(root);
printf("%s\n", pretty);
free(pretty);
}
free(allocated_script);
return has_errors ? 1 : 0;
} else {
printf("Failed to tokenize\n");
free(allocated_script);
return 1;
}
}
/* Check for --mcode-print flag to compile a script to MCODE JSON and print it */
if (argc >= 3 && strcmp(argv[1], "--mcode-print") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
cJSON *ast = JS_ASTTree(script, strlen(script), filename);
if (!ast) {
printf("Failed to parse AST\n");
free(allocated_script);
return 1;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
free(allocated_script);
return 1;
}
cJSON *mcode = JS_McodeTree(ast);
cJSON_Delete(ast);
if (!mcode) {
printf("Failed to generate MCODE\n");
free(allocated_script);
return 1;
}
if (print_tree_errors(mcode)) {
cJSON_Delete(mcode);
free(allocated_script);
return 1;
}
char *pretty = cJSON_Print(mcode);
cJSON_Delete(mcode);
printf("%s\n", pretty);
free(pretty);
free(allocated_script);
return 0;
}
/* Check for --mach flag to dump MACH bytecode */
if (argc >= 3 && strcmp(argv[1], "--mach") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
cJSON *ast = JS_ASTTree(script, strlen(script), filename);
if (!ast) {
printf("Failed to parse AST\n");
free(allocated_script);
return 1;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
free(allocated_script);
return 1;
}
JSRuntime *rt = JS_NewRuntime();
if (!rt) {
printf("Failed to create JS runtime\n");
cJSON_Delete(ast); free(allocated_script);
return 1;
}
JSContext *ctx = JS_NewContext(rt);
if (!ctx) {
printf("Failed to create JS context\n");
cJSON_Delete(ast); JS_FreeRuntime(rt); free(allocated_script);
return 1;
}
JS_DumpMachTree(ctx, ast, JS_NULL);
cJSON_Delete(ast);
JS_FreeContext(ctx);
JS_FreeRuntime(rt);
free(allocated_script);
return 0;
}
/* Check for --mach-run flag to compile and run through MACH VM */
if (argc >= 3 && strcmp(argv[1], "--mach-run") == 0) {
const char *script_name = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = script_name;
struct stat st;
if (stat(script_name, &st) == 0 && S_ISREG(st.st_mode)) {
/* Exact name found */
} else {
/* Try .ce then .cm extension */
static char pathbuf[4096];
snprintf(pathbuf, sizeof(pathbuf), "%s.ce", script_name);
if (stat(pathbuf, &st) == 0 && S_ISREG(st.st_mode)) {
script_name = pathbuf;
filename = pathbuf;
} else {
snprintf(pathbuf, sizeof(pathbuf), "%s.cm", script_name);
if (stat(pathbuf, &st) == 0 && S_ISREG(st.st_mode)) {
script_name = pathbuf;
filename = pathbuf;
} else {
printf("Failed to find file: %s\n", argv[2]);
return 1;
}
}
}
FILE *f = fopen(script_name, "r");
if (!f) {
printf("Failed to open file: %s\n", script_name);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
cJSON *ast = JS_ASTTree(script, read_size, filename);
free(allocated_script);
if (!ast) {
printf("Failed to parse %s\n", filename);
return 1;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
return 1;
}
JSRuntime *rt = JS_NewRuntime();
if (!rt) {
printf("Failed to create JS runtime\n");
cJSON_Delete(ast);
return 1;
}
JSContext *ctx = JS_NewContextWithHeapSize(rt, 16 * 1024 * 1024);
if (!ctx) {
printf("Failed to create JS context\n");
cJSON_Delete(ast); JS_FreeRuntime(rt);
return 1;
}
JS_FreeValue(ctx, js_blob_use(ctx));
JSValue result = JS_RunMachTree(ctx, ast, JS_NULL);
cJSON_Delete(ast);
int exit_code = 0;
if (JS_IsException(result)) {
JS_GetException(ctx);
exit_code = 1;
} else if (!JS_IsNull(result)) {
const char *str = JS_ToCString(ctx, result);
if (str) {
printf("%s\n", str);
JS_FreeCString(ctx, str);
}
}
JS_FreeContext(ctx);
JS_FreeRuntime(rt);
return exit_code;
}
/* Default: run script through mach-run bootstrap pipeline */
/* Default: run script through bootstrap pipeline */
int use_mcode = 0;
int arg_start = 1;
if (argc >= 3 && strcmp(argv[1], "--mcode") == 0) {
@@ -652,21 +286,16 @@ int cell_init(int argc, char **argv)
if (!find_cell_shop()) return 1;
size_t boot_size;
char *boot_data = load_core_file("internal/bootstrap.cm", &boot_size);
if (!boot_data) {
printf("ERROR: Could not load internal/bootstrap.cm from %s\n", core_path);
char *boot_json = load_core_file(BOOTSTRAP_AST, &boot_size);
if (!boot_json) {
printf("ERROR: Could not load %s from %s\n", BOOTSTRAP_AST, core_path);
return 1;
}
cJSON *boot_ast = JS_ASTTree(boot_data, boot_size, "internal/bootstrap.cm");
free(boot_data);
cJSON *boot_ast = cJSON_Parse(boot_json);
free(boot_json);
if (!boot_ast) {
printf("Failed to parse internal/bootstrap.cm\n");
return 1;
}
if (print_tree_errors(boot_ast)) {
cJSON_Delete(boot_ast);
printf("Failed to parse %s\n", BOOTSTRAP_AST);
return 1;
}
@@ -748,7 +377,7 @@ double cell_random() {
void cell_trace_sethook(cell_hook)
{
}
int uncaught_exception(JSContext *js, JSValue v)

1671
source/mcode.c
View File

File diff suppressed because it is too large Load Diff

2026
source/parse.c
View File

File diff suppressed because it is too large Load Diff

59
source/quickjs-internal.h
View File

@@ -1707,45 +1707,6 @@ typedef struct PPretext {
int cap;
} PPretext;
/* === AST Parse State (shared by tokenize.c and parse.c) === */
typedef struct ASTParseState {
const char *filename;
const uint8_t *buf_start;
const uint8_t *buf_ptr;
const uint8_t *buf_end;
const uint8_t *token_ptr;
int token_val;
BOOL got_lf;
int function_nr;
cJSON *errors; /* array of error objects */
int has_error;
int error_count;
int in_disruption;
char *decoded_str; /* allocated buffer for decoded string escapes */
GetLineColCache lc_cache;
union {
struct {
const char *str;
size_t len;
} str;
struct {
double val;
} num;
struct {
const char *str;
size_t len;
BOOL has_escape;
BOOL is_reserved;
} ident;
struct {
const char *body;
size_t body_len;
const char *flags;
size_t flags_len;
} regexp;
} token_u;
} ASTParseState;
#define JS_CALL_FLAG_COPY_ARGV (1 << 1)
extern JSClassID js_class_id_alloc;
@@ -1935,25 +1896,5 @@ int reg_vm_check_interrupt(JSContext *ctx);
/* mcode.c exports */
JSValue mcode_exec(JSContext *ctx, JSMCode *code, JSValue this_obj, int argc, JSValue *argv, JSValue outer_frame);
/* tokenize.c exports (called by parse.c) */
void cjson_add_strn (cJSON *obj, const char *key, const char *str, size_t len);
cJSON *ast_node (ASTParseState *s, const char *kind, const uint8_t *start_ptr);
void ast_node_end (ASTParseState *s, cJSON *node, const uint8_t *end_ptr);
int ast_next_token (ASTParseState *s);
void ast_free_token (ASTParseState *s);
void ast_get_line_col (ASTParseState *s, const uint8_t *ptr, int *line, int *col);
BOOL tok_eq (const char *str, size_t len, const char *lit);
BOOL ast_is_arrow_function (ASTParseState *s);
int tokenize_next (ASTParseState *s);
void ast_error (ASTParseState *s, const uint8_t *ptr, const char *fmt, ...);
/* parse.c forward declarations */
cJSON *ast_parse_expr (ASTParseState *s);
cJSON *ast_parse_assign_expr (ASTParseState *s);
cJSON *ast_parse_statement (ASTParseState *s);
void ast_sync_to_statement (ASTParseState *s);
cJSON *ast_parse_block_statements (ASTParseState *s);
cJSON *ast_parse_function_inner (ASTParseState *s, BOOL is_expr);
cJSON *ast_parse_arrow_function (ASTParseState *s);
#endif /* QUICKJS_INTERNAL_H */

20
source/quickjs.h
View File

@@ -1068,17 +1068,7 @@ void *js_malloc_rt (size_t size);
void *js_mallocz_rt (size_t size);
void js_free_rt (void *ptr);
/* Parse source code and return AST as cJSON tree.
Caller must call cJSON_Delete() on result. */
struct cJSON *JS_ASTTree (const char *source, size_t len, const char *filename);
/* Parse source code and return AST as JSON string.
Returns malloc'd JSON string (caller must free), or NULL on error. */
char *JS_AST (const char *source, size_t len, const char *filename);
/* Tokenize source code and return token array as JSON string.
Returns malloc'd JSON string (caller must free), or NULL on error. */
char *JS_Tokenize (const char *source, size_t len, const char *filename);
struct cJSON;
/* Compiled bytecode (context-free, serializable) */
typedef struct MachCode MachCode;
@@ -1107,14 +1097,6 @@ JSValue JS_RunMachTree (JSContext *ctx, struct cJSON *ast, JSValue env);
/* Compile and execute MACH bytecode from AST JSON string. */
JSValue JS_RunMach (JSContext *ctx, const char *ast_json, JSValue env);
/* Compile AST cJSON tree to MCODE cJSON tree.
Caller must call cJSON_Delete() on result. */
struct cJSON *JS_McodeTree (struct cJSON *ast);
/* Compile AST JSON string to MCODE JSON string.
Returns malloc'd JSON string, or NULL on error. Caller must free. */
char *JS_Mcode (const char *ast_json);
/* Execute MCODE from cJSON tree. Takes ownership of root. */
JSValue JS_CallMcodeTree (JSContext *ctx, struct cJSON *root);

63
source/runtime.c
View File

@@ -2611,7 +2611,7 @@ JS_HasException (JSContext *ctx) {
}
/* Relocated from cell_js.c — used by tokenize.c via ast_get_line_col */
/* get_line_col — compute line and column from a byte offset */
int get_line_col (int *pcol_num, const uint8_t *buf, size_t len) {
int line_num, col_num, c;
size_t i;
@@ -10391,63 +10391,6 @@ int js_is_blob (JSContext *js, JSValue v) {
* ============================================================================
*/
/* eval(text, env) - evaluate code with optional environment record
* text: string to compile and execute
* env: optional stone record for variable bindings (checked first before intrinsics)
*/
static JSValue js_cell_eval (JSContext *ctx, JSValue this_val, int argc, JSValue *argv) {
(void)this_val;
if (argc < 1 || !JS_IsText (argv[0]))
return JS_ThrowTypeError (ctx, "eval requires a text argument");
const char *source = JS_ToCString (ctx, argv[0]);
if (!source) return JS_EXCEPTION;
cJSON *ast = JS_ASTTree (source, strlen (source), "<eval>");
JS_FreeCString (ctx, source);
if (!ast)
return JS_ThrowSyntaxError (ctx, "eval: failed to parse");
JSValue env = (argc > 1 && JS_IsObject (argv[1])) ? argv[1] : JS_NULL;
JSValue result = JS_RunMachTree (ctx, ast, env);
cJSON_Delete (ast);
return result;
}
/* ============================================================================
* mach_eval() function - compile and execute via MACH VM
* ============================================================================
*/
/* mach_eval(name, source) - parse to AST and run through MACH VM */
static JSValue js_mach_eval (JSContext *ctx, JSValue this_val, int argc, JSValue *argv) {
if (argc < 2 || !JS_IsText (argv[0]) || !JS_IsText (argv[1]))
return JS_ThrowTypeError (ctx, "mach_eval requires (name, source) text arguments");
const char *name = JS_ToCString (ctx, argv[0]);
if (!name) return JS_EXCEPTION;
const char *source = JS_ToCString (ctx, argv[1]);
if (!source) {
JS_FreeCString (ctx, name);
return JS_EXCEPTION;
}
cJSON *ast = JS_ASTTree (source, strlen (source), name);
JS_FreeCString (ctx, source);
if (!ast) {
JS_FreeCString (ctx, name);
return JS_ThrowSyntaxError (ctx, "mach_eval: failed to parse AST");
}
JSValue env = (argc >= 3 && JS_IsObject (argv[2])) ? argv[2] : JS_NULL;
JSValue result = JS_RunMachTree (ctx, ast, env);
cJSON_Delete (ast);
JS_FreeCString (ctx, name);
return result;
}
/* mach_eval_ast(name, ast_json, env?) - compile pre-parsed AST and run */
static JSValue js_mach_eval_ast (JSContext *ctx, JSValue this_val, int argc, JSValue *argv) {
if (argc < 2 || !JS_IsText (argv[0]) || !JS_IsText (argv[1]))
@@ -11601,8 +11544,6 @@ static void JS_AddIntrinsicBaseObjects (JSContext *ctx) {
}
/* Core functions - using GC-safe helper */
js_set_global_cfunc(ctx, "eval", js_cell_eval, 2);
js_set_global_cfunc(ctx, "mach_eval", js_mach_eval, 3);
js_set_global_cfunc(ctx, "mach_eval_ast", js_mach_eval_ast, 3);
js_set_global_cfunc(ctx, "mcode_run", js_mcode_run, 3);
js_set_global_cfunc(ctx, "stone", js_cell_stone, 1);
@@ -11667,7 +11608,7 @@ static void JS_AddIntrinsicBaseObjects (JSContext *ctx) {
js_set_global_cfunc(ctx, "pop", js_cell_pop, 1);
js_set_global_cfunc(ctx, "meme", js_cell_meme, 2);
/* Engine builtins (normally from engine.cm, needed for --mach-run) */
/* Additional builtins */
js_set_global_cfunc(ctx, "logical", js_cell_logical, 1);
js_set_global_cfunc(ctx, "starts_with", js_cell_starts_with, 2);
js_set_global_cfunc(ctx, "ends_with", js_cell_ends_with, 2);

312
source/suite.c
View File

@@ -2004,290 +2004,11 @@ TEST(wota_encode_blob) {
return 1;
}
/* ============================================================================
ERROR RECOVERY TESTS - Helper macros
============================================================================ */
#define ASSERT_HAS_ERRORS(json_str, min_count) do { \
cJSON *_root = cJSON_Parse(json_str); \
ASSERT_MSG(_root != NULL, "failed to parse JSON output"); \
cJSON *_errs = cJSON_GetObjectItem(_root, "errors"); \
if (!_errs || !cJSON_IsArray(_errs) || cJSON_GetArraySize(_errs) < (min_count)) { \
printf("[line %d: expected at least %d error(s), got %d] ", __LINE__, (min_count), \
_errs && cJSON_IsArray(_errs) ? cJSON_GetArraySize(_errs) : 0); \
cJSON_Delete(_root); \
return 0; \
} \
cJSON_Delete(_root); \
} while(0)
#define ASSERT_NO_ERRORS(json_str) do { \
cJSON *_root = cJSON_Parse(json_str); \
ASSERT_MSG(_root != NULL, "failed to parse JSON output"); \
cJSON *_errs = cJSON_GetObjectItem(_root, "errors"); \
if (_errs && cJSON_IsArray(_errs) && cJSON_GetArraySize(_errs) > 0) { \
cJSON *_first = cJSON_GetArrayItem(_errs, 0); \
const char *_msg = cJSON_GetStringValue(cJSON_GetObjectItem(_first, "message")); \
printf("[line %d: expected no errors, got: %s] ", __LINE__, _msg ? _msg : "?"); \
cJSON_Delete(_root); \
return 0; \
} \
cJSON_Delete(_root); \
} while(0)
#define ASSERT_ERROR_MSG_CONTAINS(json_str, substring) do { \
cJSON *_root = cJSON_Parse(json_str); \
ASSERT_MSG(_root != NULL, "failed to parse JSON output"); \
cJSON *_errs = cJSON_GetObjectItem(_root, "errors"); \
int _found = 0; \
if (_errs && cJSON_IsArray(_errs)) { \
cJSON *_e; \
cJSON_ArrayForEach(_e, _errs) { \
const char *_msg = cJSON_GetStringValue(cJSON_GetObjectItem(_e, "message")); \
if (_msg && strstr(_msg, (substring))) { _found = 1; break; } \
} \
} \
if (!_found) { \
printf("[line %d: no error containing '%s'] ", __LINE__, (substring)); \
cJSON_Delete(_root); \
return 0; \
} \
cJSON_Delete(_root); \
} while(0)
/* ============================================================================
TOKENIZER ERROR TESTS
============================================================================ */
TEST(tokenize_unterminated_string) {
const char *src = "var x = \"hello";
char *json = JS_Tokenize(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_Tokenize returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "unterminated string");
free(json);
return 1;
}
TEST(tokenize_unterminated_template) {
const char *src = "var x = `hello";
char *json = JS_Tokenize(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_Tokenize returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "unterminated template");
free(json);
return 1;
}
TEST(tokenize_unterminated_block_comment) {
const char *src = "var x /* comment";
char *json = JS_Tokenize(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_Tokenize returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "unterminated block comment");
free(json);
return 1;
}
TEST(tokenize_malformed_hex) {
const char *src = "var x = 0x";
char *json = JS_Tokenize(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_Tokenize returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "malformed hex");
free(json);
return 1;
}
TEST(tokenize_malformed_binary) {
const char *src = "var x = 0b";
char *json = JS_Tokenize(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_Tokenize returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "malformed binary");
free(json);
return 1;
}
TEST(tokenize_malformed_exponent) {
const char *src = "var x = 1e+";
char *json = JS_Tokenize(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_Tokenize returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "no digits after exponent");
free(json);
return 1;
}
TEST(tokenize_valid_no_errors) {
const char *src = "var x = 42";
char *json = JS_Tokenize(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_Tokenize returned NULL");
ASSERT_NO_ERRORS(json);
free(json);
return 1;
}
/* ============================================================================
PARSER ERROR TESTS
============================================================================ */
TEST(ast_missing_identifier_after_var) {
const char *src = "var = 1";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "expected identifier");
free(json);
return 1;
}
TEST(ast_missing_initializer_def) {
const char *src = "def x";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "missing initializer");
free(json);
return 1;
}
TEST(ast_recovery_continues_after_error) {
const char *src = "var = 1; var y = 2";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_HAS_ERRORS(json, 1);
/* Check that 'y' statement is present in the AST */
ASSERT_MSG(strstr(json, "\"y\"") != NULL, "recovery failed: 'y' not in AST");
free(json);
return 1;
}
TEST(ast_valid_no_errors) {
const char *src = "var x = 1; var y = 2";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_NO_ERRORS(json);
free(json);
return 1;
}
/* ============================================================================
AST SEMANTIC ERROR TESTS
============================================================================ */
TEST(ast_sem_assign_to_const) {
const char *src = "def x = 5; x = 3";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "cannot assign to constant");
free(json);
return 1;
}
TEST(ast_sem_assign_to_arg) {
const char *src = "function(x) { x = 5; }";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "cannot assign to constant");
free(json);
return 1;
}
TEST(ast_sem_redeclare_const) {
const char *src = "def x = 1; def x = 2";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "cannot redeclare constant");
free(json);
return 1;
}
TEST(ast_sem_break_outside_loop) {
const char *src = "break";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "outside of loop");
free(json);
return 1;
}
TEST(ast_sem_continue_outside_loop) {
const char *src = "continue";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "outside of loop");
free(json);
return 1;
}
TEST(ast_sem_break_inside_loop_ok) {
const char *src = "while (true) { break; }";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_NO_ERRORS(json);
free(json);
return 1;
}
TEST(ast_sem_increment_const) {
const char *src = "def x = 1; x++";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_ERROR_MSG_CONTAINS(json, "cannot assign to constant");
free(json);
return 1;
}
TEST(ast_sem_shadow_var_ok) {
const char *src = "var array = []; array";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_NO_ERRORS(json);
free(json);
return 1;
}
TEST(ast_sem_var_assign_ok) {
const char *src = "var x = 1; x = x + 1";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_NO_ERRORS(json);
free(json);
return 1;
}
TEST(ast_sem_nested_function_scope) {
const char *src = "var x = 1; function f(x) { return x + 1; }";
char *json = JS_AST(src, strlen(src), "<test>");
ASSERT_MSG(json != NULL, "JS_AST returned NULL");
ASSERT_NO_ERRORS(json);
free(json);
return 1;
}
/* ============================================================================
CODEGEN TESTS (updated for new direct AST-to-bytecode compiler)
============================================================================ */
TEST(mach_compile_basic) {
const char *src = "var x = 1; x = x + 1";
cJSON *ast = JS_ASTTree(src, strlen(src), "<test>");
ASSERT_MSG(ast != NULL, "JS_ASTTree returned NULL");
MachCode *mc = JS_CompileMachTree(ast);
cJSON_Delete(ast);
ASSERT_MSG(mc != NULL, "JS_CompileMachTree returned NULL");
JS_FreeMachCode(mc);
return 1;
}
TEST(mach_compile_function) {
const char *src = "function f(x) { return x + 1 }";
cJSON *ast = JS_ASTTree(src, strlen(src), "<test>");
ASSERT_MSG(ast != NULL, "JS_ASTTree returned NULL");
MachCode *mc = JS_CompileMachTree(ast);
cJSON_Delete(ast);
ASSERT_MSG(mc != NULL, "JS_CompileMachTree returned NULL");
JS_FreeMachCode(mc);
return 1;
}
/* ============================================================================
MAIN TEST RUNNER
============================================================================ */
int run_c_test_suite(JSContext *ctx)
{
printf("\n=== Cell Runtime C Test Suite ===\n\n");
@@ -2494,37 +2215,6 @@ int run_c_test_suite(JSContext *ctx)
RUN_TEST(wota_encode_blob);
printf("\nTokenizer Errors:\n");
RUN_TEST(tokenize_unterminated_string);
RUN_TEST(tokenize_unterminated_template);
RUN_TEST(tokenize_unterminated_block_comment);
RUN_TEST(tokenize_malformed_hex);
RUN_TEST(tokenize_malformed_binary);
RUN_TEST(tokenize_malformed_exponent);
RUN_TEST(tokenize_valid_no_errors);
printf("\nParser Errors:\n");
RUN_TEST(ast_missing_identifier_after_var);
RUN_TEST(ast_missing_initializer_def);
RUN_TEST(ast_recovery_continues_after_error);
RUN_TEST(ast_valid_no_errors);
printf("\nAST Semantic Errors:\n");
RUN_TEST(ast_sem_assign_to_const);
RUN_TEST(ast_sem_assign_to_arg);
RUN_TEST(ast_sem_redeclare_const);
RUN_TEST(ast_sem_break_outside_loop);
RUN_TEST(ast_sem_continue_outside_loop);
RUN_TEST(ast_sem_break_inside_loop_ok);
RUN_TEST(ast_sem_increment_const);
RUN_TEST(ast_sem_shadow_var_ok);
RUN_TEST(ast_sem_var_assign_ok);
RUN_TEST(ast_sem_nested_function_scope);
printf("\nCodegen:\n");
RUN_TEST(mach_compile_basic);
RUN_TEST(mach_compile_function);
printf("\n=================================\n");
printf("Results: %d passed, %d failed\n", tests_passed, tests_failed);
printf("=================================\n\n");

1432
source/tokenize.c
View File

File diff suppressed because it is too large Load Diff