john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

rm tokenizer/parser/mcode generators from C

Browse Source
This commit is contained in:
John Alanbrook
2026-02-09 20:05:50 -06:00
parent 27e852af5b
commit 3e42c57479
16 changed files with 198175 additions and 5998 deletions
Download Patch File Download Diff File Expand all files Collapse all files

423
source/cell.c
View File

@@ -11,7 +11,7 @@
#include "cell_internal.h"
#include "cJSON.h"
#define ENGINE "internal/engine.cm"
#define BOOTSTRAP_AST "internal/bootstrap.ast.json"
#define CELL_SHOP_DIR ".cell"
#define CELL_CORE_DIR "packages/core"
@@ -71,81 +71,41 @@ int find_cell_shop(void)
// Load a file from the core directory
static char* load_core_file(const char *filename, size_t *out_size) {
if (!core_path) return NULL;
size_t path_len = strlen(core_path) + 1 + strlen(filename) + 1;
char *full_path = malloc(path_len);
if (!full_path) return NULL;
snprintf(full_path, path_len, "%s/%s", core_path, filename);
FILE *fh = fopen(full_path, "rb");
free(full_path);
if (!fh) return NULL;
fseek(fh, 0, SEEK_END);
long file_size = ftell(fh);
fseek(fh, 0, SEEK_SET);
char *data = malloc(file_size + 1);
if (!data) {
fclose(fh);
return NULL;
}
if (fread(data, 1, file_size, fh) != (size_t)file_size) {
free(data);
fclose(fh);
return NULL;
}
fclose(fh);
data[file_size] = 0;
if (out_size) *out_size = file_size;
return data;
}
static int print_tree_errors(cJSON *root) {
if (!root) return 0;
cJSON *errors = cJSON_GetObjectItemCaseSensitive(root, "errors");
if (!cJSON_IsArray(errors) || cJSON_GetArraySize(errors) == 0)
return 0;
const char *filename = "<unknown>";
cJSON *fname = cJSON_GetObjectItemCaseSensitive(root, "filename");
if (cJSON_IsString(fname))
filename = fname->valuestring;
int prev_line = -1;
const char *prev_msg = NULL;
cJSON *e;
cJSON_ArrayForEach(e, errors) {
const char *msg = cJSON_GetStringValue(
cJSON_GetObjectItemCaseSensitive(e, "message"));
cJSON *line = cJSON_GetObjectItemCaseSensitive(e, "line");
cJSON *col = cJSON_GetObjectItemCaseSensitive(e, "column");
int cur_line = cJSON_IsNumber(line) ? (int)line->valuedouble : -1;
if (prev_msg && msg && cur_line == prev_line && strcmp(msg, prev_msg) == 0)
continue;
prev_line = cur_line;
prev_msg = msg;
if (msg && cJSON_IsNumber(line) && cJSON_IsNumber(col))
fprintf(stderr, "%s:%d:%d: error: %s\n",
filename, (int)line->valuedouble, (int)col->valuedouble, msg);
else if (msg)
fprintf(stderr, "%s: error: %s\n", filename, msg);
}
return 1;
}
static int print_json_errors(const char *json) {
if (!json) return 0;
cJSON *root = cJSON_Parse(json);
if (!root) return 0;
int result = print_tree_errors(root);
cJSON_Delete(root);
return result;
}
// Get the core path for use by scripts
const char* cell_get_core_path(void) {
return core_path;
@@ -190,23 +150,18 @@ void script_startup(cell_rt *prt)
cell_rt *crt = JS_GetContextOpaque(js);
JS_FreeValue(js, js_blob_use(js));
// Load and parse engine.cm to AST
size_t engine_size;
char *data = load_core_file(ENGINE, &engine_size);
if (!data) {
printf("ERROR: Could not load %s from %s!\n", ENGINE, core_path);
// Load pre-compiled bootstrap AST
size_t boot_size;
char *boot_json = load_core_file(BOOTSTRAP_AST, &boot_size);
if (!boot_json) {
printf("ERROR: Could not load %s from %s!\n", BOOTSTRAP_AST, core_path);
return;
}
cJSON *ast = JS_ASTTree(data, engine_size, ENGINE);
free(data);
cJSON *ast = cJSON_Parse(boot_json);
free(boot_json);
if (!ast) {
printf("ERROR: Failed to parse %s\n", ENGINE);
return;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
printf("ERROR: Failed to parse %s\n", BOOTSTRAP_AST);
return;
}
@@ -287,18 +242,12 @@ static int run_test_suite(size_t heap_size)
return result;
}
/* Run an immediate script string */
static void print_usage(const char *prog)
{
printf("Usage: %s [options] <script> [args...]\n\n", prog);
printf("Run a cell script (.ce actor or .cm module).\n\n");
printf("Options:\n");
printf(" --ast <code|file> Output AST as JSON\n");
printf(" --tokenize <code|file> Output token array as JSON\n");
printf(" --mcode <script> [args] Run through mcode compilation pipeline\n");
printf(" --mcode-print <code|file> Compile to MCODE JSON and print it\n");
printf(" --mach <code|file> Output MACH bytecode\n");
printf(" --mach-run <code|file> Compile and run through MACH VM\n");
printf(" --test [heap_size] Run C test suite\n");
printf(" -h, --help Show this help message\n");
printf("\nRecompile after changes: make\n");
@@ -326,322 +275,7 @@ int cell_init(int argc, char **argv)
return run_test_suite(heap_size);
}
/* Check for --ast flag to output AST JSON */
if (argc >= 3 && strcmp(argv[1], "--ast") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
cJSON *ast = JS_ASTTree(script, strlen(script), filename);
if (ast) {
int has_errors = print_tree_errors(ast);
char *pretty = cJSON_Print(ast);
cJSON_Delete(ast);
printf("%s\n", pretty);
free(pretty);
free(allocated_script);
return has_errors ? 1 : 0;
} else {
printf("Failed to parse AST\n");
free(allocated_script);
return 1;
}
}
/* Check for --tokenize flag to output token array JSON */
if (argc >= 3 && strcmp(argv[1], "--tokenize") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
char *json = JS_Tokenize(script, strlen(script), filename);
if (json) {
int has_errors = print_json_errors(json);
cJSON *root = cJSON_Parse(json);
free(json);
if (root) {
char *pretty = cJSON_Print(root);
cJSON_Delete(root);
printf("%s\n", pretty);
free(pretty);
}
free(allocated_script);
return has_errors ? 1 : 0;
} else {
printf("Failed to tokenize\n");
free(allocated_script);
return 1;
}
}
/* Check for --mcode-print flag to compile a script to MCODE JSON and print it */
if (argc >= 3 && strcmp(argv[1], "--mcode-print") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
cJSON *ast = JS_ASTTree(script, strlen(script), filename);
if (!ast) {
printf("Failed to parse AST\n");
free(allocated_script);
return 1;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
free(allocated_script);
return 1;
}
cJSON *mcode = JS_McodeTree(ast);
cJSON_Delete(ast);
if (!mcode) {
printf("Failed to generate MCODE\n");
free(allocated_script);
return 1;
}
if (print_tree_errors(mcode)) {
cJSON_Delete(mcode);
free(allocated_script);
return 1;
}
char *pretty = cJSON_Print(mcode);
cJSON_Delete(mcode);
printf("%s\n", pretty);
free(pretty);
free(allocated_script);
return 0;
}
/* Check for --mach flag to dump MACH bytecode */
if (argc >= 3 && strcmp(argv[1], "--mach") == 0) {
const char *script_or_file = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = "<eval>";
struct stat st;
if (stat(script_or_file, &st) == 0 && S_ISREG(st.st_mode)) {
FILE *f = fopen(script_or_file, "r");
if (!f) {
printf("Failed to open file: %s\n", script_or_file);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
filename = script_or_file;
} else {
script = (char *)script_or_file;
}
cJSON *ast = JS_ASTTree(script, strlen(script), filename);
if (!ast) {
printf("Failed to parse AST\n");
free(allocated_script);
return 1;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
free(allocated_script);
return 1;
}
JSRuntime *rt = JS_NewRuntime();
if (!rt) {
printf("Failed to create JS runtime\n");
cJSON_Delete(ast); free(allocated_script);
return 1;
}
JSContext *ctx = JS_NewContext(rt);
if (!ctx) {
printf("Failed to create JS context\n");
cJSON_Delete(ast); JS_FreeRuntime(rt); free(allocated_script);
return 1;
}
JS_DumpMachTree(ctx, ast, JS_NULL);
cJSON_Delete(ast);
JS_FreeContext(ctx);
JS_FreeRuntime(rt);
free(allocated_script);
return 0;
}
/* Check for --mach-run flag to compile and run through MACH VM */
if (argc >= 3 && strcmp(argv[1], "--mach-run") == 0) {
const char *script_name = argv[2];
char *script = NULL;
char *allocated_script = NULL;
const char *filename = script_name;
struct stat st;
if (stat(script_name, &st) == 0 && S_ISREG(st.st_mode)) {
/* Exact name found */
} else {
/* Try .ce then .cm extension */
static char pathbuf[4096];
snprintf(pathbuf, sizeof(pathbuf), "%s.ce", script_name);
if (stat(pathbuf, &st) == 0 && S_ISREG(st.st_mode)) {
script_name = pathbuf;
filename = pathbuf;
} else {
snprintf(pathbuf, sizeof(pathbuf), "%s.cm", script_name);
if (stat(pathbuf, &st) == 0 && S_ISREG(st.st_mode)) {
script_name = pathbuf;
filename = pathbuf;
} else {
printf("Failed to find file: %s\n", argv[2]);
return 1;
}
}
}
FILE *f = fopen(script_name, "r");
if (!f) {
printf("Failed to open file: %s\n", script_name);
return 1;
}
allocated_script = malloc(st.st_size + 1);
if (!allocated_script) {
fclose(f);
printf("Failed to allocate memory for script\n");
return 1;
}
size_t read_size = fread(allocated_script, 1, st.st_size, f);
fclose(f);
allocated_script[read_size] = '\0';
script = allocated_script;
cJSON *ast = JS_ASTTree(script, read_size, filename);
free(allocated_script);
if (!ast) {
printf("Failed to parse %s\n", filename);
return 1;
}
if (print_tree_errors(ast)) {
cJSON_Delete(ast);
return 1;
}
JSRuntime *rt = JS_NewRuntime();
if (!rt) {
printf("Failed to create JS runtime\n");
cJSON_Delete(ast);
return 1;
}
JSContext *ctx = JS_NewContextWithHeapSize(rt, 16 * 1024 * 1024);
if (!ctx) {
printf("Failed to create JS context\n");
cJSON_Delete(ast); JS_FreeRuntime(rt);
return 1;
}
JS_FreeValue(ctx, js_blob_use(ctx));
JSValue result = JS_RunMachTree(ctx, ast, JS_NULL);
cJSON_Delete(ast);
int exit_code = 0;
if (JS_IsException(result)) {
JS_GetException(ctx);
exit_code = 1;
} else if (!JS_IsNull(result)) {
const char *str = JS_ToCString(ctx, result);
if (str) {
printf("%s\n", str);
JS_FreeCString(ctx, str);
}
}
JS_FreeContext(ctx);
JS_FreeRuntime(rt);
return exit_code;
}
/* Default: run script through mach-run bootstrap pipeline */
/* Default: run script through bootstrap pipeline */
int use_mcode = 0;
int arg_start = 1;
if (argc >= 3 && strcmp(argv[1], "--mcode") == 0) {
@@ -652,21 +286,16 @@ int cell_init(int argc, char **argv)
if (!find_cell_shop()) return 1;
size_t boot_size;
char *boot_data = load_core_file("internal/bootstrap.cm", &boot_size);
if (!boot_data) {
printf("ERROR: Could not load internal/bootstrap.cm from %s\n", core_path);
char *boot_json = load_core_file(BOOTSTRAP_AST, &boot_size);
if (!boot_json) {
printf("ERROR: Could not load %s from %s\n", BOOTSTRAP_AST, core_path);
return 1;
}
cJSON *boot_ast = JS_ASTTree(boot_data, boot_size, "internal/bootstrap.cm");
free(boot_data);
cJSON *boot_ast = cJSON_Parse(boot_json);
free(boot_json);
if (!boot_ast) {
printf("Failed to parse internal/bootstrap.cm\n");
return 1;
}
if (print_tree_errors(boot_ast)) {
cJSON_Delete(boot_ast);
printf("Failed to parse %s\n", BOOTSTRAP_AST);
return 1;
}
@@ -748,7 +377,7 @@ double cell_random() {
void cell_trace_sethook(cell_hook)
{
}
int uncaught_exception(JSContext *js, JSValue v)