diff --git a/build.cm b/build.cm index 1c7bbbcd..73c29df2 100644 --- a/build.cm +++ b/build.cm @@ -26,16 +26,18 @@ function get_local_dir() { } // Replace sigils in a string -// Currently supports: $LOCAL -> .cell/local full path -function replace_sigils(str) { - return replace(str, '$LOCAL', get_local_dir()) +// Supports: $LOCAL -> .cell/local, $PACKAGE -> package dir (if provided) +function replace_sigils(str, pkg_dir) { + var r = replace(str, '$LOCAL', get_local_dir()) + if (pkg_dir) r = replace(r, '$PACKAGE', pkg_dir) + return r } // Replace sigils in an array of flags -function replace_sigils_array(flags) { +function replace_sigils_array(flags, pkg_dir) { var result = [] arrfor(flags, function(flag) { - push(result, replace_sigils(flag)) + push(result, replace_sigils(flag, pkg_dir)) }) return result } @@ -101,8 +103,9 @@ Build.ensure_dir = ensure_dir // Compile a single C file for a package // Returns the object file path (content-addressed in .cell/build) -Build.compile_file = function(pkg, file, target, buildtype) { - var _buildtype = buildtype || 'release' +Build.compile_file = function(pkg, file, target, opts) { + var _opts = opts || {} + var _buildtype = _opts.buildtype || 'release' var pkg_dir = shop.get_package_dir(pkg) var src_path = pkg_dir + '/' + file var core_dir = null @@ -112,8 +115,8 @@ Build.compile_file = function(pkg, file, target, buildtype) { return null } - // Get flags (with sigil replacement) - var cflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'CFLAGS', target)) + // Use pre-fetched cflags if provided, otherwise fetch them + var cflags = _opts.cflags || replace_sigils_array(pkg_tools.get_flags(pkg, 'CFLAGS', target), pkg_dir) var target_cflags = toolchains[target].c_args || [] var cc = toolchains[target].c @@ -144,8 +147,12 @@ Build.compile_file = function(pkg, file, target, buildtype) { // Add package CFLAGS (resolve relative -I paths) arrfor(cflags, function(flag) { var f = flag + var ipath = null if (starts_with(f, '-I') && !starts_with(f, '-I/')) { - f = '-I"' + pkg_dir + '/' + text(f, 2) + '"' + ipath = text(f, 2) + if (!starts_with(ipath, pkg_dir)) { + f = '-I"' + pkg_dir + '/' + ipath + '"' + } } push(cmd_parts, f) }) @@ -179,6 +186,7 @@ Build.compile_file = function(pkg, file, target, buildtype) { var ret = os.system(full_cmd) if (ret != 0) { print('Compilation failed: ' + file) + print('Command: ' + full_cmd) return null } @@ -193,8 +201,12 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) { var c_files = pkg_tools.get_c_files(pkg, _target, exclude_main) var objects = [] + // Pre-fetch cflags once + var pkg_dir = shop.get_package_dir(pkg) + var cached_cflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'CFLAGS', _target), pkg_dir) + arrfor(c_files, function(file) { - var obj = Build.compile_file(pkg, file, _target, _buildtype) + var obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: cached_cflags}) push(objects, obj) }) @@ -235,7 +247,7 @@ Build.build_module_dylib = function(pkg, file, target, opts) { var _target = target || Build.detect_host_target() var _buildtype = _opts.buildtype || 'release' var _extra = _opts.extra_objects || [] - var obj = Build.compile_file(pkg, file, _target, _buildtype) + var obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags}) if (!obj) return null var tc = toolchains[_target] @@ -245,13 +257,17 @@ Build.build_module_dylib = function(pkg, file, target, opts) { var pkg_dir = shop.get_package_dir(pkg) // Get link flags - var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target)) + var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target), pkg_dir) var target_ldflags = tc.c_link_args || [] var resolved_ldflags = [] arrfor(ldflags, function(flag) { var f = flag + var lpath = null if (starts_with(f, '-L') && !starts_with(f, '-L/')) { - f = '-L"' + pkg_dir + '/' + text(f, 2) + '"' + lpath = text(f, 2) + if (!starts_with(lpath, pkg_dir)) { + f = '-L"' + pkg_dir + '/' + lpath + '"' + } } push(resolved_ldflags, f) }) @@ -333,17 +349,23 @@ Build.build_dynamic = function(pkg, target, buildtype) { var c_files = pkg_tools.get_c_files(pkg, _target, true) var results = [] + // Pre-fetch cflags once to avoid repeated TOML reads + var pkg_dir = shop.get_package_dir(pkg) + var cached_cflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'CFLAGS', _target), pkg_dir) + + log.console(`CFLAGS ${pkg}: ${text(cached_cflags, '|')}`) + // Compile support sources to cached objects var sources = pkg_tools.get_sources(pkg) var support_objects = [] arrfor(sources, function(src_file) { - var obj = Build.compile_file(pkg, src_file, _target, _buildtype) + var obj = Build.compile_file(pkg, src_file, _target, {buildtype: _buildtype, cflags: cached_cflags}) push(support_objects, obj) }) arrfor(c_files, function(file) { var sym_name = shop.c_symbol_for_file(pkg, file) - var dylib = Build.build_module_dylib(pkg, file, _target, {buildtype: _buildtype, extra_objects: support_objects}) + var dylib = Build.build_module_dylib(pkg, file, _target, {buildtype: _buildtype, extra_objects: support_objects, cflags: cached_cflags}) if (dylib) { push(results, {file: file, symbol: sym_name, dylib: dylib}) } @@ -378,8 +400,8 @@ Build.build_static = function(packages, target, output, buildtype) { }) // Collect LDFLAGS (with sigil replacement) - var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target)) var pkg_dir = shop.get_package_dir(pkg) + var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target), pkg_dir) // Deduplicate based on the entire LDFLAGS string for this package var ldflags_key = pkg + ':' + text(ldflags, ' ') @@ -387,8 +409,12 @@ Build.build_static = function(packages, target, output, buildtype) { seen_flags[ldflags_key] = true arrfor(ldflags, function(flag) { var f = flag + var lpath = null if (starts_with(f, '-L') && !starts_with(f, '-L/')) { - f = '-L"' + pkg_dir + '/' + text(f, 2) + '"' + lpath = text(f, 2) + if (!starts_with(lpath, pkg_dir)) { + f = '-L"' + pkg_dir + '/' + lpath + '"' + } } push(all_ldflags, f) }) diff --git a/cell.toml b/cell.toml index dc36b553..fb940045 100644 --- a/cell.toml +++ b/cell.toml @@ -1,8 +1,2 @@ -[dependencies] -cell-steam = "/Users/johnalanbrook/work/cell-steam" -cell-image = "/Users/johnalanbrook/work/cell-image" -cell-sdl3 = "/Users/johnalanbrook/work/cell-sdl3" -[compilation] -[compilation] [compilation.playdate] -CFLAGS = "-DMINIZ_NO_TIME -DTARGET_EXTENSION -DTARGET_PLAYDATE -I$LOCAL/PlaydateSDK/C_API" \ No newline at end of file +CFLAGS = "-DMINIZ_NO_TIME -DTARGET_EXTENSION -DTARGET_PLAYDATE -I$LOCAL/PlaydateSDK/C_API" diff --git a/docs/c-modules.md b/docs/c-modules.md index e74a8c25..17dd9071 100644 --- a/docs/c-modules.md +++ b/docs/c-modules.md @@ -228,11 +228,105 @@ var d = vector.dot(1, 0, 0, 1) // 0 C files are automatically compiled when you run: ```bash -pit build -pit update +cell --dev build ``` -Each C file is compiled into a per-file dynamic library at `~/.pit/lib//.dylib`. +Each C file is compiled into a per-file dynamic library at `.cell/lib//.dylib`. + +## Compilation Flags (cell.toml) + +Use the `[compilation]` section in `cell.toml` to pass compiler and linker flags: + +```toml +[compilation] +CFLAGS = "-Isrc -Ivendor/include" +LDFLAGS = "-lz -lm" +``` + +### Include paths + +Relative `-I` paths are resolved from the package root: + +```toml +CFLAGS = "-Isdk/public" +``` + +If your package is at `/path/to/mypkg`, this becomes `-I/path/to/mypkg/sdk/public`. + +Absolute paths are passed through unchanged. + +### Library paths + +Relative `-L` paths work the same way: + +```toml +LDFLAGS = "-Lsdk/lib -lmylib" +``` + +### Target-specific flags + +Add sections named `[compilation.]` for platform-specific flags: + +```toml +[compilation] +CFLAGS = "-Isdk/public" + +[compilation.macos_arm64] +LDFLAGS = "-Lsdk/lib/osx -lmylib" + +[compilation.linux] +LDFLAGS = "-Lsdk/lib/linux64 -lmylib" + +[compilation.windows] +LDFLAGS = "-Lsdk/lib/win64 -lmylib64" +``` + +Available targets: `macos_arm64`, `macos_x86_64`, `linux`, `linux_arm64`, `windows`. + +### Sigils + +Use `$LOCAL` in flags to refer to the `.cell/local` directory (for prebuilt libraries): + +```toml +LDFLAGS = "-L$LOCAL -lmyprebuilt" +``` + +### Example: vendored SDK + +A package wrapping an external SDK with platform-specific shared libraries: + +``` +mypkg/ +├── cell.toml +├── wrapper.cpp +└── sdk/ + ├── public/ + │ └── mylib/ + │ └── api.h + └── lib/ + ├── osx/ + │ └── libmylib.dylib + └── linux64/ + └── libmylib.so +``` + +```toml +[compilation] +CFLAGS = "-Isdk/public" + +[compilation.macos_arm64] +LDFLAGS = "-Lsdk/lib/osx -lmylib" + +[compilation.linux] +LDFLAGS = "-Lsdk/lib/linux64 -lmylib" +``` + +```cpp +// wrapper.cpp +#include "cell.h" +#include +// ... +``` ## Platform-Specific Code diff --git a/docs/cli.md b/docs/cli.md index c4ac7d88..80466131 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -13,7 +13,7 @@ type: "docs" pit [arguments] ``` -## Commands +## General ### pit version @@ -24,57 +24,32 @@ pit version # 0.1.0 ``` -### pit install +### pit help -Install a package to the shop. +Display help information. ```bash -pit install gitea.pockle.world/john/prosperon -pit install /Users/john/local/mypackage # local path +pit help +pit help ``` -### pit update +## Package Commands -Update packages from remote sources. +These commands operate on a package's `cell.toml`, source files, or build artifacts. + +### pit add + +Add a dependency to the current package. Installs the package to the shop, builds any C modules, and updates `cell.toml`. ```bash -pit update # update all packages -pit update # update specific package +pit add gitea.pockle.world/john/prosperon # remote, default alias +pit add gitea.pockle.world/john/prosperon myalias # remote, custom alias +pit add /Users/john/work/mylib # local path (symlinked) +pit add . # current directory +pit add ../sibling-package # relative path ``` -### pit remove - -Remove a package from the shop. Removes the lock entry, the package directory (or symlink), and any built dylibs. - -```bash -pit remove gitea.pockle.world/john/oldpackage -pit remove /Users/john/work/mylib # local path -pit remove . # current directory -pit remove mypackage --dry-run # show what would be removed -pit remove mypackage --prune # also remove orphaned dependencies -``` - -Options: -- `--prune` — also remove packages that are no longer needed by any remaining root -- `--dry-run` — show what would be removed without removing anything - -### pit list - -List installed packages. - -```bash -pit list # list all installed packages -pit list # list dependencies of a package -``` - -### pit ls - -List modules and actors in a package. - -```bash -pit ls # list files in current project -pit ls # list files in specified package -``` +For local paths, the package is symlinked into the shop rather than copied. Changes to the source directory are immediately visible. ### pit build @@ -106,85 +81,23 @@ pit test package all # run tests from all packages pit test suite --verify --diff # with IR verification and differential testing ``` -### pit link +### pit ls -Manage local package links for development. +List modules and actors in a package. ```bash -pit link add # link a package -pit link list # show all links -pit link delete # remove a link -pit link clear # remove all links +pit ls # list files in current project +pit ls # list files in specified package ``` -### pit fetch +### pit audit -Fetch package sources without extracting. +Test-compile all `.ce` and `.cm` scripts in package(s). Continues past failures and reports all errors at the end. ```bash -pit fetch -``` - -### pit upgrade - -Upgrade the ƿit installation itself. - -```bash -pit upgrade -``` - -### pit clean - -Clean build artifacts. - -```bash -pit clean -``` - -### pit add - -Add a dependency to the current package. Installs the package to the shop, builds any C modules, and updates `cell.toml`. - -```bash -pit add gitea.pockle.world/john/prosperon # remote, default alias -pit add gitea.pockle.world/john/prosperon myalias # remote, custom alias -pit add /Users/john/work/mylib # local path (symlinked) -pit add . # current directory -pit add ../sibling-package # relative path -``` - -For local paths, the package is symlinked into the shop rather than copied. Changes to the source directory are immediately visible. - -### pit clone - -Clone a package to a local path and link it for development. - -```bash -pit clone gitea.pockle.world/john/prosperon ./prosperon -``` - -### pit unlink - -Remove a link created by `pit link` or `pit clone` and restore the original package. - -```bash -pit unlink gitea.pockle.world/john/prosperon -``` - -### pit search - -Search for packages, actors, or modules matching a query. - -```bash -pit search math -``` - -### pit why - -Show which installed packages depend on a given package (reverse dependency lookup). - -```bash -pit why gitea.pockle.world/john/prosperon +pit audit # audit all installed packages +pit audit # audit specific package +pit audit . # audit current directory ``` ### pit resolve @@ -220,16 +133,6 @@ pit verify --deep # traverse full dependency closure pit verify --target ``` -### pit audit - -Test-compile all `.ce` and `.cm` scripts in package(s). Continues past failures and reports all errors at the end. - -```bash -pit audit # audit all installed packages -pit audit # audit specific package -pit audit . # audit current directory -``` - ### pit pack Build a statically linked binary from a package and all its dependencies. @@ -253,13 +156,294 @@ pit config actor get # get actor config pit config actor set # set actor config ``` -### pit help +### pit bench -Display help information. +Run benchmarks with statistical analysis. Benchmark files are `.cm` modules in a package's `benches/` directory. ```bash -pit help -pit help +pit bench # run all benchmarks in current package +pit bench all # same as above +pit bench # run specific benchmark file +pit bench package # benchmark a named package +pit bench package # specific benchmark in a package +pit bench package all # benchmark all packages +``` + +Output includes median, mean, standard deviation, and percentiles for each benchmark. + +## Shop Commands + +These commands operate on the global shop (`~/.pit/`) or system-level state. + +### pit install + +Install a package to the shop. + +```bash +pit install gitea.pockle.world/john/prosperon +pit install /Users/john/local/mypackage # local path +``` + +### pit remove + +Remove a package from the shop. Removes the lock entry, the package directory (or symlink), and any built dylibs. + +```bash +pit remove gitea.pockle.world/john/oldpackage +pit remove /Users/john/work/mylib # local path +pit remove . # current directory +pit remove mypackage --dry-run # show what would be removed +pit remove mypackage --prune # also remove orphaned dependencies +``` + +Options: +- `--prune` — also remove packages that are no longer needed by any remaining root +- `--dry-run` — show what would be removed without removing anything + +### pit update + +Update packages from remote sources. + +```bash +pit update # update all packages +pit update # update specific package +``` + +### pit list + +List installed packages. + +```bash +pit list # list all installed packages +pit list # list dependencies of a package +``` + +### pit link + +Manage local package links for development. + +```bash +pit link add # link a package +pit link list # show all links +pit link delete # remove a link +pit link clear # remove all links +``` + +### pit unlink + +Remove a link created by `pit link` or `pit clone` and restore the original package. + +```bash +pit unlink gitea.pockle.world/john/prosperon +``` + +### pit clone + +Clone a package to a local path and link it for development. + +```bash +pit clone gitea.pockle.world/john/prosperon ./prosperon +``` + +### pit fetch + +Fetch package sources without extracting. + +```bash +pit fetch +``` + +### pit search + +Search for packages, actors, or modules matching a query. + +```bash +pit search math +``` + +### pit why + +Show which installed packages depend on a given package (reverse dependency lookup). + +```bash +pit why gitea.pockle.world/john/prosperon +``` + +### pit upgrade + +Upgrade the ƿit installation itself. + +```bash +pit upgrade +``` + +### pit clean + +Clean build artifacts. + +```bash +pit clean +``` + +## Developer Commands + +Compiler pipeline tools, analysis, and testing. These are primarily useful for developing the ƿit compiler and runtime. + +### Compiler Pipeline + +Each of these commands runs the compilation pipeline up to a specific stage and prints the intermediate output. They take a source file as input. + +### pit tokenize + +Tokenize a source file and output the token stream as JSON. + +```bash +pit tokenize +``` + +### pit parse + +Parse a source file and output the AST as JSON. + +```bash +pit parse +``` + +### pit fold + +Run constant folding and semantic analysis on a source file and output the simplified AST as JSON. + +```bash +pit fold +``` + +### pit mcode + +Compile a source file to mcode (machine-independent intermediate representation) and output as JSON. + +```bash +pit mcode +``` + +### pit streamline + +Apply the full optimization pipeline to a source file and output optimized mcode as JSON. + +```bash +pit streamline +``` + +### pit qbe + +Compile a source file to QBE intermediate language (for native code generation). + +```bash +pit qbe +``` + +### pit compile + +Compile a source file to a native dynamic library. + +```bash +pit compile # outputs .dylib to .cell/lib/ +pit compile +``` + +### pit run_native + +Compile a module natively and compare execution against interpreted mode, showing timing differences. + +```bash +pit run_native # compare interpreted vs native +pit run_native # pass argument to module function +``` + +### pit run_aot + +Ahead-of-time compile and execute a program natively. + +```bash +pit run_aot +``` + +### Analysis + +### pit explain + +Query the semantic index for symbol information at a specific source location or by name. + +```bash +pit explain --span # find symbol at position +pit explain --symbol [files...] # find symbol by name +pit explain --help # show usage +``` + +### pit index + +Build a semantic index for a source file and output symbol information as JSON. + +```bash +pit index # output to stdout +pit index -o index.json # output to file +pit index --help # show usage +``` + +### pit ir_report + +Optimizer flight recorder — capture detailed information about IR transformations during optimization. + +```bash +pit ir_report # per-pass JSON summaries (default) +pit ir_report --events # include rewrite events +pit ir_report --types # include type deltas +pit ir_report --ir-before=PASS # print IR before specific pass +pit ir_report --ir-after=PASS # print IR after specific pass +pit ir_report --ir-all # print IR before/after every pass +pit ir_report --full # all options combined +``` + +Output is NDJSON (newline-delimited JSON). + +### Testing + +### pit diff + +Differential testing — run tests with and without optimizations and compare results. + +```bash +pit diff # diff all test files in current package +pit diff # diff specific test file +pit diff tests/ # diff by path +``` + +### pit fuzz + +Random program fuzzer — generates random programs and checks for optimization correctness by comparing optimized vs unoptimized execution. + +```bash +pit fuzz # 100 iterations with random seed +pit fuzz # specific number of iterations +pit fuzz --seed # start at specific seed +pit fuzz --seed +``` + +Failures are saved to `tests/fuzz_failures/`. + +### pit vm_suite + +Run the VM stability test suite (641 tests covering arithmetic, strings, control flow, closures, objects, and more). + +```bash +pit vm_suite +``` + +### pit syntax_suite + +Run the syntax feature test suite (covers all literal types, operators, control flow, functions, prototypes, and more). + +```bash +pit syntax_suite ``` ## Package Locators diff --git a/internal/shop.cm b/internal/shop.cm index c80bebf8..6f4539f5 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -495,6 +495,7 @@ function resolve_mod_fn(path, pkg) { if (!fd.is_file(path)) { print(`path ${path} is not a file`); disrupt } var content = text(fd.slurp(path)) + if (length(content) == 0) { print(`${path}: empty file`); disrupt } var content_key = stone(blob(content)) var native_result = null var cached = null @@ -1051,6 +1052,7 @@ function fetch_remote_hash(pkg) { // Returns the zip blob or null on failure function download_zip(pkg, commit_hash) { var cache_path = get_cache_path(pkg, commit_hash) + ensure_dir(global_shop_path + '/cache') var download_url = Shop.get_download_url(pkg, commit_hash) if (!download_url) { @@ -1179,8 +1181,10 @@ Shop.extract = function(pkg) { var zip_blob = get_package_zip(pkg) - if (!zip_blob) - print("No zip blob available for " + pkg); disrupt + if (!zip_blob) { + print("No zip blob available for " + pkg) + disrupt + } // Extract zip for remote package install_zip(zip_blob, target_dir) diff --git a/package.cm b/package.cm index a4c7e0fa..a2b3d2b1 100644 --- a/package.cm +++ b/package.cm @@ -40,8 +40,13 @@ function get_path(name) return global_shop_path + '/packages/' + replace(name, '@', '_') } +var config_cache = {} + package.load_config = function(name) { + var cache_key = name || '_project_' + if (config_cache[cache_key]) return config_cache[cache_key] + var config_path = get_path(name) + '/cell.toml' if (!fd.is_file(config_path)) { @@ -54,9 +59,31 @@ package.load_config = function(name) var result = toml.decode(content) if (!result) { + print(`TOML decode returned null for ${config_path}`) return {} } + // Validate: if content has [compilation] but decode result doesn't, retry + var has_compilation = search(content, '[compilation') != null + if (has_compilation && !result.compilation) { + print(`TOML decode missing compilation for ${config_path}, retrying`) + var retry = 0 + while (retry < 3 && (!result || !result.compilation)) { + result = toml.decode(content) + retry = retry + 1 + } + if (!result) return {} + } + + if (has_compilation && result.compilation) { + var cf = result.compilation.CFLAGS + if (cf == null && search(content, 'CFLAGS') != null) { + print(`TOML has CFLAGS text but decode missing it for ${config_path}`) + print(`compilation keys: ${text(array(result.compilation), ',')}`) + } + } + + config_cache[cache_key] = result return result } diff --git a/source/cell.c b/source/cell.c index ebabea08..ee47027d 100644 --- a/source/cell.c +++ b/source/cell.c @@ -503,6 +503,7 @@ int cell_init(int argc, char **argv) core_override = "."; mkdir(".cell", 0755); mkdir(".cell/build", 0755); + mkdir(".cell/cache", 0755); mkdir(".cell/packages", 0755); /* Ensure .cell/packages/core -> . symlink exists */ struct stat lst;