john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'master' into quicken_build

Browse Source
This commit is contained in:
John Alanbrook
2026-02-20 18:00:18 -06:00
parent 680b257a44 c722b9d648
commit 55fae8b5d0
2 changed files with 210 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

334
build.cm
View File

@@ -236,32 +236,23 @@ function hash_all_deps(cmd_str, deps) {
// Compilation
// ============================================================================
// Compile a single C file for a package
// Returns the object file path (content-addressed in .cell/build)
Build.compile_file = function(pkg, file, target, opts) {
// Build the compile command string and common flags for a C file.
// Returns {cmd_str, src_path, cc, common_flags, pkg_dir} or null if source missing.
function compile_setup(pkg, file, target, opts) {
var _opts = opts || {}
var _buildtype = _opts.buildtype || 'release'
var pkg_dir = shop.get_package_dir(pkg)
var src_path = pkg_dir + '/' + file
var core_dir = null
if (!fd.is_file(src_path)) {
log.error('Source file not found: ' + src_path)
return null
}
if (!fd.is_file(src_path)) return null
// Use pre-fetched cflags if provided, otherwise fetch them
var cflags = _opts.cflags || replace_sigils_array(pkg_tools.get_flags(pkg, 'CFLAGS', target), pkg_dir)
var target_cflags = toolchains[target].c_args || []
var cc = toolchains[target].c
// Symbol name for this file
var sym_name = shop.c_symbol_for_file(pkg, file)
// Build common flags (shared between dep scan and compilation)
var common_flags = []
// Add buildtype-specific flags
if (_buildtype == 'release') {
common_flags = array(common_flags, ['-O3', '-DNDEBUG'])
} else if (_buildtype == 'debug') {
@@ -273,18 +264,15 @@ Build.compile_file = function(pkg, file, target, opts) {
push(common_flags, '-DCELL_USE_NAME=' + sym_name)
push(common_flags, '-I"' + pkg_dir + '"')
// Auto-discover include/ directory
if (fd.is_dir(pkg_dir + '/include')) {
push(common_flags, '-I"' + pkg_dir + '/include"')
}
// External packages need core's source dir for cell.h, quickjs.h, blob.h
if (pkg != 'core') {
core_dir = shop.get_package_dir('core')
push(common_flags, '-I"' + core_dir + '/source"')
}
// Add package CFLAGS (resolve relative -I paths)
arrfor(cflags, function(flag) {
var f = flag
var ipath = null
@@ -297,27 +285,64 @@ Build.compile_file = function(pkg, file, target, opts) {
push(common_flags, f)
})
// Add target CFLAGS
arrfor(target_cflags, function(flag) {
push(common_flags, flag)
})
// Build full compilation command
var cmd_parts = [cc, '-c', '-fPIC']
cmd_parts = array(cmd_parts, common_flags)
push(cmd_parts, '"' + src_path + '"')
var cmd_str = text(cmd_parts, ' ')
return {
cmd_str: text(cmd_parts, ' '),
src_path: src_path,
cc: cc,
common_flags: common_flags,
pkg_dir: pkg_dir
}
}
// Probe for the full content key (source + all deps + compile flags).
// Returns {full_content, deps, fail} or null if deps not cached yet (cold path).
function probe_source_key(setup, file) {
var file_content = memo_read(setup.src_path)
var quick_content = setup.cmd_str + '\n' + file_content
var fail_path = cache_path(quick_content, SALT_FAIL)
var deps_path = cache_path(quick_content, SALT_DEPS)
var deps = null
var full_content = null
if (fd.is_file(fail_path)) return {fail: true}
if (fd.is_file(deps_path)) {
deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
return length(p) > 0
})
full_content = hash_all_deps(setup.cmd_str, deps)
return {full_content: full_content, deps: deps}
}
return null
}
// Compile a single C file for a package
// Returns the object file path (content-addressed in .cell/build)
Build.compile_file = function(pkg, file, target, opts) {
var _opts = opts || {}
var setup = compile_setup(pkg, file, target, _opts)
if (!setup) {
log.error('Source file not found: ' + shop.get_package_dir(pkg) + '/' + file)
return null
}
if (_opts.verbose) {
log.build('[verbose] CFLAGS: ' + text(cflags, ' '))
log.build('[verbose] compile: ' + cmd_str)
log.build('[verbose] compile: ' + setup.cmd_str)
}
// Layer 2: stat-based manifest probe (zero file reads on warm cache)
var mf_obj = null
if (!_opts.force) {
mf_obj = bmfst_probe(cmd_str, src_path)
mf_obj = bmfst_probe(setup.cmd_str, setup.src_path)
if (mf_obj) {
if (_opts.verbose) log.build('[verbose] manifest hit: ' + file)
log.shop('manifest hit ' + file)
@@ -325,67 +350,71 @@ Build.compile_file = function(pkg, file, target, opts) {
}
}
// Two-level cache: quick hash for deps file, full hash for object
var file_content = memo_read(src_path)
var quick_content = cmd_str + '\n' + file_content
var deps_path = cache_path(quick_content, SALT_DEPS)
var fail_path = cache_path(quick_content, SALT_FAIL)
var build_dir = get_build_dir()
fd.ensure_dir(build_dir)
// Check for cached failure (skip files that previously failed to compile)
if (fd.is_file(fail_path)) {
var probe = probe_source_key(setup, file)
// Check for cached failure
if (probe && probe.fail) {
if (_opts.verbose) log.build('[verbose] skipping ' + file + ' (cached failure)')
log.shop('skip ' + file + ' (cached failure)')
return null
}
var deps = null
var full_content = null
var deps = null
var obj_path = null
// Warm path: read cached dep list, verify by hashing all deps
if (fd.is_file(deps_path)) {
deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
return length(p) > 0
})
full_content = hash_all_deps(cmd_str, deps)
// Warm path: deps cached, check object
if (probe && probe.full_content) {
full_content = probe.full_content
obj_path = cache_path(full_content, SALT_OBJ)
if (fd.is_file(obj_path)) {
if (_opts.verbose) log.build('[verbose] cache hit: ' + file)
log.shop('cache hit ' + file)
bmfst_save(cmd_str, src_path, deps, obj_path)
bmfst_save(setup.cmd_str, setup.src_path, probe.deps, obj_path)
return obj_path
}
log.shop('cache stale ' + file + ' (header changed)')
deps = probe.deps
}
// Cold path: run cc -MM to discover deps
log.shop('dep scan ' + file)
deps = get_c_deps(cc, common_flags, src_path)
full_content = hash_all_deps(cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
var file_content = null
var quick_content = null
var err_path = null
var full_cmd = null
var err_text = null
var missing = null
var err_lines = null
var first_err = null
var ret = null
// Check if object exists (might exist from previous build with same deps)
if (fd.is_file(obj_path)) {
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
if (_opts.verbose) log.build('[verbose] cache hit: ' + file + ' (after dep scan)')
log.shop('cache hit ' + file + ' (after dep scan)')
bmfst_save(cmd_str, src_path, deps, obj_path)
return obj_path
// Cold path: run cc -MM to discover deps
if (!deps) {
log.shop('dep scan ' + file)
deps = get_c_deps(setup.cc, setup.common_flags, setup.src_path)
full_content = hash_all_deps(setup.cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
// Check if object exists (might exist from previous build with same deps)
if (fd.is_file(obj_path)) {
file_content = memo_read(setup.src_path)
quick_content = setup.cmd_str + '\n' + file_content
fd.slurpwrite(cache_path(quick_content, SALT_DEPS), stone(blob(text(deps, '\n'))))
if (_opts.verbose) log.build('[verbose] cache hit: ' + file + ' (after dep scan)')
log.shop('cache hit ' + file + ' (after dep scan)')
bmfst_save(setup.cmd_str, setup.src_path, deps, obj_path)
return obj_path
}
}
// Compile
log.shop('compiling ' + file)
log.console('Compiling ' + file)
var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log'
var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
var err_text = null
var missing = null
var err_lines = null
var first_err = null
var ret = os.system(full_cmd)
err_path = '/tmp/cell_build_err_' + content_hash(setup.src_path) + '.log'
full_cmd = setup.cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
ret = os.system(full_cmd)
if (ret != 0) {
if (fd.is_file(err_path)) {
err_text = text(fd.slurp(err_path))
@@ -403,14 +432,17 @@ Build.compile_file = function(pkg, file, target, opts) {
if (err_text) log.error(err_text)
else log.error('Command: ' + full_cmd)
}
// Cache the failure so we don't retry on every build
fd.slurpwrite(fail_path, stone(blob(err_text || 'compilation failed')))
file_content = memo_read(setup.src_path)
quick_content = setup.cmd_str + '\n' + file_content
fd.slurpwrite(cache_path(quick_content, SALT_FAIL), stone(blob(err_text || 'compilation failed')))
return null
}
// Save deps for future warm-path lookups
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
bmfst_save(cmd_str, src_path, deps, obj_path)
file_content = memo_read(setup.src_path)
quick_content = setup.cmd_str + '\n' + file_content
fd.slurpwrite(cache_path(quick_content, SALT_DEPS), stone(blob(text(deps, '\n'))))
bmfst_save(setup.cmd_str, setup.src_path, deps, obj_path)
return obj_path
}
@@ -438,46 +470,42 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) {
// Dynamic library building
// ============================================================================
// Compute link content string from all inputs that affect the dylib output
function compute_link_content(objects, ldflags, target_ldflags, opts) {
// Sort objects for deterministic hash
var sorted_objects = sort(objects)
// Build a string representing all link inputs
var parts = []
push(parts, 'target:' + text(opts.target))
push(parts, 'cc:' + text(opts.cc))
arrfor(sorted_objects, function(obj) {
// Object paths are content-addressed, so the path itself is the hash
push(parts, 'obj:' + text(obj))
// Compute dylib content key from source content key + link info
// link_opts: {extra_objects, ldflags, target_ldflags, target, cc}
function compute_dylib_content(full_content, link_opts) {
var parts = [full_content]
push(parts, 'target:' + text(link_opts.target))
push(parts, 'cc:' + text(link_opts.cc))
arrfor(link_opts.extra_objects, function(obj) {
if (obj != null) push(parts, 'extra:' + text(obj))
})
arrfor(ldflags, function(flag) {
arrfor(link_opts.ldflags, function(flag) {
push(parts, 'ldflag:' + text(flag))
})
arrfor(target_ldflags, function(flag) {
arrfor(link_opts.target_ldflags, function(flag) {
push(parts, 'target_ldflag:' + text(flag))
})
return text(parts, '\n')
}
// Build a per-module dynamic library for a single C file
// Returns the content-addressed dylib path in .cell/build/<hash>.<target>.dylib
// Returns the content-addressed dylib path in .cell/build/<hash>
// Checks dylib cache first; only compiles the object if the dylib is stale.
Build.build_module_dylib = function(pkg, file, target, opts) {
var _opts = opts || {}
var _target = target || Build.detect_host_target()
var _buildtype = _opts.buildtype || 'release'
var _extra = _opts.extra_objects || []
var obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags, force: _opts.force})
if (!obj) return null
var setup = compile_setup(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags})
if (!setup) return null
var tc = toolchains[_target]
var cc = tc.cpp || tc.c
var local_dir = get_local_dir()
var pkg_dir = shop.get_package_dir(pkg)
// Get link flags
var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target), pkg_dir)
var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target), setup.pkg_dir)
var target_ldflags = tc.c_link_args || []
var resolved_ldflags = []
arrfor(ldflags, function(flag) {
@@ -485,70 +513,122 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
var lpath = null
if (starts_with(f, '-L') && !starts_with(f, '-L/')) {
lpath = text(f, 2)
if (!starts_with(lpath, pkg_dir)) {
f = '-L"' + pkg_dir + '/' + lpath + '"'
if (!starts_with(lpath, setup.pkg_dir)) {
f = '-L"' + setup.pkg_dir + '/' + lpath + '"'
}
}
push(resolved_ldflags, f)
})
// Content-addressed output: hash of (all objects + link flags + target)
var all_objects = [obj]
all_objects = array(all_objects, _extra)
var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
var build_dir = get_build_dir()
fd.ensure_dir(build_dir)
var dylib_path = cache_path(link_content, SALT_DYLIB)
var link_info = {extra_objects: _extra, ldflags: resolved_ldflags, target_ldflags: target_ldflags, target: _target, cc: cc}
// Probe source key — check dylib cache before compiling
var probe = probe_source_key(setup, file)
var dylib_path = null
var dylib_content = null
var obj = null
var obj_path = null
var cmd_parts = null
var cmd_str = null
var ret = null
var post_probe = null
var fallback_probe = null
if (probe && probe.fail) {
log.shop('skip ' + file + ' (cached failure)')
return null
}
if (probe && probe.full_content) {
dylib_content = compute_dylib_content(probe.full_content, link_info)
dylib_path = cache_path(dylib_content, SALT_DYLIB)
if (!_opts.force && fd.is_file(dylib_path)) {
log.shop('cache hit ' + file)
return dylib_path
}
// Dylib stale but object might be cached — check before compiling
obj_path = cache_path(probe.full_content, SALT_OBJ)
if (fd.is_file(obj_path)) {
obj = obj_path
}
}
// Object not cached — compile it
if (!obj) {
obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags, force: _opts.force})
if (!obj) return null
// Recompute dylib key with the now-known source key
if (!dylib_path) {
post_probe = probe_source_key(setup, file)
if (post_probe && post_probe.full_content) {
dylib_content = compute_dylib_content(post_probe.full_content, link_info)
dylib_path = cache_path(dylib_content, SALT_DYLIB)
if (fd.is_file(dylib_path)) return dylib_path
}
}
}
// Need dylib_path for output
if (!dylib_path) {
// Fallback: probe should succeed now since compile_file cached deps
fallback_probe = probe_source_key(setup, file)
if (fallback_probe && fallback_probe.full_content) {
dylib_content = compute_dylib_content(fallback_probe.full_content, link_info)
dylib_path = cache_path(dylib_content, SALT_DYLIB)
} else {
log.error('Cannot compute dylib key for ' + file)
return null
}
}
if (_opts.verbose) {
log.build('[verbose] LDFLAGS: ' + text(resolved_ldflags, ' '))
}
if (!fd.is_file(dylib_path)) {
cmd_parts = [cc, '-shared', '-fPIC']
// Link
cmd_parts = [cc, '-shared', '-fPIC']
if (tc.system == 'darwin') {
cmd_parts = array(cmd_parts, [
'-undefined', 'dynamic_lookup',
'-Wl,-dead_strip',
'-Wl,-rpath,@loader_path/../local',
'-Wl,-rpath,' + local_dir
])
} else if (tc.system == 'linux') {
cmd_parts = array(cmd_parts, [
'-Wl,--allow-shlib-undefined',
'-Wl,--gc-sections',
'-Wl,-rpath,$ORIGIN/../local',
'-Wl,-rpath,' + local_dir
])
} else if (tc.system == 'windows') {
push(cmd_parts, '-Wl,--allow-shlib-undefined')
}
if (tc.system == 'darwin') {
cmd_parts = array(cmd_parts, [
'-undefined', 'dynamic_lookup',
'-Wl,-dead_strip',
'-Wl,-rpath,@loader_path/../local',
'-Wl,-rpath,' + local_dir
])
} else if (tc.system == 'linux') {
cmd_parts = array(cmd_parts, [
'-Wl,--allow-shlib-undefined',
'-Wl,--gc-sections',
'-Wl,-rpath,$ORIGIN/../local',
'-Wl,-rpath,' + local_dir
])
} else if (tc.system == 'windows') {
push(cmd_parts, '-Wl,--allow-shlib-undefined')
}
push(cmd_parts, '-L"' + local_dir + '"')
push(cmd_parts, '"' + text(obj) + '"')
arrfor(_extra, function(extra_obj) {
if (extra_obj != null) push(cmd_parts, '"' + text(extra_obj) + '"')
})
cmd_parts = array(cmd_parts, resolved_ldflags)
cmd_parts = array(cmd_parts, target_ldflags)
push(cmd_parts, '-o')
push(cmd_parts, '"' + dylib_path + '"')
push(cmd_parts, '-L"' + local_dir + '"')
push(cmd_parts, '"' + text(obj) + '"')
arrfor(_extra, function(extra_obj) {
if (extra_obj != null) push(cmd_parts, '"' + text(extra_obj) + '"')
})
cmd_parts = array(cmd_parts, resolved_ldflags)
cmd_parts = array(cmd_parts, target_ldflags)
push(cmd_parts, '-o')
push(cmd_parts, '"' + dylib_path + '"')
cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) log.build('[verbose] link: ' + cmd_str)
log.shop('linking ' + file)
log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
ret = os.system(cmd_str)
if (ret != 0) {
log.error('Linking failed: ' + file)
return null
}
} else {
log.shop('link cache hit ' + file)
cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) log.build('[verbose] link: ' + cmd_str)
log.shop('linking ' + file)
log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
ret = os.system(cmd_str)
if (ret != 0) {
log.error('Linking failed: ' + file)
return null
}
return dylib_path

8
link.cm
View File

@@ -86,7 +86,7 @@ Link.add = function(canonical, target, shop) {
}
} else {
// Remote package target - ensure it's installed
shop.get(target)
shop.sync(target)
}
var links = Link.load()
@@ -115,8 +115,7 @@ Link.add = function(canonical, target, shop) {
}
// Install the dependency if not already in shop
var _get_dep = function() {
shop.get(dep_locator)
shop.extract(dep_locator)
shop.sync(dep_locator)
} disruption {
log.build(` Warning: Could not install dependency ${dep_locator}`)
}
@@ -233,8 +232,7 @@ Link.sync_all = function(shop) {
}
// Install the dependency if not already in shop
var _get = function() {
shop.get(dep_locator)
shop.extract(dep_locator)
shop.sync(dep_locator)
} disruption {
// Silently continue - dependency may already be installed
}