From 680b257a449f51f5fa1c376660b5e71cd94243bb Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Fri, 20 Feb 2026 17:52:25 -0600 Subject: [PATCH] fingerprint hash --- build.ce | 4 +-- build.cm | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/build.ce b/build.ce index 3f7aebc4..1358d016 100644 --- a/build.ce +++ b/build.ce @@ -100,7 +100,7 @@ if (target_package) { // Build single package log.console('Building ' + target_package + '...') _build = function() { - lib = build.build_dynamic(target_package, target, buildtype, {verbose: verbose}) + lib = build.build_dynamic(target_package, target, buildtype, {verbose: verbose, force: force_rebuild}) if (lib) { log.console(`Built ${text(length(lib))} module(s)`) } @@ -112,7 +112,7 @@ if (target_package) { } else { // Build all packages log.console('Building all packages...') - results = build.build_all_dynamic(target, buildtype, {verbose: verbose}) + results = build.build_all_dynamic(target, buildtype, {verbose: verbose, force: force_rebuild}) success = 0 failed = 0 diff --git a/build.cm b/build.cm index 277b6e95..7439f09c 100644 --- a/build.cm +++ b/build.cm @@ -13,9 +13,36 @@ var os = use('internal/os') var toolchains = use('toolchains') var shop = use('internal/shop') var pkg_tools = use('package') +var json = use('json') var Build = {} +// ============================================================================ +// Per-run memoization caches (reset when process exits) +// ============================================================================ + +var _stat_done = {} +var _stat_fp = {} +var _read_cache = {} + +function memo_stat(path) { + var st = null + if (!_stat_done[path]) { + _stat_done[path] = true + st = fd.stat(path) + if (st.mtime != null) + _stat_fp[path] = {m: st.mtime, s: st.size} + } + return _stat_fp[path] +} + +function memo_read(path) { + if (_read_cache[path] != null) return _read_cache[path] + if (!memo_stat(path)) return null + _read_cache[path] = text(fd.slurp(path)) + return _read_cache[path] +} + // ============================================================================ // Sigil replacement // ============================================================================ @@ -100,6 +127,7 @@ var SALT_MACH = 'mach' // mach bytecode blob var SALT_MCODE = 'mcode' // mcode IR (JSON) var SALT_DEPS = 'deps' // cached cc -MM dependency list var SALT_FAIL = 'fail' // cached compilation failure +var SALT_BMFST = 'bmfst' // stat-based build manifest function cache_path(content, salt) { return get_build_dir() + '/' + content_hash(content + '\n' + salt) @@ -120,6 +148,43 @@ function get_build_dir() { Build.ensure_dir = fd.ensure_dir +// ============================================================================ +// Stat-based build manifest (zero-read warm cache) +// ============================================================================ + +function bmfst_path(cmd_str, src_path) { + return cache_path(cmd_str + '\n' + src_path, SALT_BMFST) +} + +function bmfst_probe(cmd_str, src_path) { + var mf_path = bmfst_path(cmd_str, src_path) + if (!fd.is_file(mf_path)) return null + var mf = json.decode(text(fd.slurp(mf_path))) + if (!mf || !mf.d || !mf.o) return null + if (!fd.is_file(mf.o)) return null + var ok = true + arrfor(mf.d, function(entry) { + if (!ok) return + var st = memo_stat(entry.p) + if (!st || st.m != entry.m || st.s != entry.s) + ok = false + }) + if (!ok) return null + return mf.o +} + +function bmfst_save(cmd_str, src_path, deps, obj_path) { + var entries = [] + arrfor(deps, function(dep_path) { + var st = memo_stat(dep_path) + if (st) + push(entries, {p: dep_path, m: st.m, s: st.s}) + }) + var mf = {o: obj_path, d: entries} + var mf_path = bmfst_path(cmd_str, src_path) + fd.slurpwrite(mf_path, stone(blob(json.encode(mf)))) +} + // ============================================================================ // Dependency scanning helpers // ============================================================================ @@ -158,8 +223,9 @@ function get_c_deps(cc, flags, src_path) { function hash_all_deps(cmd_str, deps) { var parts = [cmd_str] arrfor(deps, function(dep_path) { - if (fd.is_file(dep_path)) - push(parts, dep_path + '\n' + text(fd.slurp(dep_path))) + var content = memo_read(dep_path) + if (content != null) + push(parts, dep_path + '\n' + content) else push(parts, dep_path + '\n') }) @@ -248,9 +314,20 @@ Build.compile_file = function(pkg, file, target, opts) { log.build('[verbose] compile: ' + cmd_str) } + // Layer 2: stat-based manifest probe (zero file reads on warm cache) + var mf_obj = null + if (!_opts.force) { + mf_obj = bmfst_probe(cmd_str, src_path) + if (mf_obj) { + if (_opts.verbose) log.build('[verbose] manifest hit: ' + file) + log.shop('manifest hit ' + file) + return mf_obj + } + } + // Two-level cache: quick hash for deps file, full hash for object - var file_content = fd.slurp(src_path) - var quick_content = cmd_str + '\n' + text(file_content) + var file_content = memo_read(src_path) + var quick_content = cmd_str + '\n' + file_content var deps_path = cache_path(quick_content, SALT_DEPS) var fail_path = cache_path(quick_content, SALT_FAIL) @@ -278,6 +355,7 @@ Build.compile_file = function(pkg, file, target, opts) { if (fd.is_file(obj_path)) { if (_opts.verbose) log.build('[verbose] cache hit: ' + file) log.shop('cache hit ' + file) + bmfst_save(cmd_str, src_path, deps, obj_path) return obj_path } log.shop('cache stale ' + file + ' (header changed)') @@ -294,6 +372,7 @@ Build.compile_file = function(pkg, file, target, opts) { fd.slurpwrite(deps_path, stone(blob(text(deps, '\n')))) if (_opts.verbose) log.build('[verbose] cache hit: ' + file + ' (after dep scan)') log.shop('cache hit ' + file + ' (after dep scan)') + bmfst_save(cmd_str, src_path, deps, obj_path) return obj_path } @@ -331,6 +410,7 @@ Build.compile_file = function(pkg, file, target, opts) { // Save deps for future warm-path lookups fd.slurpwrite(deps_path, stone(blob(text(deps, '\n')))) + bmfst_save(cmd_str, src_path, deps, obj_path) return obj_path } @@ -388,7 +468,7 @@ Build.build_module_dylib = function(pkg, file, target, opts) { var _target = target || Build.detect_host_target() var _buildtype = _opts.buildtype || 'release' var _extra = _opts.extra_objects || [] - var obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags}) + var obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags, force: _opts.force}) if (!obj) return null var tc = toolchains[_target] @@ -493,21 +573,20 @@ Build.build_dynamic = function(pkg, target, buildtype, opts) { var support_objects = [] if (pkg != 'core') { arrfor(sources, function(src_file) { - var obj = Build.compile_file(pkg, src_file, _target, {buildtype: _buildtype, cflags: cached_cflags, verbose: _opts.verbose}) + var obj = Build.compile_file(pkg, src_file, _target, {buildtype: _buildtype, cflags: cached_cflags, verbose: _opts.verbose, force: _opts.force}) if (obj != null) push(support_objects, obj) }) } arrfor(c_files, function(file) { var sym_name = shop.c_symbol_for_file(pkg, file) - var dylib = Build.build_module_dylib(pkg, file, _target, {buildtype: _buildtype, extra_objects: support_objects, cflags: cached_cflags, verbose: _opts.verbose}) + var dylib = Build.build_module_dylib(pkg, file, _target, {buildtype: _buildtype, extra_objects: support_objects, cflags: cached_cflags, verbose: _opts.verbose, force: _opts.force}) if (dylib) { push(results, {file: file, symbol: sym_name, dylib: dylib}) } }) // Write manifest so runtime can find dylibs without the build module - var json = use('json') var mpath = manifest_path(pkg) fd.slurpwrite(mpath, stone(blob(json.encode(results)))) @@ -819,7 +898,6 @@ Build.compile_cm_to_mach = function(src_path) { if (!fd.is_file(src_path)) { log.error('Source file not found: ' + src_path); disrupt } - var json = use('json') var optimized = shop.compile_file(src_path) return mach_compile_mcode_bin(src_path, json.encode(optimized)) } @@ -829,7 +907,6 @@ Build.compile_cm_to_mach = function(src_path) { // output: path to write the generated .c file Build.generate_module_table = function(modules, output) { var lines = [] - var json = use('json') push(lines, '/* Generated module table — do not edit */') push(lines, '#include ') push(lines, '#include ') @@ -935,6 +1012,7 @@ Build.SALT_MACH = SALT_MACH Build.SALT_MCODE = SALT_MCODE Build.SALT_DEPS = SALT_DEPS Build.SALT_FAIL = SALT_FAIL +Build.SALT_BMFST = SALT_BMFST Build.cache_path = cache_path Build.manifest_path = manifest_path Build.native_sanitize_flags = native_sanitize_flags