cell/build.cm

// build.cm - Simplified build utilities for Cell
//
// Key functions:
//   Build.compile_file(pkg, file, target)     - Compile a C file, returns object path
//   Build.build_package(pkg, target)          - Build all C files for a package
//   Build.build_dynamic(pkg, target)          - Build dynamic library for a package
//   Build.build_static(packages, target, output) - Build static binary

var fd = use('fd')
var crypto = use('crypto')
var blob = use('blob')
var os = use('os')
var toolchains = use('toolchains')
var shop = use('internal/shop')
var pkg_tools = use('package')

var Build = {}

// ============================================================================
// Sigil replacement
// ============================================================================

// Get the local directory for prebuilt libraries
function get_local_dir() {
  return shop.get_local_dir()
}

// Replace sigils in a string
// Currently supports: $LOCAL -> .cell/local full path
function replace_sigils(str) {
  return replace(str, '$LOCAL', get_local_dir())
}

// Replace sigils in an array of flags
function replace_sigils_array(flags) {
  var result = []
  arrfor(flags, function(flag) {
    push(result, replace_sigils(flag))
  })
  return result
}

Build.get_local_dir = get_local_dir

// ============================================================================
// Toolchain helpers
// ============================================================================

Build.list_targets = function() {
  return array(toolchains)
}

Build.has_target = function(target) {
  return toolchains[target] != null
}

Build.detect_host_target = function() {
  var platform = os.platform()
  var arch = os.arch ? os.arch() : 'arm64'
  
  if (platform == 'macOS' || platform == 'darwin') {
    return arch == 'x86_64' ? 'macos_x86_64' : 'macos_arm64'
  } else if (platform == 'Linux' || platform == 'linux') {
    return arch == 'x86_64' ? 'linux' : 'linux_arm64'
  } else if (platform == 'Windows' || platform == 'windows') {
    return 'windows'
  }
  return null
}

// ============================================================================
// Content-addressed build cache
// ============================================================================

function content_hash(str) {
  var bb = stone(blob(str))
  return text(crypto.blake2(bb, 32), 'h')
}

function get_build_dir() {
  return shop.get_build_dir()
}

function ensure_dir(path) {
  if (fd.stat(path).isDirectory) return
  var parts = array(path, '/')
  var current = starts_with(path, '/') ? '/' : ''
  for (var i = 0; i < length(parts); i++) {
    if (parts[i] == '') continue
    current += parts[i] + '/'
    if (!fd.stat(current).isDirectory) fd.mkdir(current)
  }
}

Build.ensure_dir = ensure_dir

// ============================================================================
// Compilation
// ============================================================================

// Compile a single C file for a package
// Returns the object file path (content-addressed in .cell/build)
Build.compile_file = function(pkg, file, target, buildtype = 'release') {
  var pkg_dir = shop.get_package_dir(pkg)
  var src_path = pkg_dir + '/' + file
  
  if (!fd.is_file(src_path)) {
    throw Error('Source file not found: ' + src_path)
  }
  
  // Get flags (with sigil replacement)
  var cflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'CFLAGS', target))
  var target_cflags = toolchains[target].c_args || []
  var cc = toolchains[target].c
  
  // Symbol name for this file
  var sym_name = shop.c_symbol_for_file(pkg, file)
  
  // Build command
  var cmd_parts = [cc, '-c', '-fPIC']
  
  // Add buildtype-specific flags
  if (buildtype == 'release') {
    cmd_parts = array(cmd_parts, ['-O3', '-DNDEBUG'])
  } else if (buildtype == 'debug') {
    cmd_parts = array(cmd_parts, ['-O2', '-g'])
  } else if (buildtype == 'minsize') {
    cmd_parts = array(cmd_parts, ['-Os', '-DNDEBUG'])
  }
  
  push(cmd_parts, '-DCELL_USE_NAME=' + sym_name)
  push(cmd_parts, '-I"' + pkg_dir + '"')
  
  // Add package CFLAGS (resolve relative -I paths)
  arrfor(cflags, function(flag) {
    if (starts_with(flag, '-I') && !starts_with(flag, '-I/')) {
      flag = '-I"' + pkg_dir + '/' + text(flag, 2) + '"'
    }
    push(cmd_parts, flag)
  })
  
  // Add target CFLAGS
  arrfor(target_cflags, function(flag) {
    push(cmd_parts, flag)
  })
  
  push(cmd_parts, '"' + src_path + '"')
  
  var cmd_str = text(cmd_parts, ' ')
  
  // Content hash: command + file content
  var file_content = fd.slurp(src_path)
  var hash_input = cmd_str + '\n' + text(file_content)
  var hash = content_hash(hash_input)
  
  var build_dir = get_build_dir()
  ensure_dir(build_dir)
  var obj_path = build_dir + '/' + hash
  
  // Check if already compiled
  if (fd.is_file(obj_path)) {
    return obj_path
  }
  
  // Compile
  var full_cmd = cmd_str + ' -o "' + obj_path + '"'
  log.console('Compiling ' + file)
  var ret = os.system(full_cmd)
  if (ret != 0) {
    throw Error('Compilation failed: ' + file)
  }
  
  return obj_path
}

// Build all C files for a package
// Returns array of object file paths
Build.build_package = function(pkg, target = Build.detect_host_target(), exclude_main, buildtype = 'release') {
  var c_files = pkg_tools.get_c_files(pkg, target, exclude_main)
  var objects = []
  
  arrfor(c_files, function(file) {
    var obj = Build.compile_file(pkg, file, target, buildtype)
    push(objects, obj)
  })
  
  return objects
}

// ============================================================================
// Dynamic library building
// ============================================================================

// Compute link key from all inputs that affect the dylib output
function compute_link_key(objects, ldflags, target_ldflags, target, cc) {
  // Sort objects for deterministic hash
  var sorted_objects = sort(objects)

  // Build a string representing all link inputs
  var parts = []
  push(parts, 'target:' + target)
  push(parts, 'cc:' + cc)
  arrfor(sorted_objects, function(obj) {
    // Object paths are content-addressed, so the path itself is the hash
    push(parts, 'obj:' + obj)
  })
  arrfor(ldflags, function(flag) {
    push(parts, 'ldflag:' + flag)
  })
  arrfor(target_ldflags, function(flag) {
    push(parts, 'target_ldflag:' + flag)
  })

  return content_hash(text(parts, '\n'))
}

// Build a per-module dynamic library for a single C file
// Returns the content-addressed dylib path in .cell/build/<hash>.<target>.dylib
Build.build_module_dylib = function(pkg, file, target, buildtype) {
  var _target = target || Build.detect_host_target()
  var _buildtype = buildtype || 'release'
  var obj = Build.compile_file(pkg, file, _target, _buildtype)

  var tc = toolchains[_target]
  var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
  var cc = tc.cpp || tc.c
  var local_dir = get_local_dir()
  var pkg_dir = shop.get_package_dir(pkg)

  // Get link flags
  var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target))
  var target_ldflags = tc.c_link_args || []
  var resolved_ldflags = []
  arrfor(ldflags, function(flag) {
    if (starts_with(flag, '-L') && !starts_with(flag, '-L/')) {
      flag = '-L"' + pkg_dir + '/' + text(flag, 2) + '"'
    }
    push(resolved_ldflags, flag)
  })

  // Content-addressed output: hash of (object + link flags + target)
  var link_key = compute_link_key([obj], resolved_ldflags, target_ldflags, _target, cc)
  var build_dir = get_build_dir()
  ensure_dir(build_dir)
  var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext

  if (fd.is_file(dylib_path))
    return dylib_path

  var cmd_parts = [cc, '-shared', '-fPIC']

  if (tc.system == 'darwin') {
    cmd_parts = array(cmd_parts, [
      '-undefined', 'dynamic_lookup',
      '-Wl,-dead_strip',
      '-Wl,-rpath,@loader_path/../local',
      '-Wl,-rpath,' + local_dir
    ])
  } else if (tc.system == 'linux') {
    cmd_parts = array(cmd_parts, [
      '-Wl,--allow-shlib-undefined',
      '-Wl,--gc-sections',
      '-Wl,-rpath,$ORIGIN/../local',
      '-Wl,-rpath,' + local_dir
    ])
  } else if (tc.system == 'windows') {
    push(cmd_parts, '-Wl,--allow-shlib-undefined')
  }

  push(cmd_parts, '-L"' + local_dir + '"')
  push(cmd_parts, '"' + obj + '"')
  cmd_parts = array(cmd_parts, resolved_ldflags)
  cmd_parts = array(cmd_parts, target_ldflags)
  push(cmd_parts, '-o')
  push(cmd_parts, '"' + dylib_path + '"')

  var cmd_str = text(cmd_parts, ' ')
  log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
  var ret = os.system(cmd_str)
  if (ret != 0) {
    print('Linking failed: ' + file); disrupt
  }

  return dylib_path
}

// Build a dynamic library for a package (one dylib per C file)
// Returns array of {file, symbol, dylib} for each module
// Also writes a manifest mapping symbols to dylib paths
Build.build_dynamic = function(pkg, target, buildtype) {
  var _target = target || Build.detect_host_target()
  var _buildtype = buildtype || 'release'
  var c_files = pkg_tools.get_c_files(pkg, _target, true)
  var results = []
  var manifest = {}

  arrfor(c_files, function(file) {
    var sym_name = shop.c_symbol_for_file(pkg, file)
    var dylib = Build.build_module_dylib(pkg, file, _target, _buildtype)
    push(results, {file: file, symbol: sym_name, dylib: dylib})
    manifest[sym_name] = dylib
  })

  // Write manifest so the loader can find per-module dylibs by symbol
  if (length(results) > 0) {
    var lib_dir = shop.get_lib_dir()
    ensure_dir(lib_dir)
    var lib_name = shop.lib_name_for_package(pkg)
    var manifest_path = lib_dir + '/' + lib_name + '.manifest.json'
    var json = use('json')
    fd.slurpwrite(manifest_path, stone(blob(json.encode(manifest))))
  }

  return results
}

// ============================================================================
// Static binary building
// ============================================================================

// Build a static binary from multiple packages
// packages: array of package names
// output: output binary path
Build.build_static = function(packages, target = Build.detect_host_target(), output, buildtype = 'release') {
  var all_objects = []
  var all_ldflags = []
  var seen_flags = {}
  
  // Compile all packages
  arrfor(packages, function(pkg) {
    var is_core = (pkg == 'core')
    
    // For core, include main.c; for others, exclude it
    var objects = Build.build_package(pkg, target, !is_core, buildtype)
    
    arrfor(objects, function(obj) {
      push(all_objects, obj)
    })
    
    // Collect LDFLAGS (with sigil replacement)
    var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', target))
    var pkg_dir = shop.get_package_dir(pkg)
    
    // Deduplicate based on the entire LDFLAGS string for this package
    var ldflags_key = pkg + ':' + text(ldflags, ' ')
    if (!seen_flags[ldflags_key]) {
      seen_flags[ldflags_key] = true
      arrfor(ldflags, function(flag) {
        // Resolve relative -L paths
        if (starts_with(flag, '-L') && !starts_with(flag, '-L/')) {
          flag = '-L"' + pkg_dir + '/' + text(flag, 2) + '"'
        }
        push(all_ldflags, flag)
      })
    }
  })
  
  if (length(all_objects) == 0) {
    throw Error('No object files to link')
  }
  
  // Link
  var cc = toolchains[target].c
  var target_ldflags = toolchains[target].c_link_args || []
  var exe_ext = toolchains[target].system == 'windows' ? '.exe' : ''
  
  if (!ends_with(output, exe_ext) && exe_ext) {
    output = output + exe_ext
  }
  
  var cmd_parts = [cc]
  
  arrfor(all_objects, function(obj) {
    push(cmd_parts, '"' + obj + '"')
  })
  
  arrfor(all_ldflags, function(flag) {
    push(cmd_parts, flag)
  })
  
  arrfor(target_ldflags, function(flag) {
    push(cmd_parts, flag)
  })
  
  push(cmd_parts, '-o', '"' + output + '"')
  
  var cmd_str = text(cmd_parts, ' ')
  
  log.console('Linking ' + output)
  var ret = os.system(cmd_str)
  if (ret != 0) {
    throw Error('Linking failed with command: ' + cmd_str)
  }
  
  log.console('Built ' + output)
  return output
}

// ============================================================================
// Native .cm compilation (source → mcode → QBE IL → .o → .dylib)
// ============================================================================

// Post-process QBE IL: insert dead labels after ret/jmp (QBE requirement)
function qbe_insert_dead_labels(il_text) {
  var lines = array(il_text, "\n")
  var result = []
  var dead_id = 0
  var need_label = false
  var i = 0
  var line = null
  var trimmed = null
  while (i < length(lines)) {
    line = lines[i]
    trimmed = trim(line)
    if (need_label && !starts_with(trimmed, '@') && !starts_with(trimmed, '}') && length(trimmed) > 0) {
      push(result, "@_dead_" + text(dead_id))
      dead_id = dead_id + 1
      need_label = false
    }
    if (starts_with(trimmed, '@') || starts_with(trimmed, '}') || length(trimmed) == 0) {
      need_label = false
    }
    if (starts_with(trimmed, 'ret ') || starts_with(trimmed, 'jmp ')) {
      need_label = true
    }
    push(result, line)
    i = i + 1
  }
  return text(result, "\n")
}

// Compile a .cm source file to a native .dylib via QBE
// Returns the content-addressed dylib path
Build.compile_native = function(src_path, target, buildtype) {
  var _target = target || Build.detect_host_target()
  var _buildtype = buildtype || 'release'

  if (!fd.is_file(src_path)) {
    print('Source file not found: ' + src_path); disrupt
  }

  var tc = toolchains[_target]
  var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
  var cc = tc.c

  // Step 1: Read source and compile through pipeline
  var content = fd.slurp(src_path)
  var src = text(content)
  var tokenize = use('tokenize')
  var parse = use('parse')
  var fold = use('fold')
  var mcode_mod = use('mcode')
  var streamline_mod = use('streamline')
  var qbe_macros = use('qbe')
  var qbe_emit = use('qbe_emit')

  var tok_result = tokenize(src, src_path)
  var ast = parse(tok_result.tokens, src, src_path, tokenize)
  var folded = fold(ast)
  var compiled = mcode_mod(folded)
  var optimized = streamline_mod(compiled)

  // Step 2: Generate QBE IL
  var il = qbe_emit(optimized, qbe_macros)

  // Step 3: Post-process (insert dead labels)
  il = qbe_insert_dead_labels(il)

  // Content hash for cache key
  var hash = content_hash(src + '\n' + _target + '\nnative')
  var build_dir = get_build_dir()
  ensure_dir(build_dir)

  var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
  if (fd.is_file(dylib_path))
    return dylib_path

  // Step 4: Write QBE IL to temp file
  var tmp = '/tmp/cell_native_' + hash
  var ssa_path = tmp + '.ssa'
  var s_path = tmp + '.s'
  var o_path = tmp + '.o'
  var rt_o_path = '/tmp/cell_qbe_rt.o'

  fd.slurpwrite(ssa_path, stone(blob(il)))

  // Step 5: QBE compile to assembly
  var rc = os.system('qbe -o ' + s_path + ' ' + ssa_path)
  if (rc != 0) {
    print('QBE compilation failed for: ' + src_path); disrupt
  }

  // Step 6: Assemble
  rc = os.system(cc + ' -c ' + s_path + ' -o ' + o_path)
  if (rc != 0) {
    print('Assembly failed for: ' + src_path); disrupt
  }

  // Step 7: Compile QBE runtime stubs if needed
  if (!fd.is_file(rt_o_path)) {
    var qbe_rt_path = shop.get_package_dir('core') + '/qbe_rt.c'
    rc = os.system(cc + ' -c ' + qbe_rt_path + ' -o ' + rt_o_path + ' -fPIC')
    if (rc != 0) {
      print('QBE runtime stubs compilation failed'); disrupt
    }
  }

  // Step 8: Link dylib
  var link_cmd = cc + ' -shared -fPIC'
  if (tc.system == 'darwin') {
    link_cmd = link_cmd + ' -undefined dynamic_lookup'
  } else if (tc.system == 'linux') {
    link_cmd = link_cmd + ' -Wl,--allow-shlib-undefined'
  }
  link_cmd = link_cmd + ' ' + o_path + ' ' + rt_o_path + ' -o ' + dylib_path

  rc = os.system(link_cmd)
  if (rc != 0) {
    print('Linking native dylib failed for: ' + src_path); disrupt
  }

  log.console('Built native: ' + fd.basename(dylib_path))
  return dylib_path
}

// ============================================================================
// Module table generation (for static builds)
// ============================================================================

// Compile a .cm module to mach bytecode blob
// Returns the raw mach bytes as a blob
Build.compile_cm_to_mach = function(src_path) {
  if (!fd.is_file(src_path)) {
    print('Source file not found: ' + src_path); disrupt
  }
  var src = text(fd.slurp(src_path))
  var tokenize = use('tokenize')
  var parse = use('parse')
  var fold = use('fold')
  var mcode_mod = use('mcode')
  var streamline_mod = use('streamline')
  var json = use('json')

  var tok_result = tokenize(src, src_path)
  var ast = parse(tok_result.tokens, src, src_path, tokenize)
  var folded = fold(ast)
  var compiled = mcode_mod(folded)
  var optimized = streamline_mod(compiled)
  return mach_compile_mcode_bin(src_path, json.encode(optimized))
}

// Generate a module_table.c file that embeds mach bytecode for .cm modules
// modules: array of {name, src_path} — name is the module name, src_path is the .cm file
// output: path to write the generated .c file
Build.generate_module_table = function(modules, output) {
  var lines = []
  var json = use('json')
  push(lines, '/* Generated module table — do not edit */')
  push(lines, '#include <stddef.h>')
  push(lines, '#include <string.h>')
  push(lines, '')
  push(lines, 'struct cell_embedded_entry {')
  push(lines, '  const char *name;')
  push(lines, '  const unsigned char *data;')
  push(lines, '  size_t size;')
  push(lines, '};')
  push(lines, '')

  var entries = []
  arrfor(modules, function(mod) {
    var safe = replace(replace(replace(mod.name, '/', '_'), '.', '_'), '-', '_')
    var mach = Build.compile_cm_to_mach(mod.src_path)
    var bytes = array(mach)
    var hex = []
    arrfor(bytes, function(b) {
      push(hex, '0x' + text(b, 'h2'))
    })
    push(lines, 'static const unsigned char mod_' + safe + '_data[] = {')
    push(lines, '  ' + text(hex, ', '))
    push(lines, '};')
    push(lines, '')
    push(entries, safe)
    log.console('Embedded: ' + mod.name + ' (' + text(length(bytes)) + ' bytes)')
  })

  // Lookup function
  push(lines, 'const struct cell_embedded_entry *cell_embedded_module_lookup(const char *name) {')
  arrfor(modules, function(mod, i) {
    var safe = entries[i]
    push(lines, '  if (strcmp(name, "' + mod.name + '") == 0) {')
    push(lines, '    static const struct cell_embedded_entry e = {"' + mod.name + '", mod_' + safe + '_data, sizeof(mod_' + safe + '_data)};')
    push(lines, '    return &e;')
    push(lines, '  }')
  })
  push(lines, '  return (void *)0;')
  push(lines, '}')

  var c_text = text(lines, '\n')
  fd.slurpwrite(output, stone(blob(c_text)))
  log.console('Generated ' + output)
  return output
}

// ============================================================================
// Convenience functions
// ============================================================================

// Build dynamic libraries for all installed packages
Build.build_all_dynamic = function(target, buildtype) {
  var _target = target || Build.detect_host_target()
  var _buildtype = buildtype || 'release'

  var packages = shop.list_packages()
  var results = []

  // Build core first
  if (find(packages, function(p) { return p == 'core' }) != null) {
    var core_mods = Build.build_dynamic('core', _target, _buildtype)
    push(results, {package: 'core', modules: core_mods})
  }

  // Build other packages
  arrfor(packages, function(pkg) {
    if (pkg == 'core') return
    var pkg_mods = Build.build_dynamic(pkg, _target, _buildtype)
    push(results, {package: pkg, modules: pkg_mods})
  })

  return results
}

return Build