john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'sem_grab'

Browse Source
This commit is contained in:
John Alanbrook
2026-02-18 10:49:00 -06:00
parent 22c0b421d2 bd7f9f34ec
commit 42f7c270e1
10 changed files with 219 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
build.cm
View File

@@ -614,23 +614,11 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
// Step 1: Read source and compile through pipeline
var content = fd.slurp(src_path)
var src = text(content)
var tokenize = use('tokenize')
var parse = use('parse')
var fold = use('fold')
var mcode_mod = use('mcode')
var streamline_mod = use('streamline')
// Step 1: Compile through pipeline
var optimized = shop.compile_file(src_path)
var qbe_macros = use('qbe')
var qbe_emit = use('qbe_emit')
var tok_result = tokenize(src, src_path)
var ast = parse(tok_result.tokens, src, src_path, tokenize)
var folded = fold(ast)
var compiled = mcode_mod(folded)
var optimized = streamline_mod(compiled)
// Step 2: Generate QBE IL
var sym_name = null
if (pkg) {
@@ -639,7 +627,7 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
// Content hash for cache key
var hash = content_hash(src + '\n' + _target + '\nnative')
var hash = content_hash(text(fd.slurp(src_path)) + '\n' + _target + '\nnative')
var build_dir = get_build_dir()
ensure_dir(build_dir)
@@ -788,19 +776,8 @@ Build.compile_cm_to_mach = function(src_path) {
if (!fd.is_file(src_path)) {
print('Source file not found: ' + src_path); disrupt
}
var src = text(fd.slurp(src_path))
var tokenize = use('tokenize')
var parse = use('parse')
var fold = use('fold')
var mcode_mod = use('mcode')
var streamline_mod = use('streamline')
var json = use('json')
var tok_result = tokenize(src, src_path)
var ast = parse(tok_result.tokens, src, src_path, tokenize)
var folded = fold(ast)
var compiled = mcode_mod(folded)
var optimized = streamline_mod(compiled)
var optimized = shop.compile_file(src_path)
return mach_compile_mcode_bin(src_path, json.encode(optimized))
}

2
compare_aot.ce
View File

@@ -33,7 +33,7 @@ if (!fd_mod.is_file(file)) {
var abs = fd_mod.realpath(file)
// Shared compilation front-end
// Shared compilation front-end — uses raw modules for per-stage timing
var tokenize = use('tokenize')
var parse_mod = use('parse')
var fold = use('fold')

18
docs/semantic-index.md
View File

@@ -217,21 +217,21 @@ This tells you: `connect` is a function taking `(from, to, label)`, declared on
The index and explain modules can be used directly from ƿit scripts:
### index.cm
### Via shop (recommended)
```javascript
var tokenize_mod = use('tokenize')
var parse_mod = use('parse')
var fold_mod = use('fold')
var index_mod = use('index')
var pipeline = {tokenize: tokenize_mod, parse: parse_mod, fold: fold_mod}
var idx = index_mod.index_file(src, filename, pipeline)
var shop = use('internal/shop')
var idx = shop.index_file(path)
```
`index_file` runs the full pipeline (tokenize, parse, fold) and returns the index. If you already have a parsed AST and tokens, use `index_ast` instead:
`shop.index_file` runs the full pipeline (tokenize, parse, index, resolve imports) and caches the result.
### index.cm (direct)
If you already have a parsed AST and tokens, use `index_ast` directly:
```javascript
var index_mod = use('index')
var idx = index_mod.index_ast(ast, tokens, filename)
```

15
dump_ir.ce
View File

@@ -1,18 +1,7 @@
var tokenize = use('tokenize')
var parse_mod = use('parse')
var fold = use('fold')
var mcode_mod = use('mcode')
var streamline_mod = use('streamline')
var shop = use('internal/shop')
var json = use('json')
var fd = use('fd')
var file = args[0]
var src = text(fd.slurp(file))
var tok = tokenize(src, file)
var ast = parse_mod(tok.tokens, src, file, tokenize)
var folded = fold(ast)
var compiled = mcode_mod(folded)
var optimized = streamline_mod(compiled)
var optimized = shop.compile_file(args[0])
var instrs = optimized.main.instructions
var i = 0

15
index.cm
View File

@@ -1,8 +1,7 @@
// index.cm — Core semantic indexing module.
// Walks AST output from parse (+ optional fold) to build a semantic index.
//
// Two entry points:
// index_file(src, filename, tokenize_mod, parse_mod, fold_mod) — full pipeline
// Entry point:
// index_ast(ast, tokens, filename) — index a pre-parsed AST
var make_span = function(node) {
@@ -638,18 +637,6 @@ var index_ast = function(ast, tokens, filename) {
}
}
// Run the full pipeline (tokenize -> parse -> fold) and index.
// pipeline is {tokenize, parse, fold} — pass fold as null to skip folding.
var index_file = function(src, filename, pipeline) {
var tok_result = pipeline.tokenize(src, filename)
var ast = pipeline.parse(tok_result.tokens, src, filename, pipeline.tokenize)
if (pipeline.fold != null) {
ast = pipeline.fold(ast)
}
return index_ast(ast, tok_result.tokens, filename)
}
return {
index_file: index_file,
index_ast: index_ast
}

36
internal/shop.cm
View File

@@ -515,11 +515,14 @@ function inject_env(inject) {
var _tokenize_mod = null
var _parse_mod = null
var _fold_mod = null
var _mcode_mod = null
var _streamline_mod = null
var _index_mod = null
var _token_cache = {}
var _ast_cache = {}
var _analyze_cache = {}
var _compile_cache = {}
var _index_cache = {}
var get_tokenize = function() {
@@ -534,6 +537,14 @@ var get_fold = function() {
if (!_fold_mod) _fold_mod = use_cache['core/fold'] || use_cache['fold']
return _fold_mod
}
var get_mcode = function() {
if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode']
return _mcode_mod
}
var get_streamline = function() {
if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline']
return _streamline_mod
}
var get_index = function() {
if (!_index_mod) {
_index_mod = use_cache['core/index'] || use_cache['index']
@@ -598,21 +609,26 @@ Shop.index_file = function(path) {
var key = content_hash(stone(blob(src)))
if (_index_cache[key]) return _index_cache[key]
var tok = Shop.tokenize_file(path)
var pipeline = {tokenize: get_tokenize(), parse: get_parse(), fold: get_fold()}
var idx = get_index().index_file(src, path, pipeline)
var ast = get_parse()(tok.tokens, src, path, get_tokenize())
var idx = get_index().index_ast(ast, tok.tokens, path)
Shop.resolve_imports(idx, path)
_index_cache[key] = idx
return idx
}
Shop.pipeline = function() {
return {
tokenize: get_tokenize(),
parse: get_parse(),
fold: get_fold(),
mcode: use_cache['core/mcode'] || use_cache['mcode'],
streamline: use_cache['core/streamline'] || use_cache['streamline']
}
Shop.mcode_file = function(path) {
var folded = Shop.analyze_file(path)
return get_mcode()(folded)
}
Shop.compile_file = function(path) {
var src = text(fd.slurp(path))
var key = content_hash(stone(blob(src)))
if (_compile_cache[key]) return _compile_cache[key]
var compiled = Shop.mcode_file(path)
var optimized = get_streamline()(compiled)
_compile_cache[key] = optimized
return optimized
}
Shop.all_script_paths = function() {

4
mcode.ce
View File

@@ -28,9 +28,7 @@ if (!filename) {
$stop()
}
var folded = shop.analyze_file(filename)
var pl = shop.pipeline()
var compiled = pl.mcode(folded)
var compiled = shop.mcode_file(filename)
if (!show_pretty) {
print(json.encode(compiled))

116
query.ce Normal file
View File

@@ -0,0 +1,116 @@
// cell query — Semantic queries across packages.
//
// Usage:
// cell query --this [<package>] Top-level this references
// cell query --intrinsic <name> [<package>] Find built-in intrinsic usage
// cell query --decl <name> [<package>] Variable declarations by name
// cell query --help Show usage
var shop = use('internal/shop')
var query_mod = use('query')
var fd = use('fd')
var mode = null
var name = null
var pkg_filter = null
var show_help = false
var i = 0
for (i = 0; i < length(args); i++) {
if (args[i] == '--this') {
mode = "this"
} else if (args[i] == '--intrinsic') {
mode = "intrinsic"
if (i + 1 < length(args) && !starts_with(args[i + 1], '-')) {
name = args[i + 1]
i = i + 1
} else {
log.error('--intrinsic requires a name')
mode = "error"
}
} else if (args[i] == '--decl') {
mode = "decl"
if (i + 1 < length(args) && !starts_with(args[i + 1], '-')) {
name = args[i + 1]
i = i + 1
} else {
log.error('--decl requires a name')
mode = "error"
}
} else if (args[i] == '--help' || args[i] == '-h') {
show_help = true
} else if (!starts_with(args[i], '-')) {
pkg_filter = args[i]
}
}
var all_files = null
var files = []
var j = 0
var idx = null
var hits = null
var hit = null
var k = 0
// Use return pattern to avoid closure-over-object issue with disruption.
var safe_index = function(path) {
return shop.index_file(path)
} disruption {
return null
}
if (show_help) {
log.console("Usage: cell query [options] [<package>]")
log.console("")
log.console("Semantic queries across packages.")
log.console("")
log.console("Options:")
log.console(" --this Top-level this references")
log.console(" --intrinsic <name> Find built-in intrinsic usage (e.g., print)")
log.console(" --decl <name> Variable declarations by name")
log.console("")
log.console("Without a package argument, searches all installed packages.")
} else if (mode == null || mode == "error") {
if (mode != "error") {
log.error('Specify --this, --intrinsic, or --decl. Use --help for usage.')
}
} else {
all_files = shop.all_script_paths()
if (pkg_filter != null) {
for (j = 0; j < length(all_files); j++) {
if (all_files[j].package == pkg_filter) {
files[] = all_files[j]
}
}
} else {
files = all_files
}
for (j = 0; j < length(files); j++) {
idx = safe_index(files[j].full_path)
if (idx == null) continue
hits = null
if (mode == "this") {
hits = query_mod.top_level_this(idx)
} else if (mode == "intrinsic") {
hits = query_mod.intrinsic(idx, name)
} else if (mode == "decl") {
hits = query_mod.find_decl(idx, name, null)
}
if (hits != null && length(hits) > 0) {
for (k = 0; k < length(hits); k++) {
hit = hits[k]
if (hit.span != null) {
log.console(`${files[j].package}:${files[j].rel_path}:${text(hit.span.from_row)}:${text(hit.span.from_col)}: ${hit.name}`)
} else if (hit.decl_span != null) {
log.console(`${files[j].package}:${files[j].rel_path}:${text(hit.decl_span.from_row)}:${text(hit.decl_span.from_col)}: ${hit.kind} ${hit.name}`)
}
}
}
}
}
$stop()

56
query.cm Normal file
View File

@@ -0,0 +1,56 @@
// query.cm — Semantic queries over index data.
//
// All functions take an index object (from index.cm) and return arrays of hits.
var query = {}
// Top-level this: references where name=="this" and enclosing==null.
query.top_level_this = function(idx) {
var hits = []
var i = 0
var ref = null
while (i < length(idx.references)) {
ref = idx.references[i]
if (ref.name == "this" && ref.enclosing == null) {
hits[] = ref
}
i = i + 1
}
return hits
}
// Intrinsic usage: find refs to a built-in name (e.g., print).
query.intrinsic = function(idx, name) {
var hits = []
var i = 0
var ref = null
if (idx.intrinsic_refs == null) return hits
while (i < length(idx.intrinsic_refs)) {
ref = idx.intrinsic_refs[i]
if (ref.name == name) {
hits[] = ref
}
i = i + 1
}
return hits
}
// Variable declarations matching a name and optional kind filter.
// kind is one of "var", "def", "fn", "param", or null (any).
query.find_decl = function(idx, name, kind) {
var hits = []
var i = 0
var sym = null
while (i < length(idx.symbols)) {
sym = idx.symbols[i]
if (sym.name == name) {
if (kind == null || sym.kind == kind) {
hits[] = sym
}
}
i = i + 1
}
return hits
}
return query

10
streamline.ce
View File

@@ -40,17 +40,13 @@ if (!filename) {
$stop()
}
var folded = shop.analyze_file(filename)
var pl = shop.pipeline()
var compiled = pl.mcode(folded)
// Deep copy for before snapshot (needed by --stats)
// Deep copy mcode for before snapshot (needed by --stats, streamline mutates)
var before = null
if (show_stats) {
before = json.decode(json.encode(compiled))
before = json.decode(json.encode(shop.mcode_file(filename)))
}
var optimized = pl.streamline(compiled)
var optimized = shop.compile_file(filename)
// If no flags, default to full JSON output
if (!show_stats && !show_ir && !show_check && !show_types) {