517 lines
15 KiB
Plaintext
517 lines
15 KiB
Plaintext
var tokenize = function(src, filename) {
|
|
var len = length(src)
|
|
var cp = []
|
|
var _i = 0
|
|
while (_i < len) {
|
|
push(cp, codepoint(src[_i]))
|
|
_i = _i + 1
|
|
}
|
|
|
|
var pos = 0
|
|
var row = 0
|
|
var col = 0
|
|
var tokens = []
|
|
|
|
// Codepoint constants
|
|
def CP_LF = 10
|
|
def CP_CR = 13
|
|
def CP_TAB = 9
|
|
def CP_SPACE = 32
|
|
def CP_BANG = 33
|
|
def CP_DQUOTE = 34
|
|
def CP_HASH = 35
|
|
def CP_DOLLAR = 36
|
|
def CP_PERCENT = 37
|
|
def CP_AMP = 38
|
|
def CP_SQUOTE = 39
|
|
def CP_LPAREN = 40
|
|
def CP_RPAREN = 41
|
|
def CP_STAR = 42
|
|
def CP_PLUS = 43
|
|
def CP_COMMA = 44
|
|
def CP_MINUS = 45
|
|
def CP_DOT = 46
|
|
def CP_SLASH = 47
|
|
def CP_0 = 48
|
|
def CP_1 = 49
|
|
def CP_7 = 55
|
|
def CP_9 = 57
|
|
def CP_COLON = 58
|
|
def CP_SEMI = 59
|
|
def CP_LT = 60
|
|
def CP_EQ = 61
|
|
def CP_GT = 62
|
|
def CP_QMARK = 63
|
|
def CP_AT = 64
|
|
def CP_A = 65
|
|
def CP_B = 66
|
|
def CP_E = 69
|
|
def CP_F = 70
|
|
def CP_O = 79
|
|
def CP_X = 88
|
|
def CP_Z = 90
|
|
def CP_LBRACKET = 91
|
|
def CP_BSLASH = 92
|
|
def CP_RBRACKET = 93
|
|
def CP_CARET = 94
|
|
def CP_UNDERSCORE = 95
|
|
def CP_BACKTICK = 96
|
|
def CP_a = 97
|
|
def CP_b = 98
|
|
def CP_e = 101
|
|
def CP_f = 102
|
|
def CP_n = 110
|
|
def CP_o = 111
|
|
def CP_r = 114
|
|
def CP_t = 116
|
|
def CP_u = 117
|
|
def CP_x = 120
|
|
def CP_z = 122
|
|
def CP_LBRACE = 123
|
|
def CP_PIPE = 124
|
|
def CP_RBRACE = 125
|
|
def CP_TILDE = 126
|
|
|
|
// Keywords lookup
|
|
var keywords = {
|
|
if: "if", in: "in", do: "do", go: "go",
|
|
var: "var", def: "def", for: "for",
|
|
else: "else", this: "this", null: "null", true: "true",
|
|
false: "false", while: "while", break: "break",
|
|
return: "return", delete: "delete",
|
|
disrupt: "disrupt", function: "function", continue: "continue",
|
|
disruption: "disruption"
|
|
}
|
|
|
|
var pk = function() {
|
|
if (pos >= len) return -1
|
|
return cp[pos]
|
|
}
|
|
|
|
var pk_at = function(n) {
|
|
var idx = pos + n
|
|
if (idx >= len) return -1
|
|
return cp[idx]
|
|
}
|
|
|
|
var adv = function() {
|
|
var c = cp[pos]
|
|
pos = pos + 1
|
|
if (c == CP_LF) {
|
|
row = row + 1
|
|
col = 0
|
|
} else {
|
|
col = col + 1
|
|
}
|
|
return c
|
|
}
|
|
|
|
var is_digit = function(c) {
|
|
return c >= CP_0 && c <= CP_9
|
|
}
|
|
|
|
var is_hex = function(c) {
|
|
return (c >= CP_0 && c <= CP_9) || (c >= CP_a && c <= CP_f) || (c >= CP_A && c <= CP_F)
|
|
}
|
|
|
|
var hex_val = function(c) {
|
|
if (c >= CP_0 && c <= CP_9) return c - CP_0
|
|
if (c >= CP_a && c <= CP_f) return c - CP_a + 10
|
|
if (c >= CP_A && c <= CP_F) return c - CP_A + 10
|
|
return 0
|
|
}
|
|
|
|
var read_unicode_escape = function() {
|
|
var cp_val = 0
|
|
var hi = 0
|
|
while (hi < 4 && pos < len && is_hex(pk())) {
|
|
cp_val = cp_val * 16 + hex_val(adv())
|
|
hi = hi + 1
|
|
}
|
|
return character(cp_val)
|
|
}
|
|
|
|
var is_alpha = function(c) {
|
|
return (c >= CP_a && c <= CP_z) || (c >= CP_A && c <= CP_Z)
|
|
}
|
|
|
|
var is_alnum = function(c) {
|
|
return is_alpha(c) || is_digit(c)
|
|
}
|
|
|
|
var is_ident_start = function(c) {
|
|
return is_alpha(c) || c == CP_UNDERSCORE || c == CP_DOLLAR
|
|
}
|
|
|
|
var is_ident_char = function(c) {
|
|
return is_alnum(c) || c == CP_UNDERSCORE || c == CP_DOLLAR || c == CP_QMARK || c == CP_BANG
|
|
}
|
|
|
|
var substr = function(start, end) {
|
|
var s = ""
|
|
var i = start
|
|
while (i < end) {
|
|
s = s + character(cp[i])
|
|
i = i + 1
|
|
}
|
|
return s
|
|
}
|
|
|
|
var read_string = function(quote_cp) {
|
|
var start = pos
|
|
var start_row = row
|
|
var start_col = col
|
|
var value = ""
|
|
var esc = 0
|
|
adv() // skip opening quote
|
|
while (pos < len && pk() != quote_cp) {
|
|
if (pk() == CP_BSLASH) {
|
|
adv()
|
|
esc = adv()
|
|
if (esc == CP_n) { value = value + "\n" }
|
|
else if (esc == CP_t) { value = value + "\t" }
|
|
else if (esc == CP_r) { value = value + "\r" }
|
|
else if (esc == CP_BSLASH) { value = value + "\\" }
|
|
else if (esc == CP_SQUOTE) { value = value + "'" }
|
|
else if (esc == CP_DQUOTE) { value = value + "\"" }
|
|
else if (esc == CP_0) { value = value + character(0) }
|
|
else if (esc == CP_BACKTICK) { value = value + "`" }
|
|
else if (esc == CP_u) { value = value + read_unicode_escape() }
|
|
else { value = value + character(esc) }
|
|
} else {
|
|
value = value + character(adv())
|
|
}
|
|
}
|
|
if (pos < len) adv() // skip closing quote
|
|
push(tokens, {
|
|
kind: "text", at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col,
|
|
value: value
|
|
})
|
|
}
|
|
|
|
var read_template = function() {
|
|
var start = pos
|
|
var start_row = row
|
|
var start_col = col
|
|
var value = ""
|
|
var depth = 0
|
|
var tc = 0
|
|
var q = 0
|
|
adv() // skip opening backtick
|
|
while (pos < len && pk() != CP_BACKTICK) {
|
|
if (pk() == CP_BSLASH && pos + 1 < len) {
|
|
value = value + character(adv())
|
|
value = value + character(adv())
|
|
} else if (pk() == CP_DOLLAR && pos + 1 < len && pk_at(1) == CP_LBRACE) {
|
|
value = value + character(adv()) // $
|
|
value = value + character(adv()) // {
|
|
depth = 1
|
|
while (pos < len && depth > 0) {
|
|
tc = pk()
|
|
if (tc == CP_LBRACE) { depth = depth + 1; value = value + character(adv()) }
|
|
else if (tc == CP_RBRACE) {
|
|
depth = depth - 1
|
|
if (depth > 0) { value = value + character(adv()) }
|
|
else { value = value + character(adv()) }
|
|
}
|
|
else if (tc == CP_SQUOTE || tc == CP_DQUOTE || tc == CP_BACKTICK) {
|
|
q = adv()
|
|
value = value + character(q)
|
|
while (pos < len && pk() != q) {
|
|
if (pk() == CP_BSLASH && pos + 1 < len) {
|
|
value = value + character(adv())
|
|
}
|
|
value = value + character(adv())
|
|
}
|
|
if (pos < len) { value = value + character(adv()) }
|
|
} else { value = value + character(adv()) }
|
|
}
|
|
} else {
|
|
value = value + character(adv())
|
|
}
|
|
}
|
|
if (pos < len) adv() // skip closing backtick
|
|
push(tokens, {
|
|
kind: "text", at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col,
|
|
value: value
|
|
})
|
|
}
|
|
|
|
var read_number = function() {
|
|
var start = pos
|
|
var start_row = row
|
|
var start_col = col
|
|
var raw = ""
|
|
if (pk() == CP_0 && (pk_at(1) == CP_x || pk_at(1) == CP_X)) {
|
|
adv(); adv()
|
|
while (pos < len && (is_hex(pk()) || pk() == CP_UNDERSCORE)) adv()
|
|
} else if (pk() == CP_0 && (pk_at(1) == CP_b || pk_at(1) == CP_B)) {
|
|
adv(); adv()
|
|
while (pos < len && (pk() == CP_0 || pk() == CP_1 || pk() == CP_UNDERSCORE)) adv()
|
|
} else if (pk() == CP_0 && (pk_at(1) == CP_o || pk_at(1) == CP_O)) {
|
|
adv(); adv()
|
|
while (pos < len && pk() >= CP_0 && pk() <= CP_7) adv()
|
|
} else {
|
|
while (pos < len && (is_digit(pk()) || pk() == CP_UNDERSCORE)) adv()
|
|
if (pos < len && pk() == CP_DOT) {
|
|
adv()
|
|
while (pos < len && (is_digit(pk()) || pk() == CP_UNDERSCORE)) adv()
|
|
}
|
|
if (pos < len && (pk() == CP_e || pk() == CP_E)) {
|
|
adv()
|
|
if (pos < len && (pk() == CP_PLUS || pk() == CP_MINUS)) adv()
|
|
while (pos < len && is_digit(pk())) adv()
|
|
}
|
|
}
|
|
raw = substr(start, pos)
|
|
push(tokens, {
|
|
kind: "number", at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col,
|
|
value: raw, number: number(raw)
|
|
})
|
|
}
|
|
|
|
var read_name = function() {
|
|
var start = pos
|
|
var start_row = row
|
|
var start_col = col
|
|
var name = ""
|
|
var kw = null
|
|
while (pos < len && is_ident_char(pk())) adv()
|
|
name = substr(start, pos)
|
|
kw = keywords[name]
|
|
if (kw != null) {
|
|
push(tokens, {
|
|
kind: kw, at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col
|
|
})
|
|
} else {
|
|
push(tokens, {
|
|
kind: "name", at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col,
|
|
value: name
|
|
})
|
|
}
|
|
}
|
|
|
|
var read_comment = function() {
|
|
var start = pos
|
|
var start_row = row
|
|
var start_col = col
|
|
var raw = ""
|
|
if (pk_at(1) == CP_SLASH) {
|
|
while (pos < len && pk() != CP_LF && pk() != CP_CR) adv()
|
|
} else {
|
|
adv(); adv() // skip /*
|
|
while (pos < len) {
|
|
if (pk() == CP_STAR && pk_at(1) == CP_SLASH) {
|
|
adv(); adv()
|
|
break
|
|
}
|
|
adv()
|
|
}
|
|
}
|
|
raw = substr(start, pos)
|
|
push(tokens, {
|
|
kind: "comment", at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col,
|
|
value: raw
|
|
})
|
|
}
|
|
|
|
var emit_op = function(kind, count) {
|
|
var start = pos
|
|
var start_row = row
|
|
var start_col = col
|
|
var i = 0
|
|
while (i < count) { adv(); i = i + 1 }
|
|
push(tokens, {
|
|
kind: kind, at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col
|
|
})
|
|
}
|
|
|
|
var emit_ident = function(count) {
|
|
var start = pos
|
|
var start_row = row
|
|
var start_col = col
|
|
var val = ""
|
|
var i = 0
|
|
while (i < count) { val = val + character(adv()); i = i + 1 }
|
|
push(tokens, {
|
|
kind: "name", at: start,
|
|
from_row: start_row, from_column: start_col,
|
|
to_row: row, to_column: col,
|
|
value: val
|
|
})
|
|
}
|
|
|
|
var tokenize_one = function() {
|
|
var c = pk()
|
|
var start = 0
|
|
var start_row = 0
|
|
var start_col = 0
|
|
var raw = ""
|
|
if (c == -1) return false
|
|
|
|
if (c == CP_LF) {
|
|
start = pos; start_row = row; start_col = col
|
|
adv()
|
|
push(tokens, { kind: "newline", at: start, from_row: start_row, from_column: start_col, to_row: row, to_column: col, value: "\n" })
|
|
return true
|
|
}
|
|
if (c == CP_CR) {
|
|
start = pos; start_row = row; start_col = col
|
|
adv()
|
|
if (pos < len && pk() == CP_LF) adv()
|
|
push(tokens, { kind: "newline", at: start, from_row: start_row, from_column: start_col, to_row: row, to_column: col, value: "\n" })
|
|
return true
|
|
}
|
|
if (c == CP_SPACE || c == CP_TAB) {
|
|
start = pos; start_row = row; start_col = col
|
|
while (pos < len && (pk() == CP_SPACE || pk() == CP_TAB)) adv()
|
|
raw = substr(start, pos)
|
|
push(tokens, { kind: "space", at: start, from_row: start_row, from_column: start_col, to_row: row, to_column: col, value: raw })
|
|
return true
|
|
}
|
|
if (c == CP_SQUOTE || c == CP_DQUOTE) { read_string(c); return true }
|
|
if (c == CP_BACKTICK) { read_template(); return true }
|
|
if (is_digit(c)) { read_number(); return true }
|
|
if (c == CP_DOT && is_digit(pk_at(1))) { read_number(); return true }
|
|
if (is_ident_start(c)) { read_name(); return true }
|
|
if (c == CP_SLASH) {
|
|
if (pk_at(1) == CP_SLASH || pk_at(1) == CP_STAR) { read_comment(); return true }
|
|
if (pk_at(1) == CP_EQ) { emit_op("/=", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("/", 1); return true
|
|
}
|
|
if (c == CP_STAR) {
|
|
if (pk_at(1) == CP_STAR) {
|
|
if (pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(2) == CP_EQ) { emit_op("**=", 3); return true }
|
|
emit_op("**", 2); return true
|
|
}
|
|
if (pk_at(1) == CP_EQ) { emit_op("*=", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("*", 1); return true
|
|
}
|
|
if (c == CP_PERCENT) {
|
|
if (pk_at(1) == CP_EQ) { emit_op("%=", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("%", 1); return true
|
|
}
|
|
if (c == CP_PLUS) {
|
|
if (pk_at(1) == CP_EQ) { emit_op("+=", 2); return true }
|
|
if (pk_at(1) == CP_PLUS) { emit_op("++", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("+", 1); return true
|
|
}
|
|
if (c == CP_MINUS) {
|
|
if (pk_at(1) == CP_EQ) { emit_op("-=", 2); return true }
|
|
if (pk_at(1) == CP_MINUS) { emit_op("--", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("-", 1); return true
|
|
}
|
|
if (c == CP_LT) {
|
|
if (pk_at(1) == CP_EQ && pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(1) == CP_EQ) { emit_op("<=", 2); return true }
|
|
if (pk_at(1) == CP_LT) {
|
|
if (pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(2) == CP_EQ) { emit_op("<<=", 3); return true }
|
|
emit_op("<<", 2); return true
|
|
}
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("<", 1); return true
|
|
}
|
|
if (c == CP_GT) {
|
|
if (pk_at(1) == CP_EQ && pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(1) == CP_EQ) { emit_op(">=", 2); return true }
|
|
if (pk_at(1) == CP_GT) {
|
|
if (pk_at(2) == CP_GT) {
|
|
if (pk_at(3) == CP_BANG) { emit_ident(4); return true }
|
|
if (pk_at(3) == CP_EQ) { emit_op(">>>=", 4); return true }
|
|
emit_op(">>>", 3); return true
|
|
}
|
|
if (pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(2) == CP_EQ) { emit_op(">>=", 3); return true }
|
|
emit_op(">>", 2); return true
|
|
}
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op(">", 1); return true
|
|
}
|
|
if (c == CP_EQ) {
|
|
if (pk_at(1) == CP_EQ) {
|
|
if (pk_at(2) == CP_EQ) { emit_op("===", 3); return true }
|
|
emit_op("==", 2); return true
|
|
}
|
|
if (pk_at(1) == CP_GT) { emit_op("=>", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("=", 1); return true
|
|
}
|
|
if (c == CP_BANG) {
|
|
if (pk_at(1) == CP_EQ) {
|
|
if (pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(2) == CP_EQ) { emit_op("!==", 3); return true }
|
|
emit_op("!=", 2); return true
|
|
}
|
|
emit_op("!", 1); return true
|
|
}
|
|
if (c == CP_AMP) {
|
|
if (pk_at(1) == CP_AMP) {
|
|
if (pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(2) == CP_EQ) { emit_op("&&=", 3); return true }
|
|
emit_op("&&", 2); return true
|
|
}
|
|
if (pk_at(1) == CP_EQ) { emit_op("&=", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("&", 1); return true
|
|
}
|
|
if (c == CP_PIPE) {
|
|
if (pk_at(1) == CP_PIPE) {
|
|
if (pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
if (pk_at(2) == CP_EQ) { emit_op("||=", 3); return true }
|
|
emit_op("||", 2); return true
|
|
}
|
|
if (pk_at(1) == CP_EQ) { emit_op("|=", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("|", 1); return true
|
|
}
|
|
if (c == CP_CARET) {
|
|
if (pk_at(1) == CP_EQ) { emit_op("^=", 2); return true }
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("^", 1); return true
|
|
}
|
|
if (c == CP_LBRACKET) {
|
|
if (pk_at(1) == CP_RBRACKET && pk_at(2) == CP_BANG) { emit_ident(3); return true }
|
|
emit_op("[", 1); return true
|
|
}
|
|
if (c == CP_TILDE) {
|
|
if (pk_at(1) == CP_BANG) { emit_ident(2); return true }
|
|
emit_op("~", 1); return true
|
|
}
|
|
emit_op(character(c), 1)
|
|
return true
|
|
}
|
|
|
|
// Main loop
|
|
while (pos < len) {
|
|
tokenize_one()
|
|
}
|
|
|
|
// EOF token
|
|
push(tokens, { kind: "eof", at: pos, from_row: row, from_column: col, to_row: row, to_column: col })
|
|
|
|
return {filename: filename, tokens: tokens, cp: cp}
|
|
}
|
|
|
|
return tokenize
|