turtle: add working tlang

Not very well tested, needs a better public API, not integrated with Minetest, ...
This is WAY bigger than a commit should be.
The next stages will be:
- unit tests
- API (allows it to be more than just a language for this project)
- integration with Minetest

clientmods/turtle/tlang.lua

@@ -0,0 +1,100 @@
+local tlang = {}
+
+tlang.lex = dofile("tlang_lex.lua")
+tlang.parse = dofile("tlang_parse.lua")
+tlang.builtins, tlang.gassign, tlang.step = dofile("tlang_vm.lua")
+
+-- TODO
+--[[
+code shouldnt require a final whitespace
+lexer should include line/character number in symbols
+error messages
+maps shouldnt require whitespace around [ and ]
+maps should be able to have out of order number indexes (like [1 2 3 10:"Out of order"])
+map.key accessing syntax
+--]]
+
+function tlang.run(state)
+    while true do
+        local more = tlang.step(state)
+        if more == true or more == nil then
+            -- continue along
+        elseif type(more) == "string" then
+            print(more) -- error
+        elseif more == false then
+            return -- done
+        else
+            print("Unknown error, tlang.step returned: " .. tostring(more))
+        end
+    end
+end
+
+local function assign_many(state, source)
+    for k, v in pairs(source) do
+        tlang.gassign(state, k, v)
+    end
+end
+
+-- convert a lua value into a tlang literal
+function tlang.valconv(value)
+    local t = type(value)
+    if t == "string" then
+        return {type = "string", value = value}
+    elseif t == "number" then
+        return {type = "number", value = value}
+    elseif t == "table" then
+        local map = {}
+
+        for k, v in pairs(value) do
+            map[k] = tlang.valconv(v)
+        end
+
+        return {type = "map", value = map}
+    end
+end
+
+function tlang.get_state(code)
+    local lexed = tlang.lex(code)
+    local parsed = tlang.parse(lexed)
+
+    return {
+        locals = {{
+            pc = {sg = 1, pos = "__ast__", elem = 1},
+            v__src__ = tlang.valconv(code),
+            v__lex__ = tlang.valconv(lexed),
+            v__ast__ = {type = "code", value = parsed}}},
+        stack = {},
+        builtins = tlang.builtins,
+        wait_target = nil,
+        nextpop = false,
+        tree = parse_state
+    }
+end
+
+function tlang.exec(code)
+    local state = tlang.get_state(code)
+    tlang.run(state)
+end
+
+
+local complex = [[{dup *} `square =
+-5.42 square
+"Hello, world!" print
+[ 1 2 3 str:"String" ]
+]]
+
+local number = [[-4.2123
+]]
+
+local simple = [[{dup *}
+]]
+
+local map = [[
+[ "thing":1 ]
+]]
+
+tlang.exec([[{dup *} `square =
+5 square print
+]])
+
+return tlang

clientmods/turtle/tlang_lex.lua

@@ -0,0 +1,306 @@
+local function in_list(value, list)
+    for k, v in ipairs(list) do
+        if v == value then
+            return true
+        end
+    end
+    return false
+end
+
+
+-- lex state
+--[[
+{
+    code = "",
+    position = int
+}
+--]]
+
+-- lex types
+--[[
+literal
+    number
+    quote
+    identifier
+    string
+symbol
+code_open
+code_close
+code_e_open
+code_e_close
+map_open
+map_close
+map_relation
+--]]
+
+
+-- yeah yeah regex im lazy in this time consuming way shush
+local whitespace = {" ", "\t", "\n", "\r", "\v"}
+local identifier_start = {
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
+    "_"
+}
+local identifier_internal = {
+    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z",
+    "_",
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"
+}
+local symbol_start = {"!", "-", "+", "=", "&", "*", "/", "^", "%", ">", "<", "?", "~"}
+local symbol_values = {
+    "!", "-", "+", "=", "&", "*", "/", "^", "%", ">", "<", "?", "~"
+}
+local string_start = {"\"", "'"}
+local number_start = {"-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
+local number_values = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
+local escape_values = {n = "\n", r = "\r", v = "\v", t = "\t", ['"'] = '"'}
+local symbols = {
+    "!", "-", "+", "=", "&", "*", "/", "^", "%", ">", "<", "?", "~",
+    "&&", "||", "==", "!=", ">=", "<="
+}
+
+local function lex_peek(state)
+    return state.code:sub(state.position, state.position)
+end
+
+local function lex_next(state)
+    local value = lex_peek(state)
+    state.position = state.position + 1
+    return value
+end
+
+local function lex_expect(state, chars)
+    if type(chars) == "string" then
+        chars = {chars}
+    end
+
+    local n = lex_next(state)
+    if in_list(n, chars) then
+        return n
+    else
+        return nil -- ERROR!
+    end
+end
+
+local function lex_whitespace(state)
+    while true do
+        local n = lex_peek(state)
+        if not in_list(n, whitespace) then
+            return
+        end
+        lex_next(state)
+    end
+end
+
+
+local function lex_identifier(state)
+    --lex_next(state) -- skip first (should be verified as in identifier_start)
+    local identifier = {}
+    local n = 1
+
+    while true do
+        local cur = lex_peek(state)
+        if in_list(cur, identifier_internal) then
+            identifier[n] = lex_next(state)
+            n = n + 1
+        else
+            break
+        end
+    end
+
+    return {type = "literal", subtype = "identifier", value = table.concat(identifier)}
+end
+
+-- `identifier
+local function lex_quote(state)
+    lex_next(state)
+    local val = lex_identifier(state)
+    val.subtype = "quote"
+    return val
+end
+
+local function lex_single_char(state, t, char)
+    lex_next(state)
+    return {type = t, value = char}
+end
+
+local function lex_code_open(state)
+    return lex_single_char(state, "code_open", "{")
+end
+
+local function lex_code_close(state)
+    return lex_single_char(state, "code_close", "}")
+end
+
+local function lex_code_e_open(state)
+    return lex_single_char(state, "code_e_open", "(")
+end
+
+local function lex_code_e_close(state)
+    return lex_single_char(state, "code_e_close", ")")
+end
+
+local function lex_map_open(state)
+    return lex_single_char(state, "map_open", "[")
+end
+
+local function lex_map_relation(state)
+    return lex_single_char(state, "map_relation", ":")
+end
+
+local function lex_map_close(state)
+    return lex_single_char(state, "map_close", "]")
+end
+
+local function lex_string_escape(state)
+    local n = lex_next(state)
+    return escape_values[n]
+end
+
+local function lex_string(state)
+    lex_next(state)
+    local escaped = false
+    local string = {}
+    local stringi = 1
+
+    while true do
+        local n = lex_next(state)
+
+        if n == "\"" then
+            return {type = "literal", subtype = "string", value = table.concat(string)}
+        elseif n == "\\" then
+            n = lex_string_escape(state)
+        end
+
+        if n == nil then
+            return nil -- ERROR
+        end
+
+        string[stringi] = n
+        stringi = stringi + 1
+    end
+end
+
+local function lex_number(state)
+    local used_sep = false
+    local num = {}
+    local numi = 1
+    
+    local n = lex_peek(state)
+    if in_list(n, number_start) then
+        num[numi] = lex_next(state)
+        numi = numi + 1
+
+        while true do
+            n = lex_peek(state)
+            if n == "." and not used_sep then
+                used_sep = true
+            elseif in_list(n, number_values) then
+
+            elseif in_list(n, whitespace) then
+                return {type = "literal", subtype = "number", value = table.concat(num)}
+            else
+                return nil -- ERROR
+            end
+
+            num[numi] = lex_next(state)
+            numi = numi + 1
+        end
+    end
+end
+
+local function lex_symbol(state)
+    local sym = {}
+    local symi = 1
+
+    while true do
+        local n = lex_peek(state)
+        if not in_list(n, symbol_values) then
+            local symbol = table.concat(sym)
+            if in_list(symbol, symbols) then
+                return {type = "symbol", value = symbol}
+            else
+                return nil -- ERROR
+            end
+        elseif n == nil then
+            return nil -- ERROR
+        else
+            sym[symi] = lex_next(state)
+            symi = symi + 1
+        end
+    end
+end
+
+local function lex_number_or_symbol(state)
+    local nextpeek = state.code:sub(state.position + 1, state.position + 1)
+    if in_list(nextpeek, number_values) then
+        return lex_number(state)
+    else
+        return lex_symbol(state)
+    end
+end
+
+local function lex_step(state)
+    local cur = lex_peek(state)
+
+    if cur == nil then
+        return nil
+    end
+
+    if in_list(cur, whitespace) then
+        lex_whitespace(state)
+    end
+    
+    cur = lex_peek(state)
+
+    if cur == "`" then
+        return lex_quote(state)
+    elseif cur == "-" then -- special case for negative numbers and the minus
+        return lex_number_or_symbol(state)
+    elseif in_list(cur, symbol_start) then
+        return lex_symbol(state)
+    elseif cur == "{" then
+        return lex_code_open(state)
+    elseif cur == "}" then
+        return lex_code_close(state)
+    elseif cur == "(" then
+        return lex_code_e_open(state)
+    elseif cur == ")" then
+        return lex_code_e_close(state)
+    elseif cur == "[" then
+        return lex_map_open(state)
+    elseif cur == "]" then
+        return lex_map_close(state)
+    elseif cur == ":" then
+        return lex_map_relation(state)
+    elseif in_list(cur, identifier_start) then
+        return lex_identifier(state)
+    elseif in_list(cur, string_start) then
+        return lex_string(state)
+    elseif in_list(cur, number_start) then
+        return lex_number(state)
+    end
+end
+
+-- lex
+return function(code)
+    local state = {code = code, position = 1}
+    local lexed = {}
+    local lexi = 1
+
+    while true do
+        local n = lex_step(state)
+
+        if n == nil then
+            if state.position <= #state.code then
+                return nil
+            else
+                return lexed
+            end
+        end
+
+        lexed[lexi] = n
+        lexi = lexi + 1
+    end
+end

clientmods/turtle/tlang_parse.lua

@@ -0,0 +1,183 @@
+-- parse types
+--[[
+quote
+identifier
+code
+map
+string
+number
+symbol
+--]]
+
+
+local function sublist(list, istart, iend, inclusive)
+    local o = {}
+    local oi = 1
+
+    inclusive = inclusive or false
+
+    for i, v in ipairs(list) do
+        iend = iend or 0 -- idk how but iend can become nil
+
+        local uninc = i > istart and i < iend
+        local incl = i >= istart and i <= iend
+
+        if (inclusive and incl) or (not inclusive and uninc) then
+            o[oi] = v
+            oi = oi + 1
+        end
+    end
+
+    return o
+end
+
+
+local parse
+
+local function parse_peek(state)
+    return state.lexed[state.position]
+end
+
+local function parse_next(state)
+    local n = parse_peek(state)
+    state.position = state.position + 1
+    return n
+end
+
+local function parse_map(state)
+    local map = {}
+    local mapi = 1
+
+    if parse_next(state).type ~= "map_open" then
+        return nil -- ERROR
+    end
+
+    while true do
+        local n = parse_next(state)
+        local skip = false -- lua has no continue, 5.1 has no goto
+
+        if n == nil then
+            return nil -- ERROR
+        end
+
+        if n.type == "map_close" then
+            break
+        elseif n.type == "literal" and (n.subtype == "identifier" or n.subtype == "string") then
+            local key = n.value
+            local mr = parse_peek(state)
+
+            if mr.type == "map_relation" then
+                parse_next(state)
+                local nval = parse({parse_next(state)})
+
+                if nval == nil then
+                    return nil -- ERROR
+                end
+
+                map[key] = nval[1]
+                skip = true
+            end
+        end
+
+        if not skip then
+            local nval = parse({n})
+
+            if nval == nil then
+                return nil -- ERROR
+            end
+
+            map[mapi] = nval[1]
+            mapi = mapi + 1
+        end
+    end
+
+    return {type = "map", value = map}
+end
+
+local function parse_find_matching(state, open, close)
+    local level = 1
+
+    parse_next(state) -- skip beginning
+
+    while level ~= 0 do
+        local n = parse_next(state)
+        if n == nil then
+            return nil -- ERROR
+        elseif n.type == open then
+            level = level + 1
+        elseif n.type == close then
+            level = level - 1
+        end
+    end
+
+    return state.position - 1
+end
+
+local function parse_code(state, open, close)
+    local istart = state.position
+    local iend = parse_find_matching(state, open, close)
+
+    return {
+        type = "code",
+        value = parse(sublist(state.lexed, istart, iend))
+    }
+end
+
+local function parse_step(state)
+    local n = parse_peek(state)
+
+    if n == nil then
+        return nil
+    elseif n.type == "code_open" then
+        return parse_code(state, "code_open", "code_close")
+    elseif n.type == "code_e_open" then
+        return parse_code(state, "code_e_open", "code_e_close")
+        -- also return run
+    elseif n.type == "map_open" then
+        local istart = state.position
+        local iend = parse_find_matching(state, "map_open", "map_close")
+        return parse_map({lexed = sublist(state.lexed, istart, iend, true), position = 1})
+    elseif n.type == "literal" then
+        if n.subtype == "number" then
+            parse_next(state)
+            return {type = "number", value = tonumber(n.value)}
+        elseif n.subtype == "string" then
+            parse_next(state)
+            return {type = "string", value = n.value}
+        elseif n.subtype == "identifier" then
+            parse_next(state)
+            return {type = "identifier", value = n.value}
+        elseif n.subtype == "quote" then
+            parse_next(state)
+            return {type = "quote", value = n.value}
+        end
+    elseif n.type == "symbol" then
+        parse_next(state)
+        return {type = "symbol", value = n.value}
+    end
+end
+
+
+-- parse
+parse = function(lexed)
+    local state = {lexed = lexed, position = 1}
+    local tree = {}
+    local treei = 1
+
+    while true do
+        local n = parse_step(state)
+
+        if n == nil then
+            if state.position <= #state.lexed then
+                return nil
+            else
+                return tree
+            end
+        end
+
+        tree[treei] = n
+        treei = treei + 1
+    end
+end
+
+return parse

clientmods/turtle/tlang_vm.lua

@@ -0,0 +1,205 @@
+
+local function in_list(value, list)
+    for k, v in ipairs(list) do
+        if v == value then
+            return true
+        end
+    end
+    return false
+end
+
+local function in_keys(value, list)
+    for k, v in pairs(list) do
+        if k == value then
+            return true
+        end
+    end
+    return false
+end
+
+-- state
+--[[
+    {
+        locals = {},
+        stack = {},
+        tree = {}
+    }
+--]]
+
+-- program counter
+--[[
+    sg = 0/1,
+    pos = int/string,
+    elem = int,
+    wait_target = float
+--]]
+
+local literals = {
+    "quote",
+    "code",
+    "map",
+    "string",
+    "number"
+}
+
+
+local function call(state, target)
+    state.locals[#state.locals + 1] = {pc = target}
+end
+
+local function access(state, name)
+    name = "v" .. name
+    for i, v in ipairs(state.locals) do
+        if in_keys(name, v) then
+            return v[name]
+        end
+    end
+end
+
+local function gassign(state, name, value)
+    state.locals[0]["v" .. name] = value
+end
+
+local function assign(state, name, value)
+    state.locals[#state.locals]["v" .. name] = value
+end
+
+local function getpc(state)
+    return state.locals[#state.locals].pc
+end
+
+local function accesspc(state, pc)
+    local code
+    if pc.sg == 0 then -- stack
+        code = state.stack[pc.pos]
+    elseif pc.sg == 1 then -- global
+        code = access(state, pc.pos)
+    end
+
+    if code then
+        return code.value[pc.elem]
+    end
+end
+
+local function incpc(state, pc)
+    local next_pc = {sg = pc.sg, pos = pc.pos, elem = pc.elem + 1}
+
+    if accesspc(state, next_pc) then
+        return next_pc
+    end
+end
+
+local function getnext(state)
+    if state.nextpop then
+        state.locals[#state.locals] = nil
+        if #state.locals == 0 then
+            return nil
+        end
+        state.nextpop = false
+    end
+
+    local current = accesspc(state, getpc(state))
+
+    local incd = incpc(state, getpc(state))
+    state.locals[#state.locals].pc = incd
+    if not incd then
+        state.nextpop = true
+    end
+
+    return current
+end
+
+
+local function statepeek(state)
+    return state.stack[#state.stack]
+end
+
+local function statepop(state)
+    local tos = statepeek(state)
+    state.stack[#state.stack] = nil
+    return tos
+end
+
+local function statepop_type(state, t)
+    local tos = statepeek(state)
+
+    if tos.type == t then
+        return statepop(state)
+    else
+        return nil -- ERROR
+    end
+end
+
+local function statepush(state, value)
+    state.stack[#state.stack + 1] = value
+end
+
+
+
+local builtins = {}
+
+builtins["="] = function(state)
+    local name = statepop_type(state, "quote")
+    local value = statepop(state)
+
+    assign(state, name.value, value)
+end
+
+builtins["*"] = function(state)
+    local tos = statepop_type(state, "number")
+    local tos1 = statepop_type(state, "number")
+
+    statepush(state, {type = "number", value = tos.value * tos.value})
+end
+
+function builtins.print(state)
+    local value = statepop(state)
+
+    print(value.value)
+end
+
+function builtins.dup(state)
+    statepush(state, statepeek(state))
+end
+
+function builtins.popoff(state)
+    state.stack[#state.stack] = nil
+end
+
+
+-- returns:
+-- true - more to do
+-- nil - more to do but waiting
+-- false - finished
+-- string - error
+local step = function(state)
+    if state.wait_target and os.clock() < state.wait_target then
+        return nil
+    end
+
+    local cur = getnext(state)
+
+    if cur == nil then
+        return false
+    elseif in_list(cur.type, literals) then
+        state.stack[#state.stack + 1] = cur
+    elseif cur.type == "identifier" or cur.type == "symbol" then
+        if in_keys(cur.value, state.builtins) then
+            local f = state.builtins[cur.value]
+            f(state)
+        else
+            local var = access(state, cur.value)
+            if var == nil then
+                return "Undefined identifier: " .. cur.value
+            elseif var.type == "code" then
+                call(state, {sg = 1, pos = cur.value, elem = 1})
+            else
+                state.stack[#state.stack + 1] = var
+            end
+        end
+    end
+
+    return true
+end
+
+return builtins, gassign, step