From cc6ac779139434d4f1cc342d36fd2ef5f120ded5 Mon Sep 17 00:00:00 2001 From: Tesla Ice Zhang Date: Sat, 17 Mar 2018 23:17:07 +0800 Subject: [PATCH 001/132] Fix some explicit errors --- doc/langref.html.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 55d3acc59..fca0a342a 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -5733,7 +5733,7 @@ VariableDeclaration = ("var" | "const") Symbol option(":" TypeExpr) option("alig ContainerMember = (ContainerField | FnDef | GlobalVarDecl) -ContainerField = Symbol option(":" PrefixOpExpression option("=" PrefixOpExpression "," +ContainerField = Symbol option(":" PrefixOpExpression option("=" PrefixOpExpression)) "," UseDecl = "use" Expression ";" @@ -5833,7 +5833,7 @@ BinaryAndExpression = BitShiftExpression "&" BinaryAndExpression | BitShiftE BitShiftExpression = AdditionExpression BitShiftOperator BitShiftExpression | AdditionExpression -BitShiftOperator = "<<" | ">>" | "<<" +BitShiftOperator = "<<" | ">>" AdditionExpression = MultiplyExpression AdditionOperator AdditionExpression | MultiplyExpression From 4a921b2eba294518c19b7ad71f0b3bd9a4c6522b Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Sat, 17 Mar 2018 18:39:58 +0100 Subject: [PATCH 002/132] add CLANG_LIBDIRS cmake build variable Mirrors LLVM_LIBDIRS, tells cmake where to look for libclang libraries. --- cmake/Findclang.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/Findclang.cmake b/cmake/Findclang.cmake index 26617d36f..b42abd10a 100644 --- a/cmake/Findclang.cmake +++ b/cmake/Findclang.cmake @@ -5,6 +5,7 @@ # CLANG_FOUND # CLANG_INCLUDE_DIRS # CLANG_LIBRARIES +# CLANG_LIBDIRS if(MSVC) find_package(CLANG REQUIRED CONFIG) @@ -34,6 +35,7 @@ else() string(TOUPPER ${_libname_} _prettylibname_) find_library(CLANG_${_prettylibname_}_LIB NAMES ${_libname_} PATHS + ${CLANG_LIBDIRS} /usr/lib/llvm/6/lib /usr/lib/llvm-6.0/lib /mingw64/lib @@ -60,4 +62,4 @@ endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args(CLANG DEFAULT_MSG CLANG_LIBRARIES CLANG_INCLUDE_DIRS) -mark_as_advanced(CLANG_INCLUDE_DIRS CLANG_LIBRARIES) +mark_as_advanced(CLANG_INCLUDE_DIRS CLANG_LIBRARIES CLANG_LIBDIRS) From c2c34c09b9afd8a5370dce771d5e98af5b3303a1 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Sat, 17 Mar 2018 18:39:58 +0100 Subject: [PATCH 003/132] add LLVM_LIBDIRS to link directories This seems to be the only way to get the zig build to link against llvm libraries in a non-standard location. --- cmake/Findllvm.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Findllvm.cmake b/cmake/Findllvm.cmake index d0a4a3cba..296e0a91f 100644 --- a/cmake/Findllvm.cmake +++ b/cmake/Findllvm.cmake @@ -66,7 +66,7 @@ if(NOT LLVM_LIBRARIES) endif() link_directories("${CMAKE_PREFIX_PATH}/lib") - +link_directories("${LLVM_LIBDIRS}") include(FindPackageHandleStandardArgs) find_package_handle_standard_args(LLVM DEFAULT_MSG LLVM_LIBRARIES LLVM_INCLUDE_DIRS) From bea9e9c7f8abc1e270dd2ea0065934b32627288c Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Wed, 14 Mar 2018 22:19:51 +1300 Subject: [PATCH 004/132] Add addIncludeDir to TestStep + fix build template Closes #794. --- std/build.zig | 11 +++++++++++ std/special/build_file_template.zig | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/std/build.zig b/std/build.zig index e6b667626..a4d745e45 100644 --- a/std/build.zig +++ b/std/build.zig @@ -1627,6 +1627,7 @@ pub const TestStep = struct { filter: ?[]const u8, target: Target, exec_cmd_args: ?[]const ?[]const u8, + include_dirs: ArrayList([]const u8), pub fn init(builder: &Builder, root_src: []const u8) TestStep { const step_name = builder.fmt("test {}", root_src); @@ -1641,6 +1642,7 @@ pub const TestStep = struct { .link_libs = BufSet.init(builder.allocator), .target = Target { .Native = {} }, .exec_cmd_args = null, + .include_dirs = ArrayList([]const u8).init(builder.allocator), }; } @@ -1648,6 +1650,10 @@ pub const TestStep = struct { self.verbose = value; } + pub fn addIncludeDir(self: &TestStep, path: []const u8) void { + self.include_dirs.append(path) catch unreachable; + } + pub fn setBuildMode(self: &TestStep, mode: builtin.Mode) void { self.build_mode = mode; } @@ -1746,6 +1752,11 @@ pub const TestStep = struct { } } + for (self.include_dirs.toSliceConst()) |include_path| { + try zig_args.append("-isystem"); + try zig_args.append(builder.pathFromRoot(include_path)); + } + for (builder.include_paths.toSliceConst()) |include_path| { try zig_args.append("-isystem"); try zig_args.append(builder.pathFromRoot(include_path)); diff --git a/std/special/build_file_template.zig b/std/special/build_file_template.zig index 2edfdadf5..1c06c93cd 100644 --- a/std/special/build_file_template.zig +++ b/std/special/build_file_template.zig @@ -1,6 +1,6 @@ const Builder = @import("std").build.Builder; -pub fn build(b: &Builder) !void { +pub fn build(b: &Builder) void { const mode = b.standardReleaseOptions(); const exe = b.addExecutable("YOUR_NAME_HERE", "src/main.zig"); exe.setBuildMode(mode); From d959faa4c78ded05ac457e93a16edfd1d7c58c0e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 17 Mar 2018 18:19:23 -0400 Subject: [PATCH 005/132] add test for addIncludeDir for test step --- test/build_examples.zig | 1 + test/standalone/issue_794/a_directory/foo.h | 1 + test/standalone/issue_794/build.zig | 11 +++++++++++ test/standalone/issue_794/main.zig | 7 +++++++ 4 files changed, 20 insertions(+) create mode 100644 test/standalone/issue_794/a_directory/foo.h create mode 100644 test/standalone/issue_794/build.zig create mode 100644 test/standalone/issue_794/main.zig diff --git a/test/build_examples.zig b/test/build_examples.zig index b0d939006..a3b44b913 100644 --- a/test/build_examples.zig +++ b/test/build_examples.zig @@ -14,6 +14,7 @@ pub fn addCases(cases: &tests.BuildExamplesContext) void { cases.addBuildFile("example/mix_o_files/build.zig"); } cases.addBuildFile("test/standalone/issue_339/build.zig"); + cases.addBuildFile("test/standalone/issue_794/build.zig"); cases.addBuildFile("test/standalone/pkg_import/build.zig"); cases.addBuildFile("test/standalone/use_alias/build.zig"); cases.addBuildFile("test/standalone/brace_expansion/build.zig"); diff --git a/test/standalone/issue_794/a_directory/foo.h b/test/standalone/issue_794/a_directory/foo.h new file mode 100644 index 000000000..7598ed89f --- /dev/null +++ b/test/standalone/issue_794/a_directory/foo.h @@ -0,0 +1 @@ +#define NUMBER 1234 diff --git a/test/standalone/issue_794/build.zig b/test/standalone/issue_794/build.zig new file mode 100644 index 000000000..4f5dcd7ff --- /dev/null +++ b/test/standalone/issue_794/build.zig @@ -0,0 +1,11 @@ +const Builder = @import("std").build.Builder; + +pub fn build(b: &Builder) void { + const test_artifact = b.addTest("main.zig"); + test_artifact.addIncludeDir("a_directory"); + + b.default_step.dependOn(&test_artifact.step); + + const test_step = b.step("test", "Test the program"); + test_step.dependOn(&test_artifact.step); +} diff --git a/test/standalone/issue_794/main.zig b/test/standalone/issue_794/main.zig new file mode 100644 index 000000000..356a10641 --- /dev/null +++ b/test/standalone/issue_794/main.zig @@ -0,0 +1,7 @@ +const c = @cImport(@cInclude("foo.h")); +const std = @import("std"); +const assert = std.debug.assert; + +test "c import" { + comptime assert(c.NUMBER == 1234); +} From c4544df011cf61a61538bdeb18808f67ff71043e Mon Sep 17 00:00:00 2001 From: Tesla Ice Zhang Date: Tue, 20 Mar 2018 03:00:11 +0800 Subject: [PATCH 006/132] Remove unnecessary rule and re-fix an old bug The "old bug" is cause my last pr. I'm fixing it now. --- doc/langref.html.in | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index fca0a342a..5165a7bb9 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -5733,7 +5733,7 @@ VariableDeclaration = ("var" | "const") Symbol option(":" TypeExpr) option("alig ContainerMember = (ContainerField | FnDef | GlobalVarDecl) -ContainerField = Symbol option(":" PrefixOpExpression option("=" PrefixOpExpression)) "," +ContainerField = Symbol option(":" PrefixOpExpression) option("=" PrefixOpExpression) "," UseDecl = "use" Expression ";" @@ -5751,9 +5751,7 @@ Block = option(Symbol ":") "{" many(Statement) "}" Statement = LocalVarDecl ";" | Defer(Block) | Defer(Expression) ";" | BlockExpression(Block) | Expression ";" | ";" -TypeExpr = ErrorSetExpr - -ErrorSetExpr = (PrefixOpExpression "!" PrefixOpExpression) | PrefixOpExpression +TypeExpr = (PrefixOpExpression "!" PrefixOpExpression) | PrefixOpExpression BlockOrExpression = Block | Expression @@ -5845,7 +5843,7 @@ CurlySuffixExpression = TypeExpr option(ContainerInitExpression) MultiplyOperator = "||" | "*" | "/" | "%" | "**" | "*%" -PrefixOpExpression = PrefixOp ErrorSetExpr | SuffixOpExpression +PrefixOpExpression = PrefixOp TypeExpr | SuffixOpExpression SuffixOpExpression = ("async" option("(" Expression ")") PrimaryExpression FnCallExpression) | PrimaryExpression option(FnCallExpression | ArrayAccessExpression | FieldAccessExpression | SliceExpression) From cb744f3a28cf5aa6caef797e9265b7165c903a55 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 20 Mar 2018 13:48:25 -0400 Subject: [PATCH 007/132] self-hosted build: print helpful message when libstdc++.a not found closes #843 --- build.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build.zig b/build.zig index 2e6c6dd83..88775498c 100644 --- a/build.zig +++ b/build.zig @@ -64,6 +64,14 @@ pub fn build(b: &Builder) !void { if (exe.target.getOs() == builtin.Os.linux) { const libstdcxx_path_padded = try b.exec([][]const u8{cxx_compiler, "-print-file-name=libstdc++.a"}); const libstdcxx_path = ??mem.split(libstdcxx_path_padded, "\r\n").next(); + if (mem.eql(u8, libstdcxx_path, "libstdc++.a")) { + warn( + \\Unable to determine path to libstdc++.a + \\On Fedora, install libstdc++-static and try again. + \\ + ); + return error.RequiredLibraryNotFound; + } exe.addObjectFile(libstdcxx_path); exe.linkSystemLibrary("pthread"); From 543952eb87f7b06a41dd001273a59a0dea728c39 Mon Sep 17 00:00:00 2001 From: Wander Lairson Costa Date: Tue, 20 Mar 2018 11:46:31 +0000 Subject: [PATCH 008/132] Include libxml2 and zlib as required libraries libxml2 is a required library, but we only find out that when the build fails to link against it, if it is not present. The same for zlib. We use find_library to find these two libraries and print nice fail messages if they are not found. --- CMakeLists.txt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a4f57171..979d771cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,22 @@ option(ZIG_FORCE_EXTERNAL_LLD "If your system has the LLD patches use it instead find_package(llvm) find_package(clang) +if(NOT MSVC) + find_library(LIBXML2 NAMES xml2 libxml2) + if(${LIBXML2} STREQUAL "LIBXML2-NOTFOUND") + message(FATAL_ERROR "Could not find libxml2") + else() + message("${LIBXML2} found") + endif() + + find_library(ZLIB NAMES z zlib libz) + if(${ZLIB} STREQUAL "ZLIB-NOTFOUND") + message(FATAL_ERROR "Could not find zlib") + else() + message("${ZLIB} found") + endif() +endif() + set(ZIG_CPP_LIB_DIR "${CMAKE_BINARY_DIR}/zig_cpp") if(ZIG_FORCE_EXTERNAL_LLD) @@ -710,7 +726,7 @@ target_link_libraries(zig LINK_PUBLIC ${CMAKE_THREAD_LIBS_INIT} ) if(NOT MSVC) - target_link_libraries(zig LINK_PUBLIC xml2) + target_link_libraries(zig LINK_PUBLIC ${LIBXML2}) endif() if(ZIG_DIA_GUIDS_LIB) target_link_libraries(zig LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB}) From f885a1ab61a664378c3f3062a3de1150b6a4ee07 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 21 Mar 2018 19:56:41 -0400 Subject: [PATCH 009/132] change async function call syntax * instead of `async(allocator) call()`, now it is `async call()`. * Fixes syntax ambiguity when leaving off the allocator * Fixes parse failure when call is a field access This sets a precedent for using `<` to pass arguments to a keyword. This will affect `enum`, `union`, and `fn` (see #661) --- doc/langref.html.in | 2 +- src/parser.cpp | 19 ++++++++++--------- test/cases/coroutines.zig | 25 +++++++++++++++++++------ test/compile_errors.zig | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 5165a7bb9..d227936f8 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -5845,7 +5845,7 @@ MultiplyOperator = "||" | "*" | "/" | "%" | "**" | "*%" PrefixOpExpression = PrefixOp TypeExpr | SuffixOpExpression -SuffixOpExpression = ("async" option("(" Expression ")") PrimaryExpression FnCallExpression) | PrimaryExpression option(FnCallExpression | ArrayAccessExpression | FieldAccessExpression | SliceExpression) +SuffixOpExpression = ("async" option("<" SuffixOpExpression ">") SuffixOpExpression FnCallExpression) | PrimaryExpression option(FnCallExpression | ArrayAccessExpression | FieldAccessExpression | SliceExpression) FieldAccessExpression = "." Symbol diff --git a/src/parser.cpp b/src/parser.cpp index c7675ad67..0c9b7e326 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -956,7 +956,7 @@ static AstNode *ast_parse_curly_suffix_expr(ParseContext *pc, size_t *token_inde } /* -SuffixOpExpression = ("async" option("(" Expression ")") PrimaryExpression FnCallExpression) | PrimaryExpression option(FnCallExpression | ArrayAccessExpression | FieldAccessExpression | SliceExpression) +SuffixOpExpression = ("async" option("<" SuffixOpExpression ">") SuffixOpExpression FnCallExpression) | PrimaryExpression option(FnCallExpression | ArrayAccessExpression | FieldAccessExpression | SliceExpression) FnCallExpression : token(LParen) list(Expression, token(Comma)) token(RParen) ArrayAccessExpression : token(LBracket) Expression token(RBracket) SliceExpression = "[" Expression ".." option(Expression) "]" @@ -972,19 +972,20 @@ static AstNode *ast_parse_suffix_op_expr(ParseContext *pc, size_t *token_index, AstNode *allocator_expr_node = nullptr; Token *async_lparen_tok = &pc->tokens->at(*token_index); - if (async_lparen_tok->id == TokenIdLParen) { + if (async_lparen_tok->id == TokenIdCmpLessThan) { *token_index += 1; - allocator_expr_node = ast_parse_expression(pc, token_index, true); - ast_eat_token(pc, token_index, TokenIdRParen); + allocator_expr_node = ast_parse_prefix_op_expr(pc, token_index, true); + ast_eat_token(pc, token_index, TokenIdCmpGreaterThan); } - AstNode *fn_ref_expr_node = ast_parse_primary_expr(pc, token_index, true); - Token *lparen_tok = ast_eat_token(pc, token_index, TokenIdLParen); - AstNode *node = ast_create_node(pc, NodeTypeFnCallExpr, lparen_tok); + Token *fncall_token = &pc->tokens->at(*token_index); + AstNode *node = ast_parse_suffix_op_expr(pc, token_index, true); + if (node->type != NodeTypeFnCallExpr) { + ast_error(pc, fncall_token, "expected function call, found '%s'", token_name(fncall_token->id)); + } node->data.fn_call_expr.is_async = true; node->data.fn_call_expr.async_allocator = allocator_expr_node; - node->data.fn_call_expr.fn_ref_expr = fn_ref_expr_node; - ast_parse_fn_call_param_list(pc, token_index, &node->data.fn_call_expr.params); + assert(node->data.fn_call_expr.fn_ref_expr != nullptr); primary_expr = node; } else { diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 12235cf40..9e98276e0 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -4,7 +4,7 @@ const assert = std.debug.assert; var x: i32 = 1; test "create a coroutine and cancel it" { - const p = try async(std.debug.global_allocator) simpleAsyncFn(); + const p = try async simpleAsyncFn(); cancel p; assert(x == 2); } @@ -17,7 +17,7 @@ async fn simpleAsyncFn() void { test "coroutine suspend, resume, cancel" { seq('a'); - const p = try async(std.debug.global_allocator) testAsyncSeq(); + const p = try async testAsyncSeq(); seq('c'); resume p; seq('f'); @@ -43,7 +43,7 @@ fn seq(c: u8) void { } test "coroutine suspend with block" { - const p = try async(std.debug.global_allocator) testSuspendBlock(); + const p = try async testSuspendBlock(); std.debug.assert(!result); resume a_promise; std.debug.assert(result); @@ -65,7 +65,7 @@ var await_final_result: i32 = 0; test "coroutine await" { await_seq('a'); - const p = async(std.debug.global_allocator) await_amain() catch unreachable; + const p = async await_amain() catch unreachable; await_seq('f'); resume await_a_promise; await_seq('i'); @@ -104,7 +104,7 @@ var early_final_result: i32 = 0; test "coroutine await early return" { early_seq('a'); - const p = async(std.debug.global_allocator) early_amain() catch unreachable; + const p = async early_amain() catch unreachable; early_seq('f'); assert(early_final_result == 1234); assert(std.mem.eql(u8, early_points, "abcdef")); @@ -133,7 +133,7 @@ fn early_seq(c: u8) void { test "coro allocation failure" { var failing_allocator = std.debug.FailingAllocator.init(std.debug.global_allocator, 0); - if (async(&failing_allocator.allocator) asyncFuncThatNeverGetsRun()) { + if (async<&failing_allocator.allocator> asyncFuncThatNeverGetsRun()) { @panic("expected allocation failure"); } else |err| switch (err) { error.OutOfMemory => {}, @@ -143,3 +143,16 @@ test "coro allocation failure" { async fn asyncFuncThatNeverGetsRun() void { @panic("coro frame allocation should fail"); } + +test "async function with dot syntax" { + const S = struct { + var y: i32 = 1; + async fn foo() void { + y += 1; + suspend; + } + }; + const p = try async S.foo(); + cancel p; + assert(S.y == 2); +} diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 68737eee1..ddf528633 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -17,7 +17,7 @@ pub fn addCases(cases: &tests.CompileErrorContext) void { cases.add("returning error from void async function", \\const std = @import("std"); \\export fn entry() void { - \\ const p = async(std.debug.global_allocator) amain() catch unreachable; + \\ const p = async amain() catch unreachable; \\} \\async fn amain() void { \\ return error.ShouldBeCompileError; From 53588f4f124bb713cc2c67c721d5f4e3586b7d79 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Thu, 22 Mar 2018 19:18:51 +1300 Subject: [PATCH 010/132] Add missing pub specifier to atan2 --- std/math/atan2.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/math/atan2.zig b/std/math/atan2.zig index 6f29b7b18..37c520da4 100644 --- a/std/math/atan2.zig +++ b/std/math/atan2.zig @@ -22,7 +22,7 @@ const std = @import("../index.zig"); const math = std.math; const assert = std.debug.assert; -fn atan2(comptime T: type, x: T, y: T) T { +pub fn atan2(comptime T: type, x: T, y: T) T { return switch (T) { f32 => atan2_32(x, y), f64 => atan2_64(x, y), From 7a99d63c764f3d5d92370c90f932b1bf156269f6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 22 Mar 2018 16:56:03 -0400 Subject: [PATCH 011/132] ability to use async function pointers closes #817 --- doc/langref.html.in | 2 +- src/all_types.hpp | 1 + src/analyze.cpp | 3 +- src/ir.cpp | 22 ++++++- src/parser.cpp | 126 +++++++++++++++++++++----------------- test/cases/coroutines.zig | 20 ++++++ 6 files changed, 113 insertions(+), 61 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index d227936f8..f4aa42eb8 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -5739,7 +5739,7 @@ UseDecl = "use" Expression ";" ExternDecl = "extern" option(String) (FnProto | VariableDeclaration) ";" -FnProto = option("nakedcc" | "stdcallcc" | "extern" | ("async" option("(" Expression ")"))) "fn" option(Symbol) ParamDeclList option("align" "(" Expression ")") option("section" "(" Expression ")") option("!") (TypeExpr | "var") +FnProto = option("nakedcc" | "stdcallcc" | "extern" | ("async" option("<" Expression ">"))) "fn" option(Symbol) ParamDeclList option("align" "(" Expression ")") option("section" "(" Expression ")") option("!") (TypeExpr | "var") FnDef = option("inline" | "export") FnProto Block diff --git a/src/all_types.hpp b/src/all_types.hpp index 6afa7ccf2..c1dec80d7 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2673,6 +2673,7 @@ struct IrInstructionFnProto { IrInstruction **param_types; IrInstruction *align_value; IrInstruction *return_type; + IrInstruction *async_allocator_type_value; bool is_var_args; }; diff --git a/src/analyze.cpp b/src/analyze.cpp index 395df229c..eb1850c8e 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -985,7 +985,8 @@ TypeTableEntry *get_fn_type(CodeGen *g, FnTypeId *fn_type_id) { // populate the name of the type buf_resize(&fn_type->name, 0); if (fn_type->data.fn.fn_type_id.cc == CallingConventionAsync) { - buf_appendf(&fn_type->name, "async(%s) ", buf_ptr(&fn_type_id->async_allocator_type->name)); + assert(fn_type_id->async_allocator_type != nullptr); + buf_appendf(&fn_type->name, "async<%s> ", buf_ptr(&fn_type_id->async_allocator_type->name)); } else { const char *cc_str = calling_convention_fn_type_str(fn_type->data.fn.fn_type_id.cc); buf_appendf(&fn_type->name, "%s", cc_str); diff --git a/src/ir.cpp b/src/ir.cpp index db9a2b24c..8f418e2c2 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2141,12 +2141,14 @@ static IrInstruction *ir_build_unwrap_err_payload_from(IrBuilder *irb, IrInstruc } static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, bool is_var_args) + IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, + IrInstruction *async_allocator_type_value, bool is_var_args) { IrInstructionFnProto *instruction = ir_build_instruction(irb, scope, source_node); instruction->param_types = param_types; instruction->align_value = align_value; instruction->return_type = return_type; + instruction->async_allocator_type_value = async_allocator_type_value; instruction->is_var_args = is_var_args; assert(source_node->type == NodeTypeFnProto); @@ -2156,6 +2158,7 @@ static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *s if (param_types[i] != nullptr) ir_ref_instruction(param_types[i], irb->current_basic_block); } if (align_value != nullptr) ir_ref_instruction(align_value, irb->current_basic_block); + if (async_allocator_type_value != nullptr) ir_ref_instruction(async_allocator_type_value, irb->current_basic_block); ir_ref_instruction(return_type, irb->current_basic_block); return &instruction->base; @@ -5989,7 +5992,15 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return_type = nullptr; } - return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, is_var_args); + IrInstruction *async_allocator_type_value = nullptr; + if (node->data.fn_proto.async_allocator_type != nullptr) { + async_allocator_type_value = ir_gen_node(irb, node->data.fn_proto.async_allocator_type, parent_scope); + if (async_allocator_type_value == irb->codegen->invalid_instruction) + return irb->codegen->invalid_instruction; + } + + return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, + async_allocator_type_value, is_var_args); } static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *parent_scope, AstNode *node) { @@ -16561,6 +16572,13 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc if (type_is_invalid(fn_type_id.return_type)) return ira->codegen->builtin_types.entry_invalid; + if (fn_type_id.cc == CallingConventionAsync) { + IrInstruction *async_allocator_type_value = instruction->async_allocator_type_value->other; + fn_type_id.async_allocator_type = ir_resolve_type(ira, async_allocator_type_value); + if (type_is_invalid(fn_type_id.async_allocator_type)) + return ira->codegen->builtin_types.entry_invalid; + } + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); out_val->data.x_type = get_fn_type(ira->codegen, &fn_type_id); return ira->codegen->builtin_types.entry_type; diff --git a/src/parser.cpp b/src/parser.cpp index 0c9b7e326..666b9da3c 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -955,6 +955,66 @@ static AstNode *ast_parse_curly_suffix_expr(ParseContext *pc, size_t *token_inde } } +static AstNode *ast_parse_fn_proto_partial(ParseContext *pc, size_t *token_index, Token *fn_token, + AstNode *async_allocator_type_node, CallingConvention cc, bool is_extern, VisibMod visib_mod) +{ + AstNode *node = ast_create_node(pc, NodeTypeFnProto, fn_token); + node->data.fn_proto.visib_mod = visib_mod; + node->data.fn_proto.cc = cc; + node->data.fn_proto.is_extern = is_extern; + node->data.fn_proto.async_allocator_type = async_allocator_type_node; + + Token *fn_name = &pc->tokens->at(*token_index); + + if (fn_name->id == TokenIdSymbol) { + *token_index += 1; + node->data.fn_proto.name = token_buf(fn_name); + } else { + node->data.fn_proto.name = nullptr; + } + + ast_parse_param_decl_list(pc, token_index, &node->data.fn_proto.params, &node->data.fn_proto.is_var_args); + + Token *next_token = &pc->tokens->at(*token_index); + if (next_token->id == TokenIdKeywordAlign) { + *token_index += 1; + ast_eat_token(pc, token_index, TokenIdLParen); + + node->data.fn_proto.align_expr = ast_parse_expression(pc, token_index, true); + ast_eat_token(pc, token_index, TokenIdRParen); + next_token = &pc->tokens->at(*token_index); + } + if (next_token->id == TokenIdKeywordSection) { + *token_index += 1; + ast_eat_token(pc, token_index, TokenIdLParen); + + node->data.fn_proto.section_expr = ast_parse_expression(pc, token_index, true); + ast_eat_token(pc, token_index, TokenIdRParen); + next_token = &pc->tokens->at(*token_index); + } + if (next_token->id == TokenIdKeywordVar) { + node->data.fn_proto.return_var_token = next_token; + *token_index += 1; + next_token = &pc->tokens->at(*token_index); + } else { + if (next_token->id == TokenIdKeywordError) { + Token *maybe_lbrace_tok = &pc->tokens->at(*token_index + 1); + if (maybe_lbrace_tok->id == TokenIdLBrace) { + *token_index += 1; + node->data.fn_proto.return_type = ast_create_node(pc, NodeTypeErrorType, next_token); + return node; + } + } else if (next_token->id == TokenIdBang) { + *token_index += 1; + node->data.fn_proto.auto_err_set = true; + next_token = &pc->tokens->at(*token_index); + } + node->data.fn_proto.return_type = ast_parse_type_expr(pc, token_index, true); + } + + return node; +} + /* SuffixOpExpression = ("async" option("<" SuffixOpExpression ">") SuffixOpExpression FnCallExpression) | PrimaryExpression option(FnCallExpression | ArrayAccessExpression | FieldAccessExpression | SliceExpression) FnCallExpression : token(LParen) list(Expression, token(Comma)) token(RParen) @@ -979,6 +1039,11 @@ static AstNode *ast_parse_suffix_op_expr(ParseContext *pc, size_t *token_index, } Token *fncall_token = &pc->tokens->at(*token_index); + if (fncall_token->id == TokenIdKeywordFn) { + *token_index += 1; + return ast_parse_fn_proto_partial(pc, token_index, fncall_token, allocator_expr_node, CallingConventionAsync, + false, VisibModPrivate); + } AstNode *node = ast_parse_suffix_op_expr(pc, token_index, true); if (node->type != NodeTypeFnCallExpr) { ast_error(pc, fncall_token, "expected function call, found '%s'", token_name(fncall_token->id)); @@ -2434,9 +2499,10 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc, size_t *token_index, bool m } else if (first_token->id == TokenIdKeywordAsync) { *token_index += 1; Token *next_token = &pc->tokens->at(*token_index); - if (next_token->id == TokenIdLParen) { + if (next_token->id == TokenIdCmpLessThan) { + *token_index += 1; async_allocator_type_node = ast_parse_type_expr(pc, token_index, true); - ast_eat_token(pc, token_index, TokenIdRParen); + ast_eat_token(pc, token_index, TokenIdCmpGreaterThan); } fn_token = ast_eat_token(pc, token_index, TokenIdKeywordFn); cc = CallingConventionAsync; @@ -2470,61 +2536,7 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc, size_t *token_index, bool m return nullptr; } - AstNode *node = ast_create_node(pc, NodeTypeFnProto, fn_token); - node->data.fn_proto.visib_mod = visib_mod; - node->data.fn_proto.cc = cc; - node->data.fn_proto.is_extern = is_extern; - node->data.fn_proto.async_allocator_type = async_allocator_type_node; - - Token *fn_name = &pc->tokens->at(*token_index); - - if (fn_name->id == TokenIdSymbol) { - *token_index += 1; - node->data.fn_proto.name = token_buf(fn_name); - } else { - node->data.fn_proto.name = nullptr; - } - - ast_parse_param_decl_list(pc, token_index, &node->data.fn_proto.params, &node->data.fn_proto.is_var_args); - - Token *next_token = &pc->tokens->at(*token_index); - if (next_token->id == TokenIdKeywordAlign) { - *token_index += 1; - ast_eat_token(pc, token_index, TokenIdLParen); - - node->data.fn_proto.align_expr = ast_parse_expression(pc, token_index, true); - ast_eat_token(pc, token_index, TokenIdRParen); - next_token = &pc->tokens->at(*token_index); - } - if (next_token->id == TokenIdKeywordSection) { - *token_index += 1; - ast_eat_token(pc, token_index, TokenIdLParen); - - node->data.fn_proto.section_expr = ast_parse_expression(pc, token_index, true); - ast_eat_token(pc, token_index, TokenIdRParen); - next_token = &pc->tokens->at(*token_index); - } - if (next_token->id == TokenIdKeywordVar) { - node->data.fn_proto.return_var_token = next_token; - *token_index += 1; - next_token = &pc->tokens->at(*token_index); - } else { - if (next_token->id == TokenIdKeywordError) { - Token *maybe_lbrace_tok = &pc->tokens->at(*token_index + 1); - if (maybe_lbrace_tok->id == TokenIdLBrace) { - *token_index += 1; - node->data.fn_proto.return_type = ast_create_node(pc, NodeTypeErrorType, next_token); - return node; - } - } else if (next_token->id == TokenIdBang) { - *token_index += 1; - node->data.fn_proto.auto_err_set = true; - next_token = &pc->tokens->at(*token_index); - } - node->data.fn_proto.return_type = ast_parse_type_expr(pc, token_index, true); - } - - return node; + return ast_parse_fn_proto_partial(pc, token_index, fn_token, async_allocator_type_node, cc, is_extern, visib_mod); } /* diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 9e98276e0..f8ad58f70 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -156,3 +156,23 @@ test "async function with dot syntax" { cancel p; assert(S.y == 2); } + +test "async fn pointer in a struct field" { + var data: i32 = 1; + const Foo = struct { + bar: async<&std.mem.Allocator> fn(&i32) void, + }; + var foo = Foo { + .bar = simpleAsyncFn2, + }; + const p = (async foo.bar(&data)) catch unreachable; + assert(data == 2); + cancel p; + assert(data == 4); +} + +async<&std.mem.Allocator> fn simpleAsyncFn2(y: &i32) void { + defer *y += 2; + *y += 1; + suspend; +} From 3d1732ef6c8ebc7edf10485f61b4ec905303cd8a Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 23 Mar 2018 20:27:11 +1300 Subject: [PATCH 012/132] Fix OpqaueType usage in exported c functions We prefer `struct typename`. If a typedef is required, this must be done manually after generation. --- src/codegen.cpp | 6 +----- test/gen_h.zig | 11 +++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index a44091bdb..de8cfa31e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6592,6 +6592,7 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf } } case TypeTableEntryIdStruct: + case TypeTableEntryIdOpaque: { buf_init_from_str(out_buf, "struct "); buf_append_buf(out_buf, &type_entry->name); @@ -6609,11 +6610,6 @@ static void get_c_type(CodeGen *g, GenH *gen_h, TypeTableEntry *type_entry, Buf buf_append_buf(out_buf, &type_entry->name); return; } - case TypeTableEntryIdOpaque: - { - buf_init_from_buf(out_buf, &type_entry->name); - return; - } case TypeTableEntryIdArray: { TypeTableEntryArray *array_data = &type_entry->data.array; diff --git a/test/gen_h.zig b/test/gen_h.zig index 5f28326ff..30d168cf2 100644 --- a/test/gen_h.zig +++ b/test/gen_h.zig @@ -51,6 +51,16 @@ pub fn addCases(cases: &tests.GenHContext) void { \\ ); + cases.add("declare opaque type", + \\export const Foo = @OpaqueType(); + \\ + \\export fn entry(foo: ?&Foo) void { } + , + \\struct Foo; + \\ + \\TEST_EXPORT void entry(struct Foo * foo); + ); + cases.add("array field-type", \\const Foo = extern struct { \\ A: [2]i32, @@ -66,4 +76,5 @@ pub fn addCases(cases: &tests.GenHContext) void { \\TEST_EXPORT void entry(struct Foo foo, uint8_t bar[]); \\ ); + } From 7350181a4a778f9d03186e5123beffdf80f58606 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 23 Mar 2018 22:41:08 +1300 Subject: [PATCH 013/132] Fix os.File.mode function --- std/os/file.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/os/file.zig b/std/os/file.zig index 772fbf7c7..eed3a443b 100644 --- a/std/os/file.zig +++ b/std/os/file.zig @@ -233,7 +233,7 @@ pub const File = struct { Unexpected, }; - fn mode(self: &File) ModeError!FileMode { + fn mode(self: &File) ModeError!os.FileMode { if (is_posix) { var stat: posix.Stat = undefined; const err = posix.getErrno(posix.fstat(self.handle, &stat)); From 2cff31937f6008769ad1034f9451d136d03c06bb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 15:57:36 -0400 Subject: [PATCH 014/132] std.os.linux exposes syscall functions and syscall numbers --- std/os/linux/index.zig | 158 +++++++++++++++++------------------------ 1 file changed, 67 insertions(+), 91 deletions(-) diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 646b1ef30..8fd8bcbe7 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -1,7 +1,7 @@ const std = @import("../../index.zig"); const assert = std.debug.assert; const builtin = @import("builtin"); -const arch = switch (builtin.arch) { +pub use switch (builtin.arch) { builtin.Arch.x86_64 => @import("x86_64.zig"), builtin.Arch.i386 => @import("i386.zig"), else => @compileError("unsupported arch"), @@ -93,27 +93,6 @@ pub const O_RDONLY = 0o0; pub const O_WRONLY = 0o1; pub const O_RDWR = 0o2; -pub const O_CREAT = arch.O_CREAT; -pub const O_EXCL = arch.O_EXCL; -pub const O_NOCTTY = arch.O_NOCTTY; -pub const O_TRUNC = arch.O_TRUNC; -pub const O_APPEND = arch.O_APPEND; -pub const O_NONBLOCK = arch.O_NONBLOCK; -pub const O_DSYNC = arch.O_DSYNC; -pub const O_SYNC = arch.O_SYNC; -pub const O_RSYNC = arch.O_RSYNC; -pub const O_DIRECTORY = arch.O_DIRECTORY; -pub const O_NOFOLLOW = arch.O_NOFOLLOW; -pub const O_CLOEXEC = arch.O_CLOEXEC; - -pub const O_ASYNC = arch.O_ASYNC; -pub const O_DIRECT = arch.O_DIRECT; -pub const O_LARGEFILE = arch.O_LARGEFILE; -pub const O_NOATIME = arch.O_NOATIME; -pub const O_PATH = arch.O_PATH; -pub const O_TMPFILE = arch.O_TMPFILE; -pub const O_NDELAY = arch.O_NDELAY; - pub const SEEK_SET = 0; pub const SEEK_CUR = 1; pub const SEEK_END = 2; @@ -394,65 +373,65 @@ pub fn getErrno(r: usize) usize { } pub fn dup2(old: i32, new: i32) usize { - return arch.syscall2(arch.SYS_dup2, usize(old), usize(new)); + return syscall2(SYS_dup2, usize(old), usize(new)); } pub fn chdir(path: &const u8) usize { - return arch.syscall1(arch.SYS_chdir, @ptrToInt(path)); + return syscall1(SYS_chdir, @ptrToInt(path)); } pub fn execve(path: &const u8, argv: &const ?&const u8, envp: &const ?&const u8) usize { - return arch.syscall3(arch.SYS_execve, @ptrToInt(path), @ptrToInt(argv), @ptrToInt(envp)); + return syscall3(SYS_execve, @ptrToInt(path), @ptrToInt(argv), @ptrToInt(envp)); } pub fn fork() usize { - return arch.syscall0(arch.SYS_fork); + return syscall0(SYS_fork); } pub fn getcwd(buf: &u8, size: usize) usize { - return arch.syscall2(arch.SYS_getcwd, @ptrToInt(buf), size); + return syscall2(SYS_getcwd, @ptrToInt(buf), size); } pub fn getdents(fd: i32, dirp: &u8, count: usize) usize { - return arch.syscall3(arch.SYS_getdents, usize(fd), @ptrToInt(dirp), count); + return syscall3(SYS_getdents, usize(fd), @ptrToInt(dirp), count); } pub fn isatty(fd: i32) bool { var wsz: winsize = undefined; - return arch.syscall3(arch.SYS_ioctl, usize(fd), TIOCGWINSZ, @ptrToInt(&wsz)) == 0; + return syscall3(SYS_ioctl, usize(fd), TIOCGWINSZ, @ptrToInt(&wsz)) == 0; } pub fn readlink(noalias path: &const u8, noalias buf_ptr: &u8, buf_len: usize) usize { - return arch.syscall3(arch.SYS_readlink, @ptrToInt(path), @ptrToInt(buf_ptr), buf_len); + return syscall3(SYS_readlink, @ptrToInt(path), @ptrToInt(buf_ptr), buf_len); } pub fn mkdir(path: &const u8, mode: u32) usize { - return arch.syscall2(arch.SYS_mkdir, @ptrToInt(path), mode); + return syscall2(SYS_mkdir, @ptrToInt(path), mode); } pub fn mmap(address: ?&u8, length: usize, prot: usize, flags: usize, fd: i32, offset: isize) usize { - return arch.syscall6(arch.SYS_mmap, @ptrToInt(address), length, prot, flags, usize(fd), + return syscall6(SYS_mmap, @ptrToInt(address), length, prot, flags, usize(fd), @bitCast(usize, offset)); } pub fn munmap(address: &u8, length: usize) usize { - return arch.syscall2(arch.SYS_munmap, @ptrToInt(address), length); + return syscall2(SYS_munmap, @ptrToInt(address), length); } pub fn read(fd: i32, buf: &u8, count: usize) usize { - return arch.syscall3(arch.SYS_read, usize(fd), @ptrToInt(buf), count); + return syscall3(SYS_read, usize(fd), @ptrToInt(buf), count); } pub fn rmdir(path: &const u8) usize { - return arch.syscall1(arch.SYS_rmdir, @ptrToInt(path)); + return syscall1(SYS_rmdir, @ptrToInt(path)); } pub fn symlink(existing: &const u8, new: &const u8) usize { - return arch.syscall2(arch.SYS_symlink, @ptrToInt(existing), @ptrToInt(new)); + return syscall2(SYS_symlink, @ptrToInt(existing), @ptrToInt(new)); } pub fn pread(fd: i32, buf: &u8, count: usize, offset: usize) usize { - return arch.syscall4(arch.SYS_pread, usize(fd), @ptrToInt(buf), count, offset); + return syscall4(SYS_pread, usize(fd), @ptrToInt(buf), count, offset); } pub fn pipe(fd: &[2]i32) usize { @@ -460,84 +439,84 @@ pub fn pipe(fd: &[2]i32) usize { } pub fn pipe2(fd: &[2]i32, flags: usize) usize { - return arch.syscall2(arch.SYS_pipe2, @ptrToInt(fd), flags); + return syscall2(SYS_pipe2, @ptrToInt(fd), flags); } pub fn write(fd: i32, buf: &const u8, count: usize) usize { - return arch.syscall3(arch.SYS_write, usize(fd), @ptrToInt(buf), count); + return syscall3(SYS_write, usize(fd), @ptrToInt(buf), count); } pub fn pwrite(fd: i32, buf: &const u8, count: usize, offset: usize) usize { - return arch.syscall4(arch.SYS_pwrite, usize(fd), @ptrToInt(buf), count, offset); + return syscall4(SYS_pwrite, usize(fd), @ptrToInt(buf), count, offset); } pub fn rename(old: &const u8, new: &const u8) usize { - return arch.syscall2(arch.SYS_rename, @ptrToInt(old), @ptrToInt(new)); + return syscall2(SYS_rename, @ptrToInt(old), @ptrToInt(new)); } pub fn open(path: &const u8, flags: u32, perm: usize) usize { - return arch.syscall3(arch.SYS_open, @ptrToInt(path), flags, perm); + return syscall3(SYS_open, @ptrToInt(path), flags, perm); } pub fn create(path: &const u8, perm: usize) usize { - return arch.syscall2(arch.SYS_creat, @ptrToInt(path), perm); + return syscall2(SYS_creat, @ptrToInt(path), perm); } pub fn openat(dirfd: i32, path: &const u8, flags: usize, mode: usize) usize { - return arch.syscall4(arch.SYS_openat, usize(dirfd), @ptrToInt(path), flags, mode); + return syscall4(SYS_openat, usize(dirfd), @ptrToInt(path), flags, mode); } pub fn close(fd: i32) usize { - return arch.syscall1(arch.SYS_close, usize(fd)); + return syscall1(SYS_close, usize(fd)); } pub fn lseek(fd: i32, offset: isize, ref_pos: usize) usize { - return arch.syscall3(arch.SYS_lseek, usize(fd), @bitCast(usize, offset), ref_pos); + return syscall3(SYS_lseek, usize(fd), @bitCast(usize, offset), ref_pos); } pub fn exit(status: i32) noreturn { - _ = arch.syscall1(arch.SYS_exit, @bitCast(usize, isize(status))); + _ = syscall1(SYS_exit, @bitCast(usize, isize(status))); unreachable; } pub fn getrandom(buf: &u8, count: usize, flags: u32) usize { - return arch.syscall3(arch.SYS_getrandom, @ptrToInt(buf), count, usize(flags)); + return syscall3(SYS_getrandom, @ptrToInt(buf), count, usize(flags)); } pub fn kill(pid: i32, sig: i32) usize { - return arch.syscall2(arch.SYS_kill, @bitCast(usize, isize(pid)), usize(sig)); + return syscall2(SYS_kill, @bitCast(usize, isize(pid)), usize(sig)); } pub fn unlink(path: &const u8) usize { - return arch.syscall1(arch.SYS_unlink, @ptrToInt(path)); + return syscall1(SYS_unlink, @ptrToInt(path)); } pub fn waitpid(pid: i32, status: &i32, options: i32) usize { - return arch.syscall4(arch.SYS_wait4, @bitCast(usize, isize(pid)), @ptrToInt(status), @bitCast(usize, isize(options)), 0); + return syscall4(SYS_wait4, @bitCast(usize, isize(pid)), @ptrToInt(status), @bitCast(usize, isize(options)), 0); } pub fn nanosleep(req: &const timespec, rem: ?×pec) usize { - return arch.syscall2(arch.SYS_nanosleep, @ptrToInt(req), @ptrToInt(rem)); + return syscall2(SYS_nanosleep, @ptrToInt(req), @ptrToInt(rem)); } pub fn setuid(uid: u32) usize { - return arch.syscall1(arch.SYS_setuid, uid); + return syscall1(SYS_setuid, uid); } pub fn setgid(gid: u32) usize { - return arch.syscall1(arch.SYS_setgid, gid); + return syscall1(SYS_setgid, gid); } pub fn setreuid(ruid: u32, euid: u32) usize { - return arch.syscall2(arch.SYS_setreuid, ruid, euid); + return syscall2(SYS_setreuid, ruid, euid); } pub fn setregid(rgid: u32, egid: u32) usize { - return arch.syscall2(arch.SYS_setregid, rgid, egid); + return syscall2(SYS_setregid, rgid, egid); } pub fn sigprocmask(flags: u32, noalias set: &const sigset_t, noalias oldset: ?&sigset_t) usize { - return arch.syscall4(arch.SYS_rt_sigprocmask, flags, @ptrToInt(set), @ptrToInt(oldset), NSIG/8); + return syscall4(SYS_rt_sigprocmask, flags, @ptrToInt(set), @ptrToInt(oldset), NSIG/8); } pub fn sigaction(sig: u6, noalias act: &const Sigaction, noalias oact: ?&Sigaction) usize { @@ -548,11 +527,11 @@ pub fn sigaction(sig: u6, noalias act: &const Sigaction, noalias oact: ?&Sigacti .handler = act.handler, .flags = act.flags | SA_RESTORER, .mask = undefined, - .restorer = @ptrCast(extern fn()void, arch.restore_rt), + .restorer = @ptrCast(extern fn()void, restore_rt), }; var ksa_old: k_sigaction = undefined; @memcpy(@ptrCast(&u8, &ksa.mask), @ptrCast(&const u8, &act.mask), 8); - const result = arch.syscall4(arch.SYS_rt_sigaction, sig, @ptrToInt(&ksa), @ptrToInt(&ksa_old), @sizeOf(@typeOf(ksa.mask))); + const result = syscall4(SYS_rt_sigaction, sig, @ptrToInt(&ksa), @ptrToInt(&ksa_old), @sizeOf(@typeOf(ksa.mask))); const err = getErrno(result); if (err != 0) { return result; @@ -592,22 +571,22 @@ pub const empty_sigset = []usize{0} ** sigset_t.len; pub fn raise(sig: i32) usize { var set: sigset_t = undefined; blockAppSignals(&set); - const tid = i32(arch.syscall0(arch.SYS_gettid)); - const ret = arch.syscall2(arch.SYS_tkill, usize(tid), usize(sig)); + const tid = i32(syscall0(SYS_gettid)); + const ret = syscall2(SYS_tkill, usize(tid), usize(sig)); restoreSignals(&set); return ret; } fn blockAllSignals(set: &sigset_t) void { - _ = arch.syscall4(arch.SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&all_mask), @ptrToInt(set), NSIG/8); + _ = syscall4(SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&all_mask), @ptrToInt(set), NSIG/8); } fn blockAppSignals(set: &sigset_t) void { - _ = arch.syscall4(arch.SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&app_mask), @ptrToInt(set), NSIG/8); + _ = syscall4(SYS_rt_sigprocmask, SIG_BLOCK, @ptrToInt(&app_mask), @ptrToInt(set), NSIG/8); } fn restoreSignals(set: &sigset_t) void { - _ = arch.syscall4(arch.SYS_rt_sigprocmask, SIG_SETMASK, @ptrToInt(set), 0, NSIG/8); + _ = syscall4(SYS_rt_sigprocmask, SIG_SETMASK, @ptrToInt(set), 0, NSIG/8); } pub fn sigaddset(set: &sigset_t, sig: u6) void { @@ -653,61 +632,61 @@ pub const iovec = extern struct { }; pub fn getsockname(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { - return arch.syscall3(arch.SYS_getsockname, usize(fd), @ptrToInt(addr), @ptrToInt(len)); + return syscall3(SYS_getsockname, usize(fd), @ptrToInt(addr), @ptrToInt(len)); } pub fn getpeername(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { - return arch.syscall3(arch.SYS_getpeername, usize(fd), @ptrToInt(addr), @ptrToInt(len)); + return syscall3(SYS_getpeername, usize(fd), @ptrToInt(addr), @ptrToInt(len)); } pub fn socket(domain: i32, socket_type: i32, protocol: i32) usize { - return arch.syscall3(arch.SYS_socket, usize(domain), usize(socket_type), usize(protocol)); + return syscall3(SYS_socket, usize(domain), usize(socket_type), usize(protocol)); } pub fn setsockopt(fd: i32, level: i32, optname: i32, optval: &const u8, optlen: socklen_t) usize { - return arch.syscall5(arch.SYS_setsockopt, usize(fd), usize(level), usize(optname), usize(optval), @ptrToInt(optlen)); + return syscall5(SYS_setsockopt, usize(fd), usize(level), usize(optname), usize(optval), @ptrToInt(optlen)); } pub fn getsockopt(fd: i32, level: i32, optname: i32, noalias optval: &u8, noalias optlen: &socklen_t) usize { - return arch.syscall5(arch.SYS_getsockopt, usize(fd), usize(level), usize(optname), @ptrToInt(optval), @ptrToInt(optlen)); + return syscall5(SYS_getsockopt, usize(fd), usize(level), usize(optname), @ptrToInt(optval), @ptrToInt(optlen)); } -pub fn sendmsg(fd: i32, msg: &const arch.msghdr, flags: u32) usize { - return arch.syscall3(arch.SYS_sendmsg, usize(fd), @ptrToInt(msg), flags); +pub fn sendmsg(fd: i32, msg: &const msghdr, flags: u32) usize { + return syscall3(SYS_sendmsg, usize(fd), @ptrToInt(msg), flags); } pub fn connect(fd: i32, addr: &const sockaddr, len: socklen_t) usize { - return arch.syscall3(arch.SYS_connect, usize(fd), @ptrToInt(addr), usize(len)); + return syscall3(SYS_connect, usize(fd), @ptrToInt(addr), usize(len)); } -pub fn recvmsg(fd: i32, msg: &arch.msghdr, flags: u32) usize { - return arch.syscall3(arch.SYS_recvmsg, usize(fd), @ptrToInt(msg), flags); +pub fn recvmsg(fd: i32, msg: &msghdr, flags: u32) usize { + return syscall3(SYS_recvmsg, usize(fd), @ptrToInt(msg), flags); } pub fn recvfrom(fd: i32, noalias buf: &u8, len: usize, flags: u32, noalias addr: ?&sockaddr, noalias alen: ?&socklen_t) usize { - return arch.syscall6(arch.SYS_recvfrom, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), @ptrToInt(alen)); + return syscall6(SYS_recvfrom, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), @ptrToInt(alen)); } pub fn shutdown(fd: i32, how: i32) usize { - return arch.syscall2(arch.SYS_shutdown, usize(fd), usize(how)); + return syscall2(SYS_shutdown, usize(fd), usize(how)); } pub fn bind(fd: i32, addr: &const sockaddr, len: socklen_t) usize { - return arch.syscall3(arch.SYS_bind, usize(fd), @ptrToInt(addr), usize(len)); + return syscall3(SYS_bind, usize(fd), @ptrToInt(addr), usize(len)); } pub fn listen(fd: i32, backlog: i32) usize { - return arch.syscall2(arch.SYS_listen, usize(fd), usize(backlog)); + return syscall2(SYS_listen, usize(fd), usize(backlog)); } pub fn sendto(fd: i32, buf: &const u8, len: usize, flags: u32, addr: ?&const sockaddr, alen: socklen_t) usize { - return arch.syscall6(arch.SYS_sendto, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), usize(alen)); + return syscall6(SYS_sendto, usize(fd), @ptrToInt(buf), len, flags, @ptrToInt(addr), usize(alen)); } pub fn socketpair(domain: i32, socket_type: i32, protocol: i32, fd: [2]i32) usize { - return arch.syscall4(arch.SYS_socketpair, usize(domain), usize(socket_type), usize(protocol), @ptrToInt(&fd[0])); + return syscall4(SYS_socketpair, usize(domain), usize(socket_type), usize(protocol), @ptrToInt(&fd[0])); } pub fn accept(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { @@ -715,7 +694,7 @@ pub fn accept(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) usize { } pub fn accept4(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t, flags: u32) usize { - return arch.syscall4(arch.SYS_accept4, usize(fd), @ptrToInt(addr), @ptrToInt(len), flags); + return syscall4(SYS_accept4, usize(fd), @ptrToInt(addr), @ptrToInt(len), flags); } // error NameTooLong; @@ -746,11 +725,8 @@ pub fn accept4(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t, flags: // return ifr.ifr_ifindex; // } -pub const Stat = arch.Stat; -pub const timespec = arch.timespec; - pub fn fstat(fd: i32, stat_buf: &Stat) usize { - return arch.syscall2(arch.SYS_fstat, usize(fd), @ptrToInt(stat_buf)); + return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } pub const epoll_data = extern union { @@ -770,19 +746,19 @@ pub fn epoll_create() usize { } pub fn epoll_create1(flags: usize) usize { - return arch.syscall1(arch.SYS_epoll_create1, flags); + return syscall1(SYS_epoll_create1, flags); } pub fn epoll_ctl(epoll_fd: i32, op: i32, fd: i32, ev: &epoll_event) usize { - return arch.syscall4(arch.SYS_epoll_ctl, usize(epoll_fd), usize(op), usize(fd), @ptrToInt(ev)); + return syscall4(SYS_epoll_ctl, usize(epoll_fd), usize(op), usize(fd), @ptrToInt(ev)); } pub fn epoll_wait(epoll_fd: i32, events: &epoll_event, maxevents: u32, timeout: i32) usize { - return arch.syscall4(arch.SYS_epoll_wait, usize(epoll_fd), @ptrToInt(events), usize(maxevents), usize(timeout)); + return syscall4(SYS_epoll_wait, usize(epoll_fd), @ptrToInt(events), usize(maxevents), usize(timeout)); } pub fn timerfd_create(clockid: i32, flags: u32) usize { - return arch.syscall2(arch.SYS_timerfd_create, usize(clockid), usize(flags)); + return syscall2(SYS_timerfd_create, usize(clockid), usize(flags)); } pub const itimerspec = extern struct { @@ -791,11 +767,11 @@ pub const itimerspec = extern struct { }; pub fn timerfd_gettime(fd: i32, curr_value: &itimerspec) usize { - return arch.syscall2(arch.SYS_timerfd_gettime, usize(fd), @ptrToInt(curr_value)); + return syscall2(SYS_timerfd_gettime, usize(fd), @ptrToInt(curr_value)); } pub fn timerfd_settime(fd: i32, flags: u32, new_value: &const itimerspec, old_value: ?&itimerspec) usize { - return arch.syscall4(arch.SYS_timerfd_settime, usize(fd), usize(flags), @ptrToInt(new_value), @ptrToInt(old_value)); + return syscall4(SYS_timerfd_settime, usize(fd), usize(flags), @ptrToInt(new_value), @ptrToInt(old_value)); } test "import linux test" { From b1c07c0ea9e351a43c9bc2fe747fc07c0a19e005 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 12 Mar 2018 23:03:47 -0400 Subject: [PATCH 015/132] move error ret tracing codegen to zig ir progress towards #821 --- src/all_types.hpp | 5 +++ src/codegen.cpp | 39 ++++++++++----------- src/ir.cpp | 86 ++++++++++++++++++++++++++++++++++------------- src/ir_print.cpp | 7 ++++ 4 files changed, 91 insertions(+), 46 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index c1dec80d7..b5428c959 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2034,6 +2034,7 @@ enum IrInstructionId { IrInstructionIdAtomicRmw, IrInstructionIdPromiseResultType, IrInstructionIdAwaitBookkeeping, + IrInstructionIdSaveErrRetAddr, }; struct IrInstruction { @@ -2988,6 +2989,10 @@ struct IrInstructionAwaitBookkeeping { IrInstruction *promise_result_type; }; +struct IrInstructionSaveErrRetAddr { + IrInstruction base; +}; + static const size_t slice_ptr_index = 0; static const size_t slice_len_index = 1; diff --git a/src/codegen.cpp b/src/codegen.cpp index de8cfa31e..eddadb94e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1607,32 +1607,25 @@ static LLVMValueRef ir_llvm_value(CodeGen *g, IrInstruction *instruction) { return instruction->llvm_value; } +static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *executable, + IrInstructionSaveErrRetAddr *save_err_ret_addr_instruction) +{ + assert(g->have_err_ret_tracing); + + LLVMValueRef return_err_fn = get_return_err_fn(g); + LLVMValueRef args[] = { + g->cur_err_ret_trace_val, + }; + LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, + get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); + LLVMSetTailCall(call_instruction, true); + return call_instruction; +} + static LLVMValueRef ir_render_return(CodeGen *g, IrExecutable *executable, IrInstructionReturn *return_instruction) { LLVMValueRef value = ir_llvm_value(g, return_instruction->value); TypeTableEntry *return_type = return_instruction->value->value.type; - if (g->have_err_ret_tracing) { - bool is_err_return = false; - if (return_type->id == TypeTableEntryIdErrorUnion) { - if (return_instruction->value->value.special == ConstValSpecialStatic) { - is_err_return = return_instruction->value->value.data.x_err_union.err != nullptr; - } else if (return_instruction->value->value.special == ConstValSpecialRuntime) { - is_err_return = return_instruction->value->value.data.rh_error_union == RuntimeHintErrorUnionError; - // TODO: emit a branch to check if the return value is an error - } - } else if (return_type->id == TypeTableEntryIdErrorSet) { - is_err_return = true; - } - if (is_err_return) { - LLVMValueRef return_err_fn = get_return_err_fn(g); - LLVMValueRef args[] = { - g->cur_err_ret_trace_val, - }; - LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, - get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); - LLVMSetTailCall(call_instruction, true); - } - } if (handle_is_ptr(return_type)) { if (calling_convention_does_first_arg_return(g->cur_fn->type_entry->data.fn.fn_type_id.cc)) { assert(g->cur_ret_ptr); @@ -4400,6 +4393,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_coro_alloc_helper(g, executable, (IrInstructionCoroAllocHelper *)instruction); case IrInstructionIdAtomicRmw: return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction); + case IrInstructionIdSaveErrRetAddr: + return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction); } zig_unreachable(); } diff --git a/src/ir.cpp b/src/ir.cpp index 8f418e2c2..a78be66dd 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -713,6 +713,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionAwaitBookkeeping return IrInstructionIdAwaitBookkeeping; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionSaveErrRetAddr *) { + return IrInstructionIdSaveErrRetAddr; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -2678,6 +2682,11 @@ static IrInstruction *ir_build_await_bookkeeping(IrBuilder *irb, Scope *scope, A return &instruction->base; } +static IrInstruction *ir_build_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *source_node) { + IrInstructionSaveErrRetAddr *instruction = ir_build_instruction(irb, scope, source_node); + return &instruction->base; +} + static void ir_count_defers(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, size_t *results) { results[ReturnKindUnconditional] = 0; results[ReturnKindError] = 0; @@ -2750,16 +2759,16 @@ static ScopeDeferExpr *get_scope_defer_expr(Scope *scope) { return nullptr; } +static bool exec_is_async(IrExecutable *exec) { + FnTableEntry *fn_entry = exec_fn_entry(exec); + return fn_entry != nullptr && fn_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; +} + static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *return_value, bool is_generated_code) { - FnTableEntry *fn_entry = exec_fn_entry(irb->exec); - bool is_async = fn_entry != nullptr && fn_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; + bool is_async = exec_is_async(irb->exec); if (!is_async) { - //if (irb->codegen->have_err_ret_tracing) { - // IrInstruction *stack_trace_ptr = ir_build_error_return_trace_nonnull(irb, scope, node); - // ir_build_save_err_ret_addr(irb, scope, node, stack_trace_ptr); - //} IrInstruction *return_inst = ir_build_return(irb, scope, node, return_value); return_inst->is_gen = is_generated_code; return return_inst; @@ -2781,21 +2790,33 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode // the above blocks are rendered by ir_gen after the rest of codegen } -//static void ir_gen_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *node, bool is_async) { -// if (!irb->codegen->have_err_ret_tracing) -// return; -// -// if (is_async) { -// IrInstruction *err_ret_addr_ptr = ir_build_load_ptr(irb, scope, node, irb->exec->coro_err_ret_addr_ptr); -// IrInstruction *return_address_ptr = ir_build_return_address(irb, scope, node); -// IrInstruction *return_address_usize = ir_build_ptr_to_int(irb, scope, node, return_address_ptr); -// ir_build_store_ptr(irb, scope, node, err_ret_addr_ptr, return_address_usize); -// return; -// } -// -// IrInstruction *stack_trace_ptr = ir_build_error_return_trace_nonnull(irb, scope, node); -// ir_build_save_err_ret_addr(irb, scope, node, stack_trace_ptr); -//} +static bool exec_have_err_ret_trace(CodeGen *g, IrExecutable *exec) { + if (!g->have_err_ret_tracing) + return false; + FnTableEntry *fn_entry = exec_fn_entry(exec); + if (fn_entry == nullptr) + return false; + if (exec->is_inline) + return false; + return type_can_fail(fn_entry->type_entry->data.fn.fn_type_id.return_type); +} + +static void ir_gen_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *node) { + if (!exec_have_err_ret_trace(irb->codegen, irb->exec)) + return; + + bool is_async = exec_is_async(irb->exec); + + if (is_async) { + //IrInstruction *err_ret_addr_ptr = ir_build_load_ptr(irb, scope, node, irb->exec->coro_err_ret_addr_ptr); + //IrInstruction *return_address_ptr = ir_build_instr_addr(irb, scope, node); + //IrInstruction *return_address_usize = ir_build_ptr_to_int(irb, scope, node, return_address_ptr); + //ir_build_store_ptr(irb, scope, node, err_ret_addr_ptr, return_address_usize); + return; + } + + ir_build_save_err_ret_addr(irb, scope, node); +} static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval) { assert(node->type == NodeTypeReturnExpr); @@ -2856,7 +2877,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, if (have_err_defers) { ir_gen_defers_for_block(irb, scope, outer_scope, true); } - //ir_gen_save_err_ret_addr(irb, scope, node, is_async); + ir_gen_save_err_ret_addr(irb, scope, node); ir_build_br(irb, scope, node, ret_stmt_block, is_comptime); ir_set_cursor_at_end_and_append_block(irb, ok_block); @@ -2895,6 +2916,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_set_cursor_at_end_and_append_block(irb, return_block); ir_gen_defers_for_block(irb, scope, outer_scope, true); IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); + ir_gen_save_err_ret_addr(irb, scope, node); ir_gen_async_return(irb, scope, node, err_val, false); ir_set_cursor_at_end_and_append_block(irb, continue_block); @@ -6406,6 +6428,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec return false; if (!instr_is_unreachable(result)) { + // no need for save_err_ret_addr because this cannot return error ir_gen_async_return(irb, scope, result->source_node, result, true); } @@ -11464,13 +11487,17 @@ static TypeTableEntry *ir_analyze_instruction_export(IrAnalyze *ira, IrInstructi return ira->codegen->builtin_types.entry_void; } +static bool exec_has_err_ret_trace(CodeGen *g, IrExecutable *exec) { + FnTableEntry *fn_entry = exec_fn_entry(exec); + return fn_entry != nullptr && fn_entry->calls_or_awaits_errorable_fn && g->have_err_ret_tracing; +} + static TypeTableEntry *ir_analyze_instruction_error_return_trace(IrAnalyze *ira, IrInstructionErrorReturnTrace *instruction) { - FnTableEntry *fn_entry = exec_fn_entry(ira->new_irb.exec); TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(ira->codegen); TypeTableEntry *nullable_type = get_maybe_type(ira->codegen, ptr_to_stack_trace_type); - if (fn_entry == nullptr || !fn_entry->calls_or_awaits_errorable_fn || !ira->codegen->have_err_ret_tracing) { + if (!exec_has_err_ret_trace(ira->codegen, ira->new_irb.exec)) { ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); out_val->data.x_maybe = nullptr; return nullable_type; @@ -17775,6 +17802,14 @@ static TypeTableEntry *ir_analyze_instruction_await_bookkeeping(IrAnalyze *ira, return out_val->type; } +static TypeTableEntry *ir_analyze_instruction_save_err_ret_addr(IrAnalyze *ira, IrInstructionSaveErrRetAddr *instruction) { + IrInstruction *result = ir_build_save_err_ret_addr(&ira->new_irb, instruction->base.scope, + instruction->base.source_node); + ir_link_new_instruction(result, &instruction->base); + result->value.type = ira->codegen->builtin_types.entry_void; + return result->value.type; +} + static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstruction *instruction) { switch (instruction->id) { case IrInstructionIdInvalid: @@ -18012,6 +18047,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_promise_result_type(ira, (IrInstructionPromiseResultType *)instruction); case IrInstructionIdAwaitBookkeeping: return ir_analyze_instruction_await_bookkeeping(ira, (IrInstructionAwaitBookkeeping *)instruction); + case IrInstructionIdSaveErrRetAddr: + return ir_analyze_instruction_save_err_ret_addr(ira, (IrInstructionSaveErrRetAddr *)instruction); } zig_unreachable(); } @@ -18137,6 +18174,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdCoroSave: case IrInstructionIdCoroAllocHelper: case IrInstructionIdAwaitBookkeeping: + case IrInstructionIdSaveErrRetAddr: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 167bd2083..709cb4394 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1161,6 +1161,10 @@ static void ir_print_await_bookkeeping(IrPrint *irp, IrInstructionAwaitBookkeepi fprintf(irp->f, ")"); } +static void ir_print_save_err_ret_addr(IrPrint *irp, IrInstructionSaveErrRetAddr *instruction) { + fprintf(irp->f, "@saveErrRetAddr()"); +} + static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { ir_print_prefix(irp, instruction); switch (instruction->id) { @@ -1532,6 +1536,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdAwaitBookkeeping: ir_print_await_bookkeeping(irp, (IrInstructionAwaitBookkeeping *)instruction); break; + case IrInstructionIdSaveErrRetAddr: + ir_print_save_err_ret_addr(irp, (IrInstructionSaveErrRetAddr *)instruction); + break; } fprintf(irp->f, "\n"); } From 18af2f9a2764cc340571578d58cb2575faeccdc6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 18:21:51 -0400 Subject: [PATCH 016/132] fix async fns with inferred error sets closes #856 --- src/all_types.hpp | 12 ++++++++++- src/analyze.cpp | 10 ++++----- src/ast_render.cpp | 11 +++++++++- src/codegen.cpp | 1 + src/ir.cpp | 45 ++++++++++++++++++++++++++++++++++----- src/ir_print.cpp | 13 +++++++++-- test/cases/coroutines.zig | 11 ++++++++++ 7 files changed, 89 insertions(+), 14 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index b5428c959..f5afecfbd 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1251,7 +1251,10 @@ struct FnTableEntry { ScopeBlock *def_scope; // parent is child_scope Buf symbol_name; TypeTableEntry *type_entry; // function type - TypeTableEntry *implicit_return_type; + // in the case of normal functions this is the implicit return type + // in the case of async functions this is the implicit return type according to the + // zig source code, not according to zig ir + TypeTableEntry *src_implicit_return_type; bool is_test; FnInline fn_inline; FnAnalState anal_state; @@ -2035,6 +2038,7 @@ enum IrInstructionId { IrInstructionIdPromiseResultType, IrInstructionIdAwaitBookkeeping, IrInstructionIdSaveErrRetAddr, + IrInstructionIdAddImplicitReturnType, }; struct IrInstruction { @@ -2993,6 +2997,12 @@ struct IrInstructionSaveErrRetAddr { IrInstruction base; }; +struct IrInstructionAddImplicitReturnType { + IrInstruction base; + + IrInstruction *value; +}; + static const size_t slice_ptr_index = 0; static const size_t slice_len_index = 1; diff --git a/src/analyze.cpp b/src/analyze.cpp index eb1850c8e..5f2162b2c 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3865,7 +3865,7 @@ void analyze_fn_ir(CodeGen *g, FnTableEntry *fn_table_entry, AstNode *return_typ TypeTableEntry *block_return_type = ir_analyze(g, &fn_table_entry->ir_executable, &fn_table_entry->analyzed_executable, fn_type_id->return_type, return_type_node); - fn_table_entry->implicit_return_type = block_return_type; + fn_table_entry->src_implicit_return_type = block_return_type; if (type_is_invalid(block_return_type) || fn_table_entry->analyzed_executable.invalid) { assert(g->errors.length > 0); @@ -3877,10 +3877,10 @@ void analyze_fn_ir(CodeGen *g, FnTableEntry *fn_table_entry, AstNode *return_typ TypeTableEntry *return_err_set_type = fn_type_id->return_type->data.error_union.err_set_type; if (return_err_set_type->data.error_set.infer_fn != nullptr) { TypeTableEntry *inferred_err_set_type; - if (fn_table_entry->implicit_return_type->id == TypeTableEntryIdErrorSet) { - inferred_err_set_type = fn_table_entry->implicit_return_type; - } else if (fn_table_entry->implicit_return_type->id == TypeTableEntryIdErrorUnion) { - inferred_err_set_type = fn_table_entry->implicit_return_type->data.error_union.err_set_type; + if (fn_table_entry->src_implicit_return_type->id == TypeTableEntryIdErrorSet) { + inferred_err_set_type = fn_table_entry->src_implicit_return_type; + } else if (fn_table_entry->src_implicit_return_type->id == TypeTableEntryIdErrorUnion) { + inferred_err_set_type = fn_table_entry->src_implicit_return_type->data.error_union.err_set_type; } else { add_node_error(g, return_type_node, buf_sprintf("function with inferred error set must return at least one possible error")); diff --git a/src/ast_render.cpp b/src/ast_render.cpp index f88feee85..432489c4d 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -658,6 +658,15 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { if (node->data.fn_call_expr.is_builtin) { fprintf(ar->f, "@"); } + if (node->data.fn_call_expr.is_async) { + fprintf(ar->f, "async"); + if (node->data.fn_call_expr.async_allocator != nullptr) { + fprintf(ar->f, "<"); + render_node_extra(ar, node->data.fn_call_expr.async_allocator, true); + fprintf(ar->f, ">"); + } + fprintf(ar->f, " "); + } AstNode *fn_ref_node = node->data.fn_call_expr.fn_ref_expr; bool grouped = (fn_ref_node->type != NodeTypePrefixOpExpr && fn_ref_node->type != NodeTypeAddrOfExpr); render_node_extra(ar, fn_ref_node, grouped); @@ -1023,7 +1032,7 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { case NodeTypeUnwrapErrorExpr: { render_node_ungrouped(ar, node->data.unwrap_err_expr.op1); - fprintf(ar->f, " %%%% "); + fprintf(ar->f, " catch "); if (node->data.unwrap_err_expr.symbol) { Buf *var_name = node->data.unwrap_err_expr.symbol->data.symbol_expr.symbol; fprintf(ar->f, "|%s| ", buf_ptr(var_name)); diff --git a/src/codegen.cpp b/src/codegen.cpp index eddadb94e..d67501053 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4245,6 +4245,7 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdErrorUnion: case IrInstructionIdPromiseResultType: case IrInstructionIdAwaitBookkeeping: + case IrInstructionIdAddImplicitReturnType: zig_unreachable(); case IrInstructionIdReturn: diff --git a/src/ir.cpp b/src/ir.cpp index a78be66dd..d896153d0 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -34,7 +34,7 @@ struct IrAnalyze { size_t old_bb_index; size_t instruction_index; TypeTableEntry *explicit_return_type; - ZigList implicit_return_type_list; + ZigList src_implicit_return_type_list; IrBasicBlock *const_predecessor_bb; }; @@ -717,6 +717,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSaveErrRetAddr * return IrInstructionIdSaveErrRetAddr; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionAddImplicitReturnType *) { + return IrInstructionIdAddImplicitReturnType; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -2687,6 +2691,17 @@ static IrInstruction *ir_build_save_err_ret_addr(IrBuilder *irb, Scope *scope, A return &instruction->base; } +static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *value) +{ + IrInstructionAddImplicitReturnType *instruction = ir_build_instruction(irb, scope, source_node); + instruction->value = value; + + ir_ref_instruction(value, irb->current_basic_block); + + return &instruction->base; +} + static void ir_count_defers(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, size_t *results) { results[ReturnKindUnconditional] = 0; results[ReturnKindError] = 0; @@ -2767,6 +2782,8 @@ static bool exec_is_async(IrExecutable *exec) { static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *return_value, bool is_generated_code) { + ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, return_value)); + bool is_async = exec_is_async(irb->exec); if (!is_async) { IrInstruction *return_inst = ir_build_return(irb, scope, node, return_value); @@ -6399,6 +6416,8 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec ir_build_cond_br(irb, scope, node, alloc_result_is_ok, alloc_ok_block, alloc_err_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, alloc_err_block); + // we can return undefined here, because the caller passes a pointer to the error struct field + // in the error union result, and we populate it in case of allocation failure. IrInstruction *undef = ir_build_const_undefined(irb, scope, node); ir_build_return(irb, scope, node, undef); @@ -10108,13 +10127,26 @@ static Buf *ir_resolve_str(IrAnalyze *ira, IrInstruction *value) { return result; } +static TypeTableEntry *ir_analyze_instruction_add_implicit_return_type(IrAnalyze *ira, + IrInstructionAddImplicitReturnType *instruction) +{ + IrInstruction *value = instruction->value->other; + if (type_is_invalid(value->value.type)) + return ir_unreach_error(ira); + + ira->src_implicit_return_type_list.append(value); + + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->type = ira->codegen->builtin_types.entry_void; + return out_val->type; +} + static TypeTableEntry *ir_analyze_instruction_return(IrAnalyze *ira, IrInstructionReturn *return_instruction) { IrInstruction *value = return_instruction->value->other; if (type_is_invalid(value->value.type)) return ir_unreach_error(ira); - ira->implicit_return_type_list.append(value); IrInstruction *casted_value = ir_implicit_cast(ira, value, ira->explicit_return_type); if (casted_value == ira->codegen->invalid_instruction) @@ -18049,6 +18081,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_await_bookkeeping(ira, (IrInstructionAwaitBookkeeping *)instruction); case IrInstructionIdSaveErrRetAddr: return ir_analyze_instruction_save_err_ret_addr(ira, (IrInstructionSaveErrRetAddr *)instruction); + case IrInstructionIdAddImplicitReturnType: + return ir_analyze_instruction_add_implicit_return_type(ira, (IrInstructionAddImplicitReturnType *)instruction); } zig_unreachable(); } @@ -18122,11 +18156,11 @@ TypeTableEntry *ir_analyze(CodeGen *codegen, IrExecutable *old_exec, IrExecutabl if (new_exec->invalid) { return ira->codegen->builtin_types.entry_invalid; - } else if (ira->implicit_return_type_list.length == 0) { + } else if (ira->src_implicit_return_type_list.length == 0) { return codegen->builtin_types.entry_unreachable; } else { - return ir_resolve_peer_types(ira, expected_type_source_node, ira->implicit_return_type_list.items, - ira->implicit_return_type_list.length); + return ir_resolve_peer_types(ira, expected_type_source_node, ira->src_implicit_return_type_list.items, + ira->src_implicit_return_type_list.length); } } @@ -18175,6 +18209,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdCoroAllocHelper: case IrInstructionIdAwaitBookkeeping: case IrInstructionIdSaveErrRetAddr: + case IrInstructionIdAddImplicitReturnType: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 709cb4394..5008d3564 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -201,9 +201,9 @@ static void ir_print_call(IrPrint *irp, IrInstructionCall *call_instruction) { if (call_instruction->is_async) { fprintf(irp->f, "async"); if (call_instruction->async_allocator != nullptr) { - fprintf(irp->f, "("); + fprintf(irp->f, "<"); ir_print_other_instruction(irp, call_instruction->async_allocator); - fprintf(irp->f, ")"); + fprintf(irp->f, ">"); } fprintf(irp->f, " "); } @@ -1165,6 +1165,12 @@ static void ir_print_save_err_ret_addr(IrPrint *irp, IrInstructionSaveErrRetAddr fprintf(irp->f, "@saveErrRetAddr()"); } +static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImplicitReturnType *instruction) { + fprintf(irp->f, "@addImplicitReturnType("); + ir_print_other_instruction(irp, instruction->value); + fprintf(irp->f, ")"); +} + static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { ir_print_prefix(irp, instruction); switch (instruction->id) { @@ -1539,6 +1545,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdSaveErrRetAddr: ir_print_save_err_ret_addr(irp, (IrInstructionSaveErrRetAddr *)instruction); break; + case IrInstructionIdAddImplicitReturnType: + ir_print_add_implicit_return_type(irp, (IrInstructionAddImplicitReturnType *)instruction); + break; } fprintf(irp->f, "\n"); } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index f8ad58f70..087d56171 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -176,3 +176,14 @@ async<&std.mem.Allocator> fn simpleAsyncFn2(y: &i32) void { *y += 1; suspend; } + +test "async fn with inferred error set" { + const p = (async failing()) catch unreachable; + resume p; + cancel p; +} + +async fn failing() !void { + suspend; + return error.Fail; +} From 897e783763d60449ad1b9514cb5ba86a38f7ae4a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 19:25:53 -0400 Subject: [PATCH 017/132] add promise->T syntax parsing closes #857 --- doc/langref.html.in | 4 ++- src/all_types.hpp | 13 ++++++++++ src/analyze.cpp | 1 + src/ast_render.cpp | 11 ++++++++ src/codegen.cpp | 1 + src/ir.cpp | 54 +++++++++++++++++++++++++++++++++++++++ src/ir_print.cpp | 11 ++++++++ src/parser.cpp | 14 +++++++++- src/tokenizer.cpp | 2 ++ src/tokenizer.hpp | 1 + test/cases/coroutines.zig | 1 + 11 files changed, 111 insertions(+), 2 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index f4aa42eb8..b2e14ac19 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -5863,7 +5863,9 @@ StructLiteralField = "." Symbol "=" Expression PrefixOp = "!" | "-" | "~" | "*" | ("&" option("align" "(" Expression option(":" Integer ":" Integer) ")" ) option("const") option("volatile")) | "?" | "??" | "-%" | "try" | "await" -PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl +PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType + +PromiseType = "promise" option("->" TypeExpr) ArrayType : "[" option(Expression) "]" option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") TypeExpr diff --git a/src/all_types.hpp b/src/all_types.hpp index f5afecfbd..64b9b3166 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -409,6 +409,7 @@ enum NodeType { NodeTypeResume, NodeTypeAwaitExpr, NodeTypeSuspend, + NodeTypePromiseType, }; struct AstNodeRoot { @@ -879,6 +880,10 @@ struct AstNodeSuspend { AstNode *promise_symbol; }; +struct AstNodePromiseType { + AstNode *payload_type; // can be NULL +}; + struct AstNode { enum NodeType type; size_t line; @@ -939,6 +944,7 @@ struct AstNode { AstNodeResumeExpr resume_expr; AstNodeAwaitExpr await_expr; AstNodeSuspend suspend; + AstNodePromiseType promise_type; } data; }; @@ -1947,6 +1953,7 @@ enum IrInstructionId { IrInstructionIdSetRuntimeSafety, IrInstructionIdSetFloatMode, IrInstructionIdArrayType, + IrInstructionIdPromiseType, IrInstructionIdSliceType, IrInstructionIdAsm, IrInstructionIdSizeOf, @@ -2365,6 +2372,12 @@ struct IrInstructionArrayType { IrInstruction *child_type; }; +struct IrInstructionPromiseType { + IrInstruction base; + + IrInstruction *payload_type; +}; + struct IrInstructionSliceType { IrInstruction base; diff --git a/src/analyze.cpp b/src/analyze.cpp index 5f2162b2c..0f4728f82 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3254,6 +3254,7 @@ void scan_decls(CodeGen *g, ScopeDecls *decls_scope, AstNode *node) { case NodeTypeResume: case NodeTypeAwaitExpr: case NodeTypeSuspend: + case NodeTypePromiseType: zig_unreachable(); } } diff --git a/src/ast_render.cpp b/src/ast_render.cpp index 432489c4d..7b5fc03ea 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -250,6 +250,8 @@ static const char *node_type_str(NodeType node_type) { return "AwaitExpr"; case NodeTypeSuspend: return "Suspend"; + case NodeTypePromiseType: + return "PromiseType"; } zig_unreachable(); } @@ -781,6 +783,15 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { render_node_ungrouped(ar, node->data.array_type.child_type); break; } + case NodeTypePromiseType: + { + fprintf(ar->f, "promise"); + if (node->data.promise_type.payload_type != nullptr) { + fprintf(ar->f, "->"); + render_node_grouped(ar, node->data.promise_type.payload_type); + } + break; + } case NodeTypeErrorType: fprintf(ar->f, "error"); break; diff --git a/src/codegen.cpp b/src/codegen.cpp index d67501053..786bd0398 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4205,6 +4205,7 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, case IrInstructionIdSetRuntimeSafety: case IrInstructionIdSetFloatMode: case IrInstructionIdArrayType: + case IrInstructionIdPromiseType: case IrInstructionIdSliceType: case IrInstructionIdSizeOf: case IrInstructionIdSwitchTarget: diff --git a/src/ir.cpp b/src/ir.cpp index d896153d0..95142c88d 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -349,6 +349,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionArrayType *) { return IrInstructionIdArrayType; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionPromiseType *) { + return IrInstructionIdPromiseType; +} + static constexpr IrInstructionId ir_instruction_id(IrInstructionSliceType *) { return IrInstructionIdSliceType; } @@ -1469,6 +1473,17 @@ static IrInstruction *ir_build_array_type(IrBuilder *irb, Scope *scope, AstNode return &instruction->base; } +static IrInstruction *ir_build_promise_type(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *payload_type) +{ + IrInstructionPromiseType *instruction = ir_build_instruction(irb, scope, source_node); + instruction->payload_type = payload_type; + + if (payload_type != nullptr) ir_ref_instruction(payload_type, irb->current_basic_block); + + return &instruction->base; +} + static IrInstruction *ir_build_slice_type(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *child_type, bool is_const, bool is_volatile, IrInstruction *align_value) { @@ -5074,6 +5089,22 @@ static IrInstruction *ir_gen_array_type(IrBuilder *irb, Scope *scope, AstNode *n } } +static IrInstruction *ir_gen_promise_type(IrBuilder *irb, Scope *scope, AstNode *node) { + assert(node->type == NodeTypePromiseType); + + AstNode *payload_type_node = node->data.promise_type.payload_type; + IrInstruction *payload_type_value = nullptr; + + if (payload_type_node != nullptr) { + payload_type_value = ir_gen_node(irb, payload_type_node, scope); + if (payload_type_value == irb->codegen->invalid_instruction) + return payload_type_value; + + } + + return ir_build_promise_type(irb, scope, node, payload_type_value); +} + static IrInstruction *ir_gen_undefined_literal(IrBuilder *irb, Scope *scope, AstNode *node) { assert(node->type == NodeTypeUndefinedLiteral); return ir_build_const_undefined(irb, scope, node); @@ -6282,6 +6313,8 @@ static IrInstruction *ir_gen_node_raw(IrBuilder *irb, AstNode *node, Scope *scop return ir_lval_wrap(irb, scope, ir_gen_bool_literal(irb, scope, node), lval); case NodeTypeArrayType: return ir_lval_wrap(irb, scope, ir_gen_array_type(irb, scope, node), lval); + case NodeTypePromiseType: + return ir_lval_wrap(irb, scope, ir_gen_promise_type(irb, scope, node), lval); case NodeTypeStringLiteral: return ir_lval_wrap(irb, scope, ir_gen_string_literal(irb, scope, node), lval); case NodeTypeUndefinedLiteral: @@ -14069,6 +14102,24 @@ static TypeTableEntry *ir_analyze_instruction_array_type(IrAnalyze *ira, zig_unreachable(); } +static TypeTableEntry *ir_analyze_instruction_promise_type(IrAnalyze *ira, IrInstructionPromiseType *instruction) { + TypeTableEntry *promise_type; + + if (instruction->payload_type == nullptr) { + promise_type = ira->codegen->builtin_types.entry_promise; + } else { + TypeTableEntry *payload_type = ir_resolve_type(ira, instruction->payload_type->other); + if (type_is_invalid(payload_type)) + return ira->codegen->builtin_types.entry_invalid; + + promise_type = get_promise_type(ira->codegen, payload_type); + } + + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->data.x_type = promise_type; + return ira->codegen->builtin_types.entry_type; +} + static TypeTableEntry *ir_analyze_instruction_size_of(IrAnalyze *ira, IrInstructionSizeOf *size_of_instruction) { @@ -17907,6 +17958,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_asm(ira, (IrInstructionAsm *)instruction); case IrInstructionIdArrayType: return ir_analyze_instruction_array_type(ira, (IrInstructionArrayType *)instruction); + case IrInstructionIdPromiseType: + return ir_analyze_instruction_promise_type(ira, (IrInstructionPromiseType *)instruction); case IrInstructionIdSizeOf: return ir_analyze_instruction_size_of(ira, (IrInstructionSizeOf *)instruction); case IrInstructionIdTestNonNull: @@ -18232,6 +18285,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdStructFieldPtr: case IrInstructionIdUnionFieldPtr: case IrInstructionIdArrayType: + case IrInstructionIdPromiseType: case IrInstructionIdSliceType: case IrInstructionIdSizeOf: case IrInstructionIdTestNonNull: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 5008d3564..b14d49a4c 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -404,6 +404,14 @@ static void ir_print_array_type(IrPrint *irp, IrInstructionArrayType *instructio ir_print_other_instruction(irp, instruction->child_type); } +static void ir_print_promise_type(IrPrint *irp, IrInstructionPromiseType *instruction) { + fprintf(irp->f, "promise"); + if (instruction->payload_type != nullptr) { + fprintf(irp->f, "->"); + ir_print_other_instruction(irp, instruction->payload_type); + } +} + static void ir_print_slice_type(IrPrint *irp, IrInstructionSliceType *instruction) { const char *const_kw = instruction->is_const ? "const " : ""; fprintf(irp->f, "[]%s", const_kw); @@ -1263,6 +1271,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdArrayType: ir_print_array_type(irp, (IrInstructionArrayType *)instruction); break; + case IrInstructionIdPromiseType: + ir_print_promise_type(irp, (IrInstructionPromiseType *)instruction); + break; case IrInstructionIdSliceType: ir_print_slice_type(irp, (IrInstructionSliceType *)instruction); break; diff --git a/src/parser.cpp b/src/parser.cpp index 666b9da3c..d6faf4c98 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -705,7 +705,7 @@ static AstNode *ast_parse_comptime_expr(ParseContext *pc, size_t *token_index, b } /* -PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl +PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType KeywordLiteral = "true" | "false" | "null" | "undefined" | "error" | "this" | "unreachable" | "suspend" ErrorSetDecl = "error" "{" list(Symbol, ",") "}" */ @@ -774,6 +774,15 @@ static AstNode *ast_parse_primary_expr(ParseContext *pc, size_t *token_index, bo AstNode *node = ast_create_node(pc, NodeTypeSuspend, token); *token_index += 1; return node; + } else if (token->id == TokenIdKeywordPromise) { + AstNode *node = ast_create_node(pc, NodeTypePromiseType, token); + *token_index += 1; + Token *arrow_tok = &pc->tokens->at(*token_index); + if (arrow_tok->id == TokenIdArrow) { + *token_index += 1; + node->data.promise_type.payload_type = ast_parse_type_expr(pc, token_index, true); + } + return node; } else if (token->id == TokenIdKeywordError) { Token *next_token = &pc->tokens->at(*token_index + 1); if (next_token->id == TokenIdLBrace) { @@ -3081,6 +3090,9 @@ void ast_visit_node_children(AstNode *node, void (*visit)(AstNode **, void *cont visit_field(&node->data.array_type.child_type, visit, context); visit_field(&node->data.array_type.align_expr, visit, context); break; + case NodeTypePromiseType: + visit_field(&node->data.promise_type.payload_type, visit, context); + break; case NodeTypeErrorType: // none break; diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index dc17829c0..365b35cdf 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -135,6 +135,7 @@ static const struct ZigKeyword zig_keywords[] = { {"null", TokenIdKeywordNull}, {"or", TokenIdKeywordOr}, {"packed", TokenIdKeywordPacked}, + {"promise", TokenIdKeywordPromise}, {"pub", TokenIdKeywordPub}, {"resume", TokenIdKeywordResume}, {"return", TokenIdKeywordReturn}, @@ -1558,6 +1559,7 @@ const char * token_name(TokenId id) { case TokenIdKeywordNull: return "null"; case TokenIdKeywordOr: return "or"; case TokenIdKeywordPacked: return "packed"; + case TokenIdKeywordPromise: return "promise"; case TokenIdKeywordPub: return "pub"; case TokenIdKeywordReturn: return "return"; case TokenIdKeywordSection: return "section"; diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index 2d7142799..b71929370 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -76,6 +76,7 @@ enum TokenId { TokenIdKeywordNull, TokenIdKeywordOr, TokenIdKeywordPacked, + TokenIdKeywordPromise, TokenIdKeywordPub, TokenIdKeywordResume, TokenIdKeywordReturn, diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 087d56171..25a75dca5 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -5,6 +5,7 @@ var x: i32 = 1; test "create a coroutine and cancel it" { const p = try async simpleAsyncFn(); + comptime assert(@typeOf(p) == promise->void); cancel p; assert(x == 2); } From a43c7af3d1e41ccee73678f114bb3844febcaad6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 19:31:00 -0400 Subject: [PATCH 018/132] add comptime test for the type of suspend promise --- test/cases/coroutines.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 25a75dca5..922c1a7e5 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -56,6 +56,7 @@ var result = false; async fn testSuspendBlock() void { suspend |p| { + comptime assert(@typeOf(p) == promise->void); a_promise = p; } result = true; From aa2995ee395b2c1329a61513debcac6225fcb8a8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 24 Mar 2018 22:05:29 -0400 Subject: [PATCH 019/132] fix invalid codegen for error return traces across suspend points See #821 Now the code works correctly, but error return traces are missing the frames from coroutines. --- src/all_types.hpp | 10 +++++- src/analyze.cpp | 7 +++++ src/analyze.hpp | 1 + src/codegen.cpp | 65 +++++++++++++++++++++++++++++--------- src/ir.cpp | 69 +++++++++++++++++++++-------------------- test/runtime_safety.zig | 30 ++++++++++++++++++ 6 files changed, 132 insertions(+), 50 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 64b9b3166..6951230aa 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1621,7 +1621,8 @@ struct CodeGen { FnTableEntry *panic_fn; LLVMValueRef cur_ret_ptr; LLVMValueRef cur_fn_val; - LLVMValueRef cur_err_ret_trace_val; + LLVMValueRef cur_err_ret_trace_val_arg; + LLVMValueRef cur_err_ret_trace_val_stack; bool c_want_stdint; bool c_want_stdbool; AstNode *root_export_decl; @@ -1760,6 +1761,7 @@ enum ScopeId { ScopeIdLoop, ScopeIdFnDef, ScopeIdCompTime, + ScopeIdCoroPrelude, }; struct Scope { @@ -1867,6 +1869,12 @@ struct ScopeFnDef { FnTableEntry *fn_entry; }; +// This scope is created to indicate that the code in the scope +// is auto-generated coroutine prelude stuff. +struct ScopeCoroPrelude { + Scope base; +}; + // synchronized with code in define_builtin_compile_vars enum AtomicOrder { AtomicOrderUnordered, diff --git a/src/analyze.cpp b/src/analyze.cpp index 0f4728f82..7ee1de78a 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -170,6 +170,12 @@ Scope *create_comptime_scope(AstNode *node, Scope *parent) { return &scope->base; } +Scope *create_coro_prelude_scope(AstNode *node, Scope *parent) { + ScopeCoroPrelude *scope = allocate(1); + init_scope(&scope->base, ScopeIdCoroPrelude, node, parent); + return &scope->base; +} + ImportTableEntry *get_scope_import(Scope *scope) { while (scope) { if (scope->id == ScopeIdDecls) { @@ -3592,6 +3598,7 @@ FnTableEntry *scope_get_fn_if_root(Scope *scope) { case ScopeIdCImport: case ScopeIdLoop: case ScopeIdCompTime: + case ScopeIdCoroPrelude: scope = scope->parent; continue; case ScopeIdFnDef: diff --git a/src/analyze.hpp b/src/analyze.hpp index 936134030..aa4557666 100644 --- a/src/analyze.hpp +++ b/src/analyze.hpp @@ -107,6 +107,7 @@ ScopeLoop *create_loop_scope(AstNode *node, Scope *parent); ScopeFnDef *create_fndef_scope(AstNode *node, Scope *parent, FnTableEntry *fn_entry); ScopeDecls *create_decls_scope(AstNode *node, Scope *parent, TypeTableEntry *container_type, ImportTableEntry *import); Scope *create_comptime_scope(AstNode *node, Scope *parent); +Scope *create_coro_prelude_scope(AstNode *node, Scope *parent); void init_const_str_lit(CodeGen *g, ConstExprValue *const_val, Buf *str); ConstExprValue *create_const_str_lit(CodeGen *g, Buf *str); diff --git a/src/codegen.cpp b/src/codegen.cpp index 786bd0398..25b2ffbf1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -653,6 +653,7 @@ static ZigLLVMDIScope *get_di_scope(CodeGen *g, Scope *scope) { case ScopeIdDeferExpr: case ScopeIdLoop: case ScopeIdCompTime: + case ScopeIdCoroPrelude: return get_di_scope(g, scope->parent); } zig_unreachable(); @@ -1318,9 +1319,34 @@ static LLVMValueRef get_safety_crash_err_fn(CodeGen *g) { return fn_val; } -static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val) { +static bool is_coro_prelude_scope(Scope *scope) { + while (scope != nullptr) { + if (scope->id == ScopeIdCoroPrelude) { + return true; + } else if (scope->id == ScopeIdFnDef) { + break; + } + scope = scope->parent; + } + return false; +} + +static LLVMValueRef get_cur_err_ret_trace_val(CodeGen *g, Scope *scope) { + if (!g->have_err_ret_tracing) { + return nullptr; + } + if (g->cur_fn->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) { + return is_coro_prelude_scope(scope) ? g->cur_err_ret_trace_val_arg : g->cur_err_ret_trace_val_stack; + } + if (g->cur_err_ret_trace_val_stack != nullptr) { + return g->cur_err_ret_trace_val_stack; + } + return g->cur_err_ret_trace_val_arg; +} + +static void gen_safety_crash_for_err(CodeGen *g, LLVMValueRef err_val, Scope *scope) { LLVMValueRef safety_crash_err_fn = get_safety_crash_err_fn(g); - LLVMValueRef err_ret_trace_val = g->cur_err_ret_trace_val; + LLVMValueRef err_ret_trace_val = get_cur_err_ret_trace_val(g, scope); if (err_ret_trace_val == nullptr) { TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g); err_ret_trace_val = LLVMConstNull(ptr_to_stack_trace_type->type_ref); @@ -1614,7 +1640,7 @@ static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *execut LLVMValueRef return_err_fn = get_return_err_fn(g); LLVMValueRef args[] = { - g->cur_err_ret_trace_val, + get_cur_err_ret_trace_val(g, save_err_ret_addr_instruction->base.scope), }; LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); @@ -2725,7 +2751,7 @@ static LLVMValueRef ir_render_call(CodeGen *g, IrExecutable *executable, IrInstr gen_param_index += 1; } if (prefix_arg_err_ret_stack) { - gen_param_values[gen_param_index] = g->cur_err_ret_trace_val; + gen_param_values[gen_param_index] = get_cur_err_ret_trace_val(g, instruction->base.scope); gen_param_index += 1; } if (instruction->is_async) { @@ -3292,11 +3318,12 @@ static LLVMValueRef ir_render_align_cast(CodeGen *g, IrExecutable *executable, I static LLVMValueRef ir_render_error_return_trace(CodeGen *g, IrExecutable *executable, IrInstructionErrorReturnTrace *instruction) { - if (g->cur_err_ret_trace_val == nullptr) { + LLVMValueRef cur_err_ret_trace_val = get_cur_err_ret_trace_val(g, instruction->base.scope); + if (cur_err_ret_trace_val == nullptr) { TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(g); return LLVMConstNull(ptr_to_stack_trace_type->type_ref); } - return g->cur_err_ret_trace_val; + return cur_err_ret_trace_val; } static LLVMValueRef ir_render_cancel(CodeGen *g, IrExecutable *executable, IrInstructionCancel *instruction) { @@ -3726,7 +3753,7 @@ static LLVMValueRef ir_render_unwrap_err_payload(CodeGen *g, IrExecutable *execu LLVMBuildCondBr(g->builder, cond_val, ok_block, err_block); LLVMPositionBuilderAtEnd(g->builder, err_block); - gen_safety_crash_for_err(g, err_val); + gen_safety_crash_for_err(g, err_val, instruction->base.scope); LLVMPositionBuilderAtEnd(g->builder, ok_block); } @@ -3918,7 +3945,7 @@ static LLVMValueRef ir_render_container_init_list(CodeGen *g, IrExecutable *exec } static LLVMValueRef ir_render_panic(CodeGen *g, IrExecutable *executable, IrInstructionPanic *instruction) { - gen_panic(g, ir_llvm_value(g, instruction->msg), g->cur_err_ret_trace_val); + gen_panic(g, ir_llvm_value(g, instruction->msg), get_cur_err_ret_trace_val(g, instruction->base.scope)); return nullptr; } @@ -5279,9 +5306,17 @@ static void do_code_gen(CodeGen *g) { clear_debug_source_node(g); uint32_t err_ret_trace_arg_index = get_err_ret_trace_arg_index(g, fn_table_entry); - if (err_ret_trace_arg_index != UINT32_MAX) { - g->cur_err_ret_trace_val = LLVMGetParam(fn, err_ret_trace_arg_index); - } else if (g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn) { + bool have_err_ret_trace_arg = err_ret_trace_arg_index != UINT32_MAX; + if (have_err_ret_trace_arg) { + g->cur_err_ret_trace_val_arg = LLVMGetParam(fn, err_ret_trace_arg_index); + } else { + g->cur_err_ret_trace_val_arg = nullptr; + } + + bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; + bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && + (is_async || !have_err_ret_trace_arg); + if (have_err_ret_trace_stack) { // TODO call graph analysis to find out what this number needs to be for every function static const size_t stack_trace_ptr_count = 30; @@ -5289,13 +5324,13 @@ static void do_code_gen(CodeGen *g) { TypeTableEntry *array_type = get_array_type(g, usize, stack_trace_ptr_count); LLVMValueRef err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); - g->cur_err_ret_trace_val = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); + g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; - LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val, (unsigned)index_field_index, ""); + LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false); size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; - LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val, (unsigned)addresses_field_index, ""); + LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, ""); TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; @@ -5311,7 +5346,7 @@ static void do_code_gen(CodeGen *g) { LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } else { - g->cur_err_ret_trace_val = nullptr; + g->cur_err_ret_trace_val_stack = nullptr; } // allocate temporary stack data diff --git a/src/ir.cpp b/src/ir.cpp index 95142c88d..d86b7f5b2 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -6412,60 +6412,61 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec VariableTableEntry *coro_size_var; if (is_async) { // create the coro promise - const_bool_false = ir_build_const_bool(irb, scope, node, false); - VariableTableEntry *promise_var = ir_create_var(irb, node, scope, nullptr, false, false, true, const_bool_false); + Scope *coro_scope = create_coro_prelude_scope(node, scope); + const_bool_false = ir_build_const_bool(irb, coro_scope, node, false); + VariableTableEntry *promise_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); return_type = fn_entry->type_entry->data.fn.fn_type_id.return_type; - IrInstruction *promise_init = ir_build_const_promise_init(irb, scope, node, return_type); - ir_build_var_decl(irb, scope, node, promise_var, nullptr, nullptr, promise_init); - IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, scope, node, promise_var, false, false); + IrInstruction *promise_init = ir_build_const_promise_init(irb, coro_scope, node, return_type); + ir_build_var_decl(irb, coro_scope, node, promise_var, nullptr, nullptr, promise_init); + IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); - VariableTableEntry *await_handle_var = ir_create_var(irb, node, scope, nullptr, false, false, true, const_bool_false); - IrInstruction *null_value = ir_build_const_null(irb, scope, node); - IrInstruction *await_handle_type_val = ir_build_const_type(irb, scope, node, + VariableTableEntry *await_handle_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); + IrInstruction *null_value = ir_build_const_null(irb, coro_scope, node); + IrInstruction *await_handle_type_val = ir_build_const_type(irb, coro_scope, node, get_maybe_type(irb->codegen, irb->codegen->builtin_types.entry_promise)); - ir_build_var_decl(irb, scope, node, await_handle_var, await_handle_type_val, nullptr, null_value); - irb->exec->await_handle_var_ptr = ir_build_var_ptr(irb, scope, node, + ir_build_var_decl(irb, coro_scope, node, await_handle_var, await_handle_type_val, nullptr, null_value); + irb->exec->await_handle_var_ptr = ir_build_var_ptr(irb, coro_scope, node, await_handle_var, false, false); - u8_ptr_type = ir_build_const_type(irb, scope, node, + u8_ptr_type = ir_build_const_type(irb, coro_scope, node, get_pointer_to_type(irb->codegen, irb->codegen->builtin_types.entry_u8, false)); - IrInstruction *promise_as_u8_ptr = ir_build_ptr_cast(irb, scope, node, u8_ptr_type, coro_promise_ptr); - coro_id = ir_build_coro_id(irb, scope, node, promise_as_u8_ptr); - coro_size_var = ir_create_var(irb, node, scope, nullptr, false, false, true, const_bool_false); - IrInstruction *coro_size = ir_build_coro_size(irb, scope, node); - ir_build_var_decl(irb, scope, node, coro_size_var, nullptr, nullptr, coro_size); - IrInstruction *implicit_allocator_ptr = ir_build_get_implicit_allocator(irb, scope, node, + IrInstruction *promise_as_u8_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, coro_promise_ptr); + coro_id = ir_build_coro_id(irb, coro_scope, node, promise_as_u8_ptr); + coro_size_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); + IrInstruction *coro_size = ir_build_coro_size(irb, coro_scope, node); + ir_build_var_decl(irb, coro_scope, node, coro_size_var, nullptr, nullptr, coro_size); + IrInstruction *implicit_allocator_ptr = ir_build_get_implicit_allocator(irb, coro_scope, node, ImplicitAllocatorIdArg); - irb->exec->coro_allocator_var = ir_create_var(irb, node, scope, nullptr, true, true, true, const_bool_false); - ir_build_var_decl(irb, scope, node, irb->exec->coro_allocator_var, nullptr, nullptr, implicit_allocator_ptr); + irb->exec->coro_allocator_var = ir_create_var(irb, node, coro_scope, nullptr, true, true, true, const_bool_false); + ir_build_var_decl(irb, coro_scope, node, irb->exec->coro_allocator_var, nullptr, nullptr, implicit_allocator_ptr); Buf *alloc_field_name = buf_create_from_str(ASYNC_ALLOC_FIELD_NAME); - IrInstruction *alloc_fn_ptr = ir_build_field_ptr(irb, scope, node, implicit_allocator_ptr, alloc_field_name); - IrInstruction *alloc_fn = ir_build_load_ptr(irb, scope, node, alloc_fn_ptr); - IrInstruction *maybe_coro_mem_ptr = ir_build_coro_alloc_helper(irb, scope, node, alloc_fn, coro_size); - IrInstruction *alloc_result_is_ok = ir_build_test_nonnull(irb, scope, node, maybe_coro_mem_ptr); - IrBasicBlock *alloc_err_block = ir_create_basic_block(irb, scope, "AllocError"); - IrBasicBlock *alloc_ok_block = ir_create_basic_block(irb, scope, "AllocOk"); - ir_build_cond_br(irb, scope, node, alloc_result_is_ok, alloc_ok_block, alloc_err_block, const_bool_false); + IrInstruction *alloc_fn_ptr = ir_build_field_ptr(irb, coro_scope, node, implicit_allocator_ptr, alloc_field_name); + IrInstruction *alloc_fn = ir_build_load_ptr(irb, coro_scope, node, alloc_fn_ptr); + IrInstruction *maybe_coro_mem_ptr = ir_build_coro_alloc_helper(irb, coro_scope, node, alloc_fn, coro_size); + IrInstruction *alloc_result_is_ok = ir_build_test_nonnull(irb, coro_scope, node, maybe_coro_mem_ptr); + IrBasicBlock *alloc_err_block = ir_create_basic_block(irb, coro_scope, "AllocError"); + IrBasicBlock *alloc_ok_block = ir_create_basic_block(irb, coro_scope, "AllocOk"); + ir_build_cond_br(irb, coro_scope, node, alloc_result_is_ok, alloc_ok_block, alloc_err_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, alloc_err_block); // we can return undefined here, because the caller passes a pointer to the error struct field // in the error union result, and we populate it in case of allocation failure. - IrInstruction *undef = ir_build_const_undefined(irb, scope, node); - ir_build_return(irb, scope, node, undef); + IrInstruction *undef = ir_build_const_undefined(irb, coro_scope, node); + ir_build_return(irb, coro_scope, node, undef); ir_set_cursor_at_end_and_append_block(irb, alloc_ok_block); - IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, scope, node, u8_ptr_type, maybe_coro_mem_ptr); - irb->exec->coro_handle = ir_build_coro_begin(irb, scope, node, coro_id, coro_mem_ptr); + IrInstruction *coro_mem_ptr = ir_build_ptr_cast(irb, coro_scope, node, u8_ptr_type, maybe_coro_mem_ptr); + irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr); Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); - irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, + irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, awaiter_handle_field_name); Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); - irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); + irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_field_name); result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); - irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name); - ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); + irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_ptr_field_name); + ir_build_store_ptr(irb, coro_scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); irb->exec->coro_early_final = ir_create_basic_block(irb, scope, "CoroEarlyFinal"); diff --git a/test/runtime_safety.zig b/test/runtime_safety.zig index 8b8f61205..1fea6347a 100644 --- a/test/runtime_safety.zig +++ b/test/runtime_safety.zig @@ -281,4 +281,34 @@ pub fn addCases(cases: &tests.CompareOutputContext) void { \\ f.float = 12.34; \\} ); + + // This case makes sure that the code compiles and runs. There is not actually a special + // runtime safety check having to do specifically with error return traces across suspend points. + cases.addRuntimeSafety("error return trace across suspend points", + \\const std = @import("std"); + \\ + \\pub fn panic(message: []const u8, stack_trace: ?&@import("builtin").StackTrace) noreturn { + \\ std.os.exit(126); + \\} + \\ + \\pub fn main() void { + \\ const p = nonFailing(); + \\ resume p; + \\ const p2 = async printTrace(p) catch unreachable; + \\ cancel p2; + \\} + \\ + \\fn nonFailing() promise->error!void { + \\ return async failing() catch unreachable; + \\} + \\ + \\async fn failing() error!void { + \\ suspend; + \\ return error.Fail; + \\} + \\ + \\async fn printTrace(p: promise->error!void) void { + \\ (await p) catch unreachable; + \\} + ); } From 3e836f5516c48bcfa16e0b630e53822af80d4b3b Mon Sep 17 00:00:00 2001 From: Jay Weisskopf Date: Sun, 25 Mar 2018 18:48:07 -0400 Subject: [PATCH 020/132] doc: fix typo and tighten wording in error sections Changes: - Removed superfluous "when possible" - Fixed typo in "documentationt" - Added missing comma - Moved definition of error union type up to first sentence --- doc/langref.html.in | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index b2e14ac19..04ca5314c 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2864,18 +2864,18 @@ const err = (error {FileNotFound}).FileNotFound; assert to make sure the error value is in fact in the destination error set.

- The global error set should generally be avoided when possible, because it prevents - the compiler from knowing what errors are possible at compile-time. Knowing - the error set at compile-time is better for generated documentationt and for - helpful error messages such as forgetting a possible error value in a {#link|switch#}. + The global error set should generally be avoided because it prevents the + compiler from knowing what errors are possible at compile-time. Knowing + the error set at compile-time is better for generated documentation and + helpful error messages, such as forgetting a possible error value in a {#link|switch#}.

{#header_close#} {#header_close#} {#header_open|Error Union Type#}

- Most of the time you will not find yourself using an error set type. Instead, - likely you will be using the error union type. This is when you take an error set - and a normal type, and create an error union with the ! binary operator. + An error set type and normal type can be combined with the ! + binary operator to form an error union type. You are likely to use an + error union type more often than an error set type by itself.

Here is a function to parse a string into a 64-bit integer: @@ -6033,4 +6033,3 @@ hljs.registerLanguage("zig", function(t) { - From 0b7b3190fd1121aa4e349740cff1faf213c94411 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 27 Mar 2018 10:43:51 -0400 Subject: [PATCH 021/132] fix bitrotted code in unexpected error tracing --- std/os/index.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/std/os/index.zig b/std/os/index.zig index e472908c6..09409d6d3 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1690,7 +1690,7 @@ const unexpected_error_tracing = false; pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { if (unexpected_error_tracing) { debug.warn("unexpected errno: {}\n", errno); - debug.dumpStackTrace(); + debug.dumpCurrentStackTrace(null); } return error.Unexpected; } @@ -1700,7 +1700,7 @@ pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { pub fn unexpectedErrorWindows(err: windows.DWORD) (error{Unexpected}) { if (unexpected_error_tracing) { debug.warn("unexpected GetLastError(): {}\n", err); - debug.dumpStackTrace(); + debug.dumpCurrentStackTrace(null); } return error.Unexpected; } From 6cb99fdac3fb02d7e4e5a363295bae0dd3641a45 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 27 Mar 2018 15:07:45 -0400 Subject: [PATCH 022/132] fix crash when compile error in analyzing @panic call --- src/ir.cpp | 6 +++--- test/compile_errors.zig | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index d86b7f5b2..ea001adb9 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -16918,18 +16918,18 @@ static TypeTableEntry *ir_analyze_instruction_can_implicit_cast(IrAnalyze *ira, static TypeTableEntry *ir_analyze_instruction_panic(IrAnalyze *ira, IrInstructionPanic *instruction) { IrInstruction *msg = instruction->msg->other; if (type_is_invalid(msg->value.type)) - return ira->codegen->builtin_types.entry_invalid; + return ir_unreach_error(ira); if (ir_should_inline(ira->new_irb.exec, instruction->base.scope)) { ir_add_error(ira, &instruction->base, buf_sprintf("encountered @panic at compile-time")); - return ira->codegen->builtin_types.entry_invalid; + return ir_unreach_error(ira); } TypeTableEntry *u8_ptr_type = get_pointer_to_type(ira->codegen, ira->codegen->builtin_types.entry_u8, true); TypeTableEntry *str_type = get_slice_type(ira->codegen, u8_ptr_type); IrInstruction *casted_msg = ir_implicit_cast(ira, msg, str_type); if (type_is_invalid(casted_msg->value.type)) - return ira->codegen->builtin_types.entry_invalid; + return ir_unreach_error(ira); IrInstruction *new_instruction = ir_build_panic(&ira->new_irb, instruction->base.scope, instruction->base.source_node, casted_msg); diff --git a/test/compile_errors.zig b/test/compile_errors.zig index ddf528633..bed5aa1b6 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -1,6 +1,15 @@ const tests = @import("tests.zig"); pub fn addCases(cases: &tests.CompileErrorContext) void { + cases.add("wrong type passed to @panic", + \\export fn entry() void { + \\ var e = error.Foo; + \\ @panic(e); + \\} + , + ".tmp_source.zig:3:12: error: expected type '[]const u8', found 'error{Foo}'"); + + cases.add("@tagName used on union with no associated enum tag", \\const FloatInt = extern union { \\ Float: f32, From 5b00dee0c27ae5a8547d6a4063331302cdc7b0be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 27 Mar 2018 15:20:07 -0400 Subject: [PATCH 023/132] std.math.cast handles signed integers --- std/math/index.zig | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/std/math/index.zig b/std/math/index.zig index f8668cc00..477dafcbc 100644 --- a/std/math/index.zig +++ b/std/math/index.zig @@ -515,15 +515,28 @@ test "math.negateCast" { /// Cast an integer to a different integer type. If the value doesn't fit, /// return an error. -pub fn cast(comptime T: type, x: var) !T { +pub fn cast(comptime T: type, x: var) (error{Overflow}!T) { comptime assert(@typeId(T) == builtin.TypeId.Int); // must pass an integer - if (x > @maxValue(T)) { + comptime assert(@typeId(@typeOf(x)) == builtin.TypeId.Int); // must pass an integer + if (@maxValue(@typeOf(x)) > @maxValue(T) and x > @maxValue(T)) { + return error.Overflow; + } else if (@minValue(@typeOf(x)) < @minValue(T) and x < @minValue(T)) { return error.Overflow; } else { return T(x); } } +test "math.cast" { + if (cast(u8, u32(300))) |_| @panic("fail") else |err| assert(err == error.Overflow); + if (cast(i8, i32(-200))) |_| @panic("fail") else |err| assert(err == error.Overflow); + if (cast(u8, i8(-1))) |_| @panic("fail") else |err| assert(err == error.Overflow); + if (cast(u64, i8(-1))) |_| @panic("fail") else |err| assert(err == error.Overflow); + + assert((try cast(u8, u32(255))) == u8(255)); + assert(@typeOf(try cast(u8, u32(255))) == u8); +} + pub fn floorPowerOfTwo(comptime T: type, value: T) T { var x = value; From f5b43ada469c94ba4968898a7102d27f5c8188f2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 5 Mar 2018 00:57:02 -0500 Subject: [PATCH 024/132] std/os: getting dir entries works on OS X --- std/c/darwin.zig | 10 ++++ std/c/index.zig | 1 + std/os/darwin.zig | 32 +++++++++++++ std/os/index.zig | 104 +++++++++++++++++++++++++++++++++++----- std/os/linux/x86_64.zig | 8 ++++ std/os/test.zig | 12 +++++ 6 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 std/os/test.zig diff --git a/std/c/darwin.zig b/std/c/darwin.zig index f0890d4ec..aa49dfa3d 100644 --- a/std/c/darwin.zig +++ b/std/c/darwin.zig @@ -1,6 +1,7 @@ extern "c" fn __error() &c_int; pub extern "c" fn _NSGetExecutablePath(buf: &u8, bufsize: &u32) c_int; +pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: &u8, buf_len: usize, basep: &i64) usize; pub use @import("../os/darwin_errno.zig"); @@ -45,3 +46,12 @@ pub const Sigaction = extern struct { sa_mask: sigset_t, sa_flags: c_int, }; + +pub const dirent = extern struct { + d_ino: usize, + d_seekoff: usize, + d_reclen: u16, + d_namlen: u16, + d_type: u8, + d_name: u8, // field address is address of first byte of name +}; diff --git a/std/c/index.zig b/std/c/index.zig index ce9f3c473..369ea2b35 100644 --- a/std/c/index.zig +++ b/std/c/index.zig @@ -44,6 +44,7 @@ pub extern "c" fn sigaction(sig: c_int, noalias act: &const Sigaction, noalias o pub extern "c" fn nanosleep(rqtp: &const timespec, rmtp: ?×pec) c_int; pub extern "c" fn setreuid(ruid: c_uint, euid: c_uint) c_int; pub extern "c" fn setregid(rgid: c_uint, egid: c_uint) c_int; +pub extern "c" fn rmdir(path: &const u8) c_int; pub extern "c" fn aligned_alloc(alignment: usize, size: usize) ?&c_void; pub extern "c" fn malloc(usize) ?&c_void; diff --git a/std/os/darwin.zig b/std/os/darwin.zig index ebc6f65f5..f8b1fbed3 100644 --- a/std/os/darwin.zig +++ b/std/os/darwin.zig @@ -56,10 +56,32 @@ pub const O_SYMLINK = 0x200000; /// allow open of symlinks pub const O_EVTONLY = 0x8000; /// descriptor requested for event notifications only pub const O_CLOEXEC = 0x1000000; /// mark as close-on-exec +pub const O_ACCMODE = 3; +pub const O_ALERT = 536870912; +pub const O_ASYNC = 64; +pub const O_DIRECTORY = 1048576; +pub const O_DP_GETRAWENCRYPTED = 1; +pub const O_DP_GETRAWUNENCRYPTED = 2; +pub const O_DSYNC = 4194304; +pub const O_FSYNC = O_SYNC; +pub const O_NOCTTY = 131072; +pub const O_POPUP = 2147483648; +pub const O_SYNC = 128; + pub const SEEK_SET = 0x0; pub const SEEK_CUR = 0x1; pub const SEEK_END = 0x2; +pub const DT_UNKNOWN = 0; +pub const DT_FIFO = 1; +pub const DT_CHR = 2; +pub const DT_DIR = 4; +pub const DT_BLK = 6; +pub const DT_REG = 8; +pub const DT_LNK = 10; +pub const DT_SOCK = 12; +pub const DT_WHT = 14; + pub const SIG_BLOCK = 1; /// block specified signal set pub const SIG_UNBLOCK = 2; /// unblock specified signal set pub const SIG_SETMASK = 3; /// set specified signal set @@ -192,6 +214,11 @@ pub fn pipe(fds: &[2]i32) usize { return errnoWrap(c.pipe(@ptrCast(&c_int, fds))); } + +pub fn getdirentries64(fd: i32, buf_ptr: &u8, buf_len: usize, basep: &i64) usize { + return errnoWrap(@bitCast(isize, c.__getdirentries64(fd, buf_ptr, buf_len, basep))); +} + pub fn mkdir(path: &const u8, mode: u32) usize { return errnoWrap(c.mkdir(path, mode)); } @@ -204,6 +231,10 @@ pub fn rename(old: &const u8, new: &const u8) usize { return errnoWrap(c.rename(old, new)); } +pub fn rmdir(path: &const u8) usize { + return errnoWrap(c.rmdir(path)); +} + pub fn chdir(path: &const u8) usize { return errnoWrap(c.chdir(path)); } @@ -268,6 +299,7 @@ pub const empty_sigset = sigset_t(0); pub const timespec = c.timespec; pub const Stat = c.Stat; +pub const dirent = c.dirent; /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = struct { diff --git a/std/os/index.zig b/std/os/index.zig index 09409d6d3..eb753db9b 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1055,10 +1055,11 @@ pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError! return; } else |err| switch (err) { error.FileNotFound => return, + + error.AccessDenied, error.IsDir => {}, error.OutOfMemory, - error.AccessDenied, error.SymLinkLoop, error.NameTooLong, error.SystemResources, @@ -1109,18 +1110,16 @@ pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError! } pub const Dir = struct { - // See man getdents fd: i32, + darwin_seek: darwin_seek_t, allocator: &Allocator, buf: []u8, index: usize, end_index: usize, - const LinuxEntry = extern struct { - d_ino: usize, - d_off: usize, - d_reclen: u16, - d_name: u8, // field address is the address of first byte of name + const darwin_seek_t = switch (builtin.os) { + Os.macosx, Os.ios => i64, + else => void, }; pub const Entry = struct { @@ -1135,15 +1134,26 @@ pub const Dir = struct { SymLink, File, UnixDomainSocket, + Wht, // TODO wtf is this Unknown, }; }; pub fn open(allocator: &Allocator, dir_path: []const u8) !Dir { - const fd = try posixOpen(allocator, dir_path, posix.O_RDONLY|posix.O_DIRECTORY|posix.O_CLOEXEC, 0); + const fd = switch (builtin.os) { + Os.windows => @compileError("TODO support Dir.open for windows"), + Os.linux => try posixOpen(allocator, dir_path, posix.O_RDONLY|posix.O_DIRECTORY|posix.O_CLOEXEC, 0), + Os.macosx, Os.ios => try posixOpen(allocator, dir_path, posix.O_RDONLY|posix.O_NONBLOCK|posix.O_DIRECTORY|posix.O_CLOEXEC, 0), + else => @compileError("Dir.open is not supported for this platform"), + }; + const darwin_seek_init = switch (builtin.os) { + Os.macosx, Os.ios => 0, + else => {}, + }; return Dir { .allocator = allocator, .fd = fd, + .darwin_seek = darwin_seek_init, .index = 0, .end_index = 0, .buf = []u8{}, @@ -1158,6 +1168,15 @@ pub const Dir = struct { /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to next, as well as when this ::Dir is deinitialized. pub fn next(self: &Dir) !?Entry { + switch (builtin.os) { + Os.linux => return self.nextLinux(), + Os.macosx, Os.ios => return self.nextDarwin(), + Os.windows => return self.nextWindows(), + else => @compileError("Dir.next not supported on " ++ @tagName(builtin.os)), + } + } + + fn nextDarwin(self: &Dir) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { if (self.buf.len == 0) { @@ -1165,8 +1184,9 @@ pub const Dir = struct { } while (true) { - const result = posix.getdents(self.fd, self.buf.ptr, self.buf.len); - const err = linux.getErrno(result); + const result = posix.getdirentries64(self.fd, self.buf.ptr, self.buf.len, + &self.darwin_seek); + const err = posix.getErrno(result); if (err > 0) { switch (err) { posix.EBADF, posix.EFAULT, posix.ENOTDIR => unreachable, @@ -1184,7 +1204,67 @@ pub const Dir = struct { break; } } - const linux_entry = @ptrCast(& align(1) LinuxEntry, &self.buf[self.index]); + const darwin_entry = @ptrCast(& align(1) posix.dirent, &self.buf[self.index]); + const next_index = self.index + darwin_entry.d_reclen; + self.index = next_index; + + const name = (&darwin_entry.d_name)[0..darwin_entry.d_namlen]; + + // skip . and .. entries + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (darwin_entry.d_type) { + posix.DT_BLK => Entry.Kind.BlockDevice, + posix.DT_CHR => Entry.Kind.CharacterDevice, + posix.DT_DIR => Entry.Kind.Directory, + posix.DT_FIFO => Entry.Kind.NamedPipe, + posix.DT_LNK => Entry.Kind.SymLink, + posix.DT_REG => Entry.Kind.File, + posix.DT_SOCK => Entry.Kind.UnixDomainSocket, + posix.DT_WHT => Entry.Kind.Wht, + else => Entry.Kind.Unknown, + }; + return Entry { + .name = name, + .kind = entry_kind, + }; + } + } + + fn nextWindows(self: &Dir) !?Entry { + @compileError("TODO support Dir.next for windows"); + } + + fn nextLinux(self: &Dir) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + if (self.buf.len == 0) { + self.buf = try self.allocator.alloc(u8, page_size); + } + + while (true) { + const result = posix.getdents(self.fd, self.buf.ptr, self.buf.len); + const err = posix.getErrno(result); + if (err > 0) { + switch (err) { + posix.EBADF, posix.EFAULT, posix.ENOTDIR => unreachable, + posix.EINVAL => { + self.buf = try self.allocator.realloc(u8, self.buf, self.buf.len * 2); + continue; + }, + else => return unexpectedErrorPosix(err), + } + } + if (result == 0) + return null; + self.index = 0; + self.end_index = result; + break; + } + } + const linux_entry = @ptrCast(& align(1) posix.dirent, &self.buf[self.index]); const next_index = self.index + linux_entry.d_reclen; self.index = next_index; @@ -1683,7 +1763,7 @@ test "std.os" { // TODO make this a build variable that you can set -const unexpected_error_tracing = false; +const unexpected_error_tracing = true; /// Call this when you made a syscall or something that sets errno /// and you get an unexpected error. diff --git a/std/os/linux/x86_64.zig b/std/os/linux/x86_64.zig index 3a76ca4f8..cfb2231df 100644 --- a/std/os/linux/x86_64.zig +++ b/std/os/linux/x86_64.zig @@ -488,3 +488,11 @@ pub const timespec = extern struct { tv_sec: isize, tv_nsec: isize, }; + +pub const dirent = extern struct { + d_ino: usize, + d_off: usize, + d_reclen: u16, + d_name: u8, // field address is the address of first byte of name +}; + diff --git a/std/os/test.zig b/std/os/test.zig new file mode 100644 index 000000000..2606dc1c5 --- /dev/null +++ b/std/os/test.zig @@ -0,0 +1,12 @@ +const std = @import("../index.zig"); +const os = std.os; +const io = std.io; + +const a = std.debug.global_allocator; + +test "makePath, put some files in it, deleteTree" { + try os.makePath(a, "os_test_tmp/b/c"); + try io.writeFile(a, "os_test_tmp/b/c/file.txt", "nonsense"); + try io.writeFile(a, "os_test_tmp/b/file2.txt", "blah"); + try os.deleteTree(a, "os_test_tmp"); +} From db70b909a0c29ff50984ce63535a6ba9e2ca4e07 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Wed, 28 Mar 2018 18:30:41 +0200 Subject: [PATCH 025/132] non-zero exit when build.zig cannot be created Make the stage 1 compiler exit with a non-zero status code when `zig build --init` cannot create a new build.zig file. --- src/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index eab7f29b1..791cb3b1b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -452,10 +452,11 @@ int main(int argc, char **argv) { if ((err = os_copy_file(build_template_path, &build_file_abs))) { fprintf(stderr, "Unable to write build.zig template: %s\n", err_str(err)); + return EXIT_FAILURE; } else { fprintf(stderr, "Wrote build.zig template\n"); + return EXIT_SUCCESS; } - return EXIT_SUCCESS; } fprintf(stderr, From b60b01ce9780684dd861c54099204c2578c0c898 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Wed, 28 Mar 2018 18:30:41 +0200 Subject: [PATCH 026/132] skeleton stage 2 support for 'zig build' Initial port of the 'zig build' logic from the stage 1 compiler to the stage 2 compiler sans code generation and BUILD_INFO support. --- src-self-hosted/main.zig | 159 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 742bd0391..a04faaec4 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -41,6 +41,10 @@ pub fn main() !void { const args = try os.argsAlloc(allocator); defer os.argsFree(allocator, args); + if (args.len >= 2 and mem.eql(u8, args[1], "build")) { + return buildMain(allocator, args[2..]); + } + if (args.len >= 2 and mem.eql(u8, args[1], "fmt")) { return fmtMain(allocator, args[2..]); } @@ -560,6 +564,161 @@ fn printZen() !void { ); } +fn buildMain(allocator: &mem.Allocator, argv: []const []const u8) !void { + var build_file: [] const u8 = "build.zig"; + var cache_dir: ?[] const u8 = null; + var zig_install_prefix: ?[] const u8 = null; + var asked_for_help = false; + var asked_for_init = false; + + var args = ArrayList([] const u8).init(allocator); + defer args.deinit(); + + var zig_exe_path = try os.selfExePath(allocator); + defer allocator.free(zig_exe_path); + + try args.append(""); // Placeholder for zig-cache/build + try args.append(""); // Placeholder for zig_exe_path + try args.append(""); // Placeholder for build_file_dirname + try args.append(""); // Placeholder for full_cache_dir + + var i: usize = 0; + while (i < argv.len) : (i += 1) { + var arg = argv[i]; + if (mem.eql(u8, arg, "--help")) { + asked_for_help = true; + try args.append(argv[i]); + } else if (mem.eql(u8, arg, "--init")) { + asked_for_init = true; + try args.append(argv[i]); + } else if (i + 1 < argv.len and mem.eql(u8, arg, "--build-file")) { + build_file = argv[i + 1]; + i += 1; + } else if (i + 1 < argv.len and mem.eql(u8, arg, "--cache-dir")) { + cache_dir = argv[i + 1]; + i += 1; + } else if (i + 1 < argv.len and mem.eql(u8, arg, "--zig-install-prefix")) { + try args.append(arg); + i += 1; + zig_install_prefix = argv[i]; + try args.append(argv[i]); + } else { + try args.append(arg); + } + } + + const zig_lib_dir = try resolveZigLibDir(allocator, zig_install_prefix); + defer allocator.free(zig_lib_dir); + + const zig_std_dir = try os.path.join(allocator, zig_lib_dir, "std"); + defer allocator.free(zig_std_dir); + + const special_dir = try os.path.join(allocator, zig_std_dir, "special"); + defer allocator.free(special_dir); + + const build_runner_path = try os.path.join(allocator, special_dir, "build_runner.zig"); + defer allocator.free(build_runner_path); + + // g = codegen_create(build_runner_path, ...) + // codegen_set_out_name(g, "build") + + const build_file_abs = try os.path.resolve(allocator, ".", build_file); + defer allocator.free(build_file_abs); + + const build_file_basename = os.path.basename(build_file_abs); + const build_file_dirname = os.path.dirname(build_file_abs); + + var full_cache_dir: []u8 = undefined; + if (cache_dir == null) { + full_cache_dir = try os.path.join(allocator, build_file_dirname, "zig-cache"); + } else { + full_cache_dir = try os.path.resolve(allocator, ".", ??cache_dir, full_cache_dir); + } + defer allocator.free(full_cache_dir); + + const path_to_build_exe = try os.path.join(allocator, full_cache_dir, "build"); + defer allocator.free(path_to_build_exe); + // codegen_set_cache_dir(g, full_cache_dir) + + args.items[0] = path_to_build_exe; + args.items[1] = zig_exe_path; + args.items[2] = build_file_dirname; + args.items[3] = full_cache_dir; + + var build_file_exists: bool = undefined; + if (os.File.openRead(allocator, build_file_abs)) |*file| { + file.close(); + build_file_exists = true; + } else |_| { + build_file_exists = false; + } + + if (!build_file_exists and asked_for_help) { + // TODO(bnoordhuis) Print help message from std/special/build_runner.zig + return; + } + + if (!build_file_exists and asked_for_init) { + const build_template_path = try os.path.join(allocator, special_dir, "build_file_template.zig"); + defer allocator.free(build_template_path); + + var srcfile = try os.File.openRead(allocator, build_template_path); + defer srcfile.close(); + + var dstfile = try os.File.openWrite(allocator, build_file_abs); + defer dstfile.close(); + + while (true) { + var buffer: [4096]u8 = undefined; + const n = try srcfile.read(buffer[0..]); + if (n == 0) break; + try dstfile.write(buffer[0..n]); + } + + return; + } + + if (!build_file_exists) { + warn( + \\No 'build.zig' file found. + \\Initialize a 'build.zig' template file with `zig build --init`, + \\or build an executable directly with `zig build-exe $FILENAME.zig`. + \\See: `zig build --help` or `zig help` for more options. + \\ + ); + os.exit(1); + } + + // codegen_build(g) + // codegen_link(g, path_to_build_exe) + // codegen_destroy(g) + + var proc = try os.ChildProcess.init(args.toSliceConst(), allocator); + defer proc.deinit(); + + var term = try proc.spawnAndWait(); + switch (term) { + os.ChildProcess.Term.Exited => |status| { + if (status != 0) { + warn("{} exited with status {}\n", args.at(0), status); + os.exit(1); + } + }, + os.ChildProcess.Term.Signal => |signal| { + warn("{} killed by signal {}\n", args.at(0), signal); + os.exit(1); + }, + os.ChildProcess.Term.Stopped => |signal| { + warn("{} stopped by signal {}\n", args.at(0), signal); + os.exit(1); + }, + os.ChildProcess.Term.Unknown => |status| { + warn("{} encountered unknown failure {}\n", args.at(0), status); + os.exit(1); + }, + } +} + fn fmtMain(allocator: &mem.Allocator, file_paths: []const []const u8) !void { for (file_paths) |file_path| { var file = try os.File.openRead(allocator, file_path); From 72ce1462938c5821936e7b98f95c70c572fda47f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 00:53:06 +0200 Subject: [PATCH 027/132] Fixed looking for windows sdk when targeting linux --- src/analyze.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 7ee1de78a..212833972 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -4287,17 +4287,15 @@ static ZigWindowsSDK *get_windows_sdk(CodeGen *g) { void find_libc_include_path(CodeGen *g) { if (!g->libc_include_dir || buf_len(g->libc_include_dir) == 0) { - ZigWindowsSDK *sdk = get_windows_sdk(g); if (g->zig_target.os == OsWindows) { + ZigWindowsSDK *sdk = get_windows_sdk(g); if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) { zig_panic("Unable to determine libc include path."); } } - } - // TODO find libc at runtime for other operating systems - if(!g->libc_include_dir || buf_len(g->libc_include_dir) == 0) { + // TODO find libc at runtime for other operating systems zig_panic("Unable to determine libc include path."); } } From 032fccf6151ff201ce4b8c7ab28ca460fed794c0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 28 Mar 2018 23:25:12 -0400 Subject: [PATCH 028/132] fix compile time array concatenation for slices closes #866 --- src/ir.cpp | 18 ++++++++++++++++++ test/cases/eval.zig | 12 +++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/ir.cpp b/src/ir.cpp index ea001adb9..18fd02c29 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -11058,6 +11058,24 @@ static TypeTableEntry *ir_analyze_array_cat(IrAnalyze *ira, IrInstructionBinOp * result_type = get_array_type(ira->codegen, child_type, new_len); out_array_val = out_val; + } else if (is_slice(op1_type) || is_slice(op2_type)) { + TypeTableEntry *ptr_type = get_pointer_to_type(ira->codegen, child_type, true); + result_type = get_slice_type(ira->codegen, ptr_type); + out_array_val = create_const_vals(1); + out_array_val->special = ConstValSpecialStatic; + out_array_val->type = get_array_type(ira->codegen, child_type, new_len); + + out_val->data.x_struct.fields = create_const_vals(2); + + out_val->data.x_struct.fields[slice_ptr_index].type = ptr_type; + out_val->data.x_struct.fields[slice_ptr_index].special = ConstValSpecialStatic; + out_val->data.x_struct.fields[slice_ptr_index].data.x_ptr.special = ConstPtrSpecialBaseArray; + out_val->data.x_struct.fields[slice_ptr_index].data.x_ptr.data.base_array.array_val = out_array_val; + out_val->data.x_struct.fields[slice_ptr_index].data.x_ptr.data.base_array.elem_index = 0; + + out_val->data.x_struct.fields[slice_len_index].type = ira->codegen->builtin_types.entry_usize; + out_val->data.x_struct.fields[slice_len_index].special = ConstValSpecialStatic; + bigint_init_unsigned(&out_val->data.x_struct.fields[slice_len_index].data.x_bigint, new_len); } else { new_len += 1; // null byte diff --git a/test/cases/eval.zig b/test/cases/eval.zig index a5b41275b..d6f7afe86 100644 --- a/test/cases/eval.zig +++ b/test/cases/eval.zig @@ -1,4 +1,5 @@ -const assert = @import("std").debug.assert; +const std = @import("std"); +const assert = std.debug.assert; const builtin = @import("builtin"); test "compile time recursion" { @@ -503,3 +504,12 @@ test "const ptr to comptime mutable data is not memoized" { assert(foo.read_x() == 2); } } + +test "array concat of slices gives slice" { + comptime { + var a: []const u8 = "aoeu"; + var b: []const u8 = "asdf"; + const c = a ++ b; + assert(std.mem.eql(u8, c, "aoeuasdf")); + } +} From 7e951e504302dc2a7b0efa3e2ea0dcde5524ac60 Mon Sep 17 00:00:00 2001 From: hellerve Date: Thu, 29 Mar 2018 10:23:44 +0200 Subject: [PATCH 029/132] st/os: address @andrewrk concerns --- std/os/index.zig | 7 ++++--- std/os/test.zig | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/std/os/index.zig b/std/os/index.zig index eb753db9b..d8e2fd700 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1134,7 +1134,7 @@ pub const Dir = struct { SymLink, File, UnixDomainSocket, - Wht, // TODO wtf is this + Whiteout, Unknown, }; }; @@ -1223,7 +1223,7 @@ pub const Dir = struct { posix.DT_LNK => Entry.Kind.SymLink, posix.DT_REG => Entry.Kind.File, posix.DT_SOCK => Entry.Kind.UnixDomainSocket, - posix.DT_WHT => Entry.Kind.Wht, + posix.DT_WHT => Entry.Kind.Whiteout, else => Entry.Kind.Unknown, }; return Entry { @@ -1759,11 +1759,12 @@ test "std.os" { _ = @import("linux/index.zig"); _ = @import("path.zig"); _ = @import("windows/index.zig"); + _ = @import("test.zig"); } // TODO make this a build variable that you can set -const unexpected_error_tracing = true; +const unexpected_error_tracing = false; /// Call this when you made a syscall or something that sets errno /// and you get an unexpected error. diff --git a/std/os/test.zig b/std/os/test.zig index 2606dc1c5..20d439cc4 100644 --- a/std/os/test.zig +++ b/std/os/test.zig @@ -1,5 +1,6 @@ const std = @import("../index.zig"); const os = std.os; +const debug = std.debug; const io = std.io; const a = std.debug.global_allocator; @@ -9,4 +10,9 @@ test "makePath, put some files in it, deleteTree" { try io.writeFile(a, "os_test_tmp/b/c/file.txt", "nonsense"); try io.writeFile(a, "os_test_tmp/b/file2.txt", "blah"); try os.deleteTree(a, "os_test_tmp"); + if (os.Dir.open(a, "os_test_tmp")) |dir| { + debug.assert(false); // this should not happen! + } else |err| { + debug.assert(err == error.PathNotFound); + } } From 9df2a6a5027be17cba7f11ac9d7c106c3497647b Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 13:43:17 +0200 Subject: [PATCH 030/132] std.zig.parser can now parse top level test declarations --- std/zig/ast.zig | 29 +++++++++++++++++++++++++ std/zig/parser.zig | 53 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 903dc051e..180a0a930 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,7 @@ pub const Node = struct { StringLiteral, BuiltinCall, LineComment, + TestDecl, }; pub fn iterate(base: &Node, index: usize) ?&Node { @@ -39,6 +40,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), + Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; } @@ -57,6 +59,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), + Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; } @@ -75,6 +78,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), + Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; } }; @@ -476,3 +480,28 @@ pub const NodeLineComment = struct { return self.lines.at(self.lines.len - 1); } }; + +pub const NodeTestDecl = struct { + base: Node, + test_token: Token, + name_token: Token, + body_node: &Node, + + pub fn iterate(self: &NodeTestDecl, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.body_node; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeTestDecl) Token { + return self.test_token; + } + + pub fn lastToken(self: &NodeTestDecl) Token { + return self.body_node.lastToken(); + } +}; + diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 533ad754a..06b7d35a4 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -171,6 +171,22 @@ pub const Parser = struct { stack.append(State { .TopLevelExtern = token }) catch unreachable; continue; }, + Token.Id.Keyword_test => { + stack.append(State.TopLevel) catch unreachable; + + const name_token = self.getNextToken(); + if (name_token.id != Token.Id.StringLiteral) + return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + + const lbrace = self.getNextToken(); + if (lbrace.id != Token.Id.LBrace) + return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + + const block = try self.createBlock(arena, token); + const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, name_token, block); + try stack.append(State { .Block = block }); + continue; + }, Token.Id.Eof => { root_node.eof_token = token; return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; @@ -733,6 +749,20 @@ pub const Parser = struct { return node; } + fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name_token: &const Token, + block: &ast.NodeBlock) !&ast.NodeTestDecl + { + const node = try arena.create(ast.NodeTestDecl); + + *node = ast.NodeTestDecl { + .base = self.initNode(ast.Node.Id.TestDecl), + .test_token = *test_token, + .name_token = *name_token, + .body_node = &block.base, + }; + return node; + } + fn createFnProto(self: &Parser, arena: &mem.Allocator, fn_token: &const Token, extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { @@ -867,6 +897,14 @@ pub const Parser = struct { return node; } + fn createAttachTestDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), + test_token: &const Token, name_token: &const Token, block: &ast.NodeBlock) !&ast.NodeTestDecl + { + const node = try self.createTestDecl(arena, test_token, name_token, block); + try list.append(&node.base); + return node; + } + fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { const loc = self.tokenizer.getTokenLocation(token); warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args); @@ -1032,7 +1070,11 @@ pub const Parser = struct { ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); try stack.append(RenderState { .VarDecl = var_decl}); - + }, + ast.Node.Id.TestDecl => { + const test_decl = @fieldParentPtr(ast.NodeTestDecl, "base", decl); + try stream.print("test {} ", self.tokenizer.getTokenSlice(test_decl.name_token)); + try stack.append(RenderState { .Expression = test_decl.body_node }); }, else => unreachable, } @@ -1201,6 +1243,7 @@ pub const Parser = struct { ast.Node.Id.Root, ast.Node.Id.VarDecl, + ast.Node.Id.TestDecl, ast.Node.Id.ParamDecl => unreachable, }, RenderState.FnProtoRParen => |fn_proto| { @@ -1422,4 +1465,12 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "test name" { + \\ const a = 1; + \\ var b = 1; + \\} + \\ + ); } From 0fd0f6fd1f4b9e02cc33eac304b9f6db242cbb67 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 30 Mar 2018 01:36:04 +1300 Subject: [PATCH 031/132] Rewrite Rand functions We now use a generic Rand structure which abstracts the core functions from the backing engine. The old Mersenne Twister engine is removed and replaced instead with three alternatives: - Pcg32 - Xoroshiro128+ - Isaac64 These should provide sufficient coverage for most purposes, including a CSPRNG using Isaac64. Consumers of the library that do not care about the actual engine implementation should use DefaultPrng and DefaultCsprng. --- CMakeLists.txt | 2 +- std/index.zig | 4 +- std/io_test.zig | 6 +- std/rand.zig | 240 ----------------- std/rand/index.zig | 652 +++++++++++++++++++++++++++++++++++++++++++++ std/rand_test.zig | 507 ----------------------------------- std/sort.zig | 154 +++++------ 7 files changed, 735 insertions(+), 830 deletions(-) delete mode 100644 std/rand.zig create mode 100644 std/rand/index.zig delete mode 100644 std/rand_test.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 979d771cd..fbaf929e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -509,7 +509,7 @@ set(ZIG_STD_FILES "os/windows/index.zig" "os/windows/util.zig" "os/zen.zig" - "rand.zig" + "rand/index.zig" "sort.zig" "special/bootstrap.zig" "special/bootstrap_lib.zig" diff --git a/std/index.zig b/std/index.zig index 179eae159..4bc1444ac 100644 --- a/std/index.zig +++ b/std/index.zig @@ -26,7 +26,7 @@ pub const math = @import("math/index.zig"); pub const mem = @import("mem.zig"); pub const net = @import("net.zig"); pub const os = @import("os/index.zig"); -pub const rand = @import("rand.zig"); +pub const rand = @import("rand/index.zig"); pub const sort = @import("sort.zig"); pub const unicode = @import("unicode.zig"); pub const zig = @import("zig/index.zig"); @@ -58,7 +58,7 @@ test "std" { _ = @import("heap.zig"); _ = @import("net.zig"); _ = @import("os/index.zig"); - _ = @import("rand.zig"); + _ = @import("rand/index.zig"); _ = @import("sort.zig"); _ = @import("unicode.zig"); _ = @import("zig/index.zig"); diff --git a/std/io_test.zig b/std/io_test.zig index 993ec84d2..89959b7b5 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -1,7 +1,7 @@ const std = @import("index.zig"); const io = std.io; const allocator = std.debug.global_allocator; -const Rand = std.rand.Rand; +const DefaultPrng = std.rand.DefaultPrng; const assert = std.debug.assert; const mem = std.mem; const os = std.os; @@ -9,8 +9,8 @@ const builtin = @import("builtin"); test "write a file, read it, then delete it" { var data: [1024]u8 = undefined; - var rng = Rand.init(1234); - rng.fillBytes(data[0..]); + var prng = DefaultPrng.init(1234); + prng.random.bytes(data[0..]); const tmp_file_name = "temp_test_file.txt"; { var file = try os.File.openWrite(allocator, tmp_file_name); diff --git a/std/rand.zig b/std/rand.zig deleted file mode 100644 index 96f0a385a..000000000 --- a/std/rand.zig +++ /dev/null @@ -1,240 +0,0 @@ -const std = @import("index.zig"); -const builtin = @import("builtin"); -const assert = std.debug.assert; -const rand_test = @import("rand_test.zig"); -const mem = std.mem; -const math = std.math; - -pub const MT19937_32 = MersenneTwister( - u32, 624, 397, 31, - 0x9908B0DF, - 11, 0xFFFFFFFF, - 7, 0x9D2C5680, - 15, 0xEFC60000, - 18, 1812433253); - -pub const MT19937_64 = MersenneTwister( - u64, 312, 156, 31, - 0xB5026F5AA96619E9, - 29, 0x5555555555555555, - 17, 0x71D67FFFEDA60000, - 37, 0xFFF7EEE000000000, - 43, 6364136223846793005); - -/// Use `init` to initialize this state. -pub const Rand = struct { - const Rng = if (@sizeOf(usize) >= 8) MT19937_64 else MT19937_32; - - rng: Rng, - - /// Initialize random state with the given seed. - pub fn init(seed: usize) Rand { - return Rand { - .rng = Rng.init(seed), - }; - } - - /// Get an integer or boolean with random bits. - pub fn scalar(r: &Rand, comptime T: type) T { - if (T == usize) { - return r.rng.get(); - } else if (T == bool) { - return (r.rng.get() & 0b1) == 0; - } else { - var result: [@sizeOf(T)]u8 = undefined; - r.fillBytes(result[0..]); - return mem.readInt(result, T, builtin.Endian.Little); - } - } - - /// Fill `buf` with randomness. - pub fn fillBytes(r: &Rand, buf: []u8) void { - var bytes_left = buf.len; - while (bytes_left >= @sizeOf(usize)) { - mem.writeInt(buf[buf.len - bytes_left..], r.rng.get(), builtin.Endian.Little); - bytes_left -= @sizeOf(usize); - } - if (bytes_left > 0) { - var rand_val_array: [@sizeOf(usize)]u8 = undefined; - mem.writeInt(rand_val_array[0..], r.rng.get(), builtin.Endian.Little); - while (bytes_left > 0) { - buf[buf.len - bytes_left] = rand_val_array[@sizeOf(usize) - bytes_left]; - bytes_left -= 1; - } - } - } - - /// Get a random unsigned integer with even distribution between `start` - /// inclusive and `end` exclusive. - pub fn range(r: &Rand, comptime T: type, start: T, end: T) T { - assert(start <= end); - if (T.is_signed) { - const uint = @IntType(false, T.bit_count); - if (start >= 0 and end >= 0) { - return T(r.range(uint, uint(start), uint(end))); - } else if (start < 0 and end < 0) { - // Can't overflow because the range is over signed ints - return math.negateCast(r.range(uint, math.absCast(end), math.absCast(start)) + 1) catch unreachable; - } else if (start < 0 and end >= 0) { - const end_uint = uint(end); - const total_range = math.absCast(start) + end_uint; - const value = r.range(uint, 0, total_range); - const result = if (value < end_uint) x: { - break :x T(value); - } else if (value == end_uint) x: { - break :x start; - } else x: { - // Can't overflow because the range is over signed ints - break :x math.negateCast(value - end_uint) catch unreachable; - }; - return result; - } else { - unreachable; - } - } else { - const total_range = end - start; - const leftover = @maxValue(T) % total_range; - const upper_bound = @maxValue(T) - leftover; - var rand_val_array: [@sizeOf(T)]u8 = undefined; - - while (true) { - r.fillBytes(rand_val_array[0..]); - const rand_val = mem.readInt(rand_val_array, T, builtin.Endian.Little); - if (rand_val < upper_bound) { - return start + (rand_val % total_range); - } - } - } - } - - /// Get a floating point value in the range 0.0..1.0. - pub fn float(r: &Rand, comptime T: type) T { - // TODO Implement this way instead: - // const int = @int_type(false, @sizeOf(T) * 8); - // const mask = ((1 << @float_mantissa_bit_count(T)) - 1); - // const rand_bits = r.rng.scalar(int) & mask; - // return @float_compose(T, false, 0, rand_bits) - 1.0 - const int_type = @IntType(false, @sizeOf(T) * 8); - const precision = if (T == f32) - 16777216 - else if (T == f64) - 9007199254740992 - else - @compileError("unknown floating point type") - ; - return T(r.range(int_type, 0, precision)) / T(precision); - } -}; - -fn MersenneTwister( - comptime int: type, comptime n: usize, comptime m: usize, comptime r: int, - comptime a: int, - comptime u: math.Log2Int(int), comptime d: int, - comptime s: math.Log2Int(int), comptime b: int, - comptime t: math.Log2Int(int), comptime c: int, - comptime l: math.Log2Int(int), comptime f: int) type -{ - return struct { - const Self = this; - - array: [n]int, - index: usize, - - pub fn init(seed: int) Self { - var mt = Self { - .array = undefined, - .index = n, - }; - - var prev_value = seed; - mt.array[0] = prev_value; - var i: usize = 1; - while (i < n) : (i += 1) { - prev_value = int(i) +% f *% (prev_value ^ (prev_value >> (int.bit_count - 2))); - mt.array[i] = prev_value; - } - return mt; - } - - pub fn get(mt: &Self) int { - const mag01 = []int{0, a}; - const LM: int = (1 << r) - 1; - const UM = ~LM; - - if (mt.index >= n) { - var i: usize = 0; - - while (i < n - m) : (i += 1) { - const x = (mt.array[i] & UM) | (mt.array[i + 1] & LM); - mt.array[i] = mt.array[i + m] ^ (x >> 1) ^ mag01[usize(x & 0x1)]; - } - - while (i < n - 1) : (i += 1) { - const x = (mt.array[i] & UM) | (mt.array[i + 1] & LM); - mt.array[i] = mt.array[i + m - n] ^ (x >> 1) ^ mag01[usize(x & 0x1)]; - - } - const x = (mt.array[i] & UM) | (mt.array[0] & LM); - mt.array[i] = mt.array[m - 1] ^ (x >> 1) ^ mag01[usize(x & 0x1)]; - - mt.index = 0; - } - - var x = mt.array[mt.index]; - mt.index += 1; - - x ^= ((x >> u) & d); - x ^= ((x << s) & b); - x ^= ((x << t) & c); - x ^= (x >> l); - - return x; - } - }; -} - -test "rand float 32" { - var r = Rand.init(42); - var i: usize = 0; - while (i < 1000) : (i += 1) { - const val = r.float(f32); - assert(val >= 0.0); - assert(val < 1.0); - } -} - -test "rand.MT19937_64" { - var rng = MT19937_64.init(rand_test.mt64_seed); - for (rand_test.mt64_data) |value| { - assert(value == rng.get()); - } -} - -test "rand.MT19937_32" { - var rng = MT19937_32.init(rand_test.mt32_seed); - for (rand_test.mt32_data) |value| { - assert(value == rng.get()); - } -} - -test "rand.Rand.range" { - var r = Rand.init(42); - testRange(&r, -4, 3); - testRange(&r, -4, -1); - testRange(&r, 10, 14); -} - -fn testRange(r: &Rand, start: i32, end: i32) void { - const count = usize(end - start); - var values_buffer = []bool{false} ** 20; - const values = values_buffer[0..count]; - var i: usize = 0; - while (i < count) { - const value = r.range(i32, start, end); - const index = usize(value - start); - if (!values[index]) { - i += 1; - values[index] = true; - } - } -} diff --git a/std/rand/index.zig b/std/rand/index.zig new file mode 100644 index 000000000..ce6956547 --- /dev/null +++ b/std/rand/index.zig @@ -0,0 +1,652 @@ +// The engines provided here should be initialized from an external source. For now, getRandomBytes +// from the os package is the most suitable. Be sure to use a CSPRNG when required, otherwise using +// a normal PRNG will be faster and use substantially less stack space. +// +// ``` +// var buf: [8]u8 = undefined; +// try std.os.getRandomBytes(buf[0..]); +// const seed = mem.readInt(buf[0..8], u64, builtin.Endian.Little); +// +// var r = DefaultPrng.init(seed); +// +// const s = r.random.scalar(u64); +// ``` +// +// TODO(tiehuis): Benchmark these against other reference implementations. + +const std = @import("../index.zig"); +const builtin = @import("builtin"); +const assert = std.debug.assert; +const mem = std.mem; +const math = std.math; + +// When you need fast unbiased random numbers +pub const DefaultPrng = Xoroshiro128; + +// When you need cryptographically secure random numbers +pub const DefaultCsprng = Isaac64; + +pub const Rand = struct { + fillFn: fn(r: &Rand, buf: []u8) void, + + /// Read random bytes into the specified buffer until fill. + pub fn bytes(r: &Rand, buf: []u8) void { + r.fillFn(r, buf); + } + + /// Return a random integer/boolean type. + pub fn scalar(r: &Rand, comptime T: type) T { + var rand_bytes: [@sizeOf(T)]u8 = undefined; + r.bytes(rand_bytes[0..]); + + if (T == bool) { + return rand_bytes[0] & 0b1 == 0; + } else { + // NOTE: Cannot @bitCast array to integer type. + return mem.readInt(rand_bytes, T, builtin.Endian.Little); + } + } + + /// Get a random unsigned integer with even distribution between `start` + /// inclusive and `end` exclusive. + pub fn range(r: &Rand, comptime T: type, start: T, end: T) T { + assert(start <= end); + if (T.is_signed) { + const uint = @IntType(false, T.bit_count); + if (start >= 0 and end >= 0) { + return T(r.range(uint, uint(start), uint(end))); + } else if (start < 0 and end < 0) { + // Can't overflow because the range is over signed ints + return math.negateCast(r.range(uint, math.absCast(end), math.absCast(start)) + 1) catch unreachable; + } else if (start < 0 and end >= 0) { + const end_uint = uint(end); + const total_range = math.absCast(start) + end_uint; + const value = r.range(uint, 0, total_range); + const result = if (value < end_uint) x: { + break :x T(value); + } else if (value == end_uint) x: { + break :x start; + } else x: { + // Can't overflow because the range is over signed ints + break :x math.negateCast(value - end_uint) catch unreachable; + }; + return result; + } else { + unreachable; + } + } else { + const total_range = end - start; + const leftover = @maxValue(T) % total_range; + const upper_bound = @maxValue(T) - leftover; + var rand_val_array: [@sizeOf(T)]u8 = undefined; + + while (true) { + r.bytes(rand_val_array[0..]); + const rand_val = mem.readInt(rand_val_array, T, builtin.Endian.Little); + if (rand_val < upper_bound) { + return start + (rand_val % total_range); + } + } + } + } + + /// Return a floating point value evenly distributed in the range [0, 1). + pub fn float(r: &Rand, comptime T: type) T { + // Generate a uniform value between [1, 2) and scale down to [0, 1). + // Note: The lowest mantissa bit is always set to 0 so we only use half the available range. + switch (T) { + f32 => { + const s = r.scalar(u32); + const repr = (0x7f << 23) | (s >> 9); + return @bitCast(f32, repr) - 1.0; + }, + f64 => { + const s = r.scalar(u64); + const repr = (0x3ff << 52) | (s >> 12); + return @bitCast(f64, repr) - 1.0; + }, + else => @compileError("unknown floating point type"), + } + } + + /// Return a floating point value normally distributed in the range [0, 1]. + pub fn floatNorm(r: &Rand, comptime T: type) T { + // TODO(tiehuis): See https://www.doornik.com/research/ziggurat.pdf + @compileError("floatNorm is unimplemented"); + } + + /// Return a exponentially distributed float between (0, @maxValue(f64)) + pub fn floatExp(r: &Rand, comptime T: type) T { + @compileError("floatExp is unimplemented"); + } + + /// Shuffle a slice into a random order. + pub fn shuffle(r: &Rand, comptime T: type, buf: []T) void { + if (buf.len < 2) { + return; + } + + var i: usize = 0; + while (i < buf.len - 1) : (i += 1) { + const j = r.range(usize, i, buf.len); + mem.swap(T, &buf[i], &buf[j]); + } + } +}; + +// Generator to extend 64-bit seed values into longer sequences. +// +// The number of cycles is thus limited to 64-bits regardless of the engine, but this +// is still plenty for practical purposes. +const SplitMix64 = struct { + s: u64, + + pub fn init(seed: u64) SplitMix64 { + return SplitMix64 { .s = seed }; + } + + pub fn next(self: &SplitMix64) u64 { + self.s +%= 0x9e3779b97f4a7c15; + + var z = self.s; + z = (z ^ (z >> 30)) *% 0xbf58476d1ce4e5b9; + z = (z ^ (z >> 27)) *% 0x94d049bb133111eb; + return z ^ (z >> 31); + } +}; + +test "splitmix64 sequence" { + var r = SplitMix64.init(0xaeecf86f7878dd75); + + const seq = []const u64 { + 0x5dbd39db0178eb44, + 0xa9900fb66b397da3, + 0x5c1a28b1aeebcf5c, + 0x64a963238f776912, + 0xc6d4177b21d1c0ab, + 0xb2cbdbdb5ea35394, + }; + + for (seq) |s| { + std.debug.assert(s == r.next()); + } +} + +// PCG32 - http://www.pcg-random.org/ +// +// PRNG +pub const Pcg = struct { + const default_multiplier = 6364136223846793005; + + random: Rand, + + s: u64, + i: u64, + + pub fn init(init_s: u64) Pcg { + var pcg = Pcg { + .random = Rand { .fillFn = fill }, + .s = undefined, + .i = undefined, + }; + + pcg.seed(init_s); + return pcg; + } + + fn next(self: &Pcg) u32 { + const l = self.s; + self.s = l *% default_multiplier +% (self.i | 1); + + const xor_s = @truncate(u32, ((l >> 18) ^ l) >> 27); + const rot = u32(l >> 59); + + return (xor_s >> u5(rot)) | (xor_s << u5((0 -% rot) & 31)); + } + + fn seed(self: &Pcg, init_s: u64) void { + // Pcg requires 128-bits of seed. + var gen = SplitMix64.init(init_s); + self.seedTwo(gen.next(), gen.next()); + } + + fn seedTwo(self: &Pcg, init_s: u64, init_i: u64) void { + self.s = 0; + self.i = (init_s << 1) | 1; + self.s = self.s *% default_multiplier +% self.i; + self.s +%= init_i; + self.s = self.s *% default_multiplier +% self.i; + } + + fn fill(r: &Rand, buf: []u8) void { + const self = @fieldParentPtr(Pcg, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Complete 4 byte segments. + while (i < aligned_len) : (i += 4) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 4) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Remaining. (cuts the stream) + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 4; + } + } + } +}; + +test "pcg sequence" { + var r = Pcg.init(0); + const s0: u64 = 0x9394bf54ce5d79de; + const s1: u64 = 0x84e9c579ef59bbf7; + r.seedTwo(s0, s1); + + const seq = []const u32 { + 2881561918, + 3063928540, + 1199791034, + 2487695858, + 1479648952, + 3247963454, + }; + + for (seq) |s| { + std.debug.assert(s == r.next()); + } +} + +// Xoroshiro128+ - http://xoroshiro.di.unimi.it/ +// +// PRNG +pub const Xoroshiro128 = struct { + random: Rand, + + s: [2]u64, + + pub fn init(init_s: u64) Xoroshiro128 { + var x = Xoroshiro128 { + .random = Rand { .fillFn = fill }, + .s = undefined, + }; + + x.seed(init_s); + return x; + } + + fn next(self: &Xoroshiro128) u64 { + const s0 = self.s[0]; + var s1 = self.s[1]; + const r = s0 +% s1; + + s1 ^= s0; + self.s[0] = math.rotl(u64, s0, u8(55)) ^ s1 ^ (s1 << 14); + self.s[1] = math.rotl(u64, s1, u8(36)); + + return r; + } + + // Skip 2^64 places ahead in the sequence + fn jump(self: &Xoroshiro128) void { + var s0: u64 = 0; + var s1: u64 = 0; + + const table = []const u64 { + 0xbeac0467eba5facb, + 0xd86b048b86aa9922 + }; + + inline for (table) |entry| { + var b: usize = 0; + while (b < 64) : (b += 1) { + if ((entry & (u64(1) << u6(b))) != 0) { + s0 ^= self.s[0]; + s1 ^= self.s[1]; + } + _ = self.next(); + } + } + + self.s[0] = s0; + self.s[1] = s1; + } + + fn seed(self: &Xoroshiro128, init_s: u64) void { + // Xoroshiro requires 128-bits of seed. + var gen = SplitMix64.init(init_s); + + self.s[0] = gen.next(); + self.s[1] = gen.next(); + } + + fn fill(r: &Rand, buf: []u8) void { + const self = @fieldParentPtr(Xoroshiro128, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Complete 8 byte segments. + while (i < aligned_len) : (i += 8) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Remaining. (cuts the stream) + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 8; + } + } + } +}; + +test "xoroshiro sequence" { + var r = Xoroshiro128.init(0); + r.s[0] = 0xaeecf86f7878dd75; + r.s[1] = 0x01cd153642e72622; + + const seq1 = []const u64 { + 0xb0ba0da5bb600397, + 0x18a08afde614dccc, + 0xa2635b956a31b929, + 0xabe633c971efa045, + 0x9ac19f9706ca3cac, + 0xf62b426578c1e3fb, + }; + + for (seq1) |s| { + std.debug.assert(s == r.next()); + } + + + r.jump(); + + const seq2 = []const u64 { + 0x95344a13556d3e22, + 0xb4fb32dafa4d00df, + 0xb2011d9ccdcfe2dd, + 0x05679a9b2119b908, + 0xa860a1da7c9cd8a0, + 0x658a96efe3f86550, + }; + + for (seq2) |s| { + std.debug.assert(s == r.next()); + } +} + +// ISAAC64 - http://www.burtleburtle.net/bob/rand/isaacafa.html +// +// CSPRNG +// +// Follows the general idea of the implementation from here with a few shortcuts. +// https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html +pub const Isaac64 = struct { + random: Rand, + + r: [256]u64, + m: [256]u64, + a: u64, + b: u64, + c: u64, + i: usize, + + pub fn init(init_s: u64) Isaac64 { + var isaac = Isaac64 { + .random = Rand { .fillFn = fill }, + .r = undefined, + .m = undefined, + .a = undefined, + .b = undefined, + .c = undefined, + .i = undefined, + }; + + // seed == 0 => same result as the unseeded reference implementation + isaac.seed(init_s, 1); + return isaac; + } + + fn step(self: &Isaac64, mix: u64, base: usize, comptime m1: usize, comptime m2: usize) void { + const x = self.m[base + m1]; + self.a = mix +% self.m[base + m2]; + + const y = self.a +% self.b +% self.m[(x >> 3) % self.m.len]; + self.m[base + m1] = y; + + self.b = x +% self.m[(y >> 11) % self.m.len]; + self.r[self.r.len - 1 - base - m1] = self.b; + } + + fn refill(self: &Isaac64) void { + const midpoint = self.r.len / 2; + + self.c +%= 1; + self.b +%= self.c; + + { + var i: usize = 0; + while (i < midpoint) : (i += 4) { + self.step( ~(self.a ^ (self.a << 21)), i + 0, 0, midpoint); + self.step( self.a ^ (self.a >> 5) , i + 1, 0, midpoint); + self.step( self.a ^ (self.a << 12) , i + 2, 0, midpoint); + self.step( self.a ^ (self.a >> 33) , i + 3, 0, midpoint); + } + } + + { + var i: usize = 0; + while (i < midpoint) : (i += 4) { + self.step( ~(self.a ^ (self.a << 21)), i + 0, midpoint, 0); + self.step( self.a ^ (self.a >> 5) , i + 1, midpoint, 0); + self.step( self.a ^ (self.a << 12) , i + 2, midpoint, 0); + self.step( self.a ^ (self.a >> 33) , i + 3, midpoint, 0); + } + } + + self.i = 0; + } + + fn next(self: &Isaac64) u64 { + if (self.i >= self.r.len) { + self.refill(); + } + + const value = self.r[self.i]; + self.i += 1; + return value; + } + + fn seed(self: &Isaac64, init_s: u64, comptime rounds: usize) void { + // We ignore the multi-pass requirement since we don't currently expose full access to + // seeding the self.m array completely. + mem.set(u64, self.m[0..], 0); + self.m[0] = init_s; + + // prescrambled golden ratio constants + var a = []const u64 { + 0x647c4677a2884b7c, + 0xb9f8b322c73ac862, + 0x8c0ea5053d4712a0, + 0xb29b2e824a595524, + 0x82f053db8355e0ce, + 0x48fe4a0fa5a09315, + 0xae985bf2cbfc89ed, + 0x98f5704f6c44c0ab, + }; + + comptime var i: usize = 0; + inline while (i < rounds) : (i += 1) { + var j: usize = 0; + while (j < self.m.len) : (j += 8) { + comptime var x1: usize = 0; + inline while (x1 < 8) : (x1 += 1) { + a[x1] +%= self.m[j + x1]; + } + + a[0] -%= a[4]; a[5] ^= a[7] >> 9; a[7] +%= a[0]; + a[1] -%= a[5]; a[6] ^= a[0] << 9; a[0] +%= a[1]; + a[2] -%= a[6]; a[7] ^= a[1] >> 23; a[1] +%= a[2]; + a[3] -%= a[7]; a[0] ^= a[2] << 15; a[2] +%= a[3]; + a[4] -%= a[0]; a[1] ^= a[3] >> 14; a[3] +%= a[4]; + a[5] -%= a[1]; a[2] ^= a[4] << 20; a[4] +%= a[5]; + a[6] -%= a[2]; a[3] ^= a[5] >> 17; a[5] +%= a[6]; + a[7] -%= a[3]; a[4] ^= a[6] << 14; a[6] +%= a[7]; + + comptime var x2: usize = 0; + inline while (x2 < 8) : (x2 += 1) { + self.m[j + x2] = a[x2]; + } + } + } + + mem.set(u64, self.r[0..], 0); + self.a = 0; + self.b = 0; + self.c = 0; + self.i = self.r.len; // trigger refill on first value + } + + fn fill(r: &Rand, buf: []u8) void { + const self = @fieldParentPtr(Isaac64, "random", r); + + var i: usize = 0; + const aligned_len = buf.len - (buf.len & 7); + + // Fill complete 64-byte segments + while (i < aligned_len) : (i += 8) { + var n = self.next(); + comptime var j: usize = 0; + inline while (j < 8) : (j += 1) { + buf[i + j] = @truncate(u8, n); + n >>= 8; + } + } + + // Fill trailing, ignoring excess (cut the stream). + if (i != buf.len) { + var n = self.next(); + while (i < buf.len) : (i += 1) { + buf[i] = @truncate(u8, n); + n >>= 8; + } + } + } +}; + +test "isaac64 sequence" { + var r = Isaac64.init(0); + + // from reference implementation + const seq = []const u64 { + 0xf67dfba498e4937c, + 0x84a5066a9204f380, + 0xfee34bd5f5514dbb, + 0x4d1664739b8f80d6, + 0x8607459ab52a14aa, + 0x0e78bc5a98529e49, + 0xfe5332822ad13777, + 0x556c27525e33d01a, + 0x08643ca615f3149f, + 0xd0771faf3cb04714, + 0x30e86f68a37b008d, + 0x3074ebc0488a3adf, + 0x270645ea7a2790bc, + 0x5601a0a8d3763c6a, + 0x2f83071f53f325dd, + 0xb9090f3d42d2d2ea, + }; + + for (seq) |s| { + std.debug.assert(s == r.next()); + } +} + +// Actual Rand helper function tests, pcg engine is assumed correct. +test "Rand float" { + var prng = DefaultPrng.init(0); + + var i: usize = 0; + while (i < 1000) : (i += 1) { + const val1 = prng.random.float(f32); + std.debug.assert(val1 >= 0.0); + std.debug.assert(val1 < 1.0); + + const val2 = prng.random.float(f64); + std.debug.assert(val2 >= 0.0); + std.debug.assert(val2 < 1.0); + } +} + +test "Rand scalar" { + var prng = DefaultPrng.init(0); + const s = prng .random.scalar(u64); +} + +test "Rand bytes" { + var prng = DefaultPrng.init(0); + var buf: [2048]u8 = undefined; + prng.random.bytes(buf[0..]); +} + +test "Rand shuffle" { + var prng = DefaultPrng.init(0); + + var seq = []const u8 { 0, 1, 2, 3, 4 }; + var seen = []bool {false} ** 5; + + var i: usize = 0; + while (i < 1000) : (i += 1) { + prng.random.shuffle(u8, seq[0..]); + seen[seq[0]] = true; + std.debug.assert(sumArray(seq[0..]) == 10); + } + + // we should see every entry at the head at least once + for (seen) |e| { + std.debug.assert(e == true); + } +} + +fn sumArray(s: []const u8) u32 { + var r: u32 = 0; + for (s) |e| r += e; + return r; +} + +test "Rand range" { + var prng = DefaultPrng.init(0); + testRange(&prng.random, -4, 3); + testRange(&prng.random, -4, -1); + testRange(&prng.random, 10, 14); +} + +fn testRange(r: &Rand, start: i32, end: i32) void { + const count = usize(end - start); + var values_buffer = []bool{false} ** 20; + const values = values_buffer[0..count]; + var i: usize = 0; + while (i < count) { + const value = r.range(i32, start, end); + const index = usize(value - start); + if (!values[index]) { + i += 1; + values[index] = true; + } + } +} diff --git a/std/rand_test.zig b/std/rand_test.zig deleted file mode 100644 index 200b50a5b..000000000 --- a/std/rand_test.zig +++ /dev/null @@ -1,507 +0,0 @@ -pub const mt64_seed = 0xb334e49d0977c37e; -pub const mt64_data = []u64 { - 0x2ad1a63fab6d25a9, 0xb7143aba12569814, 0xc1b60d8d49e53e8, 0x5652adfc8da656dc, - 0x43e3beb6d9e484a9, 0x17b09b71e9418ff7, 0x541646292686cfa4, 0x260457071268ecfc, - 0x1627af31774e1dc1, 0x362a49b34ed75bb3, 0x7acf72002fe0f733, 0x3aaaf2e7b9409452, - 0x9cfc2d9908115c2, 0x6e81a7f16ae613e9, 0xfc4da89c04acf3c7, 0x6984b6adb4feb9ae, - 0x6a128b334e27b03d, 0xcc45a2b02937871a, 0xe585b229e00b2283, 0x7a92c0664a6f678a, - 0x972735011bdc0744, 0xb494e743d658a084, 0x1eda3c4e7b1b2d0c, 0x4c7adb3831d87332, - 0x12f8c7355f5ec631, 0xfc2bcb6be7d60eba, 0x74b95b47895f8687, 0x171de6fe92b97f5a, - 0x86383730f52719ac, 0xe4e43ce0f61274f6, 0x514f7e072d96f19c, 0xabef324fbc6cb7fe, - 0x7534b945b742f14f, 0x47f9efe33265adbe, 0x7bcab027a0abf16b, 0x3312a2b34225bff7, - 0xb61455ce8c2e3e0b, 0x2b81008deeee4d94, 0x743b0b2b4974c7b6, 0xfc219101fd665f7d, - 0x863d78891cbfe5e5, 0x7531bb1839181778, 0xf614359a65356e72, 0xfcdd1f6e3f250bdd, - 0x528e10bd536eed5b, 0x9f69386ac60cd618, 0xbf5f242706817f01, 0x8e7da8070072cf64, - 0xaa318b622da0667f, 0xc9580540fb7efd66, 0xb5d996deccd02e0e, 0x81c6a799ea3f5a41, - 0xe6b896f4e21a8550, 0xde5206f177a24ceb, 0x53343e81639ec0b6, 0x5a6edb63d08be9f6, - 0x3602c2892f7da1b9, 0xda84f4259841bdea, 0x5880e169a7746e45, 0x57cddb5ffd3c2423, - 0x28fe1166fe7b8595, 0x92136c9decb42243, 0xa8c4199818ca7d62, 0x5042cd96f62854dd, - 0x22b2d38c3f21f8d0, 0x73a2bfccf1b5f7bb, 0xaba3718f40b6984e, 0x9c1f5dc3e399f5f0, - 0x9cf463f95369c149, 0xa7546d69e4232e18, 0x9ea57317d19ab7fc, 0xfb13c83d830731fb, - 0x635a123eaa099259, 0x9a2fe7d0ba6e3c5c, 0x40b903cd0d0d3b4e, 0xc8210eb2d2e941cb, - 0xd2582d4b1e016484, 0x1d048030875af39c, 0xb51c31a6c193d76f, 0x5ce9b801b8d61626, - 0x2bae30455cbb0022, 0xba54df5998b2443f, 0x927abb9342c9a90a, 0xc431eb7df3e06727, - 0x726f885d3da88d5, 0x7d85ff1ca4260280, 0xaf3fecf8f019817, 0x31d39105d6fc4fe8, - 0x262d9842dbafd4bd, 0x54c28a2876e62e39, 0x95a986e24e214dde, 0xbf677a1abd2e553, - 0x48ac890ff787b2b6, 0x2890ec1c67c539f4, 0x7ce88bf3975882c3, 0x88ef340414a29c88, - 0xe30de9c88a00805b, 0xe772225e3ee6c68a, 0xa3a7d0921c5d5816, 0x8354957227b1663f, - 0xb5b65ded7c747cbc, 0x93b4a12ff2e8fcea, 0x6359579c3c438b3c, 0x45b2c12e9722f2bd, - 0x659a604414f19e1, 0x4ec9a149d4219ca5, 0xd830290dd6aebe2b, 0xabc7874a6b4827f8, - 0xc91be5dd875847e7, 0x5b761d39f3f96aee, 0x5749dffad692b6c8, 0x86c94840cbd249d2, - 0x411a466e886ad7, 0x27dca1f51aebb9a0, 0x1cceb093fbab7a42, 0x7140c2d6706e927c, - 0x6881fdb87299a92f, 0xa81a28de171f3c47, 0x8fa9a1b3bb5dfb2a, 0xae076853e3e0abde, - 0xe76572308ecd6b54, 0x6cd926c2e2760d8a, 0xdf080266cfbe3dc3, 0xb99b961999765d7b, - 0xadb5d4e2b896ddf1, 0x8d3aaf4c83c83c56, 0x9b66e4f6eb65bef7, 0x7a81c3bf785eb1df, - 0xc53f02b3e8c38647, 0xcdfeb25ee787759d, 0xead5e734d64ab5f6, 0x7930d87af1072499, - 0xf30690a71d88ad6e, 0x73c347923c84728a, 0x3f2b588221003fe4, 0xe747052d0b453af2, - 0xabe6fa70539b5edf, 0x4db6d1530d628c2, 0x4ec929af434eb1b0, 0x15afbe39886181ff, - 0xa9141b9c89a07b80, 0x7d33f966c6232057, 0x8ddcb412f34a491d, 0xa74472b8ecc1e2f2, - 0x34d745de1cb7de2e, 0x6cf67091309e5e93, 0xfb25004efa59450a, 0x3355947066522286, - 0x5a8cffdf079dac21, 0x419445d6e6825887, 0x6e9c064f84381dcf, 0xbbcaf462a3a8ad76, - 0x75836c68d6c1a13e, 0xf38141565d5c3759, 0x8c65989142ffa802, 0x106067ec26e6463, - 0xadfac5e3a80de9c2, 0xc48b16e2df25b9f2, 0xa3257889c33669e6, 0x5bc760d4d65a1745, - 0x303cd31fead81139, 0xfb97f78cade31e1d, 0xb888b8e05820a469, 0x7ebb8e44d47f54d9, - 0xce76cdbb5ecdc529, 0xb1eb29949a099d52, 0xb6affc1b240a7eb3, 0x22977eac542906f1, - 0x9b105b391ff729df, 0x83371186b2834968, 0xd5b893f382ea9e90, 0x5d17aa80a1fd4854, - 0xe8ed8525eb29210c, 0xc789f3cd36c4dae3, 0x556e50a5f46b73fb, 0xef1d129b523dff77, - 0x851de0f53f6707f9, 0x8deeeadfb1fa8bfc, 0x3d8c89c0e08c4f2e, 0xecfaaea537123333, - 0x5bc9053d2dfd7669, 0x408c5bb2e880a9a2, 0x495726b3f3248219, 0x2b23cca4a6ea1ccc, - 0x1df3663045092d61, 0xaf977a46e965e45b, 0x43a2facfff7f97e, 0x9b7714344c7b51e, - 0x35643b24efb0559a, 0x502820785dc1af13, 0xbf82d2775b46433d, 0x1db626f2e16ca66, - 0x744b031447c1e27d, 0x99e79898612f4606, 0xda02a728d234821f, 0xcf00c6fbb637a6e9, - 0x242f2963196fd8b, 0x7aed8efc2dd562bb, 0x6204fb5d3dc6208a, 0x3e84861182fc7f6, - 0xd14c4ee5aeef5c7a, 0x3749fbef94378dc1, 0x8fe710ec5cfc8566, 0xf43d7e495d5384d7, - 0x8ff6396f1f1ce7c4, 0xf1252a6b6f86b42c, 0xddbbd098d6dca83f, 0x4e228724a227232a, - 0x92a5a52ba2b24fa8, 0xdfc172b03fde669c, 0x34ee55adf7f0711c, 0x21d181e79b8000bc, - 0xc788b1f48b37b693, 0x544fc4cfed0e0f92, 0xafca0c6de41789cc, 0xbb37bb5107ef97f8, - 0xb9d62bf1dc0f6c95, 0xc78b5a36110dfb1, 0x1d615b658f39657e, 0x2bb2cd04cabcc360, - 0xe563488ece6362f0, 0x213b56ce006fecc6, 0xc38207089fed0270, 0xa33199ff4a51d095, - 0x1802ad28fb1896b1, 0xbead8f18c164a332, 0xceb5149101aa450f, 0x39ad89851ee8b62a, - 0x317229aafabf37c4, 0xee68b8b9bf3520b3, 0xe4db499288350f04, 0xf8ea27feddb0ae7a, - 0xa17235067b489c42, 0xbf4a570245f95d78, 0x8065c67e1d1537ab, 0xbd9357fb1b30aee5, - 0xc224166ebeb24c42, 0xf9baf8ccd01b53bf, 0x5c13775c3fea8038, 0x4ea66f6d650ce62d, - 0x470592ed81c140f2, 0xc2d0eb6f7999321f, 0x85d762f20290dc0c, 0x9f7d0d13936f6e78, - 0x41f1fd2d20f2d62d, 0x891cb19ce1af2c2f, 0xe7ff34c3b29c3719, 0x246743f43126c69c, - 0xea4b2da3195ebab9, 0x4831e4de995187dc, 0x7fb8969bbee45ce5, 0xe35b483da73c44ed, - 0xf89158ca9af36227, 0x9fc7f34a35469a7a, 0xd02483ebca6564e7, 0xca00da156aaeda03, - 0x303d1514646822f1, 0x226832ae582b8eac, 0xf772d719e413504e, 0x87603b928c068ab1, - 0x4dc1552230e9b883, 0xef8c5e9db946fc87, 0x935581290bf7a4ee, 0xca632d2c7674bf2, - 0xd8a3933b80d39efd, 0xf026574d0ffee6fb, 0xe4412d0dcd2fe94f, 0x668916490a2983ec, - 0x73b1fe84a995718, 0x729bedefe21cc0e7, 0xd3a770f1c683b98f, 0x5d597a96323a10c2, - 0xfbb7834bbf5fed23, 0xf3546c805a42ccdd, 0x9ef3e2164bb31a0c, 0x388363ce6c6c2253, - 0x8120f4a949f017cb, 0x925a61942bbd3d10, 0xa03182d8599c0521, 0x2412e23004b40ebb, - 0x35010a126bf2aecc, 0x21147869a1a84ca7, 0x53cba503b6127b98, 0x10c89dd62ab3591c, - 0xf4c7f84faaf9f5f1, 0x8b4a37a2e844b97b, 0x23ddeb236a0bd9af, 0x4fd51d7207f49e62, - 0x6cdab447c27706b2, 0x9e8f54b9a2d1a790, 0x191aed85d4d77087, 0xf74ecf5015265af6, - 0x45925e25404922a1, 0xcf5467a0f5b42b98, 0x73590809c85c728c, 0xc16beeda74a1a1b8, - 0xc3bbe7999803dd6a, 0x1a368bb32eec184, 0xafad2d86b7bb574f, 0xdc7c7b8960dc921a, - 0xd9b68d854f5e0ae1, 0xe9e1a6a16efe0bea, 0x304a15bd6ca1cb14, 0x713ce3144e0af4b9, - 0xc50eb410981be1d2, 0xb0fba6119bf7a300, 0x7a107296731fd314, 0xdd764898a90042b3, - 0x8a69973262da3bc8, 0x29c6b9d048596c44, 0x62581bd20da76f1b, 0x4d1a3941d6d3e4bb, - 0x19c447306245055, 0xb2978afcd04ba357, 0x7c01cdefcfe24432, 0xc4b268314411deae, - 0x5ba56d49da714765, 0x33299186ac6dfd09, 0xede087aec096ef0d, 0xf758da2c7bcf9ddb, - 0x5ea6c40d56824cdd, 0x121ff879d6ba905b, 0xb5fed0c42b616f5c, 0x21029cdc347de152, - 0xb251d93f4cd7bf4a, 0xaedcb2dc6402cf13, 0x840e5e0d96e89407, 0x6a92fd328efcc6ef, - 0xc63f5d8f6fcadcbd, 0x405bd64d1621128e, 0xe1318888172f58ed, 0x1009c9764d49da2b, - 0x4bc0592cbfdf9f91, 0x972f0e080dfecd02, 0xa1cb961958eeb6aa, 0x6ee6467ad8c20aca, - 0xb3f1c738390d6a83, 0x6504eb7ac498650d, 0xdf7dda67f198f59f, 0x72615652e56a82c3, - 0x85e0fa2dfb51755a, 0x98b1f92a3d2ad940, 0xce81d51d875c0045, 0x437004d0be0a4d69, - 0x64065895526f896c, 0xe1e1fea920785d49, 0x7d507ffd56fda19a, 0x17309b625cecb42, - 0x67b6d83f0fd0f572, 0x5178665a5bcd38f4, 0x3c49fda2d35a8606, 0x7f058d2cb0ad351c, - 0xfb95691559245416, 0xc991b857662b1b9f, 0x9e6d0f4e19774f96, 0x26cc7502212ca578, - 0x3466110f03225e49, 0x2ae4958375eab9f3, 0x939a8d94c8871191, 0xa27356548ac6b28d, - 0xacf86d43ec3aa030, 0xe0d16c7fa0b13a8c, 0x408ec2b2f8da531b, 0xde72494115ee4e83, - 0xd26d7f79a02c5b1e, 0x2b6f835520c97f6a, 0x1f30f008b109ae7d, 0x698dbf9acaa222f1, - 0xbd55de40838376d5, 0xebe53822cec7eb80, 0x7ce900793008d2bc, 0x494fc7d10e8331bb, - 0x53509b90bdb7d588, 0x62aa920e9554b2f2, 0xe103098542011b6f, 0xeb722b9523d68af8, - 0xb71b1a6ea2c6591b, 0x97cd7da55c940270, 0x8ab70184427e45dc, 0x6cb6907427808465, - 0xf69232a42dbe7475, 0xbc9816d429fa8909, 0xef1e74244539d41b, 0x5569b6d10440d5c, - 0x7dda6817985c8ef5, 0xb270ed19cc8161b3, 0x87d80b66c8a15db0, 0x96966684c3ba47fb, - 0x996669bc87aff3bf, 0x17c015383c793f2, 0xa4b5de41fc69f61a, 0x14a0eb4e3055742f, - 0x5f0da5a7a2b8bc79, 0x5fe0353728aac023, 0x1554daf4abe92eed, 0x545722dac774b6a4, - 0x733fc54174f2e0d1, 0x3478ef85dd994316, 0x58ba2ac090ef7575, 0xa66b9b0cbc77b6e2, - 0xdc78cab5c708a3ee, 0x97a30c27be510f61, 0xb95d0b06fc0910b6, 0xdc80bdc42f79a25f, - 0x5cadc438e65b3070, 0x6263df49ce691b9c, 0xa7ce160daa64416b, 0xa4f8bfedb57288c1, - 0xa51714e187bbcfe9, 0xe25df4e9fc44644c, 0xeaf1854b3116ce11, 0xde1b8f810991a604, - 0xd4fc2e365e99be4b, 0x8d1b0d2799527e06, 0x7cac59eaf46baba, 0x4fa63c74d2dabaf1, - 0x4f6c5d5e676733a6, 0x7ab7ddb9c1b789b7, 0x5d9beb4877a37034, 0x5e96bc9de3985bcb, - 0x72bab4dcb75b3228, 0xfa40f33c4d799e1f, 0x73e6f61a69984a6b, 0x7499c9af466cf22f, - 0x42fab9136bfa64dd, 0xd5e8e39513b6fffd, 0x8eda1fd5ad8cd51d, 0x95338744859dff44, - 0x4f0c5e5ae768c729, 0x5bc92c60495ae348, 0xcbb48c170ac21168, 0x8374aa2440eeb138, - 0x70b663d6f6d70ca9, 0x11264ca6dd79e5a0, 0xf058a2c156974514, 0x36820eefc435ba63, - 0xd7b69f3d0c0c27a1, 0xe1a2eddf3b41d205, 0x80508ec93b038bc8, 0xd7e0429bd511ff37, - 0x7bf55a4e87e183b8, 0x4cb370ce7edb4bea, 0x26fbd0b31dcef45b, 0xf7acbd6781419fa6, - 0xf7849659f05c90c, 0xb686271ea57a47c6, 0x16f3f839dbfb4e1b, 0x906872b08b2c61a, - 0xc30c86d0a0203c15, 0xdaf238a6aa4fc9f7, 0x2399aa09ad2c069a, 0xf133c3aca703f545, - 0x868a10304a1c98ba, 0x60ef0607f46f7e90, 0xe4e69f26931e11a5, 0x487b8f6bd6d92941, - 0xd10cb2971798c0c7, 0x7126d81aa4bd0106, 0xcb620311dc84ab82, 0x26f8734a7a356bb2, - 0xcf9b9eeb02ee978c, 0x8a9ff0285d4d6b30, 0xe30e3b957e2cc7f8, 0x7c15a09e4d275809, - 0xdf723ae1dda5d167, 0xac212e4f6264bba2, 0xe3d60920ed983308, 0x91b403cbc91e290c, - 0xcdb905b012aff7c7, 0xb5ee73d45f900897, 0xafacc7cd7f5d52e7, 0xa8653272621165d6, - 0x90e2efc485ddd0d1, 0x56ef1ca9097b1a96, 0xc7a20f85777eb0a, 0xf4cec0271a50eae9, - 0x21acd76442024973, 0x19a46be82a4abdbe, 0x1bc12e0b8bf41fbb, 0x3766fe3d8e5119f2, - 0x7fea355c4c18e8ad, 0xd08b496a24fb8017, 0xe1a7cfaa0877aa2, 0xd37ad9e2a2a3fa96, - 0xaa362ba0e696f679, 0x7e7de89142c3aca5, 0x2dedb0842a3575b8, 0xb74c1e1d9082fe5b, - 0xb1ae74323699140f, 0x73623f80c727a6ea, 0x132ed204b0f10441, 0xc3e6ebe8ffc252bf, - 0x5f15cffb8286dce0, 0x66dab32df780fa8f, 0xeb00da7b25ea99e4, 0x113ad2448fafb671, - 0xee065c10a8f1924f, 0x2fcb7367cc01fe5, 0x484338f5c2d0aacd, 0xecfacd785e42d7a6, - 0x11513f845de8af3a, 0xd11be6b29054de0d, 0x2536e5d2856af9b7, 0x60ab519760acd4c4, - 0x6bfe010250a831ac, 0xb28a93e44b53b21b, 0x281cf9b233858583, 0x4ca6139abc79a710, - 0x5717d33616d77a95, 0x9ba52d2b7dfe71b5, 0x32e1c543476aa17d, 0xae242cc75806b7fa, - 0xa1415cb8fde770da, 0x3956c67542dc004d, 0x3a6f51518fdd20ce, 0x448c848f6c936d93, - 0x8fec38ff51bb5fab, 0x7463816cfc0754ac, 0x83b38e531ba39e73, 0xf9fc84dcf4f8c93e, - 0xdf20dbe8b91c3d6f, 0xec65939ac9516f9a, 0x888346f6c1aaa94a, 0xe42cabb108f60d95, - 0x39bb2e46b0599fa9, 0x529335ed75acba9e, 0x6a8767c5d00776c4, 0x8243346104fe61a5, - 0x7a2bd0339b3e9bac, 0xb68ccdf14473f4ba, 0xa06f389531ab553a, 0xc7c6f074fc2882d3, - 0x50a5fd6e6d0df962, 0xd7c0000d194139b7, 0x5ae27ef4033f873d, 0x4e7abe8a6d3570f8, - 0x27011ccd3885e709, 0x3dae53f7b7a8924a, 0xa9086c9b2b86fb71, 0xa3f9e534a399e62c, - 0x9f2f0379f9a33ec6, 0xceb51af95d4472bd, 0x15aa534182f8465, 0x96373b9cd28a627b, - 0x9fdce0ad99d41907, 0x2755bb0f52b8c239, 0x2f2e241b6aa7d243, 0xf040afbacb2f6001, - 0x552267c5f8b4c1b0, 0x22bfb3f0b58f9e48, 0x6bff8de368dbee3a, 0x652025c63d4069ce, - 0x743eb697b25f9c90, 0x8a3742c9dbf67b1c, 0xaf3bcfc260d7e69, 0x491facaa59b7e1d, - 0xd07d0761e6535fcd, 0x79ef09d9d3859232, 0xe0e0a013317d9207, 0x4b94baecf6fe4b4e, - 0x574576ed054cfc38, 0x90e90edd1a26f0aa, 0x616d32a371af78c6, 0x392cea9c34ffb0a8, - 0x692a6e730c33ac6f, 0x9e4b92ef425b78a6, 0x291d4c962d2f3e7c, 0x5f0f8ebb67f308fc, - 0xc1a0faf70ec747a3, 0x641da9550cd89392, 0x6adbe38115f09648, 0x3a51980fb324da0d, - 0xee894f2b5c17a380, 0x58788fa693f767c9, 0xc9f490ff5d88c4c0, 0x2ca6e8b204aa3070, - 0x7693837d7910c40c, 0x9240e04f16051720, 0xab773072d922faa2, 0xacb6892db8362872, - 0x5ac8b4d130613c0e, 0x65cd5ac259c653cb, 0x9647e0276864f3f8, 0xe207c57ee9237a53, - 0xe41f2663482a8fdb, 0xf605931fce8b95bd, 0x5b0247a9009255d2, 0xae949df3ab5a4ab5, - 0xc0e912dacbb72c42, 0x4226971e6351f33a, 0xd98d41f9f0fdb6c3, 0x64bf6a0af66782d5, - 0xb4ca689b3959bf46, 0xb4cbf621b4970922, 0xe9f27469c2412fe6, 0xb529d406d27c785e, - 0xf33be21f2672ec78, 0x34b3c4c4603656cd, 0x4a7efe099a0ae9d1, 0xa6ad5b909667945b, - 0xe20850e47ab440f7, 0x68739e1b4fa069e5, 0x4c191300a9207cf4, 0x9b613a1a38160c98, - 0x2cc631eb015081c9, 0x52af80aad7b676f4, 0x904b38943300ca8f, 0x50a515c0d302620f, - 0x52ae95b8a0c1f36c, 0x6a62ab87786774e2, 0x17ea45367ad58985, 0x92959c57166aa21b, - 0x2a9d91bd1a9716b4, 0x552c2f8528220174, 0xd665856c96b47d17, 0x14c6cba13b6dca3b, - 0x865f1c93ec9000d0, 0x23dda8b5e161910e, 0xc191ac4342717953, 0x6783fdb95e098f1d, - 0x1a4c26a5cad1e8ee, 0xde96e2a1e33e3046, 0xa57e65bedcf9047e, 0xa1cdef163fe7f80c, - 0xeb0abcc13feeb7b2, 0xaa158c61b0e44470, 0x98dc901679caf85b, 0x954046758f8d2e96, - 0x56db5e99c5d8c68, 0x7bb8d36962a4ac81, 0x2e301b4ecb03821, 0x121c1b2df70f0e1c, - 0x3fbdad1e8faa0543, 0x6efb222398a2f88b, 0x65760de4d96338e0, 0x15b2c58a67fb43fa, - 0x22f1532c89367d77, 0x1f726ffdad411d9e, 0x42572b54dcedf3ba, 0x3f8e0f6e9f0bbb7b, - 0x4b0e705c86571a1c, 0xf8c9b04f8bd75117, 0x67ed2e9e4545557, 0x57e8853f681bbf8c, - 0x6f99d1fd5fdbf582, 0xd2aa9bd48ad692e2, 0x2efc88bddecfe616, 0x8e9779a1e119abf4, - 0x52dfee4722a20b75, 0x79465be3aea146d6, 0x588997dbcbd5f005, 0x79bda8bcc5d4c650, - 0x2384e131ed3b5330, 0x229cb1d89738aa26, 0x1526d1a020a96507, 0xc7b6ac961a740cb9, - 0xe78cec14478c4f71, 0xaba61cfd8f1e2a71, 0x24123c8a37ae66f3, 0xbad8b07709aa215d, - 0x7896fa53fd2418e7, 0x72265842e8c4f955, 0x9f6331fd80527661, 0x7c649eeddb382c9d, - 0xd3b0708dd6b5ae84, 0xeda51f244551e15, 0xb7822c860b93dd44, 0xdf9afcc3c9cac88f, - 0x9244b9816573be70, 0xf3d103887cc4ce2d, 0xb3295c2e5bcb0218, 0x85243b7a2e0af441, - 0xbffd3df508d06098, 0x4908b967f6765c12, 0x8886ac94c0dabb, 0x9c7865af133eb6c0, - 0x5946fee66e64d7b2, 0x7737bc5af713087f, 0xbec815a80782dd5d, 0xe7672cfe0f7945fb, - 0xe1f80b6df5beece1, 0x5b749d3a22450fd6, 0xaf34a567bf838668, 0xa72a177d20943ceb, - 0x257c45c1601c4922, 0x3d7c68f6fe36ce59, 0x1b8ef47186e06c96, 0x8040426656154c17, - 0xc15f74f980647bda, 0x36389393336a78be, 0x15d174ed5536afe2, 0x51e702e8adb61f63, - 0xc9c95ca3f1c08f30, 0x6653094c8531c93a, 0xf7be5dfc2eb3b5b, 0xb5e21ec5c63850b0, - 0x9ab5f082e01021a5, 0x75c4ca38d7678fb1, 0xb20bd13e05e5bd67, 0x3378133e631fbaee, - 0x17cb3686511aeb6e, 0x4c0dea4c80ec7bc, 0x21fe874bf5509300, 0xece4f84e34d52e54, - 0x3f2649803ca9918b, 0xdb493adeac5c60f5, 0x7c02a1d153afedd, 0x2d08ceda0fef7967, - 0x6a32e018e432b0ea, 0x86479f3ea38ad57e, 0x84e3e04f1e5877be, 0x41b898cb02772049, - 0xd6dded3714d71084, 0x9df5b2f3a0ab275d, 0xd5ab652dfa73ee0a, 0x4633e03fa37d6edb, - 0x226314b8c4b937ff, 0x45d66a00ab031188, 0xb93ce1cfbcd1eaa0, 0xf88e0756f7e7c1c1, - 0xf3611966fed51e03, 0x81235048c662d9f7, 0xcc8275f866147b4f, 0xd3b3ca0f5033a863, - 0x970212eb8c2b3429, 0xec848dd58f3449d6, 0xa1d527af824d09f2, 0x60bd5e91448b9cf4, - 0x1210ddfac603aa88, 0xcbf4270f3407e25a, 0x212955fec55466a0, 0x3afeaef4c9ccc793, - 0xdd114286ec304817, 0x849e6ae3c2cf794f, 0x71c08228d6a05310, 0x177e77779d155b11, - 0xdc59148f219a9c04, 0xb0702a7802d5276d, 0x56085d6761aee015, 0x3f79ce06bcfb4f3, - 0x459a1f917f8f3e0, 0xe4ea5635e8bcd512, 0xa88e99b63f135a39, 0x95bd628d77d39446, - 0xe6432158ef4c7d98, 0xb349cd3d1c74369e, 0x25b1a32db58efb0a, 0xc2add3a44cecc0b5, - 0x5d676629f23010b5, 0x9890b3a62599408e, 0xef68ea8144d97805, 0x429e0fda34046a85, - 0x7723d9043053bf52, 0xbb78842d9b67ae91, 0x9155ef932192e6a3, 0xdb523ea403d39f6b, - 0xdb8fa1eee23ea58, 0x7c735492524a3448, 0xc580cb82e81505, 0xb0be6a006414841c, - 0xdec2e763b5cedaa5, 0x8da58bb23638af5f, 0xb0e6b33f6736e7d0, 0x8146bbcd3dd4df61, - 0x978080148a8989bb, 0x7f8119caa3308095, 0x4e88c318ea0604f3, 0x6ee5262f16b3cf83, - 0xc44395c7a578ace9, 0x92016ee635de27e1, 0xb8dc5ceb36e67fd2, 0x95c851d65bd35f8c, - 0xbe393620503c49fa, 0x42af183b92eac923, 0xfef0e660435135ce, 0x262d67d480451ed1, - 0x590f92e1ee0502b3, 0x18825c97d49e3700, 0x2cd8c30a848c9acf, 0x1025c7747fda0115, - 0x54109f23e82590aa, 0x93917bf1f8325981, 0xda674b183fa0e3cf, 0x2a0b2467eb8aefc5, - 0xb0085eb83468793c, 0x607cabb2c9d3a81b, 0xe7a6b6013804d665, 0x67629a769c1efede, - 0x2830ab6ef6d10166, 0xffd02b0655332bd4, 0x19bd056c3117568f, 0x385a834785662c6f, - 0x938b5d56bd5f7248, 0x969afe82dd4829c8, 0xf455d4ace41c797d, 0x23d9cd67eff27512, - 0x2b0c7037ecb322e2, 0x73df193328258bda, 0x5d7ee05cf2054f93, 0xdabfbd5b46b61cea, - 0xcea02d82546b96de, 0x2245d5e74d5f60ae, 0x842ca45f8ef2a44, 0x7505cc4e1c3060d8, - 0x869146ac8e68565e, 0x22ea711fb30e73e3, 0x53cd64736898a0c0, 0xfa88458b920684df, - 0xc3ae23f451e0616f, 0xe2dd69393141ff32, 0x98863ab129bcd866, 0x8c9756a40dd5b834, - 0x2eeeef78c36fede5, 0xe84d2eb23e22153b, 0xb0ccc2f7ac541d78, 0x151faba0f513acfa, - 0x4300e3cea0260717, 0xaba308c6d857d2d0, 0x5eb25dd325256c6a, 0x3342627b68038da4, - 0x70d7d75526da35, 0x80ff3f5ea2ac3cf1, 0xe7434f2f026394b0, 0xa7c5ff17f7d2cb07, - 0xdfed0bb33a06ff78, 0x485cc64d38ce2596, 0x88db8580147aa8fc, 0xf52a5111d693b973, - 0xeeaa031c02b370a8, 0xbb3b1678222d0e81, 0x27b569f2a6630939, 0x2b301fa3e50efeb8, - 0x4dbc1f85bf3f8972, 0xb37e4f2cb75a825b, 0x3c8848e0f1777cc7, 0x8fae2e6938ba00aa, - 0xfb4674b50884992c, 0x2b765005b85b7388, 0xaaf0007fb9662ce5, 0x684bd59009a4ad65, - 0x221ffead3d73ab35, 0xccfe6d02d46856b5, 0x54d3323359e1b114, 0xcb412202ed42f097, - 0x15d63df422771b9b, 0x71852bca1581d14f, 0xf30dcbb6cf891e63, 0x478fb1eed3cc5a10, - 0x849a3b52bc5bb196, 0x4c1a98dbc546dd81, 0x846dc8c2258ec4f8, 0xbd4da447c7340bd0, - 0x8f1ee1d6a85b9db0, 0x123ebfa8aaec07f2, 0xae34948e375d4477, 0xd466a4177842d8e4, - 0xd5108efeb19cac6, 0x3266f7db8f133bdb, 0xe69af4e5d8d767e4, 0xea0efc0331df64a2, - 0xb879052746ed72ed, 0x2c8233cc84de4144, 0xdcd5dda825186731, 0xb9e3b679d268f34c, - 0x7ce12e2fa95bda8b, 0xa34afd12e611a4c1, 0x6043d06ee7619f90, 0xca3a3d4813c0addf, - 0x6e97a61d1e4b3c4f, 0x8cdeae467a4bb292, 0xf08a6c69e70076b5, 0x97aa5c3180d3edbe, - 0xc39813e1573904d5, 0x42577549d026e8c8, 0xaf5827ffe259b62a, 0x9e1d48596c4f0b24, - 0xab9dd230ba8efb64, 0x81769493b85868d4, 0x715b45c5e3952245, 0x84735e138d228f35, - 0xf4f987da7d19c74d, 0x1bdc77979baf29b, 0x785b3640158e0278, 0x77fc9d00681bcdd, - 0xb5980cc490dcab93, 0x9062c158196b8244, 0xc16af5418cc97c4a, 0xdd4a4e9e8e00e524, - 0x870b554b629277f1, 0xf90ff72bb54322c0, 0xd4c273bc2199823, 0xeebc75970d438466, - 0xe4cad67413074a53, 0x6eccde71bd09dc0c, 0x14278b0ca6b910b4, 0x6e8895f17cdb933c, - 0xa0b3987821416e11, 0x71b7d24b81fa769f, 0x1d3b1a805b885a58, 0x1bc737b1719736a, - 0xea4d1dbb8823037, 0xe50ce48c8469adbd, 0x34c2d5e6c41a888e, 0x446a756eb06dc3a4, - 0xbac5ed8a8f90262, 0x7f1b76e0c707ab9d, 0xb31323309b94a12e, 0xf58269b9852f986e, - 0x3b74c2b338c244fc, 0x879b46f23a4deae4, 0x3f2591e34cdce1c9, 0x73c6f81eb560ed5c, - 0xd2aa923c7a5c18a9, 0x7170c1f7621cace9, 0xa18b327b11b951a, 0x2b36315510f56370, - 0x5cfed2f703dfc0bb, 0x1e43b99175054c07, 0x392dfa3210e8013b, 0x65e0c5c0454ee693, - 0x2a795f6f493349db, 0x17995ebdfe848db0, 0xf23174b823a52cf7, 0xaac6ef104bfd396e, - 0xfa0f5b29156eeccd, 0x9ccb3590a6e88c1a, 0x2edae0b1b80aafab, 0x1e1e92baae39aa30, - 0x24475af89cec7f33, 0xacdbeceaadb9936c, 0x7725948da8586e93, 0xcc05595e17947215, - 0x1f3cbde17a508faa, 0x2795f58ff5c8919b, 0x309658d1748f30d3, 0xe90c11962e5ed4bb, - 0xc22b876286d32e27, 0x495c8b667c6ea3dd, 0x84263045f7d6eab5, 0xf8d3a9ab1494a315, - 0xabc42cf769a21d9d, 0x3fef7fb40bdf3a81, 0xfd35336e188925a3, 0xd472bdaf277ab26d, - 0x4949e8ff51da2307, 0xbec86dce960ff3b, 0x1aa1ce4f70256b10, 0xd52986ff8cd700ea, - 0x364d8efc0f5afad6, 0xca3d1958f57a9050, 0x17a0a2ec122dc677, 0x7992695be3363fd6, - 0x5c66a265e0607da8, 0x7250114e050bb917, 0x30cd3a70bc7b723d, 0x7b77433392b3fbf8, - 0x295bb7bf46318b38, 0xdb15025af2a71c65, 0x26a82b21ef67f50c, 0x14a5573d4fc798c1, - 0xd8cad4642ec68e5e, 0x9276d7f60142822c, 0xf7b8efc27522e52f, 0xd0a3f36f6d340bed, - 0x341260fb11765c02, 0xcd1d394d796702cc, 0x7d483ef031eb3346, 0x74eaab49374576c1, - 0xa073bea32a71a273, 0xd65ce2552993b5cf, 0x8670afe77caaeb16, 0xb607f1455d072a47, - 0x30ed96be92809559, 0xa58380a503c23c9c, 0x916aa68fb957a30d, 0x30c5a675bf19738c, - 0xd4cbad34e4e4b886, 0xd6cb83061f2b0ebf, 0xceddb4040f535fa9, 0x778c586927b1e247, - 0xe4bb5c4b6e0f3c3d, 0x3e857d671db80667, 0x2b909dd8725f1fa2, 0x558ffd0772db7841, - 0x710f9638d3edb2c4, 0x21a4ccee53d46556, 0xf76e8e4737b9628b, 0x71cd157f23581c71, - 0x68d8fded9b66efd6, 0x7d9f5e182c0b9457, 0x2140757748a217ff, 0xdd1e5365520b77a4, - 0x644d8e4b2f30dcfa, 0xa1de42f4e9791564, 0x70e148ababfe9f86, 0xb97f463e0ac7daec, - 0x82844f729d9fa554, 0x5c8475e84470c924, 0x3a83de748eac32fd, 0x68725fbc9c202c5b, -}; - -pub const mt32_seed = 0x7dc0d160; -pub const mt32_data = []u32 { - 0x59327332, 0x200858fa, 0xab53c028, 0x5c442427, - 0xd8be0287, 0x3b69d304, 0x15fdf62f, 0x59b8ecd, - 0x6d7ab30c, 0x3a3dd6f1, 0xc1b9773e, 0xa12fb017, - 0xa805b5c1, 0x4a313ba5, 0xd82c790c, 0x8de311f2, - 0xe7cb23dd, 0x784b2efb, 0x9743487c, 0x73e2f2fb, - 0x1a7ac286, 0xaef90d, 0x6c0a4514, 0xae1d83aa, - 0x412fcca1, 0x3acd2d28, 0xde78292f, 0x13237756, - 0xdd6cdeba, 0x44ae4df9, 0x3e9902eb, 0x39e1cf20, - 0x62f561b9, 0x6cbdf531, 0x4a000673, 0xb1c82daa, - 0x896156ca, 0x75e410f2, 0x9c69e72c, 0x396b42bb, - 0x25c97ec0, 0xe12173f1, 0x8dcd42e5, 0x82aac3e3, - 0xcdc1c84d, 0x13509c0c, 0x46a696d2, 0xb89ad987, - 0x92da5e7f, 0xaa87d8a9, 0xe433ff57, 0x80a7ee49, - 0xd387cfcc, 0x7dc47d92, 0x21516140, 0x989ca465, - 0xf2a8e002, 0x73b99ddb, 0x2204108f, 0x27e84890, - 0x371c81c0, 0x9f581854, 0xc0841c35, 0x770f6804, - 0x87c55f4e, 0xd29516bb, 0x2b6d6bde, 0x5541f6ee, - 0x1ec9b182, 0x7d599729, 0x4f4b9a14, 0x6f8c8562, - 0x2d5151aa, 0xb54f5bc, 0xa252452b, 0x2a4266da, - 0x25a6b75d, 0x2d11106e, 0xc5d77943, 0xb10b6e0b, - 0xeb5cae4a, 0x43a0dd53, 0xa40bea1f, 0x63e632c2, - 0xf420b6ce, 0x8b080233, 0x7f70ae87, 0xf460f0d6, - 0x147c7e74, 0x710692ea, 0xa0a7d8fb, 0xe7f05808, - 0xa6173aaf, 0xae608de0, 0x8702036, 0xbf1bfc7b, - 0xf14cd548, 0xbbc7553d, 0x5358dd1d, 0xcc0c1fe5, - 0xfab6f78d, 0x9365c118, 0xf64216a3, 0xb4bdcf1b, - 0xc90b8a7a, 0x8b7b78a, 0x4c7b6854, 0xba7b5628, - 0xdd728c15, 0xcb1f8905, 0xa63e2342, 0xa78822, - 0xbda61b18, 0x160a59fa, 0xeccf473b, 0xc5a445b5, - 0x7aa86430, 0x362e0d7c, 0x8006a0cb, 0x8b11586f, - 0x6677bba9, 0x6208cf27, 0xeec9b5, 0x3dfedfc9, - 0x886cc0e8, 0x32ed77ca, 0x43525faf, 0x9786354a, - 0x1a2eb378, 0xf0e6b168, 0x49064b09, 0x8ab39681, - 0x7b6655fc, 0x35adb168, 0xc417d430, 0x2784288a, - 0xea17836, 0xc85006e7, 0x673dfdc3, 0x42765688, - 0xc2b9251, 0x840a45b, 0xcac98e2f, 0x1a6f9777, - 0x34959b23, 0xf0dcec81, 0xcaa2c6c8, 0x1cf93061, - 0x787e598d, 0xd5d9e31e, 0x14e08791, 0xd9d9d782, - 0xef162f23, 0x238f4113, 0x23f42107, 0x6ed5cc3f, - 0xa55e5a7c, 0x4650595, 0x5217da8b, 0x6eeaacdc, - 0xb453d7b1, 0xfa1ff004, 0xb9d17f74, 0x2bd6a53e, - 0xe4c2d9dd, 0xed66375e, 0xf8215568, 0x9bcadbb3, - 0x9c4f9d51, 0xff68312, 0x82308422, 0x83e990b0, - 0x38b6135b, 0x70e2aa13, 0xa30065d2, 0x6396a00, - 0x77d423bc, 0xa93abf0a, 0xc7bb8c31, 0x5d7bd3d3, - 0x6a374f2e, 0xe4b5bc88, 0x39f6e512, 0xd6aea995, - 0x878c1bfa, 0x4636014d, 0x9caa2c09, 0x7ac4758b, - 0xbd3b957e, 0x518c2fd6, 0xea009a2e, 0x542bf419, - 0x59090006, 0xb1d94703, 0xe0d9eefc, 0xe7fccb17, - 0x40111951, 0xf2560485, 0xb50ce9e1, 0xd7a1ee51, - 0x28dffa99, 0x41d12275, 0xdd89a365, 0xf22eda29, - 0x104f94ee, 0xe669983b, 0x6346a250, 0x86326fc5, - 0xb7f347df, 0x3849a39f, 0xf433929a, 0xeea5155, - 0x4cf9b778, 0x6bd7926a, 0xcda9496, 0xf430d7a2, - 0x41637670, 0xaf3bbad6, 0xeb66e44e, 0x2499605d, - 0x9988920d, 0xf9d652ef, 0x67aa80c0, 0x505073c9, - 0x85cd418f, 0x9f83fb65, 0xf50b3eac, 0x812ba6bd, - 0x74d61788, 0x86d64f3b, 0xb1f8fc1c, 0x3e2af667, - 0x4d118a2, 0xd028ffa9, 0x32e88a44, 0x4ed9ba35, - 0xea3c7030, 0xffe44aaf, 0x5e39c467, 0xeabcfebb, - 0x53e656ec, 0xced701d0, 0x31020b02, 0x4b4c1dc5, - 0x8744885c, 0xa8e93656, 0x3ef457e5, 0x272bde23, - 0xe541477c, 0x3ad3ac04, 0x63eaa692, 0x81055cf9, - 0x3ff5f782, 0xa8efe6bc, 0x15f37656, 0xaaaebf1d, - 0xf73d461a, 0xe8b2c0b5, 0x5035ff48, 0x3a95e34b, - 0x6f21d94f, 0x6f6d1f96, 0xdaf79f37, 0x826f69f3, - 0x209a00b8, 0x2ad1b2f2, 0x2c64fb45, 0xcf8bf26e, - 0x9befcff2, 0xc08f6951, 0x96d98205, 0xa267dcb5, - 0xbc43ec5, 0xee6a7e1c, 0x49224eae, 0x14e820e, - 0xbb340212, 0x68ed572c, 0x45e9e623, 0x1297f3af, - 0x49a98ed2, 0xddd34ae8, 0x211838ab, 0x47e7652d, - 0xb40430c6, 0xc8d3bd7, 0x4352356e, 0xf0e5cac9, - 0x21880df4, 0xc16b343a, 0xd9ed7350, 0x17fe1f65, - 0x6637192e, 0xd81c93aa, 0x7d6e17d2, 0xd407b13f, - 0x425da072, 0x380d423d, 0x6ce57b22, 0x7b17ed17, - 0x95fbf626, 0x768303d6, 0x76ab6b3e, 0x591491e3, - 0x259f79ab, 0xd4babeaf, 0x9c7de2f8, 0x4fe6cb58, - 0xf43680a9, 0x651a1266, 0x730ea3c8, 0x9188d4c5, - 0x12d01e34, 0x47afb2e9, 0xb4b76d35, 0x5e5164bc, - 0xc864fc46, 0x5d018aa7, 0x17fac975, 0x5a775fbd, - 0x40e6fa14, 0x7a00b683, 0x99e4e102, 0x2f933b90, - 0x474e14ba, 0xde1b0754, 0xe84aba2b, 0xb386cd43, - 0x17ca77c9, 0x7b4f38ef, 0x803ea1a8, 0x93553947, - 0x806c8224, 0x2608451e, 0x63157fe3, 0xaf53930e, - 0x5dfe8c16, 0x65592bda, 0x7086eb3f, 0x838e6a50, - 0xa27836d9, 0xf2f16d92, 0xdc0a981, 0xfbf8f915, - 0x2caea00d, 0x86bb3e18, 0x6d94c209, 0x3bbbeb6c, - 0x114d68f4, 0xc271e48f, 0xa3350dc1, 0xb8d55eb4, - 0x68be5ee1, 0xbf22ef29, 0xd6e0aa54, 0x48f7219, - 0x21aca253, 0xfbf07910, 0xfcdd61a8, 0x118a09b, - 0x3f2bbde6, 0x46eea63f, 0xdb51ed16, 0xf8a9fc36, - 0x31614dc0, 0xdd84f54d, 0xd2b66065, 0xdae0af99, - 0x6d071a51, 0xbdbac46c, 0x15deee25, 0xf792e64c, - 0x910194e8, 0xfc989a8f, 0x919727fd, 0x6f93a56c, - 0x2df36e9a, 0xd395b948, 0xb026b54a, 0xf0938a5, - 0xe9c64399, 0xb5cda15b, 0xb7b8dd41, 0x7146f944, - 0x8d41ce2f, 0x47c74099, 0x2e5a8e5f, 0x28f7a19c, - 0xef7a8ef9, 0x6a763eb9, 0xf13a3ec4, 0x9f352360, - 0x42317561, 0x6c6a0ca5, 0x5e40b472, 0x3ddaadd4, - 0x2f5d14eb, 0x5dd49aeb, 0xc89edb24, 0xa2da269b, - 0x5cf0a38b, 0x8e2f435c, 0x40970e54, 0xa2cb730e, - 0xf9d8c301, 0x8ef29fb1, 0xf08b1840, 0x7d45e4a2, - 0xa0fe4ce1, 0x939a21c4, 0xfdeebfea, 0x4c661550, - 0xdd304d1c, 0x3cdb078d, 0x94ae8db2, 0x4f6b4287, - 0xffe64fa8, 0x50384bb0, 0x16cf5ed3, 0xa91a8fec, - 0xdb8ebb1, 0x59c2898b, 0xd587edc9, 0xdec2e75a, - 0x496ccdd2, 0x897db91d, 0xf8ea5149, 0x6bed4bad, - 0xce76e472, 0x43c7f976, 0xb055dc01, 0x7ffd5671, - 0xe193b86a, 0xe288ce11, 0x514d531e, 0xa42fa47e, - 0xe7c0e194, 0xffc059ba, 0x26548e36, 0xe1f10d92, - 0x3ef5d95e, 0xa6e69282, 0xffacb09e, 0xf4a16ff5, - 0x9b7f03bd, 0x588c54b4, 0xc2b6eaa1, 0x2d83acdc, - 0x7fdbb606, 0x2b160650, 0x9923e57e, 0x32bd23bd, - 0x50cd6d4c, 0x205d901f, 0x810a9935, 0x27ce6e7a, - 0xe0c6c66, 0xac06c99c, 0x4326aa9b, 0xe1af1e90, - 0xe358c8b1, 0x2f601c2a, 0xefca77e7, 0x1a7ed2f8, - 0x8ad2e191, 0xe5520809, 0x27084438, 0xe4d8e782, - 0x5e8a4038, 0x87bba694, 0x65f07eba, 0x616f8f07, - 0xc5565d9, 0x555955e4, 0xf41c2caa, 0xb085fbf5, - 0xa5f9d9ff, 0x418fa0df, 0xec5a576d, 0x7fc332ab, - 0x7683ed33, 0x968ef54b, 0x834d598d, 0x6833f356, - 0x59dc7e7f, 0x779661dc, 0x58942dd4, 0x80387aab, - 0xf6dac9e5, 0xe043be04, 0x2ae4f872, 0x881f8d01, - 0x82cfd69d, 0x931f4648, 0x2a76ab31, 0xa3f1dd7c, - 0xd7f4826a, 0xe74918da, 0xe4c98636, 0x441164f, - 0x15a0e9aa, 0xce7480ad, 0xba39076b, 0x233aa8d, - 0x6c32f0e6, 0x169c62bf, 0xa2cd17f6, 0xb5590084, - 0xb2036f00, 0x18315935, 0x11e9c9c7, 0x25c77861, - 0x41596cda, 0x635e5e02, 0x8f396cc, 0x4cd00d8d, - 0xd665597e, 0x90f891ef, 0x547b93ee, 0x376959c1, - 0xdc5fa80, 0x9b4797a6, 0x53673041, 0x25ab117a, - 0x7b8b8292, 0xf4e99584, 0x5139da98, 0x30e2afeb, - 0xff2664b9, 0x591eb6f0, 0x9e87e602, 0xf5e26193, - 0x61831f07, 0xabc139f9, 0x984eda0a, 0xaea1b8da, - 0x65c7410d, 0x2b84800d, 0x1d3cfec3, 0xd05cb8a1, - 0x4529641b, 0x7d6712e6, 0xc38cbde7, 0xacad7787, - 0xd8482f3a, 0xa5662eaa, 0x24836ee9, 0xf3b5cc97, - 0x50a581ae, 0xff6004b6, 0x650fc547, 0x161898b1, - 0xa7593447, 0x325827dd, 0xf1844a1a, 0x7eb56de2, - 0x89882452, 0xfebb49a, 0xfe86ae9c, 0x7dba98b1, - 0x1d65adb5, 0xb71acffa, 0x861215af, 0xc0f1496, - 0x70967c72, 0x3803d127, 0x6c8fdd84, 0xe40991f1, - 0x1343e3a, 0xf57b4e73, 0x25f34f76, 0xaebcdee8, - 0x8752d71f, 0xfc710e54, 0x34f3af44, 0xfa7dea4e, - 0x477d4d83, 0x42640ff1, 0x2c5c31ce, 0xa82de5e4, - 0xcc813271, 0x4d40bf86, 0x4e416095, 0xb5ac332c, - 0xd2d44703, 0xe4c5ef57, 0xde193a29, 0xbf3e7974, - 0xbb313d75, 0x8dc973d5, 0x301b2657, 0x44dc5064, - 0x8c58c633, 0x83424c74, 0xb7cbf7ac, 0xa04238c2, - 0x6ceabd59, 0xd25e6fd0, 0x3409167, 0x42d6ef80, - 0x1f47c437, 0xdb21e45f, 0x2fd48e29, 0x9498cfb7, - 0xc9e4cb12, 0xc6dcf0df, 0xa1633c39, 0x1b349670, - 0xf76d4a64, 0x15ecd8dd, 0x777bb76d, 0xc46008e7, - 0x23d94e44, 0x78aa07de, 0x2eeac782, 0x3757b114, - 0x2b22de2a, 0x37726519, 0xf107546d, 0xe9847f74, - 0x449a4ea6, 0x2e31fa5a, 0xd719ea88, 0xb2115c87, - 0xfa6b7231, 0xf72fc9ff, 0xcd22bc37, 0x9080778a, - 0x93430a21, 0x97c24360, 0x6e5b1a76, 0x5e8baa7c, - 0x300c94f8, 0x2843d9da, 0xdceac0ae, 0xeeed885, - 0x1898ffd0, 0xa3bbee3c, 0xc16f8fd7, 0x82992b68, - 0x39c153b6, 0x1b3ba4c8, 0x41e7c3ac, 0xcdf8f06a, - 0xd40b8ae6, 0x4982b6c2, 0xb32f7437, 0x22ed3691, - 0x16579a2a, 0xff9de457, 0xc421e8e4, 0x17c8f6cb, - 0xa5c4a8da, 0x49bd8afa, 0xe2be081c, 0x95170f28, - 0xd679fbdf, 0xcf39d563, 0x4e2d2ee9, 0x39471096, - 0x3918bef0, 0x279b7679, 0xa5281a0f, 0x49481d6f, - 0x11f95ee1, 0xd9df649f, 0x2993eb27, 0x48ad815f, - 0x99cf306d, 0xca9457e4, 0xc27c51d2, 0xc2a838ec, - 0x537faf4c, 0x55dccddf, 0x8df5aeb8, 0xabb317ca, - 0xfc1bcf6b, 0x669c2b1b, 0x719b62d5, 0x6b9325cf, - 0xc123d0d3, 0x2ddc6ace, 0x27fdc30a, 0xd3f93cd8, - 0x704f5486, 0xd3f448ec, 0xbbd1e32c, 0x3bcd4c0b, - 0x86f8166, 0x957db888, 0x899b6a5e, 0x270dc8b7, - 0xff16222e, 0x51e139a8, 0x3d8b4b9f, 0x68d20818, - 0xa639ad00, 0x4c2e0fd2, 0xb4949cdc, 0x2ab6eb32, - 0xdd0c67ad, 0xd2208cbe, 0xcd17a0bc, 0xacc541f7, - 0xfa9e714f, 0x316d31a7, 0xed79fa91, 0xb5c0e980, - 0x412ecc9b, 0x9815753, 0xd0df1f43, 0x8e37dbb9, - 0xe640df75, 0x379c2fb6, 0xc7ed26a4, 0xc5190400, - 0x1cc81b53, 0xcb0b0cd5, 0x360f061b, 0x6d90284e, - 0x83c05bd0, 0xbd80bae9, 0xd584ef12, 0x228a46ec, - 0x657c4fbe, 0x5ca1043c, 0x852aca0f, 0x31ce950, - 0x33ee2cd8, 0x3cdecbf7, 0x787ef08c, 0xea610ee, - 0x47c1db89, 0x90eeda11, 0x74f8d429, 0x51d3a4c5, - 0x3135b401, 0x2e14783c, 0xb9af855c, 0xb66348d9, - 0xa3a47387, 0x6eb72af1, 0x7bb56088, 0xc664542d, - 0x7ed96b8, 0x995870a8, 0x385b1fd6, 0xa430680d, - 0x98a883ec, 0x2497a389, 0x7a880627, 0x8350ba9d, - 0x4cb35c33, 0x30bf6b14, 0x8695a469, 0x9a81e44b, - 0x8bb27c9a, 0xbfb6a4dd, 0xbae7cf6e, 0x4ebccc87, - 0xb712ed3d, 0x31e90365, 0xcc1fa63f, 0x32b93df6, - 0xbad4c7bc, 0xb2570e17, 0x73fa21be, 0x5c02a8d2, - 0x94446d75, 0x7265f3ad, 0xd58487a2, 0x919b7a07, - 0xbe2d0e05, 0xd36ccf4f, 0x6d5c66d7, 0x8448522f, - 0x8409c294, 0x6f1c7af7, 0x173a13bc, 0x1b3e4a0b, - 0x705b941b, 0x77eb584f, 0x85b68458, 0x8e3ad1ac, - 0x4aa99702, 0x7ae1b24c, 0x899ba29c, 0x860a3711, - 0xabe53a4f, 0x37870133, 0x1ed7cb89, 0xea539762, - 0x4ba64130, 0x48517a2d, 0xce0a869d, 0x937ba48, - 0xd0f234c4, 0xf9b2cf26, 0xc3c311f0, 0x153d09a9, - 0x404d3af9, 0x9f7edbc1, 0xbdcecded, 0x97969ba8, - 0x3379437, 0xadd3c893, 0x7c024639, 0x459390b, - 0xcb7c7320, 0xa5c63725, 0x65907e3e, 0xbf70583b, - 0xcebb601b, 0x4edfb286, 0x9350336f, 0xdfb4be76, - 0x88b56f39, 0x9937d7f9, 0xa12a286d, 0x34f141c, - 0xa2e75c15, 0xd69a7060, 0x931340c3, 0x22447f25, - 0xe8aed82c, 0xd76a9ae7, 0xc967288, 0xe572facd, - 0xbe82b0ee, 0x10f5dce1, 0x4f03ee35, 0x2340b923, - 0xf4fb6bd0, 0x64adbf01, 0x3d277a0a, 0x39e76f2c, - 0xe3c024d9, 0x57869c82, 0x743b7826, 0xf66f1574, - 0xc93965c, 0xf86a552, 0x13557069, 0x9e0845de, - 0xaee084f9, 0x5eafaedb, 0xc06f5f3, 0x9051f6ea, - 0x98fceda2, 0x2af2f8cc, 0x6c41b5a8, 0xc1af74de, - 0x57302276, 0x253923c9, 0xd79996b3, 0x8ecb3141, - 0x641387cc, 0xf87a4101, 0x96a50c76, 0xbcf24a11, - 0x87b6bb6, 0x58ee501b, 0xaa859695, 0xb2eed107, - 0x554173f0, 0xb12ec0e6, 0x57785c1b, 0x53685c9a, - 0x114c3163, 0x9383cf19, 0x31fd7cdf, 0xaeb8225c, - 0x58774fe7, 0x54700ad4, 0xad418726, 0xf055b71c, - 0x7d31237f, 0xdd97cad5, 0xcdd5325e, 0x42f2acf4, - 0x4bed262b, 0x7a8faaf2, 0x2b1eafdd, 0xa1b806ac, - 0x26965c6e, 0xb41b7168, 0x15e2e70b, 0x7daa8e13, - 0x6198aa5a, 0xc9b8b94c, 0x339b5754, 0xcd3b285c, - 0xffd1486c, 0xf224979a, 0xafb89ec5, 0x222058c, - 0xcb4814d0, 0x2b0e7c7d, 0x9eb25b84, 0x271564b0, - 0xbb72e076, 0x48251020, 0x18008023, 0x48d10005, - 0x4a452eaa, 0xb2365308, 0x19cdb632, 0x1fd56d04, - 0xffa5ff2a, 0xaba89e42, 0x388fc17d, 0xea61c00f, - 0x5156273d, 0x776f1a56, 0x8d539d28, 0x289c01cb, - 0x857aa71f, 0x348e411f, 0xc9eb3c91, 0x67a61079, - 0xe4276a0f, 0x45bdc15f, 0x8e0a698e, 0xbdefc310, - 0x82377ba6, 0x3bfbf404, 0xcbf22c79, 0x35f501bc, - 0xb16044a7, 0xeffdb8, 0xdbac383d, 0x7816663f, - 0x18f5a318, 0x3d04f1cb, 0x735da9b4, 0x75e339a1, - 0x5b6c55f, 0x1c18887e, 0xf698e14f, 0x338a6da1, - 0xdac85699, 0x1aca7768, 0x8eb0fa7a, 0xc98fa71d, - 0x3b794408, 0x92913041, 0xf8dc8827, 0x1cf706e9, - 0x3aeee292, 0x321dbaa8, 0xee1eb8d1, 0x23554be9, - 0x811c7804, 0xf0f4de6b, 0xd457e382, 0xeda56795, - 0xeffdfc71, 0xf2a52829, 0xa7460732, 0x2c1321c0, - 0x2f734db0, 0xf04ecb0b, 0xec7d777e, 0x43c54317, - 0xccaa74cd, 0xfe49dd9d, 0x4c509829, 0x278f9bd7, - 0x581dc500, 0x4ad38c2e, 0xcbee1047, 0x13302c1c, - 0xbc0cb734, 0xc1c8f234, 0x1df52b35, 0xd8815548, - 0x319edefb, 0x437cebe5, 0x3dcb6026, 0xe9d4f93f, - 0xb2661154, 0xeb8c15a0, 0xb008505, 0x5f869981, - 0xf5588ca4, 0xd6929c5b, 0xa3dd13d1, 0xdc863314, - 0x891a454f, 0x91737e49, 0x5064d4d8, 0x2fd32675, - 0xadefe9b1, 0xdde32b11, 0x741bbd6, 0x3b4363a9, - 0xb121d9e8, 0x916ca61d, 0x38c0af15, 0x5e3dfd72, -}; diff --git a/std/sort.zig b/std/sort.zig index c13e99fed..a3ebd293c 100644 --- a/std/sort.zig +++ b/std/sort.zig @@ -67,7 +67,7 @@ const Iterator = struct { self.numerator -= self.denominator; self.decimal += 1; } - + return Range {.start = start, .end = self.decimal}; } @@ -82,7 +82,7 @@ const Iterator = struct { self.numerator_step -= self.denominator; self.decimal_step += 1; } - + return (self.decimal_step < self.size); } @@ -219,7 +219,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons var B1 = iterator.nextRange(); var A2 = iterator.nextRange(); var B2 = iterator.nextRange(); - + if (lessThan(items[B1.end - 1], items[A1.start])) { // the two ranges are in reverse order, so copy them in reverse order into the cache mem.copy(T, cache[B1.length()..], items[A1.start..A1.end]); @@ -230,13 +230,13 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else { // if A1, B1, A2, and B2 are all in order, skip doing anything else if (!lessThan(items[B2.start], items[A2.end - 1]) and !lessThan(items[A2.start], items[B1.end - 1])) continue; - + // copy A1 and B1 into the cache in the same order mem.copy(T, cache[0..], items[A1.start..A1.end]); mem.copy(T, cache[A1.length()..], items[B1.start..B1.end]); } A1 = Range.init(A1.start, B1.end); - + // merge A2 and B2 into the cache if (lessThan(items[B2.end - 1], items[A2.start])) { // the two ranges are in reverse order, so copy them in reverse order into the cache @@ -251,11 +251,11 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons mem.copy(T, cache[A1.length() + A2.length()..], items[B2.start..B2.end]); } A2 = Range.init(A2.start, B2.end); - + // merge A1 and A2 from the cache into the items const A3 = Range.init(0, A1.length()); const B3 = Range.init(A1.length(), A1.length() + A2.length()); - + if (lessThan(cache[B3.end - 1], cache[A3.start])) { // the two ranges are in reverse order, so copy them in reverse order into the items mem.copy(T, items[A1.start + A2.length()..], cache[A3.start..A3.end]); @@ -269,17 +269,17 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons mem.copy(T, items[A1.start + A1.length()..], cache[B3.start..B3.end]); } } - + // we merged two levels at the same time, so we're done with this level already // (iterator.nextLevel() is called again at the bottom of this outer merge loop) _ = iterator.nextLevel(); - + } else { iterator.begin(); while (!iterator.finished()) { var A = iterator.nextRange(); var B = iterator.nextRange(); - + if (lessThan(items[B.end - 1], items[A.start])) { // the two ranges are in reverse order, so a simple rotation should fix it mem.rotate(T, items[A.start..B.end], A.length()); @@ -301,10 +301,10 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // 6. merge each A block with any B values that follow, using the cache or the second internal buffer // 7. sort the second internal buffer if it exists // 8. redistribute the two internal buffers back into the items - + var block_size: usize = math.sqrt(iterator.length()); var buffer_size = iterator.length()/block_size + 1; - + // as an optimization, we really only need to pull out the internal buffers once for each level of merges // after that we can reuse the same buffers over and over, then redistribute it when we're finished with this level var A: Range = undefined; @@ -322,11 +322,11 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons var buffer1 = Range.init(0, 0); var buffer2 = Range.init(0, 0); - + // find two internal buffers of size 'buffer_size' each find = buffer_size + buffer_size; var find_separately = false; - + if (block_size <= cache.len) { // if every A block fits into the cache then we won't need the second internal buffer, // so we really only need to find 'buffer_size' unique values @@ -336,21 +336,21 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons find = buffer_size; find_separately = true; } - + // we need to find either a single contiguous space containing 2√A unique values (which will be split up into two buffers of size √A each), // or we need to find one buffer of < 2√A unique values, and a second buffer of √A unique values, // OR if we couldn't find that many unique values, we need the largest possible buffer we can get - + // in the case where it couldn't find a single buffer of at least √A unique values, // all of the Merge steps must be replaced by a different merge algorithm (MergeInPlace) iterator.begin(); while (!iterator.finished()) { A = iterator.nextRange(); B = iterator.nextRange(); - + // just store information about where the values will be pulled from and to, // as well as how many values there are, to create the two internal buffers - + // check A for the number of unique values we need to fill an internal buffer // these values will be pulled out to the start of A last = A.start; @@ -360,7 +360,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons if (index == A.end) break; } index = last; - + if (count >= buffer_size) { // keep track of the range within the items where we'll need to "pull out" these values to create the internal buffer pull[pull_index] = Pull { @@ -370,7 +370,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons .to = A.start, }; pull_index = 1; - + if (count == buffer_size + buffer_size) { // we were able to find a single contiguous section containing 2√A unique values, // so this section can be used to contain both of the internal buffers we'll need @@ -405,7 +405,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons .to = A.start, }; } - + // check B for the number of unique values we need to fill an internal buffer // these values will be pulled out to the end of B last = B.end - 1; @@ -415,7 +415,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons if (index == B.start) break; } index = last; - + if (count >= buffer_size) { // keep track of the range within the items where we'll need to "pull out" these values to create the internal buffe pull[pull_index] = Pull { @@ -425,7 +425,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons .to = B.end, }; pull_index = 1; - + if (count == buffer_size + buffer_size) { // we were able to find a single contiguous section containing 2√A unique values, // so this section can be used to contain both of the internal buffers we'll need @@ -449,7 +449,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // buffer2 will be pulled out from a 'B' subarray, so if the first buffer was pulled out from the corresponding 'A' subarray, // we need to adjust the end point for that A subarray so it knows to stop redistributing its values before reaching buffer2 if (pull[0].range.start == A.start) pull[0].range.end -= pull[1].count; - + // we found a second buffer in an 'B' subarray containing √A unique values, so we're done! buffer2 = Range.init(B.end - count, B.end); break; @@ -465,12 +465,12 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons }; } } - + // pull out the two ranges so we can use them as internal buffers pull_index = 0; while (pull_index < 2) : (pull_index += 1) { const length = pull[pull_index].count; - + if (pull[pull_index].to < pull[pull_index].from) { // we're pulling the values out to the left, which means the start of an A subarray index = pull[pull_index].from; @@ -493,27 +493,27 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // adjust block_size and buffer_size based on the values we were able to pull out buffer_size = buffer1.length(); block_size = iterator.length()/buffer_size + 1; - + // the first buffer NEEDS to be large enough to tag each of the evenly sized A blocks, // so this was originally here to test the math for adjusting block_size above // assert((iterator.length() + 1)/block_size <= buffer_size); - + // now that the two internal buffers have been created, it's time to merge each A+B combination at this level of the merge sort! iterator.begin(); while (!iterator.finished()) { A = iterator.nextRange(); B = iterator.nextRange(); - + // remove any parts of A or B that are being used by the internal buffers start = A.start; if (start == pull[0].range.start) { if (pull[0].from > pull[0].to) { A.start += pull[0].count; - + // if the internal buffer takes up the entire A or B subarray, then there's nothing to merge // this only happens for very small subarrays, like √4 = 2, 2 * (2 internal buffers) = 4, // which also only happens when cache.len is small or 0 since it'd otherwise use MergeExternal @@ -532,25 +532,25 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons if (B.length() == 0) continue; } } - + if (lessThan(items[B.end - 1], items[A.start])) { // the two ranges are in reverse order, so a simple rotation should fix it mem.rotate(T, items[A.start..B.end], A.length()); } else if (lessThan(items[A.end], items[A.end - 1])) { // these two ranges weren't already in order, so we'll need to merge them! var findA: usize = undefined; - + // break the remainder of A into blocks. firstA is the uneven-sized first A block var blockA = Range.init(A.start, A.end); var firstA = Range.init(A.start, A.start + blockA.length() % block_size); - + // swap the first value of each A block with the value in buffer1 var indexA = buffer1.start; index = firstA.end; while (index < blockA.end) : ({indexA += 1; index += block_size;}) { mem.swap(T, &items[indexA], &items[index]); } - + // start rolling the A blocks through the B blocks! // whenever we leave an A block behind, we'll need to merge the previous A block with any B blocks that follow it, so track that information as well var lastA = firstA; @@ -558,7 +558,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons var blockB = Range.init(B.start, B.start + math.min(block_size, B.length())); blockA.start += firstA.length(); indexA = buffer1.start; - + // if the first unevenly sized A block fits into the cache, copy it there for when we go to Merge it // otherwise, if the second buffer is available, block swap the contents into that if (lastA.length() <= cache.len) { @@ -566,7 +566,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else if (buffer2.length() > 0) { blockSwap(T, items, lastA.start, buffer2.start, lastA.length()); } - + if (blockA.length() > 0) { while (true) { // if there's a previous B block and the first value of the minimum A block is <= the last value of the previous B block, @@ -575,7 +575,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // figure out where to split the previous B block, and rotate it at the split const B_split = binaryFirst(T, items, items[indexA], lastB, lessThan); const B_remaining = lastB.end - B_split; - + // swap the minimum A block to the beginning of the rolling A blocks var minA = blockA.start; findA = minA + block_size; @@ -585,16 +585,16 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } blockSwap(T, items, blockA.start, minA, block_size); - + // swap the first item of the previous A block back with its original value, which is stored in buffer1 mem.swap(T, &items[blockA.start], &items[indexA]); indexA += 1; - + // locally merge the previous A block with the B values that follow it // if lastA fits into the external cache we'll use that (with MergeExternal), // or if the second internal buffer exists we'll use that (with MergeInternal), // or failing that we'll use a strictly in-place merge algorithm (MergeInPlace) - + if (lastA.length() <= cache.len) { mergeExternal(T, items, lastA, Range.init(lastA.end, B_split), lessThan, cache[0..]); } else if (buffer2.length() > 0) { @@ -602,7 +602,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else { mergeInPlace(T, items, lastA, Range.init(lastA.end, B_split), lessThan); } - + if (buffer2.length() > 0 or block_size <= cache.len) { // copy the previous A block into the cache or buffer2, since that's where we need it to be when we go to merge it anyway if (block_size <= cache.len) { @@ -610,7 +610,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } else { blockSwap(T, items, blockA.start, buffer2.start, block_size); } - + // this is equivalent to rotating, but faster // the area normally taken up by the A block is either the contents of buffer2, or data we don't need anymore since we memcopied it // either way, we don't need to retain the order of those items, so instead of rotating we can just block swap B to where it belongs @@ -619,21 +619,21 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // we are unable to use the 'buffer2' trick to speed up the rotation operation since buffer2 doesn't exist, so perform a normal rotation mem.rotate(T, items[B_split..blockA.start + block_size], blockA.start - B_split); } - + // update the range for the remaining A blocks, and the range remaining from the B block after it was split lastA = Range.init(blockA.start - B_remaining, blockA.start - B_remaining + block_size); lastB = Range.init(lastA.end, lastA.end + B_remaining); - + // if there are no more A blocks remaining, this step is finished! blockA.start += block_size; if (blockA.length() == 0) break; - + } else if (blockB.length() < block_size) { // move the last B block, which is unevenly sized, to before the remaining A blocks, by using a rotation // the cache is disabled here since it might contain the contents of the previous A block mem.rotate(T, items[blockA.start..blockB.end], blockB.start - blockA.start); - + lastB = Range.init(blockA.start, blockA.start + blockB.length()); blockA.start += blockB.length(); blockA.end += blockB.length(); @@ -642,11 +642,11 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // roll the leftmost A block to the end by swapping it with the next B block blockSwap(T, items, blockA.start, blockB.start, block_size); lastB = Range.init(blockA.start, blockA.start + block_size); - + blockA.start += block_size; blockA.end += block_size; blockB.start += block_size; - + if (blockB.end > B.end - block_size) { blockB.end = B.end; } else { @@ -655,7 +655,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // merge the last A block with the remaining B values if (lastA.length() <= cache.len) { mergeExternal(T, items, lastA, Range.init(lastA.end, B.end), lessThan, cache[0..]); @@ -666,14 +666,14 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // when we're finished with this merge step we should have the one or two internal buffers left over, where the second buffer is all jumbled up // insertion sort the second buffer, then redistribute the buffers back into the items using the opposite process used for creating the buffer - + // while an unstable sort like quicksort could be applied here, in benchmarks it was consistently slightly slower than a simple insertion sort, // even for tens of millions of items. this may be because insertion sort is quite fast when the data is already somewhat sorted, like it is here insertionSort(T, items[buffer2.start..buffer2.end], lessThan); - + pull_index = 0; while (pull_index < 2) : (pull_index += 1) { var unique = pull[pull_index].count * 2; @@ -702,7 +702,7 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons } } } - + // double the size of each A and B subarray that will be merged in the next level if (!iterator.nextLevel()) break; } @@ -711,37 +711,37 @@ pub fn sort(comptime T: type, items: []T, lessThan: fn(lhs: &const T, rhs: &cons // merge operation without a buffer fn mergeInPlace(comptime T: type, items: []T, A_arg: &const Range, B_arg: &const Range, lessThan: fn(&const T,&const T)bool) void { if (A_arg.length() == 0 or B_arg.length() == 0) return; - + // this just repeatedly binary searches into B and rotates A into position. // the paper suggests using the 'rotation-based Hwang and Lin algorithm' here, // but I decided to stick with this because it had better situational performance - // + // // (Hwang and Lin is designed for merging subarrays of very different sizes, // but WikiSort almost always uses subarrays that are roughly the same size) - // + // // normally this is incredibly suboptimal, but this function is only called // when none of the A or B blocks in any subarray contained 2√A unique values, // which places a hard limit on the number of times this will ACTUALLY need // to binary search and rotate. - // + // // according to my analysis the worst case is √A rotations performed on √A items // once the constant factors are removed, which ends up being O(n) - // + // // again, this is NOT a general-purpose solution – it only works well in this case! // kind of like how the O(n^2) insertion sort is used in some places var A = *A_arg; var B = *B_arg; - + while (true) { // find the first place in B where the first item in A needs to be inserted const mid = binaryFirst(T, items, items[A.start], B, lessThan); - + // rotate A into place const amount = mid - A.end; mem.rotate(T, items[A.start..mid], A.length()); if (B.end == mid) break; - + // calculate the new A and B ranges B.start = mid; A = Range.init(A.start + amount, B.start); @@ -757,7 +757,7 @@ fn mergeInternal(comptime T: type, items: []T, A: &const Range, B: &const Range, var A_count: usize = 0; var B_count: usize = 0; var insert: usize = 0; - + if (B.length() > 0 and A.length() > 0) { while (true) { if (!lessThan(items[B.start + B_count], items[buffer.start + A_count])) { @@ -773,7 +773,7 @@ fn mergeInternal(comptime T: type, items: []T, A: &const Range, B: &const Range, } } } - + // swap the remainder of A into the final array blockSwap(T, items, buffer.start + A_count, A.start + insert, A.length() - A_count); } @@ -790,56 +790,56 @@ fn blockSwap(comptime T: type, items: []T, start1: usize, start2: usize, block_s fn findFirstForward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.start + skip; while (lessThan(items[index - 1], value)) : (index += skip) { if (index >= range.end - skip) { return binaryFirst(T, items, value, Range.init(index, range.end), lessThan); } } - + return binaryFirst(T, items, value, Range.init(index - skip, index), lessThan); } fn findFirstBackward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.end - skip; while (index > range.start and !lessThan(items[index - 1], value)) : (index -= skip) { if (index < range.start + skip) { return binaryFirst(T, items, value, Range.init(range.start, index), lessThan); } } - + return binaryFirst(T, items, value, Range.init(index, index + skip), lessThan); } fn findLastForward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.start + skip; while (!lessThan(value, items[index - 1])) : (index += skip) { if (index >= range.end - skip) { return binaryLast(T, items, value, Range.init(index, range.end), lessThan); } } - + return binaryLast(T, items, value, Range.init(index - skip, index), lessThan); } fn findLastBackward(comptime T: type, items: []T, value: &const T, range: &const Range, lessThan: fn(&const T,&const T)bool, unique: usize) usize { if (range.length() == 0) return range.start; const skip = math.max(range.length()/unique, usize(1)); - + var index = range.end - skip; while (index > range.start and lessThan(value, items[index - 1])) : (index -= skip) { if (index < range.start + skip) { return binaryLast(T, items, value, Range.init(range.start, index), lessThan); } } - + return binaryLast(T, items, value, Range.init(index, index + skip), lessThan); } @@ -885,7 +885,7 @@ fn mergeInto(comptime T: type, from: []T, A: &const Range, B: &const Range, less const A_last = A.end; const B_last = B.end; var insert_index: usize = 0; - + while (true) { if (!lessThan(from[B_index], from[A_index])) { into[insert_index] = from[A_index]; @@ -916,7 +916,7 @@ fn mergeExternal(comptime T: type, items: []T, A: &const Range, B: &const Range, var insert_index: usize = A.start; const A_last = A.length(); const B_last = B.end; - + if (B.length() > 0 and A.length() > 0) { while (true) { if (!lessThan(items[B_index], cache[A_index])) { @@ -932,7 +932,7 @@ fn mergeExternal(comptime T: type, items: []T, A: &const Range, B: &const Range, } } } - + // copy the remainder of A into the final array mem.copy(T, items[insert_index..], cache[A_index..A_last]); } @@ -1081,11 +1081,11 @@ test "another sort case" { } test "sort fuzz testing" { - var rng = std.rand.Rand.init(0x12345678); + var prng = std.rand.DefaultPrng.init(0x12345678); const test_case_count = 10; var i: usize = 0; while (i < test_case_count) : (i += 1) { - fuzzTest(&rng); + fuzzTest(&prng.random); } } From ccadcbc715c084f78dc093811d74f93b80536a0a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 29 Mar 2018 12:33:29 -0400 Subject: [PATCH 032/132] fix examples and rename std.rand.Rand to std.rand.Random --- example/guess_number/main.zig | 9 +++---- src/parser.hpp | 1 - std/rand/index.zig | 50 +++++++++++++++++------------------ std/sort.zig | 2 +- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/example/guess_number/main.zig b/example/guess_number/main.zig index d6a7b94b6..d26518a1e 100644 --- a/example/guess_number/main.zig +++ b/example/guess_number/main.zig @@ -2,7 +2,6 @@ const builtin = @import("builtin"); const std = @import("std"); const io = std.io; const fmt = std.fmt; -const Rand = std.rand.Rand; const os = std.os; pub fn main() !void { @@ -14,15 +13,15 @@ pub fn main() !void { try stdout.print("Welcome to the Guess Number Game in Zig.\n"); - var seed_bytes: [@sizeOf(usize)]u8 = undefined; + var seed_bytes: [@sizeOf(u64)]u8 = undefined; os.getRandomBytes(seed_bytes[0..]) catch |err| { std.debug.warn("unable to seed random number generator: {}", err); return err; }; - const seed = std.mem.readInt(seed_bytes, usize, builtin.Endian.Big); - var rand = Rand.init(seed); + const seed = std.mem.readInt(seed_bytes, u64, builtin.Endian.Big); + var prng = std.rand.DefaultPrng.init(seed); - const answer = rand.range(u8, 0, 100) + 1; + const answer = prng.random.range(u8, 0, 100) + 1; while (true) { try stdout.print("\nGuess a number between 1 and 100: "); diff --git a/src/parser.hpp b/src/parser.hpp index 21922db4e..224e289bf 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -16,7 +16,6 @@ ATTRIBUTE_PRINTF(2, 3) void ast_token_error(Token *token, const char *format, ...); -// This function is provided by generated code, generated by parsergen.cpp AstNode * ast_parse(Buf *buf, ZigList *tokens, ImportTableEntry *owner, ErrColor err_color); void ast_print(AstNode *node, int indent); diff --git a/std/rand/index.zig b/std/rand/index.zig index ce6956547..6a746fce9 100644 --- a/std/rand/index.zig +++ b/std/rand/index.zig @@ -26,16 +26,16 @@ pub const DefaultPrng = Xoroshiro128; // When you need cryptographically secure random numbers pub const DefaultCsprng = Isaac64; -pub const Rand = struct { - fillFn: fn(r: &Rand, buf: []u8) void, +pub const Random = struct { + fillFn: fn(r: &Random, buf: []u8) void, /// Read random bytes into the specified buffer until fill. - pub fn bytes(r: &Rand, buf: []u8) void { + pub fn bytes(r: &Random, buf: []u8) void { r.fillFn(r, buf); } /// Return a random integer/boolean type. - pub fn scalar(r: &Rand, comptime T: type) T { + pub fn scalar(r: &Random, comptime T: type) T { var rand_bytes: [@sizeOf(T)]u8 = undefined; r.bytes(rand_bytes[0..]); @@ -49,7 +49,7 @@ pub const Rand = struct { /// Get a random unsigned integer with even distribution between `start` /// inclusive and `end` exclusive. - pub fn range(r: &Rand, comptime T: type, start: T, end: T) T { + pub fn range(r: &Random, comptime T: type, start: T, end: T) T { assert(start <= end); if (T.is_signed) { const uint = @IntType(false, T.bit_count); @@ -91,7 +91,7 @@ pub const Rand = struct { } /// Return a floating point value evenly distributed in the range [0, 1). - pub fn float(r: &Rand, comptime T: type) T { + pub fn float(r: &Random, comptime T: type) T { // Generate a uniform value between [1, 2) and scale down to [0, 1). // Note: The lowest mantissa bit is always set to 0 so we only use half the available range. switch (T) { @@ -110,18 +110,18 @@ pub const Rand = struct { } /// Return a floating point value normally distributed in the range [0, 1]. - pub fn floatNorm(r: &Rand, comptime T: type) T { + pub fn floatNorm(r: &Random, comptime T: type) T { // TODO(tiehuis): See https://www.doornik.com/research/ziggurat.pdf @compileError("floatNorm is unimplemented"); } /// Return a exponentially distributed float between (0, @maxValue(f64)) - pub fn floatExp(r: &Rand, comptime T: type) T { + pub fn floatExp(r: &Random, comptime T: type) T { @compileError("floatExp is unimplemented"); } /// Shuffle a slice into a random order. - pub fn shuffle(r: &Rand, comptime T: type, buf: []T) void { + pub fn shuffle(r: &Random, comptime T: type, buf: []T) void { if (buf.len < 2) { return; } @@ -178,14 +178,14 @@ test "splitmix64 sequence" { pub const Pcg = struct { const default_multiplier = 6364136223846793005; - random: Rand, + random: Random, s: u64, i: u64, pub fn init(init_s: u64) Pcg { var pcg = Pcg { - .random = Rand { .fillFn = fill }, + .random = Random { .fillFn = fill }, .s = undefined, .i = undefined, }; @@ -218,7 +218,7 @@ pub const Pcg = struct { self.s = self.s *% default_multiplier +% self.i; } - fn fill(r: &Rand, buf: []u8) void { + fn fill(r: &Random, buf: []u8) void { const self = @fieldParentPtr(Pcg, "random", r); var i: usize = 0; @@ -269,13 +269,13 @@ test "pcg sequence" { // // PRNG pub const Xoroshiro128 = struct { - random: Rand, + random: Random, s: [2]u64, pub fn init(init_s: u64) Xoroshiro128 { var x = Xoroshiro128 { - .random = Rand { .fillFn = fill }, + .random = Random { .fillFn = fill }, .s = undefined, }; @@ -328,7 +328,7 @@ pub const Xoroshiro128 = struct { self.s[1] = gen.next(); } - fn fill(r: &Rand, buf: []u8) void { + fn fill(r: &Random, buf: []u8) void { const self = @fieldParentPtr(Xoroshiro128, "random", r); var i: usize = 0; @@ -397,7 +397,7 @@ test "xoroshiro sequence" { // Follows the general idea of the implementation from here with a few shortcuts. // https://doc.rust-lang.org/rand/src/rand/prng/isaac64.rs.html pub const Isaac64 = struct { - random: Rand, + random: Random, r: [256]u64, m: [256]u64, @@ -408,7 +408,7 @@ pub const Isaac64 = struct { pub fn init(init_s: u64) Isaac64 { var isaac = Isaac64 { - .random = Rand { .fillFn = fill }, + .random = Random { .fillFn = fill }, .r = undefined, .m = undefined, .a = undefined, @@ -522,7 +522,7 @@ pub const Isaac64 = struct { self.i = self.r.len; // trigger refill on first value } - fn fill(r: &Rand, buf: []u8) void { + fn fill(r: &Random, buf: []u8) void { const self = @fieldParentPtr(Isaac64, "random", r); var i: usize = 0; @@ -577,8 +577,8 @@ test "isaac64 sequence" { } } -// Actual Rand helper function tests, pcg engine is assumed correct. -test "Rand float" { +// Actual Random helper function tests, pcg engine is assumed correct. +test "Random float" { var prng = DefaultPrng.init(0); var i: usize = 0; @@ -593,18 +593,18 @@ test "Rand float" { } } -test "Rand scalar" { +test "Random scalar" { var prng = DefaultPrng.init(0); const s = prng .random.scalar(u64); } -test "Rand bytes" { +test "Random bytes" { var prng = DefaultPrng.init(0); var buf: [2048]u8 = undefined; prng.random.bytes(buf[0..]); } -test "Rand shuffle" { +test "Random shuffle" { var prng = DefaultPrng.init(0); var seq = []const u8 { 0, 1, 2, 3, 4 }; @@ -629,14 +629,14 @@ fn sumArray(s: []const u8) u32 { return r; } -test "Rand range" { +test "Random range" { var prng = DefaultPrng.init(0); testRange(&prng.random, -4, 3); testRange(&prng.random, -4, -1); testRange(&prng.random, 10, 14); } -fn testRange(r: &Rand, start: i32, end: i32) void { +fn testRange(r: &Random, start: i32, end: i32) void { const count = usize(end - start); var values_buffer = []bool{false} ** 20; const values = values_buffer[0..count]; diff --git a/std/sort.zig b/std/sort.zig index a3ebd293c..0f83df7bb 100644 --- a/std/sort.zig +++ b/std/sort.zig @@ -1091,7 +1091,7 @@ test "sort fuzz testing" { var fixed_buffer_mem: [100 * 1024]u8 = undefined; -fn fuzzTest(rng: &std.rand.Rand) void { +fn fuzzTest(rng: &std.rand.Random) void { const array_size = rng.range(usize, 0, 1000); var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); var array = fixed_allocator.allocator.alloc(IdAndValue, array_size) catch unreachable; From 530f79576923989928afeeb8b50b7e40b95a529f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 22:31:17 +0200 Subject: [PATCH 033/132] std.zig.parser now supports all infix operators --- std/zig/ast.zig | 61 ++++++++- std/zig/parser.zig | 249 +++++++++++++++++++++++++++--------- std/zig/tokenizer.zig | 287 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 535 insertions(+), 62 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 180a0a930..59ba0d79c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { IntegerLiteral, FloatLiteral, StringLiteral, + UndefinedLiteral, BuiltinCall, LineComment, TestDecl, @@ -38,6 +39,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), + Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), @@ -57,6 +59,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), + Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), @@ -76,6 +79,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), + Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), @@ -309,9 +313,47 @@ pub const NodeInfixOp = struct { rhs: &Node, const InfixOp = enum { - EqualEqual, + Add, + AddWrap, + ArrayCat, + ArrayMult, + Assign, + AssignBitAnd, + AssignBitOr, + AssignBitShiftLeft, + AssignBitShiftRight, + AssignBitXor, + AssignDiv, + AssignMinus, + AssignMinusWrap, + AssignMod, + AssignPlus, + AssignPlusWrap, + AssignTimes, + AssignTimesWarp, BangEqual, + BitAnd, + BitOr, + BitShiftLeft, + BitShiftRight, + BitXor, + BoolAnd, + BoolOr, + Div, + EqualEqual, + ErrorUnion, + GreaterOrEqual, + GreaterThan, + LessOrEqual, + LessThan, + MergeErrorSets, + Mod, + Mult, + MultWrap, Period, + Sub, + SubWrap, + UnwrapMaybe, }; pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node { @@ -464,6 +506,23 @@ pub const NodeStringLiteral = struct { } }; +pub const NodeUndefinedLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeUndefinedLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeUndefinedLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeUndefinedLiteral) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 06b7d35a4..a96277df0 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -371,6 +371,13 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_undefined => { + try stack.append(State { + .Operand = &(try self.createUndefined(arena, token)).base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -414,56 +421,41 @@ pub const Parser = struct { // or a postfix operator (like () or {}), // otherwise this expression is done (like on a ; or else). var token = self.getNextToken(); - switch (token.id) { - Token.Id.EqualEqual => { + if (tokenIdToInfixOp(token.id)) |infix_id| { try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.EqualEqual) + .InfixOp = try self.createInfixOp(arena, token, infix_id) }); try stack.append(State.ExpectOperand); continue; - }, - Token.Id.BangEqual => { - try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BangEqual) - }); - try stack.append(State.ExpectOperand); - continue; - }, - Token.Id.Period => { - try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period) - }); - try stack.append(State.ExpectOperand); - continue; - }, - else => { - // no postfix/infix operator after this operand. - self.putBackToken(token); - // reduce the stack - var expression: &ast.Node = stack.pop().Operand; - while (true) { - switch (stack.pop()) { - State.Expression => |dest_ptr| { - // we're done - try dest_ptr.store(expression); - break; - }, - State.InfixOp => |infix_op| { - infix_op.rhs = expression; - infix_op.lhs = stack.pop().Operand; - expression = &infix_op.base; - continue; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - expression = &prefix_op.base; - continue; - }, - else => unreachable, - } + + // TODO: Parse postfix operator + } else { + // no postfix/infix operator after this operand. + self.putBackToken(token); + // reduce the stack + var expression: &ast.Node = stack.pop().Operand; + while (true) { + switch (stack.pop()) { + State.Expression => |dest_ptr| { + // we're done + try dest_ptr.store(expression); + break; + }, + State.InfixOp => |infix_op| { + infix_op.rhs = expression; + infix_op.lhs = stack.pop().Operand; + expression = &infix_op.base; + continue; + }, + State.PrefixOp => |prefix_op| { + prefix_op.rhs = expression; + expression = &prefix_op.base; + continue; + }, + else => unreachable, } - continue; - }, + } + continue; } }, @@ -706,6 +698,53 @@ pub const Parser = struct { } } + fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + return switch (*id) { + Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd, + Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp.AssignBitAnd, + Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp.BitShiftLeft, + Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, + Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp.BitShiftRight, + Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftRight, + Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp.LessThan, + Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.LessOrEqual, + Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp.GreaterThan, + Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp.GreaterOrEqual, + Token.Id.Asterisk => ast.NodeInfixOp.InfixOp.Mult, + Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp.ArrayMult, + Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp.AssignTimes, + Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp.MultWrap, + Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp.AssignTimesWarp, + Token.Id.Bang => ast.NodeInfixOp.InfixOp.ErrorUnion, + Token.Id.BangEqual => ast.NodeInfixOp.InfixOp.BangEqual, + Token.Id.Caret => ast.NodeInfixOp.InfixOp.BitXor, + Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp.AssignBitXor, + Token.Id.Equal => ast.NodeInfixOp.InfixOp.Assign, + Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp.EqualEqual, + Token.Id.Keyword_and => ast.NodeInfixOp.InfixOp.BoolAnd, + Token.Id.Keyword_or => ast.NodeInfixOp.InfixOp.BoolOr, + Token.Id.Minus => ast.NodeInfixOp.InfixOp.Sub, + Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp.AssignMinus, + Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp.SubWrap, + Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp.AssignMinusWrap, + Token.Id.Percent => ast.NodeInfixOp.InfixOp.Mod, + Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp.AssignMod, + Token.Id.Period => ast.NodeInfixOp.InfixOp.Period, + Token.Id.Pipe => ast.NodeInfixOp.InfixOp.BitOr, + Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp.AssignBitOr, + Token.Id.PipePipe => ast.NodeInfixOp.InfixOp.MergeErrorSets, + Token.Id.Plus => ast.NodeInfixOp.InfixOp.Add, + Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp.AssignPlus, + Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp.AddWrap, + Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp.AssignPlusWrap, + Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp.ArrayCat, + Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp.UnwrapMaybe, + Token.Id.Slash => ast.NodeInfixOp.InfixOp.Div, + Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp.AssignDiv, + else => null, + }; + } + fn initNode(self: &Parser, id: ast.Node.Id) ast.Node { if (self.pending_line_comment_node) |comment_node| { self.pending_line_comment_node = null; @@ -867,6 +906,16 @@ pub const Parser = struct { return node; } + fn createUndefined(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeUndefinedLiteral { + const node = try arena.create(ast.NodeUndefinedLiteral); + + *node = ast.NodeUndefinedLiteral { + .base = self.initNode(ast.Node.Id.UndefinedLiteral), + .token = *token, + }; + return node; + } + fn createAttachIdentifier(self: &Parser, arena: &mem.Allocator, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier { const node = try self.createIdentifier(arena, name_token); try dest_ptr.store(&node.base); @@ -1173,17 +1222,51 @@ pub const Parser = struct { ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); - switch (prefix_op_node.op) { - ast.NodeInfixOp.InfixOp.EqualEqual => { - try stack.append(RenderState { .Text = " == "}); - }, - ast.NodeInfixOp.InfixOp.BangEqual => { - try stack.append(RenderState { .Text = " != "}); - }, - ast.NodeInfixOp.InfixOp.Period => { - try stack.append(RenderState { .Text = "."}); - }, - } + const text = switch (prefix_op_node.op) { + ast.NodeInfixOp.InfixOp.Add => " + ", + ast.NodeInfixOp.InfixOp.AddWrap => " +% ", + ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", + ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", + ast.NodeInfixOp.InfixOp.Assign => " = ", + ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", + ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", + ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", + ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", + ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", + ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", + ast.NodeInfixOp.InfixOp.AssignMod => " %= ", + ast.NodeInfixOp.InfixOp.AssignPlus => " += ", + ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", + ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", + ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", + ast.NodeInfixOp.InfixOp.BangEqual => " != ", + ast.NodeInfixOp.InfixOp.BitAnd => " & ", + ast.NodeInfixOp.InfixOp.BitOr => " | ", + ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", + ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", + ast.NodeInfixOp.InfixOp.BitXor => " ^ ", + ast.NodeInfixOp.InfixOp.BoolAnd => " and ", + ast.NodeInfixOp.InfixOp.BoolOr => " or ", + ast.NodeInfixOp.InfixOp.Div => " / ", + ast.NodeInfixOp.InfixOp.EqualEqual => " == ", + ast.NodeInfixOp.InfixOp.ErrorUnion => "!", + ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", + ast.NodeInfixOp.InfixOp.GreaterThan => " > ", + ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", + ast.NodeInfixOp.InfixOp.LessThan => " < ", + ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", + ast.NodeInfixOp.InfixOp.Mod => " % ", + ast.NodeInfixOp.InfixOp.Mult => " * ", + ast.NodeInfixOp.InfixOp.MultWrap => " *% ", + ast.NodeInfixOp.InfixOp.Period => ".", + ast.NodeInfixOp.InfixOp.Sub => " - ", + ast.NodeInfixOp.InfixOp.SubWrap => " -% ", + ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", + }; + + try stack.append(RenderState { .Text = text }); try stack.append(RenderState { .Expression = prefix_op_node.lhs }); }, ast.Node.Id.PrefixOp => { @@ -1224,6 +1307,10 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.UndefinedLiteral => { + const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token)); + }, ast.Node.Id.BuiltinCall => { const builtin_call = @fieldParentPtr(ast.NodeBuiltinCall, "base", base); try stream.print("{}(", self.tokenizer.getTokenSlice(builtin_call.builtin_token)); @@ -1473,4 +1560,54 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "operators" { + \\ var i = undefined; + \\ i = 2; + \\ i *= 2; + \\ i |= 2; + \\ i ^= 2; + \\ i <<= 2; + \\ i >>= 2; + \\ i &= 2; + \\ i *= 2; + \\ i *%= 2; + \\ i -= 2; + \\ i -%= 2; + \\ i += 2; + \\ i +%= 2; + \\ i /= 2; + \\ i %= 2; + \\ _ = i == i; + \\ _ = i != i; + \\ _ = i != i; + \\ _ = i.i; + \\ _ = i || i; + \\ _ = i!i; + \\ _ = i ** i; + \\ _ = i ++ i; + \\ _ = i ?? i; + \\ _ = i % i; + \\ _ = i / i; + \\ _ = i *% i; + \\ _ = i * i; + \\ _ = i -% i; + \\ _ = i - i; + \\ _ = i +% i; + \\ _ = i + i; + \\ _ = i << i; + \\ _ = i >> i; + \\ _ = i & i; + \\ _ = i ^ i; + \\ _ = i | i; + \\ _ = i >= i; + \\ _ = i <= i; + \\ _ = i > i; + \\ _ = i < i; + \\ _ = i and i; + \\ _ = i or i; + \\} + \\ + ); } diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 4af6c20ca..4b1e35f62 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -77,6 +77,7 @@ pub const Token = struct { Builtin, Bang, Pipe, + PipePipe, PipeEqual, Equal, EqualEqual, @@ -85,18 +86,45 @@ pub const Token = struct { RParen, Semicolon, Percent, + PercentEqual, LBrace, RBrace, Period, Ellipsis2, Ellipsis3, + Caret, + CaretEqual, + Plus, + PlusPlus, + PlusEqual, + PlusPercent, + PlusPercentEqual, Minus, + MinusEqual, + MinusPercent, + MinusPercentEqual, + Asterisk, + AsteriskEqual, + AsteriskAsterisk, + AsteriskPercent, + AsteriskPercentEqual, Arrow, Colon, Slash, + SlashEqual, Comma, Ampersand, AmpersandEqual, + QuestionMark, + QuestionMarkQuestionMark, + AngleBracketLeft, + AngleBracketLeftEqual, + AngleBracketAngleBracketLeft, + AngleBracketAngleBracketLeftEqual, + AngleBracketRight, + AngleBracketRightEqual, + AngleBracketAngleBracketRight, + AngleBracketAngleBracketRightEqual, IntegerLiteral, FloatLiteral, LineComment, @@ -200,6 +228,9 @@ pub const Tokenizer = struct { Bang, Pipe, Minus, + MinusPercent, + Asterisk, + AsteriskPercent, Slash, LineComment, Zero, @@ -210,6 +241,15 @@ pub const Tokenizer = struct { FloatExponentUnsigned, FloatExponentNumber, Ampersand, + Caret, + Percent, + QuestionMark, + Plus, + PlusPercent, + AngleBracketLeft, + AngleBracketAngleBracketLeft, + AngleBracketRight, + AngleBracketAngleBracketRight, Period, Period2, SawAtSign, @@ -291,9 +331,25 @@ pub const Tokenizer = struct { break; }, '%' => { - result.id = Token.Id.Percent; - self.index += 1; - break; + state = State.Percent; + }, + '*' => { + state = State.Asterisk; + }, + '+' => { + state = State.Plus; + }, + '?' => { + state = State.QuestionMark; + }, + '<' => { + state = State.AngleBracketLeft; + }, + '>' => { + state = State.AngleBracketRight; + }, + '^' => { + state = State.Caret; }, '{' => { result.id = Token.Id.LBrace; @@ -356,6 +412,107 @@ pub const Tokenizer = struct { break; }, }, + + State.Asterisk => switch (c) { + '=' => { + result.id = Token.Id.AsteriskEqual; + self.index += 1; + break; + }, + '*' => { + result.id = Token.Id.AsteriskAsterisk; + self.index += 1; + break; + }, + '%' => { + state = State.AsteriskPercent; + }, + else => { + result.id = Token.Id.Asterisk; + break; + } + }, + + State.AsteriskPercent => switch (c) { + '=' => { + result.id = Token.Id.AsteriskPercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AsteriskPercent; + break; + } + }, + + State.QuestionMark => switch (c) { + '?' => { + result.id = Token.Id.QuestionMarkQuestionMark; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.QuestionMark; + break; + }, + }, + + State.Percent => switch (c) { + '=' => { + result.id = Token.Id.PercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.Percent; + break; + }, + }, + + State.Plus => switch (c) { + '=' => { + result.id = Token.Id.PlusEqual; + self.index += 1; + break; + }, + '+' => { + result.id = Token.Id.PlusPlus; + self.index += 1; + break; + }, + '%' => { + state = State.PlusPercent; + }, + else => { + result.id = Token.Id.Plus; + break; + }, + }, + + State.PlusPercent => switch (c) { + '=' => { + result.id = Token.Id.PlusPercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.PlusPercent; + break; + }, + }, + + State.Caret => switch (c) { + '=' => { + result.id = Token.Id.CaretEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.Caret; + break; + } + }, + State.Identifier => switch (c) { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => { @@ -417,6 +574,11 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '|' => { + result.id = Token.Id.PipePipe; + self.index += 1; + break; + }, else => { result.id = Token.Id.Pipe; break; @@ -441,12 +603,86 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '=' => { + result.id = Token.Id.MinusEqual; + self.index += 1; + break; + }, + '%' => { + state = State.MinusPercent; + }, else => { result.id = Token.Id.Minus; break; }, }, + State.MinusPercent => switch (c) { + '=' => { + result.id = Token.Id.MinusPercentEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.MinusPercent; + break; + } + }, + + State.AngleBracketLeft => switch (c) { + '<' => { + state = State.AngleBracketAngleBracketLeft; + }, + '=' => { + result.id = Token.Id.AngleBracketLeftEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketLeft; + break; + }, + }, + + State.AngleBracketAngleBracketLeft => switch (c) { + '=' => { + result.id = Token.Id.AngleBracketAngleBracketLeftEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketAngleBracketLeft; + break; + }, + }, + + State.AngleBracketRight => switch (c) { + '>' => { + state = State.AngleBracketAngleBracketRight; + }, + '=' => { + result.id = Token.Id.AngleBracketRightEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketRight; + break; + }, + }, + + State.AngleBracketAngleBracketRight => switch (c) { + '=' => { + result.id = Token.Id.AngleBracketAngleBracketRightEqual; + self.index += 1; + break; + }, + else => { + result.id = Token.Id.AngleBracketAngleBracketRight; + break; + }, + }, + State.Period => switch (c) { '.' => { state = State.Period2; @@ -474,6 +710,11 @@ pub const Tokenizer = struct { result.id = Token.Id.LineComment; state = State.LineComment; }, + '=' => { + result.id = Token.Id.SlashEqual; + self.index += 1; + break; + }, else => { result.id = Token.Id.Slash; break; @@ -609,6 +850,42 @@ pub const Tokenizer = struct { State.Pipe => { result.id = Token.Id.Pipe; }, + State.AngleBracketAngleBracketRight => { + result.id = Token.Id.AngleBracketAngleBracketRight; + }, + State.AngleBracketRight => { + result.id = Token.Id.AngleBracketRight; + }, + State.AngleBracketAngleBracketLeft => { + result.id = Token.Id.AngleBracketAngleBracketLeft; + }, + State.AngleBracketLeft => { + result.id = Token.Id.AngleBracketLeft; + }, + State.PlusPercent => { + result.id = Token.Id.PlusPercent; + }, + State.Plus => { + result.id = Token.Id.Plus; + }, + State.QuestionMark => { + result.id = Token.Id.QuestionMark; + }, + State.Percent => { + result.id = Token.Id.Percent; + }, + State.Caret => { + result.id = Token.Id.Caret; + }, + State.AsteriskPercent => { + result.id = Token.Id.AsteriskPercent; + }, + State.Asterisk => { + result.id = Token.Id.Asterisk; + }, + State.MinusPercent => { + result.id = Token.Id.MinusPercent; + }, } } if (result.id == Token.Id.Eof) { @@ -752,8 +1029,8 @@ test "tokenizer - string identifier and builtin fns" { test "tokenizer - pipe and then invalid" { testTokenize("||=", []Token.Id{ - Token.Id.Pipe, - Token.Id.PipeEqual, + Token.Id.PipePipe, + Token.Id.Equal, }); } From 8ada03097117cd70800f9e5e7cce97226107f79e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 22:37:54 +0200 Subject: [PATCH 034/132] Fixed self hosted compiler compiler error from prev commit --- std/zig/ast.zig | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 59ba0d79c..36952ec3c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -363,9 +363,47 @@ pub const NodeInfixOp = struct { i -= 1; switch (self.op) { - InfixOp.EqualEqual, + InfixOp.Add, + InfixOp.AddWrap, + InfixOp.ArrayCat, + InfixOp.ArrayMult, + InfixOp.Assign, + InfixOp.AssignBitAnd, + InfixOp.AssignBitOr, + InfixOp.AssignBitShiftLeft, + InfixOp.AssignBitShiftRight, + InfixOp.AssignBitXor, + InfixOp.AssignDiv, + InfixOp.AssignMinus, + InfixOp.AssignMinusWrap, + InfixOp.AssignMod, + InfixOp.AssignPlus, + InfixOp.AssignPlusWrap, + InfixOp.AssignTimes, + InfixOp.AssignTimesWarp, InfixOp.BangEqual, - InfixOp.Period => {}, + InfixOp.BitAnd, + InfixOp.BitOr, + InfixOp.BitShiftLeft, + InfixOp.BitShiftRight, + InfixOp.BitXor, + InfixOp.BoolAnd, + InfixOp.BoolOr, + InfixOp.Div, + InfixOp.EqualEqual, + InfixOp.ErrorUnion, + InfixOp.GreaterOrEqual, + InfixOp.GreaterThan, + InfixOp.LessOrEqual, + InfixOp.LessThan, + InfixOp.MergeErrorSets, + InfixOp.Mod, + InfixOp.Mult, + InfixOp.MultWrap, + InfixOp.Period, + InfixOp.Sub, + InfixOp.SubWrap, + InfixOp.UnwrapMaybe => {}, } if (i < 1) return self.rhs; From edca1739975f69a695d1ee12ba88ebca1778fefb Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Thu, 29 Mar 2018 23:40:46 +0200 Subject: [PATCH 035/132] std.zig.parser now parses call expr --- std/zig/ast.zig | 31 ++++++++++++++++++++++++++++++ std/zig/parser.zig | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 36952ec3c..0e41836f0 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,7 @@ pub const Node = struct { StringLiteral, UndefinedLiteral, BuiltinCall, + Call, LineComment, TestDecl, }; @@ -41,6 +42,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), + Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -61,6 +63,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), + Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -81,6 +84,7 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), + Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -527,6 +531,33 @@ pub const NodeBuiltinCall = struct { } }; +pub const NodeCall = struct { + base: Node, + callee: &Node, + params: ArrayList(&Node), + rparen_token: Token, + + pub fn iterate(self: &NodeCall, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.callee; + i -= 1; + + if (i < self.params.len) return self.params.at(i); + i -= self.params.len; + + return null; + } + + pub fn firstToken(self: &NodeCall) Token { + return self.callee.firstToken(); + } + + pub fn lastToken(self: &NodeCall) Token { + return self.rparen_token; + } +}; + pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a96277df0..33bcbaeda 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -428,6 +428,29 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; + } else if (token.id == Token.Id.LParen) { + self.putBackToken(token); + + const node = try arena.create(ast.NodeCall); + *node = ast.NodeCall { + .base = self.initNode(ast.Node.Id.Call), + .callee = stack.pop().Operand, + .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + try stack.append(State {.ExprListItemOrEnd = &node.params }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LParen, + .ptr = &node.rparen_token, + }, + }); + continue; + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -1325,6 +1348,21 @@ pub const Parser = struct { } } }, + ast.Node.Id.Call => { + const call = @fieldParentPtr(ast.NodeCall, "base", base); + try stack.append(RenderState { .Text = ")"}); + var i = call.params.len; + while (i != 0) { + i -= 1; + const param_node = call.params.at(i); + try stack.append(RenderState { .Expression = param_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + try stack.append(RenderState { .Text = "("}); + try stack.append(RenderState { .Expression = call.callee }); + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), @@ -1610,4 +1648,14 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "test calls" { + \\ a(); + \\ a(1); + \\ a(1, 2); + \\ a(1, 2) + a(1, 2); + \\} + \\ + ); } From f586acabdcc4adaeda7bc278463c20e4033e8cc9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 30 Mar 2018 13:20:13 -0400 Subject: [PATCH 036/132] add ZIG_STATIC cmake option it's not compatible with glibc but it works with musl --- CMakeLists.txt | 3 +++ cmake/Findllvm.cmake | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fbaf929e8..766a78756 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ set(ZIG_LIBC_STATIC_LIB_DIR "" CACHE STRING "Default native target libc director set(ZIG_LIBC_INCLUDE_DIR "/usr/include" CACHE STRING "Default native target libc include directory") set(ZIG_DYNAMIC_LINKER "" CACHE STRING "Override dynamic linker for native target") set(ZIG_EACH_LIB_RPATH off CACHE BOOL "Add each dynamic library to rpath for native target") +set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_LIB_DIR_ESCAPED "${ZIG_LIBC_LIB_DIR}") string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_STATIC_LIB_DIR_ESCAPED "${ZIG_LIBC_STATIC_LIB_DIR}") @@ -698,6 +699,8 @@ if(MINGW) set(EXE_LDFLAGS "-static -static-libgcc -static-libstdc++") elseif(MSVC) set(EXE_LDFLAGS "/STACK:16777216") +elseif(ZIG_STATIC) + set(EXE_LDFLAGS "-static") else() set(EXE_LDFLAGS " ") endif() diff --git a/cmake/Findllvm.cmake b/cmake/Findllvm.cmake index 296e0a91f..021c3c6f2 100644 --- a/cmake/Findllvm.cmake +++ b/cmake/Findllvm.cmake @@ -15,7 +15,7 @@ find_program(LLVM_CONFIG_EXE "c:/msys64/mingw64/bin" "C:/Libraries/llvm-6.0.0/bin") -if(NOT(CMAKE_BUILD_TYPE STREQUAL "Debug")) +if(NOT(CMAKE_BUILD_TYPE STREQUAL "Debug") OR ZIG_STATIC) execute_process( COMMAND ${LLVM_CONFIG_EXE} --libfiles --link-static OUTPUT_VARIABLE LLVM_LIBRARIES_SPACES From 1dfa927a67566a043e0c1d2982333904d18c3b3a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 30 Mar 2018 20:47:09 +0200 Subject: [PATCH 037/132] std.zig.parser now treats call expr as a suffix operator --- std/zig/parser.zig | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 33bcbaeda..3a01f0398 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -86,6 +86,7 @@ pub const Parser = struct { AfterOperand, InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, + SuffixOp: &ast.Node, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -434,13 +435,11 @@ pub const Parser = struct { const node = try arena.create(ast.NodeCall); *node = ast.NodeCall { .base = self.initNode(ast.Node.Id.Call), - .callee = stack.pop().Operand, + .callee = undefined, .params = ArrayList(&ast.Node).init(arena), .rparen_token = undefined, }; - try stack.append(State { - .Operand = &node.base - }); + try stack.append(State { .SuffixOp = &node.base }); try stack.append(State.AfterOperand); try stack.append(State {.ExprListItemOrEnd = &node.params }); try stack.append(State { @@ -455,8 +454,8 @@ pub const Parser = struct { } else { // no postfix/infix operator after this operand. self.putBackToken(token); - // reduce the stack - var expression: &ast.Node = stack.pop().Operand; + + var expression = popSuffixOp(&stack); while (true) { switch (stack.pop()) { State.Expression => |dest_ptr| { @@ -466,7 +465,7 @@ pub const Parser = struct { }, State.InfixOp => |infix_op| { infix_op.rhs = expression; - infix_op.lhs = stack.pop().Operand; + infix_op.lhs = popSuffixOp(&stack); expression = &infix_op.base; continue; }, @@ -716,11 +715,39 @@ pub const Parser = struct { // These are data, not control flow. State.InfixOp => unreachable, State.PrefixOp => unreachable, + State.SuffixOp => unreachable, State.Operand => unreachable, } } } + fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { + var expression: &ast.Node = undefined; + var left_leaf_ptr: &&ast.Node = &expression; + while (true) { + switch (stack.pop()) { + State.SuffixOp => |suffix_op| { + switch (suffix_op.id) { + ast.Node.Id.Call => { + const call = @fieldParentPtr(ast.NodeCall, "base", suffix_op); + *left_leaf_ptr = &call.base; + left_leaf_ptr = &call.callee; + continue; + }, + else => unreachable, + } + }, + State.Operand => |operand| { + *left_leaf_ptr = operand; + break; + }, + else => unreachable, + } + } + + return expression; + } + fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { return switch (*id) { Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd, From 24071c6803ccc0f11b2c4d7c8f22fa8933a98a7c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 30 Mar 2018 21:45:42 +0200 Subject: [PATCH 038/132] std.zig.parser parses all prefix operators --- std/zig/ast.zig | 18 ++++++++++--- std/zig/parser.zig | 59 ++++++++++++++++++++++++++++++++++++++----- std/zig/tokenizer.zig | 6 +++++ 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 0e41836f0..715a333c0 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -432,9 +432,15 @@ pub const NodePrefixOp = struct { rhs: &Node, const PrefixOp = union(enum) { + AddrOf: AddrOfInfo, + BitNot, + BoolNot, + Deref, + Negation, + NegationWrap, Return, Try, - AddrOf: AddrOfInfo, + UnwrapMaybe, }; const AddrOfInfo = struct { align_expr: ?&Node, @@ -448,14 +454,20 @@ pub const NodePrefixOp = struct { var i = index; switch (self.op) { - PrefixOp.Return, - PrefixOp.Try => {}, PrefixOp.AddrOf => |addr_of_info| { if (addr_of_info.align_expr) |align_expr| { if (i < 1) return align_expr; i -= 1; } }, + PrefixOp.BitNot, + PrefixOp.BoolNot, + PrefixOp.Deref, + PrefixOp.Negation, + PrefixOp.NegationWrap, + PrefixOp.Return, + PrefixOp.Try, + PrefixOp.UnwrapMaybe => {}, } if (i < 1) return self.rhs; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3a01f0398..62c62ed18 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -336,6 +336,42 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.Minus => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.Negation) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.MinusPercent => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.NegationWrap) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.Tilde => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.BitNot) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.QuestionMarkQuestionMark => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.UnwrapMaybe) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.Bang => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.BoolNot) }); + try stack.append(State.ExpectOperand); + continue; + }, + Token.Id.Asterisk => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.Deref) }); + try stack.append(State.ExpectOperand); + continue; + }, Token.Id.Ampersand => { const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ .AddrOf = ast.NodePrefixOp.AddrOfInfo { @@ -1323,12 +1359,6 @@ pub const Parser = struct { const prefix_op_node = @fieldParentPtr(ast.NodePrefixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); switch (prefix_op_node.op) { - ast.NodePrefixOp.PrefixOp.Return => { - try stream.write("return "); - }, - ast.NodePrefixOp.PrefixOp.Try => { - try stream.write("try "); - }, ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| { try stream.write("&"); if (addr_of_info.volatile_token != null) { @@ -1343,6 +1373,14 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = align_expr}); } }, + ast.NodePrefixOp.PrefixOp.BitNot => try stream.write("~"), + ast.NodePrefixOp.PrefixOp.BoolNot => try stream.write("!"), + ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), + ast.NodePrefixOp.PrefixOp.Negation => try stream.write("-"), + ast.NodePrefixOp.PrefixOp.NegationWrap => try stream.write("-%"), + ast.NodePrefixOp.PrefixOp.Return => try stream.write("return "), + ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), + ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), } }, ast.Node.Id.IntegerLiteral => { @@ -1627,7 +1665,7 @@ test "zig fmt" { ); try testCanonical( - \\test "operators" { + \\test "infix operators" { \\ var i = undefined; \\ i = 2; \\ i *= 2; @@ -1676,6 +1714,13 @@ test "zig fmt" { \\ ); + try testCanonical( + \\test "prefix operators" { + \\ --%~??!*&0; + \\} + \\ + ); + try testCanonical( \\test "test calls" { \\ a(); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 4b1e35f62..7a13d8997 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -125,6 +125,7 @@ pub const Token = struct { AngleBracketRightEqual, AngleBracketAngleBracketRight, AngleBracketAngleBracketRightEqual, + Tilde, IntegerLiteral, FloatLiteral, LineComment, @@ -361,6 +362,11 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '~' => { + result.id = Token.Id.Tilde; + self.index += 1; + break; + }, '.' => { state = State.Period; }, From b01c50d6fae581050affdc438942b979ae54a8da Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 30 Mar 2018 17:10:54 -0400 Subject: [PATCH 039/132] find libc and zig std lib at runtime this removes the following configure options: * ZIG_LIBC_LIB_DIR * ZIG_LIBC_STATIC_LIB_DIR * ZIG_LIBC_INCLUDE_DIR * ZIG_DYNAMIC_LINKER * ZIG_EACH_LIB_RPATH * zig's reliance on CMAKE_INSTALL_PREFIX these options are still available as command line options, however, the default will attempt to execute the system's C compiler to collect system defaults for these values. closes #870 --- CMakeLists.txt | 5 --- README.md | 8 +--- src/analyze.cpp | 116 ++++++++++++++++++++++++++++++++++++++++++++---- src/codegen.cpp | 19 ++++---- src/config.h.in | 8 ---- src/link.cpp | 44 +++++++++++++++--- src/main.cpp | 7 --- src/os.cpp | 17 ++++--- src/target.cpp | 4 ++ 9 files changed, 171 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 766a78756..1acc789de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,11 +30,6 @@ if(GIT_EXE) endif() message("Configuring zig version ${ZIG_VERSION}") -set(ZIG_LIBC_LIB_DIR "" CACHE STRING "Default native target libc directory where crt1.o can be found") -set(ZIG_LIBC_STATIC_LIB_DIR "" CACHE STRING "Default native target libc directory where crtbeginT.o can be found") -set(ZIG_LIBC_INCLUDE_DIR "/usr/include" CACHE STRING "Default native target libc include directory") -set(ZIG_DYNAMIC_LINKER "" CACHE STRING "Override dynamic linker for native target") -set(ZIG_EACH_LIB_RPATH off CACHE BOOL "Add each dynamic library to rpath for native target") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_LIB_DIR_ESCAPED "${ZIG_LIBC_LIB_DIR}") diff --git a/README.md b/README.md index acc7e891e..d1e7e8f6d 100644 --- a/README.md +++ b/README.md @@ -138,14 +138,10 @@ libc. Create demo games using Zig. ##### POSIX -If you have gcc or clang installed, you can find out what `ZIG_LIBC_LIB_DIR`, -`ZIG_LIBC_STATIC_LIB_DIR`, and `ZIG_LIBC_INCLUDE_DIR` should be set to -(example below). - ``` mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -DZIG_LIBC_LIB_DIR=$(dirname $(cc -print-file-name=crt1.o)) -DZIG_LIBC_INCLUDE_DIR=$(echo -n | cc -E -x c - -v 2>&1 | grep -B1 "End of search list." | head -n1 | cut -c 2- | sed "s/ .*//") -DZIG_LIBC_STATIC_LIB_DIR=$(dirname $(cc -print-file-name=crtbegin.o)) +cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) make make install ./zig build --build-file ../build.zig test @@ -153,8 +149,6 @@ make install ##### MacOS -`ZIG_LIBC_LIB_DIR` and `ZIG_LIBC_STATIC_LIB_DIR` are unused. - ``` brew install cmake llvm@6 brew outdated llvm@6 || brew upgrade llvm@6 diff --git a/src/analyze.cpp b/src/analyze.cpp index 212833972..5dd15a694 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -4285,24 +4285,117 @@ static ZigWindowsSDK *get_windows_sdk(CodeGen *g) { return g->win_sdk; } + +Buf *get_linux_libc_lib_path(const char *o_file) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append(buf_ptr(buf_sprintf("-print-file-name=%s", o_file))); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + zig_panic("unable to determine libc lib path: executing C compiler: %s", err_str(err)); + } + if (term.how != TerminationIdClean || term.code != 0) { + zig_panic("unable to determine libc lib path: executing C compiler command failed"); + } + if (buf_ends_with_str(out_stdout, "\n")) { + buf_resize(out_stdout, buf_len(out_stdout) - 1); + } + if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, o_file)) { + zig_panic("unable to determine libc lib path: C compiler could not find %s", o_file); + } + Buf *result = buf_alloc(); + os_path_dirname(out_stdout, result); + return result; +} + +Buf *get_linux_libc_include_path(void) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append("-E"); + args.append("-Wp,-v"); + args.append("-xc"); + args.append("/dev/null"); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + zig_panic("unable to determine libc include path: executing C compiler: %s", err_str(err)); + } + if (term.how != TerminationIdClean || term.code != 0) { + zig_panic("unable to determine libc include path: executing C compiler command failed"); + } + char *prev_newline = buf_ptr(out_stderr); + ZigList search_paths = {}; + bool found_search_paths = false; + for (;;) { + char *newline = strchr(prev_newline, '\n'); + if (newline == nullptr) { + zig_panic("unable to determine libc include path: bad output from C compiler command"); + } + *newline = 0; + if (found_search_paths) { + if (strcmp(prev_newline, "End of search list.") == 0) { + break; + } + search_paths.append(prev_newline); + } else { + if (strcmp(prev_newline, "#include <...> search starts here:") == 0) { + found_search_paths = true; + } + } + prev_newline = newline + 1; + } + if (search_paths.length == 0) { + zig_panic("unable to determine libc include path: even C compiler does not know where libc headers are"); + } + for (size_t i = 0; i < search_paths.length; i += 1) { + // search in reverse order + const char *search_path = search_paths.items[search_paths.length - i - 1]; + // cut off spaces + while (*search_path == ' ') { + search_path += 1; + } + Buf *stdlib_path = buf_sprintf("%s/stdlib.h", search_path); + bool exists; + if ((err = os_file_exists(stdlib_path, &exists))) { + exists = false; + } + if (exists) { + return buf_create_from_str(search_path); + } + } + zig_panic("unable to determine libc include path: stdlib.h not found in C compiler search paths"); +} + void find_libc_include_path(CodeGen *g) { - if (!g->libc_include_dir || buf_len(g->libc_include_dir) == 0) { + if (g->libc_include_dir == nullptr) { if (g->zig_target.os == OsWindows) { ZigWindowsSDK *sdk = get_windows_sdk(g); if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) { zig_panic("Unable to determine libc include path."); } + } else if (g->zig_target.os == OsLinux) { + g->libc_include_dir = get_linux_libc_include_path(); + } else if (g->zig_target.os == OsMacOSX) { + g->libc_include_dir = buf_create_from_str("/usr/include"); + } else { + // TODO find libc at runtime for other operating systems + zig_panic("Unable to determine libc include path."); } - - // TODO find libc at runtime for other operating systems - zig_panic("Unable to determine libc include path."); } + assert(buf_len(g->libc_include_dir) != 0); } void find_libc_lib_path(CodeGen *g) { // later we can handle this better by reporting an error via the normal mechanism - if (!g->libc_lib_dir || buf_len(g->libc_lib_dir) == 0 || + if (g->libc_lib_dir == nullptr || (g->zig_target.os == OsWindows && (g->msvc_lib_dir == nullptr || g->kernel32_lib_dir == nullptr))) { if (g->zig_target.os == OsWindows) { @@ -4326,18 +4419,25 @@ void find_libc_lib_path(CodeGen *g) { g->msvc_lib_dir = vc_lib_dir; g->libc_lib_dir = ucrt_lib_path; g->kernel32_lib_dir = kern_lib_path; + } else if (g->zig_target.os == OsLinux) { + g->libc_lib_dir = get_linux_libc_lib_path("crt1.o"); } else { zig_panic("Unable to determine libc lib path."); } + } else { + assert(buf_len(g->libc_lib_dir) != 0); } - if (!g->libc_static_lib_dir || buf_len(g->libc_static_lib_dir) == 0) { + if (g->libc_static_lib_dir == nullptr) { if ((g->zig_target.os == OsWindows) && (g->msvc_lib_dir != NULL)) { return; - } - else { + } else if (g->zig_target.os == OsLinux) { + g->libc_static_lib_dir = get_linux_libc_lib_path("crtbegin.o"); + } else { zig_panic("Unable to determine libc static lib path."); } + } else { + assert(buf_len(g->libc_static_lib_dir) != 0); } } diff --git a/src/codegen.cpp b/src/codegen.cpp index 25b2ffbf1..0ebdf7fa3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -112,10 +112,10 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out // that's for native compilation g->zig_target = *target; resolve_target_object_format(&g->zig_target); - g->dynamic_linker = buf_create_from_str(""); - g->libc_lib_dir = buf_create_from_str(""); - g->libc_static_lib_dir = buf_create_from_str(""); - g->libc_include_dir = buf_create_from_str(""); + g->dynamic_linker = nullptr; + g->libc_lib_dir = nullptr; + g->libc_static_lib_dir = nullptr; + g->libc_include_dir = nullptr; g->msvc_lib_dir = nullptr; g->kernel32_lib_dir = nullptr; g->each_lib_rpath = false; @@ -123,16 +123,13 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out // native compilation, we can rely on the configuration stuff g->is_native_target = true; get_native_target(&g->zig_target); - g->dynamic_linker = buf_create_from_str(ZIG_DYNAMIC_LINKER); - g->libc_lib_dir = buf_create_from_str(ZIG_LIBC_LIB_DIR); - g->libc_static_lib_dir = buf_create_from_str(ZIG_LIBC_STATIC_LIB_DIR); - g->libc_include_dir = buf_create_from_str(ZIG_LIBC_INCLUDE_DIR); + g->dynamic_linker = nullptr; // find it at runtime + g->libc_lib_dir = nullptr; // find it at runtime + g->libc_static_lib_dir = nullptr; // find it at runtime + g->libc_include_dir = nullptr; // find it at runtime g->msvc_lib_dir = nullptr; // find it at runtime g->kernel32_lib_dir = nullptr; // find it at runtime - -#ifdef ZIG_EACH_LIB_RPATH g->each_lib_rpath = true; -#endif if (g->zig_target.os == OsMacOSX || g->zig_target.os == OsIOS) diff --git a/src/config.h.in b/src/config.h.in index 1fcc3fe12..16896273b 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -13,14 +13,6 @@ #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@ #define ZIG_VERSION_STRING "@ZIG_VERSION@" -#define ZIG_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@" -#define ZIG_LIBC_INCLUDE_DIR "@ZIG_LIBC_INCLUDE_DIR_ESCAPED@" -#define ZIG_LIBC_LIB_DIR "@ZIG_LIBC_LIB_DIR_ESCAPED@" -#define ZIG_LIBC_STATIC_LIB_DIR "@ZIG_LIBC_STATIC_LIB_DIR_ESCAPED@" -#define ZIG_DYNAMIC_LINKER "@ZIG_DYNAMIC_LINKER@" - -#cmakedefine ZIG_EACH_LIB_RPATH - // Only used for running tests before installing. #define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test" diff --git a/src/link.cpp b/src/link.cpp index f0537ffa0..57fa59b67 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -164,6 +164,34 @@ static void add_rpath(LinkJob *lj, Buf *rpath) { lj->rpath_table.put(rpath, true); } +static Buf *get_dynamic_linker_path(CodeGen *g) { + if (g->is_native_target && g->zig_target.arch.arch == ZigLLVM_x86_64) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append("-print-file-name=ld-linux-x86-64.so.2"); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + return target_dynamic_linker(&g->zig_target); + } + if (term.how != TerminationIdClean || term.code != 0) { + return target_dynamic_linker(&g->zig_target); + } + if (buf_ends_with_str(out_stdout, "\n")) { + buf_resize(out_stdout, buf_len(out_stdout) - 1); + } + if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, "ld-linux-x86-64.so.2")) { + return target_dynamic_linker(&g->zig_target); + } + return out_stdout; + } else { + return target_dynamic_linker(&g->zig_target); + } +} + static void construct_linker_job_elf(LinkJob *lj) { CodeGen *g = lj->codegen; @@ -259,12 +287,16 @@ static void construct_linker_job_elf(LinkJob *lj) { lj->args.append(buf_ptr(g->libc_static_lib_dir)); } - if (g->dynamic_linker && buf_len(g->dynamic_linker) > 0) { - lj->args.append("-dynamic-linker"); - lj->args.append(buf_ptr(g->dynamic_linker)); - } else { - lj->args.append("-dynamic-linker"); - lj->args.append(buf_ptr(target_dynamic_linker(&g->zig_target))); + if (!g->is_static) { + if (g->dynamic_linker != nullptr) { + assert(buf_len(g->dynamic_linker) != 0); + lj->args.append("-dynamic-linker"); + lj->args.append(buf_ptr(g->dynamic_linker)); + } else { + Buf *resolved_dynamic_linker = get_dynamic_linker_path(g); + lj->args.append("-dynamic-linker"); + lj->args.append(buf_ptr(resolved_dynamic_linker)); + } } if (shared) { diff --git a/src/main.cpp b/src/main.cpp index 791cb3b1b..f2f4cc970 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -195,13 +195,6 @@ static int find_zig_lib_dir(Buf *out_path) { } } - if (ZIG_INSTALL_PREFIX != nullptr) { - if (test_zig_install_prefix(buf_create_from_str(ZIG_INSTALL_PREFIX), out_path)) { - return 0; - } - } - - return ErrorFileNotFound; } diff --git a/src/os.cpp b/src/os.cpp index e31285461..90eecef3c 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -57,10 +57,6 @@ static clock_serv_t cclock; #include #include -// these implementations are lazy. But who cares, we'll make a robust -// implementation in the zig standard library and then this code all gets -// deleted when we self-host. it works for now. - #if defined(ZIG_OS_POSIX) static void populate_termination(Termination *term, int status) { if (WIFEXITED(status)) { @@ -929,7 +925,18 @@ int os_self_exe_path(Buf *out_path) { #elif defined(ZIG_OS_DARWIN) return ErrorFileNotFound; #elif defined(ZIG_OS_LINUX) - return ErrorFileNotFound; + buf_resize(out_path, 256); + for (;;) { + ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path)); + if (amt == -1) { + return ErrorUnexpected; + } + if (amt == (ssize_t)buf_len(out_path)) { + buf_resize(out_path, buf_len(out_path) * 2); + continue; + } + return 0; + } #endif return ErrorFileNotFound; } diff --git a/src/target.cpp b/src/target.cpp index 8e7c5ce57..5008b51a0 100644 --- a/src/target.cpp +++ b/src/target.cpp @@ -863,6 +863,10 @@ Buf *target_dynamic_linker(ZigTarget *target) { env == ZigLLVM_GNUX32) { return buf_create_from_str("/libx32/ld-linux-x32.so.2"); + } else if (arch == ZigLLVM_x86_64 && + (env == ZigLLVM_Musl || env == ZigLLVM_MuslEABI || env == ZigLLVM_MuslEABIHF)) + { + return buf_create_from_str("/lib/ld-musl-x86_64.so.1"); } else { return buf_create_from_str("/lib64/ld-linux-x86-64.so.2"); } From 5d5feb11deb797bbecb8f3676457a74056f09496 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 30 Mar 2018 17:24:56 -0400 Subject: [PATCH 040/132] appveyor and travis ci: stop passing unused configure args --- ci/appveyor/build_script.bat | 4 +--- ci/travis_linux_script | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/appveyor/build_script.bat b/ci/appveyor/build_script.bat index 5ae47df5e..9aee7a7bf 100644 --- a/ci/appveyor/build_script.bat +++ b/ci/appveyor/build_script.bat @@ -20,9 +20,7 @@ call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_ mkdir %ZIGBUILDDIR% cd %ZIGBUILDDIR% -cmake.exe .. -Thost=x64 -G"Visual Studio 14 2015 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGBUILDDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release "-DZIG_LIBC_INCLUDE_DIR=C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt" "-DZIG_LIBC_LIB_DIR=C:\Program Files (x86)\Windows Kits\10\bin\x64\ucrt" "-DZIG_LIBC_STATIC_LIB_DIR=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64" || exit /b +cmake.exe .. -Thost=x64 -G"Visual Studio 14 2015 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGBUILDDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release || exit /b msbuild /p:Configuration=Release INSTALL.vcxproj || exit /b bin\zig.exe build --build-file ..\build.zig test || exit /b - -@echo "MSVC build succeeded" diff --git a/ci/travis_linux_script b/ci/travis_linux_script index d6b9eb923..a709539bc 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -8,7 +8,7 @@ export CXX=clang++-6.0 echo $PATH mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -DZIG_LIBC_LIB_DIR=$(dirname $($CC -print-file-name=crt1.o)) -DZIG_LIBC_INCLUDE_DIR=$(echo -n | $CC -E -x c - -v 2>&1 | grep -B1 "End of search list." | head -n1 | cut -c 2- | sed "s/ .*//") -DZIG_LIBC_STATIC_LIB_DIR=$(dirname $($CC -print-file-name=crtbegin.o)) +cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) make VERBOSE=1 make install ./zig build --build-file ../build.zig test From 5118caf5ab2599ca61b57f68a89aa2d094f51981 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 00:53:00 +0200 Subject: [PATCH 041/132] Added a lot of test cases --- std/zig/parser.zig | 150 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 62c62ed18..3b532d703 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1648,6 +1648,13 @@ test "zig fmt" { \\ ); + try testCanonical( + \\extern fn f1(s: [][]align(1) []const []volatile u8) c_int; + \\extern fn f2(s: []align(1) const []align(1) volatile []const volatile u8) c_int; + \\extern fn f3(s: []align(1) const volatile u8) c_int; + \\ + ); + try testCanonical( \\fn f1(a: bool, b: bool) bool { \\ a != b; @@ -1716,7 +1723,7 @@ test "zig fmt" { try testCanonical( \\test "prefix operators" { - \\ --%~??!*&0; + \\ try return --%~??!*&0; \\} \\ ); @@ -1730,4 +1737,145 @@ test "zig fmt" { \\} \\ ); + + try testCanonical( + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\} + \\ + ); + + try testCanonical( + \\test "test array" { + \\ const a : [2]u8 = [2]u8{ 1, 2 }; + \\ const a : [2]u8 = []u8{ 1, 2 }; + \\ const a : [0]u8 = []u8{}; + \\} + \\ + ); + +// PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType + try testCanonical( + \\test "values" { + \\ 1; + \\ 1.0; + \\ "string"; + \\ c"cstring"; + \\ \\ Multi + \\ \\ line + \\ \\ string + \\ ; + \\ 'c'; + \\ true; + \\ false; + \\ null; + \\ undefined; + \\ error; + \\ this; + \\ unreachable; + \\ suspend; + \\} + \\ + ); + + try testCanonical( + \\test "percendence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{}; + \\ !(a{}); + \\ a + b{}; + \\ (a + b){}; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + ); + + try testCanonical( + \\const S = struct { + \\ const Self = this; + \\ f1: u8, + \\ + \\ fn method(self: &Self) Self { + \\ return *self; + \\ } + \\ + \\ f2: u8, + \\}; + \\ + \\const Ps = packed struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + \\const Es = extern struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8, + \\}; + \\ + ); + + try testCanonical( + \\const E = enum { + \\ Ok, + \\ SomethingElse = 0, + \\}; + \\ + \\const E2 = enum(u8) { + \\ Ok, + \\ SomethingElse = 255, + \\ SomethingThird, + \\}; + \\ + \\const Ee = extern enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + \\const Ep = packed enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird, + \\}; + \\ + ); + + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); + + try testCanonical(@embedFile("ast.zig")); + try testCanonical(@embedFile("index.zig")); + try testCanonical(@embedFile("parser.zig")); + try testCanonical(@embedFile("tokenizer.zig")); } From c3724ec506a9e3a4b23a145a733050c17321da9d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 31 Mar 2018 02:12:44 -0400 Subject: [PATCH 042/132] implement os_self_exe_path in the c++ compiler for darwin ported from the zig std lib this fixes looking for zig std lib at runtime on darwin --- src/os.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/os.cpp b/src/os.cpp index 90eecef3c..de1a1f21d 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -45,6 +45,7 @@ typedef SSIZE_T ssize_t; #if defined(__MACH__) #include #include +#include #endif #if defined(ZIG_OS_WINDOWS) @@ -923,7 +924,13 @@ int os_self_exe_path(Buf *out_path) { } #elif defined(ZIG_OS_DARWIN) - return ErrorFileNotFound; + uint32_t u32_len = 0; + int ret1 = _NSGetExecutablePath(nullptr, &u32_len); + assert(ret1 != 0); + buf_resize(out_path, u32_len); + int ret2 = _NSGetExecutablePath(buf_ptr(out_path), &u32_len); + assert(ret2 == 0); + return 0; #elif defined(ZIG_OS_LINUX) buf_resize(out_path, 256); for (;;) { From 7d66908f294eed1138802c060185721a2e265f3b Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Sat, 31 Mar 2018 23:17:02 +1300 Subject: [PATCH 043/132] docs: fix unclosed code tag --- doc/langref.html.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 04ca5314c..7f837186b 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2873,7 +2873,7 @@ const err = (error {FileNotFound}).FileNotFound; {#header_close#} {#header_open|Error Union Type#}

- An error set type and normal type can be combined with the ! + An error set type and normal type can be combined with the ! binary operator to form an error union type. You are likely to use an error union type more often than an error set type by itself.

From 596f4b6002d3ed29adc8777b0417952cb79ad888 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 14:00:49 +0200 Subject: [PATCH 044/132] Fixed review commented code --- std/zig/parser.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3b532d703..b9246033d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1750,9 +1750,9 @@ test "zig fmt" { try testCanonical( \\test "test array" { - \\ const a : [2]u8 = [2]u8{ 1, 2 }; - \\ const a : [2]u8 = []u8{ 1, 2 }; - \\ const a : [0]u8 = []u8{}; + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; \\} \\ ); @@ -1782,7 +1782,7 @@ test "zig fmt" { ); try testCanonical( - \\test "percendence" { + \\test "precendence" { \\ a!b(); \\ (a!b)(); \\ !a!b; From 26e56f2fab7c56829d51db1afb10acb0306a101f Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 14:18:09 +0200 Subject: [PATCH 045/132] Each test now have it's own test name --- std/zig/parser.zig | 122 +++++++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index b9246033d..9b493d3b4 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1557,7 +1557,7 @@ fn testCanonical(source: []const u8) !void { } } -test "zig fmt" { +test "zig fmt: get stdout or fail" { try testCanonical( \\const std = @import("std"); \\ @@ -1568,7 +1568,9 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: preserve spacing" { try testCanonical( \\const std = @import("std"); \\ @@ -1581,25 +1583,33 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: return types" { try testCanonical( \\pub fn main() !void {} \\pub fn main() var {} \\pub fn main() i32 {} \\ ); +} +test "zig fmt: imports" { try testCanonical( \\const std = @import("std"); \\const std = @import(); \\ ); +} +test "zig fmt: extern function" { try testCanonical( \\extern fn puts(s: &const u8) c_int; \\ ); +} +test "zig fmt: global declarations" { try testCanonical( \\const a = b; \\pub const a = b; @@ -1611,66 +1621,71 @@ test "zig fmt" { \\pub var a: i32 = b; \\ ); +} +test "zig fmt: extern declaration" { try testCanonical( \\extern var foo: c_int; \\ ); +} - try testCanonical( +test "zig fmt: alignment" { + try testCanonical( \\var foo: c_int align(1); \\ ); +} +test "zig fmt: C main" { try testCanonical( \\fn main(argc: c_int, argv: &&u8) c_int { \\ const a = b; \\} \\ ); +} +test "zig fmt: return" { try testCanonical( \\fn foo(argc: c_int, argv: &&u8) c_int { \\ return 0; \\} \\ ); +} +test "zig fmt: pointer attributes" { try testCanonical( \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; \\ ); +} +test "zig fmt: slice attributes" { try testCanonical( - \\extern fn f1(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f2(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f3(s: &align(1) const volatile u8) c_int; + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; \\ ); +} - try testCanonical( - \\extern fn f1(s: [][]align(1) []const []volatile u8) c_int; - \\extern fn f2(s: []align(1) const []align(1) volatile []const volatile u8) c_int; - \\extern fn f3(s: []align(1) const volatile u8) c_int; - \\ - ); - - try testCanonical( - \\fn f1(a: bool, b: bool) bool { - \\ a != b; - \\ return a == b; - \\} - \\ - ); - - try testCanonical( +test "zig fmt: test declaration" { + try testCanonical( \\test "test name" { \\ const a = 1; \\ var b = 1; \\} \\ ); +} +test "zig fmt: infix operators" { try testCanonical( \\test "infix operators" { \\ var i = undefined; @@ -1720,14 +1735,18 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: prefix operators" { try testCanonical( \\test "prefix operators" { \\ try return --%~??!*&0; \\} \\ ); +} +test "zig fmt: call expression" { try testCanonical( \\test "test calls" { \\ a(); @@ -1737,27 +1756,9 @@ test "zig fmt" { \\} \\ ); +} - try testCanonical( - \\test "test index" { - \\ a[0]; - \\ a[0 + 5]; - \\ a[0..]; - \\ a[0..5]; - \\} - \\ - ); - - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); - -// PrimaryExpression = Integer | Float | String | CharLiteral | KeywordLiteral | GroupedExpression | BlockExpression(BlockOrExpression) | Symbol | ("@" Symbol FnCallExpression) | ArrayType | FnProto | AsmExpression | ContainerDecl | ("continue" option(":" Symbol)) | ErrorSetDecl | PromiseType +test "zig fmt: values" { try testCanonical( \\test "values" { \\ 1; @@ -1780,9 +1781,34 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: indexing" { try testCanonical( - \\test "precendence" { + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\} + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { \\ a!b(); \\ (a!b)(); \\ !a!b; @@ -1811,7 +1837,9 @@ test "zig fmt" { \\} \\ ); +} +test "zig fmt: struct declaration" { try testCanonical( \\const S = struct { \\ const Self = this; @@ -1839,8 +1867,10 @@ test "zig fmt" { \\}; \\ ); +} - try testCanonical( +test "zig fmt: enum declaration" { + try testCanonical( \\const E = enum { \\ Ok, \\ SomethingElse = 0, @@ -1865,7 +1895,9 @@ test "zig fmt" { \\}; \\ ); +} +test "zig fmt: container initializers" { try testCanonical( \\const a1 = []u8{ }; \\const a2 = []u8{ 1, 2, 3, 4 }; @@ -1873,9 +1905,11 @@ test "zig fmt" { \\const s2 = S{ .a = 1, .b = 2, }; \\ ); +} +test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); try testCanonical(@embedFile("parser.zig")); try testCanonical(@embedFile("tokenizer.zig")); -} +} \ No newline at end of file From cda35093537221abe7c148c57b3a28892046b8b2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 15:39:51 +0200 Subject: [PATCH 046/132] Added test cases to cover all of zigs syntax --- std/zig/parser.zig | 352 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 345 insertions(+), 7 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9b493d3b4..22e87dd17 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1602,13 +1602,6 @@ test "zig fmt: imports" { ); } -test "zig fmt: extern function" { - try testCanonical( - \\extern fn puts(s: &const u8) c_int; - \\ - ); -} - test "zig fmt: global declarations" { try testCanonical( \\const a = b; @@ -1758,6 +1751,21 @@ test "zig fmt: call expression" { ); } +test "zig fmt: var args" { + try testCanonical( + \\fn print(args: ...) void {} + \\ + ); +} + +test "zig fmt: extern function" { + try testCanonical( + \\extern fn puts(s: &const u8) c_int; + \\extern "c" fn puts(s: &const u8) c_int; + \\ + ); +} + test "zig fmt: values" { try testCanonical( \\test "values" { @@ -1897,6 +1905,41 @@ test "zig fmt: enum declaration" { ); } +test "zig fmt: union declaration" { + try testCanonical( + \\const U = union { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const Ue = union(enum) { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const E = enum { + \\ Int, + \\ Float, + \\ Bool, + \\}; + \\ + \\const Ue2 = union(E) { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + \\const Eu = extern union { + \\ Int: u8, + \\ Float: f32, + \\ Bool: bool, + \\}; + \\ + ); +} + test "zig fmt: container initializers" { try testCanonical( \\const a1 = []u8{ }; @@ -1907,6 +1950,301 @@ test "zig fmt: container initializers" { ); } +test "zig fmt: switch" { + try testCanonical( + \\test "switch" { + \\ switch (0) { + \\ 0 => {}, + \\ 1 => unreachable, + \\ 2, 3 => {}, + \\ 4 ... 7 => {}, + \\ 1 + 4 * 3 + 22 => {}, + \\ else => { + \\ const a = 1; + \\ const b = a; + \\ }, + \\ } + \\ + \\ const res = switch (0) { + \\ 0 => 0, + \\ 1 => 2, + \\ else => 4, + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64, + \\ }; + \\ + \\ const u = Union { .Int = 0 }; + \\ switch (u) { + \\ Union.Int => |int| {}, + \\ Union.Float => |*float| unreachable, + \\ } + \\} + \\ + ); +} + +test "zig fmt: while" { + try testCanonical( + \\test "while" { + \\ while (10 < 1) { + \\ unreachable; + \\ } + \\ + \\ while (10 < 1) + \\ unreachable; + \\ + \\ var i: usize = 0; + \\ while (i < 10) : (i += 1) { + \\ continue; + \\ } + \\ + \\ i = 0; + \\ while (i < 10) : (i += 1) + \\ continue; + \\ + \\ i = 0; + \\ var j usize = 0; + \\ while (i < 10) : ({ i += 1; j += 1; }) { + \\ continue; + \\ } + \\ + \\ var a: ?u8 = 2; + \\ while (a) |v| : (a = null) { + \\ continue; + \\ } + \\ + \\ while (a) |v| : (a = null) + \\ unreachable; + \\ + \\ label: while (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ const res = while (0 < 10) { + \\ break 7; + \\ } else { + \\ unreachable; + \\ } + \\ + \\ var a: error!u8 = 0; + \\ while (a) |v| { + \\ a = error.Err; + \\ } else |err| { + \\ i = 1; + \\ } + \\ + \\ comptime var k: usize = 0; + \\ inline while (i < 10) (i += 1) + \\ j += 2; + \\} + \\ + ); +} + +test "zig fmt: for" { + try testCanonical( + \\test "for" { + \\ const a = []u8{ 1, 2, 3 }; + \\ for (a) |v| { + \\ continue; + \\ } + \\ + \\ for (a) |v| + \\ continue; + \\ + \\ for (a) |*v| + \\ continue; + \\ + \\ for (a) |v, i| { + \\ continue; + \\ } + \\ + \\ for (a) |v, i| + \\ continue; + \\ + \\ const res = for (a) |v, i| { + \\ breal v; + \\ } else { + \\ unreachable; + \\ } + \\ + \\ var num: usize = 0; + \\ inline for (a) |v, i| { + \\ num += v; + \\ num += i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: if" { + try testCanonical( + \\test "if" { + \\ if (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ if (10 < 0) + \\ unreachable; + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else if (5 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ const is_world_broken = if (10 < 0) true else false; + \\ + \\ const a: ?u8 = 10; + \\ const b: ?u8 = null; + \\ if (a) |v| { + \\ const some = v; + \\ } else if (b) |*v| { + \\ unreachable; + \\ } else { + \\ const some = 10; + \\ } + \\ + \\ const non_null_a = if (a) |v| v else 0; + \\ + \\ const a_err: error!u8 = 0; + \\ if (a_err) |v| { + \\ const p = v; + \\ } else |err| { + \\ unreachable; + \\ } + \\} + \\ + ); +} + +test "zig fmt: defer" { + try testCanonical( + \\test "defer" { + \\ var i: usize = 0; + \\ defer i = 1; + \\ defer { + \\ i += 2; + \\ i *= i; + \\ } + \\ + \\ errdefer i += 3; + \\ errdefer { + \\ i += 2; + \\ i /= i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: comptime" { + try testCanonical( + \\fn a() u8 { + \\ return 5; + \\} + \\ + \\fn b(comptime i: u8) u8 { + \\ return i; + \\} + \\ + \\const av = comptime a(); + \\const av2 = comptime blk: { + \\ var res = a(); + \\ res *= b(2); + \\ break :blk res; + \\}; + \\ + \\comptime { + \\ _ = a(); + \\} + \\ + \\test "comptime" { + \\ const av3 = comptime a(); + \\ const av4 = comptime blk: { + \\ var res = a(); + \\ res *= a(); + \\ break :blk res; + \\ }; + \\ + \\ comptime var i = 0; + \\ comptime { + \\ i = a(); + \\ i += b(i); + \\ } + \\} + \\ + ); +} + +test "zig fmt: fn type" { + try testCanonical( + \\fn a(i: u8) u8 { + \\ return i + 1; + \\} + \\ + \\const ap: fn(u8) u8 = a; + \\ + ); +} + +test "zig fmt: inline asm" { + try testCanonical( + \\pub fn syscall1(number: usize, arg1: usize) usize { + \\ return asm volatile ("syscall" + \\ : [ret] "={rax}" (-> usize) + \\ : [number] "{rax}" (number), + \\ [arg1] "{rdi}" (arg1) + \\ : "rcx", "r11"); + \\} + \\ + ); +} + +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| { + \\ } + \\ const p = async simpleAsyncFn() cache unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 8f962a957a3645342fba8219cf8f33d0ac42e16d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 31 Mar 2018 11:26:02 -0400 Subject: [PATCH 047/132] fix regressions on windows --- src/analyze.cpp | 1 + src/link.cpp | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 5dd15a694..291e7e764 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -4378,6 +4378,7 @@ void find_libc_include_path(CodeGen *g) { if (g->zig_target.os == OsWindows) { ZigWindowsSDK *sdk = get_windows_sdk(g); + g->libc_include_dir = buf_alloc(); if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) { zig_panic("Unable to determine libc include path."); } diff --git a/src/link.cpp b/src/link.cpp index 57fa59b67..c089d6961 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -455,7 +455,9 @@ static void construct_linker_job_coff(LinkJob *lj) { lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->kernel32_lib_dir)))); lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_lib_dir)))); - lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_static_lib_dir)))); + if (g->libc_static_lib_dir != nullptr) { + lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_static_lib_dir)))); + } } if (lj->link_in_crt) { From 4793c3397e17169317d2feded9ca6901ff0e99e8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 17:46:29 +0200 Subject: [PATCH 048/132] std.zig.parser now handles lib name for extern var and fn --- std/zig/ast.zig | 3 +- std/zig/parser.zig | 162 ++++++++++++++++++++++++++++----------------- 2 files changed, 102 insertions(+), 63 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 715a333c0..e64e5931c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -624,7 +624,7 @@ pub const NodeLineComment = struct { pub const NodeTestDecl = struct { base: Node, test_token: Token, - name_token: Token, + name: &Node, body_node: &Node, pub fn iterate(self: &NodeTestDecl, index: usize) ?&Node { @@ -644,4 +644,3 @@ pub const NodeTestDecl = struct { return self.body_node.lastToken(); } }; - diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 22e87dd17..1bad72aea 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -55,6 +55,7 @@ pub const Parser = struct { const TopLevelDeclCtx = struct { visib_token: ?Token, extern_token: ?Token, + lib_name: ?&ast.Node, }; const DestPtr = union(enum) { @@ -183,8 +184,9 @@ pub const Parser = struct { if (lbrace.id != Token.Id.LBrace) return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); - const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, name_token, block); + const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); try stack.append(State { .Block = block }); continue; }, @@ -202,10 +204,22 @@ pub const Parser = struct { State.TopLevelExtern => |visib_token| { const token = self.getNextToken(); if (token.id == Token.Id.Keyword_extern) { + const lib_name_token = self.getNextToken(); + const lib_name = blk: { + if (lib_name_token.id == Token.Id.StringLiteral) { + const res = try self.createStringLiteral(arena, lib_name_token); + break :blk &res.base; + } else { + self.putBackToken(lib_name_token); + break :blk null; + } + }; + stack.append(State { .TopLevelDecl = TopLevelDeclCtx { .visib_token = visib_token, .extern_token = token, + .lib_name = lib_name, }, }) catch unreachable; continue; @@ -215,6 +229,7 @@ pub const Parser = struct { .TopLevelDecl = TopLevelDeclCtx { .visib_token = visib_token, .extern_token = null, + .lib_name = null, }, }) catch unreachable; continue; @@ -226,7 +241,7 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, &root_node.decls, ctx.visib_token, - token, (?Token)(null), ctx.extern_token); + token, (?Token)(null), ctx.extern_token, ctx.lib_name); try stack.append(State { .VarDecl = var_decl_node }); continue; }, @@ -234,20 +249,17 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, token, - ctx.extern_token, (?Token)(null), ctx.visib_token, (?Token)(null)); + ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); continue; }, - Token.Id.StringLiteral => { - @panic("TODO extern with string literal"); - }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, fn_token, - ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null)); + ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); continue; @@ -437,11 +449,7 @@ pub const Parser = struct { continue; }, Token.Id.StringLiteral => { - const node = try arena.create(ast.NodeStringLiteral); - *node = ast.NodeStringLiteral { - .base = self.initNode(ast.Node.Id.StringLiteral), - .token = token, - }; + const node = try self.createStringLiteral(arena, token); try stack.append(State { .Operand = &node.base }); @@ -722,7 +730,7 @@ pub const Parser = struct { if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(comptime_token), (?Token)(null)); + mut_token, (?Token)(comptime_token), (?Token)(null), null); try stack.append(State { .VarDecl = var_decl }); continue; } @@ -736,7 +744,7 @@ pub const Parser = struct { if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(null), (?Token)(null)); + mut_token, (?Token)(null), (?Token)(null), null); try stack.append(State { .VarDecl = var_decl }); continue; } @@ -852,7 +860,7 @@ pub const Parser = struct { } fn createVarDecl(self: &Parser, arena: &mem.Allocator, visib_token: &const ?Token, mut_token: &const Token, - comptime_token: &const ?Token, extern_token: &const ?Token) !&ast.NodeVarDecl + comptime_token: &const ?Token, extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl { const node = try arena.create(ast.NodeVarDecl); @@ -865,7 +873,7 @@ pub const Parser = struct { .type_node = null, .align_node = null, .init_node = null, - .lib_name = null, + .lib_name = lib_name, // initialized later .name_token = undefined, .eq_token = undefined, @@ -874,7 +882,18 @@ pub const Parser = struct { return node; } - fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name_token: &const Token, + fn createStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !&ast.NodeStringLiteral { + const node = try arena.create(ast.NodeStringLiteral); + + assert(token.id == Token.Id.StringLiteral); + *node = ast.NodeStringLiteral { + .base = self.initNode(ast.Node.Id.StringLiteral), + .token = *token, + }; + return node; + } + + fn createTestDecl(self: &Parser, arena: &mem.Allocator, test_token: &const Token, name: &ast.Node, block: &ast.NodeBlock) !&ast.NodeTestDecl { const node = try arena.create(ast.NodeTestDecl); @@ -882,14 +901,14 @@ pub const Parser = struct { *node = ast.NodeTestDecl { .base = self.initNode(ast.Node.Id.TestDecl), .test_token = *test_token, - .name_token = *name_token, + .name = name, .body_node = &block.base, }; return node; } fn createFnProto(self: &Parser, arena: &mem.Allocator, fn_token: &const Token, extern_token: &const ?Token, - cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto + lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { const node = try arena.create(ast.NodeFnProto); @@ -905,7 +924,7 @@ pub const Parser = struct { .inline_token = *inline_token, .cc_token = *cc_token, .body_node = null, - .lib_name = null, + .lib_name = lib_name, .align_expr = null, }; return node; @@ -1015,27 +1034,27 @@ pub const Parser = struct { } fn createAttachFnProto(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), fn_token: &const Token, - extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token, + extern_token: &const ?Token, lib_name: ?&ast.Node, cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto { - const node = try self.createFnProto(arena, fn_token, extern_token, cc_token, visib_token, inline_token); + const node = try self.createFnProto(arena, fn_token, extern_token, lib_name, cc_token, visib_token, inline_token); try list.append(&node.base); return node; } fn createAttachVarDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token, - extern_token: &const ?Token) !&ast.NodeVarDecl + extern_token: &const ?Token, lib_name: ?&ast.Node) !&ast.NodeVarDecl { - const node = try self.createVarDecl(arena, visib_token, mut_token, comptime_token, extern_token); + const node = try self.createVarDecl(arena, visib_token, mut_token, comptime_token, extern_token, lib_name); try list.append(&node.base); return node; } fn createAttachTestDecl(self: &Parser, arena: &mem.Allocator, list: &ArrayList(&ast.Node), - test_token: &const Token, name_token: &const Token, block: &ast.NodeBlock) !&ast.NodeTestDecl + test_token: &const Token, name: &ast.Node, block: &ast.NodeBlock) !&ast.NodeTestDecl { - const node = try self.createTestDecl(arena, test_token, name_token, block); + const node = try self.createTestDecl(arena, test_token, name, block); try list.append(&node.base); return node; } @@ -1169,23 +1188,6 @@ pub const Parser = struct { switch (decl.id) { ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl); - if (fn_proto.visib_token) |visib_token| { - switch (visib_token.id) { - Token.Id.Keyword_pub => try stream.print("pub "), - Token.Id.Keyword_export => try stream.print("export "), - else => unreachable, - } - } - if (fn_proto.extern_token) |extern_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token)); - } - try stream.print("fn"); - - if (fn_proto.name_token) |name_token| { - try stream.print(" {}", self.tokenizer.getTokenSlice(name_token)); - } - - try stream.print("("); if (fn_proto.body_node == null) { try stack.append(RenderState { .Text = ";" }); @@ -1201,6 +1203,27 @@ pub const Parser = struct { try stack.append(RenderState { .Text = ", " }); } } + + try stack.append(RenderState { .Text = "(" }); + if (fn_proto.name_token) |name_token| { + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); + } + + try stack.append(RenderState { .Text = "fn " }); + if (fn_proto.lib_name) |lib_name| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = lib_name }); + } + if (fn_proto.extern_token) |extern_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (fn_proto.visib_token) |visib_token| { + assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); + } }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); @@ -1208,29 +1231,16 @@ pub const Parser = struct { }, ast.Node.Id.TestDecl => { const test_decl = @fieldParentPtr(ast.NodeTestDecl, "base", decl); - try stream.print("test {} ", self.tokenizer.getTokenSlice(test_decl.name_token)); + try stream.print("test "); try stack.append(RenderState { .Expression = test_decl.body_node }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = test_decl.name }); }, else => unreachable, } }, RenderState.VarDecl => |var_decl| { - if (var_decl.visib_token) |visib_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token)); - } - if (var_decl.extern_token) |extern_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token)); - if (var_decl.lib_name != null) { - @panic("TODO"); - } - } - if (var_decl.comptime_token) |comptime_token| { - try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token)); - } - try stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token)); - try stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token)); - try stack.append(RenderState { .Text = ";" }); if (var_decl.init_node) |init_node| { try stack.append(RenderState { .Expression = init_node }); @@ -1242,8 +1252,30 @@ pub const Parser = struct { try stack.append(RenderState { .Text = " align(" }); } if (var_decl.type_node) |type_node| { - try stream.print(": "); try stack.append(RenderState { .Expression = type_node }); + try stack.append(RenderState { .Text = ": " }); + } + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(var_decl.name_token) }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(var_decl.mut_token) }); + + if (var_decl.comptime_token) |comptime_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(comptime_token) }); + } + + if (var_decl.extern_token) |extern_token| { + if (var_decl.lib_name != null) { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = ??var_decl.lib_name }); + } + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (var_decl.visib_token) |visib_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); } }, @@ -1612,6 +1644,14 @@ test "zig fmt: global declarations" { \\pub const a: i32 = b; \\var a: i32 = b; \\pub var a: i32 = b; + \\extern const a: i32 = b; + \\pub extern const a: i32 = b; + \\extern var a: i32 = b; + \\pub extern var a: i32 = b; + \\extern "a" const a: i32 = b; + \\pub extern "a" const a: i32 = b; + \\extern "a" var a: i32 = b; + \\pub extern "a" var a: i32 = b; \\ ); } @@ -2232,7 +2272,7 @@ test "zig fmt: coroutines" { \\ x += 1; \\ suspend |p| { \\ } - \\ const p = async simpleAsyncFn() cache unreachable; + \\ const p = async simpleAsyncFn() catch unreachable; \\ await p; \\} \\ From eb6ff796c17b50969a8f0171ba089f31667b7278 Mon Sep 17 00:00:00 2001 From: Raul Leal Date: Sat, 31 Mar 2018 17:21:19 +0100 Subject: [PATCH 049/132] Fix undeclared identifier error in readUntilDelimiterBuffer and incorrect number of parameters in readUntilDelimiterAlloc (#877) --- std/io.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/std/io.zig b/std/io.zig index 94685c4d0..952d9081f 100644 --- a/std/io.zig +++ b/std/io.zig @@ -144,7 +144,7 @@ pub fn InStream(comptime ReadError: type) type { /// If `buffer.len()` would exceed `max_size`, `error.StreamTooLong` is returned and the contents /// read from the stream so far are lost. pub fn readUntilDelimiterBuffer(self: &Self, buffer: &Buffer, delimiter: u8, max_size: usize) !void { - try buf.resize(0); + try buffer.resize(0); while (true) { var byte: u8 = try self.readByte(); @@ -153,11 +153,11 @@ pub fn InStream(comptime ReadError: type) type { return; } - if (buf.len() == max_size) { + if (buffer.len() == max_size) { return error.StreamTooLong; } - try buf.appendByte(byte); + try buffer.appendByte(byte); } } @@ -171,7 +171,7 @@ pub fn InStream(comptime ReadError: type) type { var buf = Buffer.initNull(allocator); defer buf.deinit(); - try self.readUntilDelimiterBuffer(self, &buf, delimiter, max_size); + try self.readUntilDelimiterBuffer(&buf, delimiter, max_size); return buf.toOwnedSlice(); } From 4d8f9e2295bb672f70550b3bb5a4cb667f68bb70 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 21:04:54 +0200 Subject: [PATCH 050/132] std.zig.parser now parses multi line strings --- std/zig/ast.zig | 21 +++++++++++++++++ std/zig/parser.zig | 52 +++++++++++++++++++++++++++++++++++++++---- std/zig/tokenizer.zig | 40 ++++++++++++++++++++++++++++++++- 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index e64e5931c..75003b06c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { IntegerLiteral, FloatLiteral, StringLiteral, + MultilineStringLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -40,6 +41,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -61,6 +63,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -82,6 +85,7 @@ pub const Node = struct { Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), + Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -587,6 +591,23 @@ pub const NodeStringLiteral = struct { } }; +pub const NodeMultilineStringLiteral = struct { + base: Node, + tokens: ArrayList(Token), + + pub fn iterate(self: &NodeMultilineStringLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeMultilineStringLiteral) Token { + return self.tokens.at(0); + } + + pub fn lastToken(self: &NodeMultilineStringLiteral) Token { + return self.tokens.at(self.tokens.len - 1); + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 1bad72aea..fa09a26db 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -456,6 +456,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.MultilineStringLiteralLine => { + const node = try arena.create(ast.NodeMultilineStringLiteral); + *node = ast.NodeMultilineStringLiteral { + .base = self.initNode(ast.Node.Id.MultilineStringLiteral), + .tokens = ArrayList(Token).init(arena), + }; + try node.tokens.append(token); + + while (true) { + const multiline_str = self.getNextToken(); + if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { + self.putBackToken(multiline_str); + break; + } + + try node.tokens.append(multiline_str); + } + + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } @@ -1427,6 +1451,20 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.MultilineStringLiteral => { + const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); + try stream.print("\n"); + + var i : usize = 0; + indent += 4; + while (i < multiline_str_literal.tokens.len) : (i += 1) { + const t = multiline_str_literal.tokens.at(i); + try stream.writeByteNTimes(' ', indent); + try stream.print("{}", self.tokenizer.getTokenSlice(t)); + } + try stream.writeByteNTimes(' ', indent); + indent -= 4; + }, ast.Node.Id.UndefinedLiteral => { const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token)); @@ -1806,6 +1844,16 @@ test "zig fmt: extern function" { ); } +test "zig fmt: multiline string" { + try testCanonical( + \\const s = + \\ \\ something + \\ \\ something else + \\ ; + \\ + ); +} + test "zig fmt: values" { try testCanonical( \\test "values" { @@ -1813,10 +1861,6 @@ test "zig fmt: values" { \\ 1.0; \\ "string"; \\ c"cstring"; - \\ \\ Multi - \\ \\ line - \\ \\ string - \\ ; \\ 'c'; \\ true; \\ false; diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 7a13d8997..5647fcb86 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -72,6 +72,7 @@ pub const Token = struct { Invalid, Identifier, StringLiteral: StrLitKind, + MultilineStringLiteralLine: StrLitKind, StringIdentifier, Eof, Builtin, @@ -225,6 +226,9 @@ pub const Tokenizer = struct { C, StringLiteral, StringLiteralBackslash, + MultilineStringLiteralLine, + MultilineStringLiteralLineBackslash, + Backslash, Equal, Bang, Pipe, @@ -352,6 +356,10 @@ pub const Tokenizer = struct { '^' => { state = State.Caret; }, + '\\' => { + state = State.Backslash; + result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.Normal }; + }, '{' => { result.id = Token.Id.LBrace; self.index += 1; @@ -532,8 +540,17 @@ pub const Tokenizer = struct { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => break, }, + State.Backslash => switch (c) { + '\\' => { + state = State.MultilineStringLiteralLine; + }, + else => break, + }, State.C => switch (c) { - '\\' => @panic("TODO"), + '\\' => { + state = State.Backslash; + result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.C }; + }, '"' => { state = State.StringLiteral; result.id = Token.Id { .StringLiteral = Token.StrLitKind.C }; @@ -562,6 +579,24 @@ pub const Tokenizer = struct { }, }, + State.MultilineStringLiteralLine => switch (c) { + '\\' => { + state = State.MultilineStringLiteralLineBackslash; + }, + '\n' => { + self.index += 1; + break; + }, + else => self.checkLiteralCharacter(), + }, + + State.MultilineStringLiteralLineBackslash => switch (c) { + '\n' => break, // Look for this error later. + else => { + state = State.MultilineStringLiteralLine; + }, + }, + State.Bang => switch (c) { '=' => { result.id = Token.Id.BangEqual; @@ -811,6 +846,7 @@ pub const Tokenizer = struct { State.FloatFraction, State.FloatExponentNumber, State.StringLiteral, // find this error later + State.MultilineStringLiteralLine, State.Builtin => {}, State.Identifier => { @@ -825,6 +861,8 @@ pub const Tokenizer = struct { State.NumberDot, State.FloatExponentUnsigned, State.SawAtSign, + State.Backslash, + State.MultilineStringLiteralLineBackslash, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, From 975dc5a390490794df367b6a9869609a8b14d8f8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 21:28:40 +0200 Subject: [PATCH 051/132] std.zig.parser now parses char literals --- std/zig/ast.zig | 21 ++++++++++++++++++++ std/zig/parser.zig | 16 +++++++++++++++ std/zig/tokenizer.zig | 46 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 75003b06c..1c84e1264 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { FloatLiteral, StringLiteral, MultilineStringLiteral, + CharLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -42,6 +43,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -64,6 +66,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -86,6 +89,7 @@ pub const Node = struct { Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), + Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct { } }; +pub const NodeCharLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeCharLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeCharLiteral) Token { + return self.token; + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index fa09a26db..a05328660 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -456,6 +456,18 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.CharLiteral => { + const node = try arena.create(ast.NodeCharLiteral); + *node = ast.NodeCharLiteral { + .base = self.initNode(ast.Node.Id.CharLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1451,6 +1463,10 @@ pub const Parser = struct { const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token)); }, + ast.Node.Id.CharLiteral => { + const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 5647fcb86..64eb62759 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -73,6 +73,7 @@ pub const Token = struct { Identifier, StringLiteral: StrLitKind, MultilineStringLiteralLine: StrLitKind, + CharLiteral, StringIdentifier, Eof, Builtin, @@ -228,6 +229,9 @@ pub const Tokenizer = struct { StringLiteralBackslash, MultilineStringLiteralLine, MultilineStringLiteralLineBackslash, + CharLiteral, + CharLiteralBackslash, + CharLiteralEnd, Backslash, Equal, Bang, @@ -294,6 +298,9 @@ pub const Tokenizer = struct { state = State.StringLiteral; result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal }; }, + '\'' => { + state = State.CharLiteral; + }, 'a'...'b', 'd'...'z', 'A'...'Z', '_' => { state = State.Identifier; result.id = Token.Id.Identifier; @@ -579,6 +586,35 @@ pub const Tokenizer = struct { }, }, + State.CharLiteral => switch (c) { + '\\' => { + state = State.CharLiteralBackslash; + }, + '\'' => break, // Look for this error later. + else => { + if (c < 0x20 or c == 0x7f) + break; // Look for this error later. + + state = State.CharLiteralEnd; + } + }, + + State.CharLiteralBackslash => switch (c) { + '\n' => break, // Look for this error later. + else => { + state = State.CharLiteralEnd; + }, + }, + + State.CharLiteralEnd => switch (c) { + '\'' => { + result.id = Token.Id.CharLiteral; + self.index += 1; + break; + }, + else => break, // Look for this error later. + }, + State.MultilineStringLiteralLine => switch (c) { '\\' => { state = State.MultilineStringLiteralLineBackslash; @@ -847,6 +883,7 @@ pub const Tokenizer = struct { State.FloatExponentNumber, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, + State.CharLiteralEnd, State.Builtin => {}, State.Identifier => { @@ -863,6 +900,8 @@ pub const Tokenizer = struct { State.SawAtSign, State.Backslash, State.MultilineStringLiteralLineBackslash, + State.CharLiteral, + State.CharLiteralBackslash, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, @@ -1006,9 +1045,16 @@ test "tokenizer" { }); } +test "tokenizer - chars" { + testTokenize("'c'", []Token.Id {Token.Id.CharLiteral}); +} + test "tokenizer - invalid token characters" { testTokenize("#", []Token.Id{Token.Id.Invalid}); testTokenize("`", []Token.Id{Token.Id.Invalid}); + testTokenize("'c", []Token.Id {Token.Id.Invalid}); + testTokenize("'", []Token.Id {Token.Id.Invalid}); + testTokenize("''", []Token.Id {Token.Id.Invalid}); } test "tokenizer - invalid literal/comment characters" { From aabf7cf57e62379d5f21aa013d0735c5427eace2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 22:10:49 +0200 Subject: [PATCH 052/132] std.zig.parser now parses null and bool literals --- std/zig/ast.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ std/zig/parser.zig | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1c84e1264..6c06b5423 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,8 @@ pub const Node = struct { StringLiteral, MultilineStringLiteral, CharLiteral, + BoolLiteral, + NullLiteral, UndefinedLiteral, BuiltinCall, Call, @@ -44,6 +46,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).iterate(index), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), @@ -67,6 +71,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).firstToken(), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), @@ -90,6 +96,8 @@ pub const Node = struct { Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(), Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(), + Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).lastToken(), + Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), @@ -629,6 +637,40 @@ pub const NodeCharLiteral = struct { } }; +pub const NodeBoolLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeBoolLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeBoolLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeBoolLiteral) Token { + return self.token; + } +}; + +pub const NodeNullLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeNullLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeNullLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeNullLiteral) Token { + return self.token; + } +}; + pub const NodeUndefinedLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a05328660..6c0a1f3f1 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -427,6 +427,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_true, Token.Id.Keyword_false => { + const node = try arena.create(ast.NodeBoolLiteral); + *node = ast.NodeBoolLiteral { + .base = self.initNode(ast.Node.Id.BoolLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_null => { + const node = try arena.create(ast.NodeNullLiteral); + *node = ast.NodeNullLiteral { + .base = self.initNode(ast.Node.Id.NullLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1467,6 +1491,14 @@ pub const Parser = struct { const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token)); }, + ast.Node.Id.BoolLiteral => { + const bool_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(bool_literal.token)); + }, + ast.Node.Id.NullLiteral => { + const null_literal = @fieldParentPtr(ast.NodeNullLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(null_literal.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); From 67f11190d10caac1d08b76d60ec28eb2f5173946 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 31 Mar 2018 16:34:55 -0400 Subject: [PATCH 053/132] musl-friendly dynamic linking --- build.zig | 5 +++++ src/link.cpp | 57 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/build.zig b/build.zig index 88775498c..b72641a2e 100644 --- a/build.zig +++ b/build.zig @@ -45,6 +45,11 @@ pub fn build(b: &Builder) !void { var exe = b.addExecutable("zig", "src-self-hosted/main.zig"); exe.setBuildMode(mode); + + // This is for finding /lib/libz.a on alpine linux. + // TODO turn this into -Dextra-lib-path=/lib option + exe.addLibPath("/lib"); + exe.addIncludeDir("src"); exe.addIncludeDir(cmake_binary_dir); addCppLib(b, exe, cmake_binary_dir, "zig_cpp"); diff --git a/src/link.cpp b/src/link.cpp index c089d6961..3c6e27e33 100644 --- a/src/link.cpp +++ b/src/link.cpp @@ -164,32 +164,45 @@ static void add_rpath(LinkJob *lj, Buf *rpath) { lj->rpath_table.put(rpath, true); } +static Buf *try_dynamic_linker_path(const char *ld_name) { + const char *cc_exe = getenv("CC"); + cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; + ZigList args = {}; + args.append(buf_ptr(buf_sprintf("-print-file-name=%s", ld_name))); + Termination term; + Buf *out_stderr = buf_alloc(); + Buf *out_stdout = buf_alloc(); + int err; + if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { + return nullptr; + } + if (term.how != TerminationIdClean || term.code != 0) { + return nullptr; + } + if (buf_ends_with_str(out_stdout, "\n")) { + buf_resize(out_stdout, buf_len(out_stdout) - 1); + } + if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, ld_name)) { + return nullptr; + } + return out_stdout; +} + static Buf *get_dynamic_linker_path(CodeGen *g) { if (g->is_native_target && g->zig_target.arch.arch == ZigLLVM_x86_64) { - const char *cc_exe = getenv("CC"); - cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe; - ZigList args = {}; - args.append("-print-file-name=ld-linux-x86-64.so.2"); - Termination term; - Buf *out_stderr = buf_alloc(); - Buf *out_stdout = buf_alloc(); - int err; - if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) { - return target_dynamic_linker(&g->zig_target); + static const char *ld_names[] = { + "ld-linux-x86-64.so.2", + "ld-musl-x86_64.so.1", + }; + for (size_t i = 0; i < array_length(ld_names); i += 1) { + const char *ld_name = ld_names[i]; + Buf *result = try_dynamic_linker_path(ld_name); + if (result != nullptr) { + return result; + } } - if (term.how != TerminationIdClean || term.code != 0) { - return target_dynamic_linker(&g->zig_target); - } - if (buf_ends_with_str(out_stdout, "\n")) { - buf_resize(out_stdout, buf_len(out_stdout) - 1); - } - if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, "ld-linux-x86-64.so.2")) { - return target_dynamic_linker(&g->zig_target); - } - return out_stdout; - } else { - return target_dynamic_linker(&g->zig_target); } + return target_dynamic_linker(&g->zig_target); } static void construct_linker_job_elf(LinkJob *lj) { From df09c01f7f141a384010d17ab23db3b36316f5b6 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 31 Mar 2018 22:48:12 +0200 Subject: [PATCH 054/132] std.zig.parser now parses error, this and unreachable --- std/zig/ast.zig | 63 ++++++++++++++++++++++++++++++++++++++++++++++ std/zig/parser.zig | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 6c06b5423..8a9a072d6 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -25,6 +25,9 @@ pub const Node = struct { BoolLiteral, NullLiteral, UndefinedLiteral, + ThisLiteral, + Unreachable, + ErrorType, BuiltinCall, Call, LineComment, @@ -49,6 +52,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).iterate(index), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), @@ -74,6 +80,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).firstToken(), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).firstToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -99,6 +108,9 @@ pub const Node = struct { Id.BoolLiteral => @fieldParentPtr(NodeBoolLiteral, "base", base).lastToken(), Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), + Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), + Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), + Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), @@ -688,6 +700,57 @@ pub const NodeUndefinedLiteral = struct { } }; +pub const NodeThisLiteral = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeThisLiteral, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeThisLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeThisLiteral) Token { + return self.token; + } +}; + +pub const NodeUnreachable = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeUnreachable, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeUnreachable) Token { + return self.token; + } + + pub fn lastToken(self: &NodeUnreachable) Token { + return self.token; + } +}; + +pub const NodeErrorType = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeErrorType, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeErrorType) Token { + return self.token; + } + + pub fn lastToken(self: &NodeErrorType) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 6c0a1f3f1..0b8a15d3d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -451,6 +451,42 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_this => { + const node = try arena.create(ast.NodeThisLiteral); + *node = ast.NodeThisLiteral { + .base = self.initNode(ast.Node.Id.ThisLiteral), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_unreachable => { + const node = try arena.create(ast.NodeUnreachable); + *node = ast.NodeUnreachable { + .base = self.initNode(ast.Node.Id.Unreachable), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, + Token.Id.Keyword_error => { + const node = try arena.create(ast.NodeErrorType); + *node = ast.NodeErrorType { + .base = self.initNode(ast.Node.Id.ErrorType), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1499,6 +1535,18 @@ pub const Parser = struct { const null_literal = @fieldParentPtr(ast.NodeNullLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(null_literal.token)); }, + ast.Node.Id.ThisLiteral => { + const this_literal = @fieldParentPtr(ast.NodeThisLiteral, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(this_literal.token)); + }, + ast.Node.Id.Unreachable => { + const unreachable_node = @fieldParentPtr(ast.NodeUnreachable, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(unreachable_node.token)); + }, + ast.Node.Id.ErrorType => { + const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); From 2e5115b0687ee9b7078dbf70da7d7070a7c80399 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Sat, 31 Mar 2018 19:04:01 +1300 Subject: [PATCH 055/132] Add run compiler command 'zig run file.zig' builds a file and stores the artifacts in the global cache. On successful compilation the binary is executed. 'zig run file.zig -- a b c' does the same, but passes the arguments a, b and c as runtime arguments to the program. Everything after an '--' are treated as runtime arguments. On a posix system, a shebang can be used to run a zig file directly. An example shebang would be '#!/usr/bin/zig run'. You may not be able pass extra compile arguments currently as part of the shebang. Linux for example treats all arguments after the first as a single argument which will result in an 'invalid command'. Currently there is no customisability for the cache path as a compile argument. For a posix system you can use `TMPDIR=. zig run file.zig` to override, in this case using the current directory for the run cache. The input file is always recompiled, even if it has changed. This is intended to be cached but further discussion/thought needs to go into this. Closes #466. --- src/codegen.cpp | 6 ++-- src/ir.cpp | 4 +-- src/main.cpp | 58 ++++++++++++++++++++++++++++++++------ src/os.cpp | 75 +++++++++++++++++++++++++++++++++++++++++++++---- src/os.hpp | 6 ++-- 5 files changed, 129 insertions(+), 20 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 0ebdf7fa3..bdd28b86f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6286,7 +6286,7 @@ static ImportTableEntry *add_special_code(CodeGen *g, PackageTableEntry *package zig_panic("unable to open '%s': %s", buf_ptr(&path_to_code_src), err_str(err)); } Buf *import_code = buf_alloc(); - if ((err = os_fetch_file_path(abs_full_path, import_code))) { + if ((err = os_fetch_file_path(abs_full_path, import_code, false))) { zig_panic("unable to open '%s': %s", buf_ptr(&path_to_code_src), err_str(err)); } @@ -6374,7 +6374,7 @@ static void gen_root_source(CodeGen *g) { } Buf *source_code = buf_alloc(); - if ((err = os_fetch_file_path(rel_full_path, source_code))) { + if ((err = os_fetch_file_path(rel_full_path, source_code, true))) { zig_panic("unable to open '%s': %s", buf_ptr(rel_full_path), err_str(err)); } @@ -6439,7 +6439,7 @@ static void gen_global_asm(CodeGen *g) { int err; for (size_t i = 0; i < g->assembly_files.length; i += 1) { Buf *asm_file = g->assembly_files.at(i); - if ((err = os_fetch_file_path(asm_file, &contents))) { + if ((err = os_fetch_file_path(asm_file, &contents, false))) { zig_panic("Unable to read %s: %s", buf_ptr(asm_file), err_str(err)); } buf_append_buf(&g->global_asm, &contents); diff --git a/src/ir.cpp b/src/ir.cpp index 18fd02c29..4fe6769f7 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -14709,7 +14709,7 @@ static TypeTableEntry *ir_analyze_instruction_import(IrAnalyze *ira, IrInstructi return ira->codegen->builtin_types.entry_namespace; } - if ((err = os_fetch_file_path(abs_full_path, import_code))) { + if ((err = os_fetch_file_path(abs_full_path, import_code, true))) { if (err == ErrorFileNotFound) { ir_add_error_node(ira, source_node, buf_sprintf("unable to find '%s'", buf_ptr(import_target_path))); @@ -15570,7 +15570,7 @@ static TypeTableEntry *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstr // load from file system into const expr Buf *file_contents = buf_alloc(); int err; - if ((err = os_fetch_file_path(&file_path, file_contents))) { + if ((err = os_fetch_file_path(&file_path, file_contents, false))) { if (err == ErrorFileNotFound) { ir_add_error(ira, instruction->name, buf_sprintf("unable to find '%s'", buf_ptr(&file_path))); return ira->codegen->builtin_types.entry_invalid; diff --git a/src/main.cpp b/src/main.cpp index f2f4cc970..63b077e83 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -23,6 +23,7 @@ static int usage(const char *arg0) { " build-exe [source] create executable from source or object files\n" " build-lib [source] create library from source or object files\n" " build-obj [source] create object from source or assembly\n" + " run [source] create executable and run immediately\n" " translate-c [source] convert c code to zig code\n" " targets list available compilation targets\n" " test [source] create and run a test build\n" @@ -220,6 +221,7 @@ static Buf *resolve_zig_lib_dir(const char *zig_install_prefix_arg) { enum Cmd { CmdInvalid, CmdBuild, + CmdRun, CmdTest, CmdVersion, CmdZen, @@ -329,6 +331,8 @@ int main(int argc, char **argv) { CliPkg *cur_pkg = allocate(1); BuildMode build_mode = BuildModeDebug; ZigList test_exec_args = {0}; + int comptime_args_end = 0; + int runtime_args_start = argc; if (argc >= 2 && strcmp(argv[1], "build") == 0) { const char *zig_exe_path = arg0; @@ -481,11 +485,15 @@ int main(int argc, char **argv) { return (term.how == TerminationIdClean) ? term.code : -1; } - for (int i = 1; i < argc; i += 1) { + for (int i = 1; i < argc; i += 1, comptime_args_end += 1) { char *arg = argv[i]; if (arg[0] == '-') { - if (strcmp(arg, "--release-fast") == 0) { + if (strcmp(arg, "--") == 0) { + // ignore -- from both compile and runtime arg sets + runtime_args_start = i + 1; + break; + } else if (strcmp(arg, "--release-fast") == 0) { build_mode = BuildModeFastRelease; } else if (strcmp(arg, "--release-safe") == 0) { build_mode = BuildModeSafeRelease; @@ -652,6 +660,9 @@ int main(int argc, char **argv) { } else if (strcmp(arg, "build-lib") == 0) { cmd = CmdBuild; out_type = OutTypeLib; + } else if (strcmp(arg, "run") == 0) { + cmd = CmdRun; + out_type = OutTypeExe; } else if (strcmp(arg, "version") == 0) { cmd = CmdVersion; } else if (strcmp(arg, "zen") == 0) { @@ -670,6 +681,7 @@ int main(int argc, char **argv) { } else { switch (cmd) { case CmdBuild: + case CmdRun: case CmdTranslateC: case CmdTest: if (!in_file) { @@ -724,8 +736,8 @@ int main(int argc, char **argv) { } } - switch (cmd) { + case CmdRun: case CmdBuild: case CmdTranslateC: case CmdTest: @@ -733,7 +745,7 @@ int main(int argc, char **argv) { if (cmd == CmdBuild && !in_file && objects.length == 0 && asm_files.length == 0) { fprintf(stderr, "Expected source file argument or at least one --object or --assembly argument.\n"); return usage(arg0); - } else if ((cmd == CmdTranslateC || cmd == CmdTest) && !in_file) { + } else if ((cmd == CmdTranslateC || cmd == CmdTest || cmd == CmdRun) && !in_file) { fprintf(stderr, "Expected source file argument.\n"); return usage(arg0); } else if (cmd == CmdBuild && out_type == OutTypeObj && objects.length != 0) { @@ -745,6 +757,10 @@ int main(int argc, char **argv) { bool need_name = (cmd == CmdBuild || cmd == CmdTranslateC); + if (cmd == CmdRun) { + out_name = "run"; + } + Buf *in_file_buf = nullptr; Buf *buf_out_name = (cmd == CmdTest) ? buf_create_from_str("test") : @@ -769,9 +785,23 @@ int main(int argc, char **argv) { Buf *zig_root_source_file = (cmd == CmdTranslateC) ? nullptr : in_file_buf; Buf *full_cache_dir = buf_alloc(); - os_path_resolve(buf_create_from_str("."), - buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir), - full_cache_dir); + Buf *run_exec_path = buf_alloc(); + if (cmd == CmdRun) { + if (buf_out_name == nullptr) { + buf_out_name = buf_create_from_str("run"); + } + + Buf *global_cache_dir = buf_alloc(); + os_get_global_cache_directory(global_cache_dir); + os_path_join(global_cache_dir, buf_out_name, run_exec_path); + os_path_resolve(buf_create_from_str("."), global_cache_dir, full_cache_dir); + + out_file = buf_ptr(run_exec_path); + } else { + os_path_resolve(buf_create_from_str("."), + buf_create_from_str((cache_dir == nullptr) ? default_zig_cache_name : cache_dir), + full_cache_dir); + } Buf *zig_lib_dir_buf = resolve_zig_lib_dir(zig_install_prefix); @@ -855,7 +885,7 @@ int main(int argc, char **argv) { add_package(g, cur_pkg, g->root_package); - if (cmd == CmdBuild) { + if (cmd == CmdBuild || cmd == CmdRun) { codegen_set_emit_file_type(g, emit_file_type); for (size_t i = 0; i < objects.length; i += 1) { @@ -868,6 +898,18 @@ int main(int argc, char **argv) { codegen_link(g, out_file); if (timing_info) codegen_print_timing_report(g, stdout); + + if (cmd == CmdRun) { + ZigList args = {0}; + for (int i = runtime_args_start; i < argc; ++i) { + args.append(argv[i]); + } + + Termination term; + os_spawn_process(buf_ptr(run_exec_path), args, &term); + return term.code; + } + return EXIT_SUCCESS; } else if (cmd == CmdTranslateC) { codegen_translate_c(g, in_file_buf); diff --git a/src/os.cpp b/src/os.cpp index de1a1f21d..e0491b21d 100644 --- a/src/os.cpp +++ b/src/os.cpp @@ -288,13 +288,39 @@ void os_path_resolve(Buf *ref_path, Buf *target_path, Buf *out_abs_path) { return; } -int os_fetch_file(FILE *f, Buf *out_buf) { +int os_fetch_file(FILE *f, Buf *out_buf, bool skip_shebang) { static const ssize_t buf_size = 0x2000; buf_resize(out_buf, buf_size); ssize_t actual_buf_len = 0; + + bool first_read = true; + for (;;) { size_t amt_read = fread(buf_ptr(out_buf) + actual_buf_len, 1, buf_size, f); actual_buf_len += amt_read; + + if (skip_shebang && first_read && buf_starts_with_str(out_buf, "#!")) { + size_t i = 0; + while (true) { + if (i > buf_len(out_buf)) { + zig_panic("shebang line exceeded %zd characters", buf_size); + } + + size_t current_pos = i; + i += 1; + + if (out_buf->list.at(current_pos) == '\n') { + break; + } + } + + ZigList *list = &out_buf->list; + memmove(list->items, list->items + i, list->length - i); + list->length -= i; + + actual_buf_len -= i; + } + if (amt_read != buf_size) { if (feof(f)) { buf_resize(out_buf, actual_buf_len); @@ -305,6 +331,7 @@ int os_fetch_file(FILE *f, Buf *out_buf) { } buf_resize(out_buf, actual_buf_len + buf_size); + first_read = false; } zig_unreachable(); } @@ -374,8 +401,8 @@ static int os_exec_process_posix(const char *exe, ZigList &args, FILE *stdout_f = fdopen(stdout_pipe[0], "rb"); FILE *stderr_f = fdopen(stderr_pipe[0], "rb"); - os_fetch_file(stdout_f, out_stdout); - os_fetch_file(stderr_f, out_stderr); + os_fetch_file(stdout_f, out_stdout, false); + os_fetch_file(stderr_f, out_stderr, false); fclose(stdout_f); fclose(stderr_f); @@ -588,7 +615,7 @@ int os_copy_file(Buf *src_path, Buf *dest_path) { } } -int os_fetch_file_path(Buf *full_path, Buf *out_contents) { +int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang) { FILE *f = fopen(buf_ptr(full_path), "rb"); if (!f) { switch (errno) { @@ -607,7 +634,7 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents) { return ErrorFileSystem; } } - int result = os_fetch_file(f, out_contents); + int result = os_fetch_file(f, out_contents, skip_shebang); fclose(f); return result; } @@ -780,6 +807,44 @@ int os_buf_to_tmp_file(Buf *contents, Buf *suffix, Buf *out_tmp_path) { #endif } +#if defined(ZIG_OS_POSIX) +int os_get_global_cache_directory(Buf *out_tmp_path) { + const char *tmp_dir = getenv("TMPDIR"); + if (!tmp_dir) { + tmp_dir = P_tmpdir; + } + + Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); + Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); + + buf_resize(out_tmp_path, 0); + os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); + + buf_deinit(tmp_dir_buf); + buf_deinit(cache_dirname_buf); + return 0; +} +#endif + +#if defined(ZIG_OS_WINDOWS) +int os_get_global_cache_directory(Buf *out_tmp_path) { + char tmp_dir[MAX_PATH + 1]; + if (GetTempPath(MAX_PATH, tmp_dir) == 0) { + zig_panic("GetTempPath failed"); + } + + Buf *tmp_dir_buf = buf_create_from_str(tmp_dir); + Buf *cache_dirname_buf = buf_create_from_str("zig-cache"); + + buf_resize(out_tmp_path, 0); + os_path_join(tmp_dir_buf, cache_dirname_buf, out_tmp_path); + + buf_deinit(tmp_dir_buf); + buf_deinit(cache_dirname_buf); + return 0; +} +#endif + int os_delete_file(Buf *path) { if (remove(buf_ptr(path))) { return ErrorFileSystem; diff --git a/src/os.hpp b/src/os.hpp index 5d29db0d0..b94e98ec3 100644 --- a/src/os.hpp +++ b/src/os.hpp @@ -51,14 +51,16 @@ int os_path_real(Buf *rel_path, Buf *out_abs_path); void os_path_resolve(Buf *ref_path, Buf *target_path, Buf *out_abs_path); bool os_path_is_absolute(Buf *path); +int os_get_global_cache_directory(Buf *out_tmp_path); + int os_make_path(Buf *path); int os_make_dir(Buf *path); void os_write_file(Buf *full_path, Buf *contents); int os_copy_file(Buf *src_path, Buf *dest_path); -int os_fetch_file(FILE *file, Buf *out_contents); -int os_fetch_file_path(Buf *full_path, Buf *out_contents); +int os_fetch_file(FILE *file, Buf *out_contents, bool skip_shebang); +int os_fetch_file_path(Buf *full_path, Buf *out_contents, bool skip_shebang); int os_get_cwd(Buf *out_cwd); From b9093185f748293064e7eeaaa07e7479099420ed Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sun, 1 Apr 2018 22:02:51 +0200 Subject: [PATCH 056/132] std.zig.parser now parses slicing and array access --- std/zig/ast.zig | 66 +++++++++++++++++++++++++++++++ std/zig/parser.zig | 92 +++++++++++++++++++++++++++++++++++++++++++ std/zig/tokenizer.zig | 12 ++++++ 3 files changed, 170 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 8a9a072d6..680e61796 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -30,6 +30,8 @@ pub const Node = struct { ErrorType, BuiltinCall, Call, + ArrayAccess, + SliceExpression, LineComment, TestDecl, }; @@ -57,6 +59,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).iterate(index), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -85,6 +89,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).firstToken(), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -113,6 +119,8 @@ pub const Node = struct { Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), + Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).lastToken(), + Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -598,6 +606,64 @@ pub const NodeCall = struct { } }; +pub const NodeArrayAccess = struct { + base: Node, + expr: &Node, + index: &Node, + rbracket_token: Token, + + pub fn iterate(self: &NodeArrayAccess, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + if (i < 1) return self.index; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeArrayAccess) Token { + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeArrayAccess) Token { + return self.rbracket_token; + } +}; + +pub const NodeSliceExpression = struct { + base: Node, + expr: &Node, + start: &Node, + end: ?&Node, + rbracket_token: Token, + + pub fn iterate(self: &NodeSliceExpression, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.callee; + i -= 1; + + if (i < 1) return self.start; + i -= 1; + + if (i < 1) return self.end; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeSliceExpression) Token { + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeSliceExpression) Token { + return self.rbracket_token; + } +}; + pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0b8a15d3d..d0ab7e17d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -88,6 +88,7 @@ pub const Parser = struct { InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, SuffixOp: &ast.Node, + SliceOrArrayAccess, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -590,6 +591,11 @@ pub const Parser = struct { }); continue; + } else if (token.id == Token.Id.LBracket) { + try stack.append(State.SliceOrArrayAccess); + try stack.append(State.ExpectOperand); + continue; + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -603,6 +609,53 @@ pub const Parser = struct { try dest_ptr.store(expression); break; }, + State.SliceOrArrayAccess => { + var rbracket_or_ellipsis2_token = self.getNextToken(); + + switch (rbracket_or_ellipsis2_token.id) { + Token.Id.Ellipsis2 => { + const node = try arena.create(ast.NodeSliceExpression); + *node = ast.NodeSliceExpression { + .base = self.initNode(ast.Node.Id.SliceExpression), + .expr = undefined, + .start = expression, + .end = null, + .rbracket_token = undefined, + }; + + try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State.AfterOperand); + + const rbracket_token = self.getNextToken(); + if (rbracket_token.id != Token.Id.RBracket) { + self.putBackToken(rbracket_token); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RBracket, + .ptr = &node.rbracket_token, + } + }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.end } }); + } else { + node.rbracket_token = rbracket_token; + } + break; + }, + Token.Id.RBracket => { + const node = try arena.create(ast.NodeArrayAccess); + *node = ast.NodeArrayAccess { + .base = self.initNode(ast.Node.Id.ArrayAccess), + .expr = undefined, + .index = expression, + .rbracket_token = token, + }; + try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State.AfterOperand); + break; + }, + else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) + } + }, State.InfixOp => |infix_op| { infix_op.rhs = expression; infix_op.lhs = popSuffixOp(&stack); @@ -857,6 +910,7 @@ pub const Parser = struct { State.PrefixOp => unreachable, State.SuffixOp => unreachable, State.Operand => unreachable, + State.SliceOrArrayAccess => unreachable, } } } @@ -874,6 +928,18 @@ pub const Parser = struct { left_leaf_ptr = &call.callee; continue; }, + ast.Node.Id.ArrayAccess => { + const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", suffix_op); + *left_leaf_ptr = &arr_access.base; + left_leaf_ptr = &arr_access.expr; + continue; + }, + ast.Node.Id.SliceExpression => { + const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", suffix_op); + *left_leaf_ptr = &slice_expr.base; + left_leaf_ptr = &slice_expr.expr; + continue; + }, else => unreachable, } }, @@ -1594,6 +1660,24 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "("}); try stack.append(RenderState { .Expression = call.callee }); }, + ast.Node.Id.ArrayAccess => { + const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", base); + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = arr_access.index}); + try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = arr_access.expr }); + }, + ast.Node.Id.SliceExpression => { + const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", base); + try stack.append(RenderState { .Text = "]"}); + if (slice_expr.end) |end| { + try stack.append(RenderState { .Expression = end}); + } + try stack.append(RenderState { .Text = ".."}); + try stack.append(RenderState { .Expression = slice_expr.start}); + try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = slice_expr.expr}); + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), @@ -1978,6 +2062,14 @@ test "zig fmt: indexing" { \\ a[0 + 5]; \\ a[0..]; \\ a[0..5]; + \\ a[a[0]]; + \\ a[a[0..]]; + \\ a[a[0..5]]; + \\ a[a[0]..]; + \\ a[a[0..5]..]; + \\ a[a[0]..a[0]]; + \\ a[a[0..5]..a[0]]; + \\ a[a[0..5]..a[0..5]]; \\} \\ ); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 64eb62759..1014bacb3 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -91,6 +91,8 @@ pub const Token = struct { PercentEqual, LBrace, RBrace, + LBracket, + RBracket, Period, Ellipsis2, Ellipsis3, @@ -327,6 +329,16 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '[' => { + result.id = Token.Id.LBracket; + self.index += 1; + break; + }, + ']' => { + result.id = Token.Id.RBracket; + self.index += 1; + break; + }, ';' => { result.id = Token.Id.Semicolon; self.index += 1; From 4eb68987d81cc4a1a92ebb4b5e83be73669801ad Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 2 Apr 2018 11:34:31 -0400 Subject: [PATCH 057/132] std.io.readLine function this provides a better input for guess number example. see #882 --- example/guess_number/main.zig | 13 +++++++------ std/io.zig | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/example/guess_number/main.zig b/example/guess_number/main.zig index d26518a1e..7178c5274 100644 --- a/example/guess_number/main.zig +++ b/example/guess_number/main.zig @@ -9,8 +9,6 @@ pub fn main() !void { var stdout_file_stream = io.FileOutStream.init(&stdout_file); const stdout = &stdout_file_stream.stream; - var stdin_file = try io.getStdIn(); - try stdout.print("Welcome to the Guess Number Game in Zig.\n"); var seed_bytes: [@sizeOf(u64)]u8 = undefined; @@ -27,12 +25,15 @@ pub fn main() !void { try stdout.print("\nGuess a number between 1 and 100: "); var line_buf : [20]u8 = undefined; - const line_len = stdin_file.read(line_buf[0..]) catch |err| { - try stdout.print("Unable to read from stdin: {}\n", @errorName(err)); - return err; + const line_len = io.readLine(line_buf[0..]) catch |err| switch (err) { + error.InputTooLong => { + try stdout.print("Input too long.\n"); + continue; + }, + error.EndOfFile, error.StdInUnavailable => return err, }; - const guess = fmt.parseUnsigned(u8, line_buf[0..line_len - 1], 10) catch { + const guess = fmt.parseUnsigned(u8, line_buf[0..line_len], 10) catch { try stdout.print("Invalid number.\n"); continue; }; diff --git a/std/io.zig b/std/io.zig index 952d9081f..93d50e670 100644 --- a/std/io.zig +++ b/std/io.zig @@ -478,3 +478,20 @@ test "import io tests" { } } +pub fn readLine(buf: []u8) !usize { + var stdin = getStdIn() catch return error.StdInUnavailable; + var adapter = FileInStream.init(&stdin); + var stream = &adapter.stream; + var index: usize = 0; + while (true) { + const byte = stream.readByte() catch return error.EndOfFile; + switch (byte) { + '\n' => return index, + else => { + if (index == buf.len) return error.InputTooLong; + buf[index] = byte; + index += 1; + }, + } + } +} From a2330d0ea366e0ced7fa917d3bf1ccd022e932c3 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 10:54:19 +0200 Subject: [PATCH 058/132] std.zig.parser now parses slice and array types --- std/zig/ast.zig | 5 +++++ std/zig/parser.zig | 50 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 680e61796..1e62bd26a 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -475,9 +475,12 @@ pub const NodePrefixOp = struct { Negation, NegationWrap, Return, + ArrayType: &Node, + SliceType: AddrOfInfo, Try, UnwrapMaybe, }; + const AddrOfInfo = struct { align_expr: ?&Node, bit_offset_start_token: ?Token, @@ -502,6 +505,8 @@ pub const NodePrefixOp = struct { PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, + PrefixOp.ArrayType, + PrefixOp.SliceType, PrefixOp.Try, PrefixOp.UnwrapMaybe => {}, } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index d0ab7e17d..671f454f5 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -385,6 +385,35 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.LBracket => { + const rbracket_token = self.getNextToken(); + if (rbracket_token.id == Token.Id.RBracket) { + const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .SliceType = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + } + }); + try stack.append(State { .PrefixOp = prefix_op }); + try stack.append(State.ExpectOperand); + try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); + continue; + } + + self.putBackToken(rbracket_token); + + const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .ArrayType = undefined, + }); + try stack.append(State { .PrefixOp = prefix_op }); + try stack.append(State.ExpectOperand); + try stack.append(State { .ExpectToken = Token.Id.RBracket }); + try stack.append(State { .Expression = DestPtr { .Field = &prefix_op.op.ArrayType } }); + + }, Token.Id.Ampersand => { const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ .AddrOf = ast.NodePrefixOp.AddrOfInfo { @@ -1567,6 +1596,25 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = align_expr}); } }, + ast.NodePrefixOp.PrefixOp.SliceType => |addr_of_info| { + try stream.write("[]"); + if (addr_of_info.volatile_token != null) { + try stack.append(RenderState { .Text = "volatile "}); + } + if (addr_of_info.const_token != null) { + try stack.append(RenderState { .Text = "const "}); + } + if (addr_of_info.align_expr) |align_expr| { + try stream.print("align("); + try stack.append(RenderState { .Text = ") "}); + try stack.append(RenderState { .Expression = align_expr}); + } + }, + ast.NodePrefixOp.PrefixOp.ArrayType => |array_index| { + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = array_index}); + try stack.append(RenderState { .Text = "["}); + }, ast.NodePrefixOp.PrefixOp.BitNot => try stream.write("~"), ast.NodePrefixOp.PrefixOp.BoolNot => try stream.write("!"), ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), @@ -2522,4 +2570,4 @@ test "zig fmt: zig fmt" { try testCanonical(@embedFile("index.zig")); try testCanonical(@embedFile("parser.zig")); try testCanonical(@embedFile("tokenizer.zig")); -} \ No newline at end of file +} From 22e38ffb54775c2e523665ac739f2a895d8757ad Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 11:18:18 +0200 Subject: [PATCH 059/132] std.zig.tokenizer fixed tokens having wrong column and line --- std/zig/tokenizer.zig | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 1014bacb3..2e4099992 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -271,6 +271,7 @@ pub const Tokenizer = struct { self.pending_invalid_token = null; return token; } + const start_index = self.index; var state = State.Start; var result = Token { .id = Token.Id.Eof, @@ -279,7 +280,7 @@ pub const Tokenizer = struct { .line = self.line, .column = self.column, }; - while (self.index < self.buffer.len) { + while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; switch (state) { State.Start => switch (c) { @@ -877,14 +878,6 @@ pub const Tokenizer = struct { else => break, }, } - - self.index += 1; - if (c == '\n') { - self.line += 1; - self.column = 0; - } else { - self.column += 1; - } } else if (self.index == self.buffer.len) { switch (state) { State.Start, @@ -983,6 +976,16 @@ pub const Tokenizer = struct { }, } } + + for (self.buffer[start_index..self.index]) |c| { + if (c == '\n') { + self.line += 1; + self.column = 0; + } else { + self.column += 1; + } + } + if (result.id == Token.Id.Eof) { if (self.pending_invalid_token) |token| { self.pending_invalid_token = null; From b424cd75ab9df69f44fe47c10ded3fd25c2d27bb Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 12:33:06 +0200 Subject: [PATCH 060/132] std.zig.parser refactored call, slice and array access to be suffix op --- std/zig/ast.zig | 183 +++++++++++++++++++++------------------------ std/zig/parser.zig | 161 +++++++++++++++++++-------------------- 2 files changed, 160 insertions(+), 184 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1e62bd26a..380dd95ae 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -17,6 +17,7 @@ pub const Node = struct { Block, InfixOp, PrefixOp, + SuffixOp, IntegerLiteral, FloatLiteral, StringLiteral, @@ -29,9 +30,6 @@ pub const Node = struct { Unreachable, ErrorType, BuiltinCall, - Call, - ArrayAccess, - SliceExpression, LineComment, TestDecl, }; @@ -46,6 +44,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), @@ -58,9 +57,6 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), - Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).iterate(index), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), }; @@ -76,6 +72,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), @@ -88,9 +85,6 @@ pub const Node = struct { Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), - Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).firstToken(), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), }; @@ -106,6 +100,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), + Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), @@ -118,9 +113,6 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), - Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(), - Id.ArrayAccess => @fieldParentPtr(NodeArrayAccess, "base", base).lastToken(), - Id.SliceExpression => @fieldParentPtr(NodeSliceExpression, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), }; @@ -493,20 +485,28 @@ pub const NodePrefixOp = struct { var i = index; switch (self.op) { + PrefixOp.SliceType => |addr_of_info| { + if (addr_of_info.align_expr) |align_expr| { + if (i < 1) return align_expr; + i -= 1; + } + }, PrefixOp.AddrOf => |addr_of_info| { if (addr_of_info.align_expr) |align_expr| { if (i < 1) return align_expr; i -= 1; } }, + PrefixOp.ArrayType => |size_expr| { + if (i < 1) return size_expr; + i -= 1; + }, PrefixOp.BitNot, PrefixOp.BoolNot, PrefixOp.Deref, PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, - PrefixOp.ArrayType, - PrefixOp.SliceType, PrefixOp.Try, PrefixOp.UnwrapMaybe => {}, } @@ -526,6 +526,76 @@ pub const NodePrefixOp = struct { } }; +pub const NodeSuffixOp = struct { + base: Node, + lhs: &Node, + op: SuffixOp, + rtoken: Token, + + const SuffixOp = union(enum) { + Call: CallInfo, + ArrayAccess: &Node, + Slice: SliceRange, + ArrayInitializer: ArrayList(&Node), + StructInitializer: ArrayList(&Node), + }; + + const CallInfo = struct { + params: ArrayList(&Node), + is_async: bool, + }; + + const SliceRange = struct { + start: &Node, + end: ?&Node, + }; + + pub fn iterate(self: &NodeSuffixOp, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.lhs; + i -= 1; + + switch (self.op) { + SuffixOp.Call => |call_info| { + if (i < call_info.params.len) return call_info.params.at(i); + i -= call_info.params.len; + }, + SuffixOp.ArrayAccess => |index_expr| { + if (i < 1) return index_expr; + i -= 1; + }, + SuffixOp.Slice => |range| { + if (i < 1) return range.start; + i -= 1; + + if (range.end) |end| { + if (i < 1) return end; + i -= 1; + } + }, + SuffixOp.ArrayInitializer => |exprs| { + if (i < exprs.len) return exprs.at(i); + i -= exprs.len; + }, + SuffixOp.StructInitializer => |fields| { + if (i < fields.len) return fields.at(i); + i -= fields.len; + }, + } + + return null; + } + + pub fn firstToken(self: &NodeSuffixOp) Token { + return self.lhs.firstToken(); + } + + pub fn lastToken(self: &NodeSuffixOp) Token { + return self.rtoken; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, @@ -584,91 +654,6 @@ pub const NodeBuiltinCall = struct { } }; -pub const NodeCall = struct { - base: Node, - callee: &Node, - params: ArrayList(&Node), - rparen_token: Token, - - pub fn iterate(self: &NodeCall, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.callee; - i -= 1; - - if (i < self.params.len) return self.params.at(i); - i -= self.params.len; - - return null; - } - - pub fn firstToken(self: &NodeCall) Token { - return self.callee.firstToken(); - } - - pub fn lastToken(self: &NodeCall) Token { - return self.rparen_token; - } -}; - -pub const NodeArrayAccess = struct { - base: Node, - expr: &Node, - index: &Node, - rbracket_token: Token, - - pub fn iterate(self: &NodeArrayAccess, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.expr; - i -= 1; - - if (i < 1) return self.index; - i -= 1; - - return null; - } - - pub fn firstToken(self: &NodeArrayAccess) Token { - return self.expr.firstToken(); - } - - pub fn lastToken(self: &NodeArrayAccess) Token { - return self.rbracket_token; - } -}; - -pub const NodeSliceExpression = struct { - base: Node, - expr: &Node, - start: &Node, - end: ?&Node, - rbracket_token: Token, - - pub fn iterate(self: &NodeSliceExpression, index: usize) ?&Node { - var i = index; - - if (i < 1) return self.callee; - i -= 1; - - if (i < 1) return self.start; - i -= 1; - - if (i < 1) return self.end; - i -= 1; - - return null; - } - - pub fn firstToken(self: &NodeSliceExpression) Token { - return self.expr.firstToken(); - } - - pub fn lastToken(self: &NodeSliceExpression) Token { - return self.rbracket_token; - } -}; - pub const NodeStringLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 671f454f5..6d5ec54bc 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -87,7 +87,7 @@ pub const Parser = struct { AfterOperand, InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, - SuffixOp: &ast.Node, + SuffixOp: &ast.NodeSuffixOp, SliceOrArrayAccess, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, @@ -602,20 +602,19 @@ pub const Parser = struct { } else if (token.id == Token.Id.LParen) { self.putBackToken(token); - const node = try arena.create(ast.NodeCall); - *node = ast.NodeCall { - .base = self.initNode(ast.Node.Id.Call), - .callee = undefined, - .params = ArrayList(&ast.Node).init(arena), - .rparen_token = undefined, - }; - try stack.append(State { .SuffixOp = &node.base }); + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Call = ast.NodeSuffixOp.CallInfo { + .params = ArrayList(&ast.Node).init(arena), + .is_async = false, // TODO: ASYNC + } + }); + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.params }); + try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.LParen, - .ptr = &node.rparen_token, + .ptr = &node.rtoken, }, }); continue; @@ -643,16 +642,14 @@ pub const Parser = struct { switch (rbracket_or_ellipsis2_token.id) { Token.Id.Ellipsis2 => { - const node = try arena.create(ast.NodeSliceExpression); - *node = ast.NodeSliceExpression { - .base = self.initNode(ast.Node.Id.SliceExpression), - .expr = undefined, - .start = expression, - .end = null, - .rbracket_token = undefined, - }; + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Slice = ast.NodeSuffixOp.SliceRange { + .start = expression, + .end = null, + } + }); - try stack.append(State { .SuffixOp = &node.base }); + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); const rbracket_token = self.getNextToken(); @@ -661,24 +658,21 @@ pub const Parser = struct { try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RBracket, - .ptr = &node.rbracket_token, + .ptr = &node.rtoken, } }); - try stack.append(State { .Expression = DestPtr { .NullableField = &node.end } }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); } else { - node.rbracket_token = rbracket_token; + node.rtoken = rbracket_token; } break; }, Token.Id.RBracket => { - const node = try arena.create(ast.NodeArrayAccess); - *node = ast.NodeArrayAccess { - .base = self.initNode(ast.Node.Id.ArrayAccess), - .expr = undefined, - .index = expression, - .rbracket_token = token, - }; - try stack.append(State { .SuffixOp = &node.base }); + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = expression + }); + node.rtoken = token; + try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); break; }, @@ -950,27 +944,8 @@ pub const Parser = struct { while (true) { switch (stack.pop()) { State.SuffixOp => |suffix_op| { - switch (suffix_op.id) { - ast.Node.Id.Call => { - const call = @fieldParentPtr(ast.NodeCall, "base", suffix_op); - *left_leaf_ptr = &call.base; - left_leaf_ptr = &call.callee; - continue; - }, - ast.Node.Id.ArrayAccess => { - const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", suffix_op); - *left_leaf_ptr = &arr_access.base; - left_leaf_ptr = &arr_access.expr; - continue; - }, - ast.Node.Id.SliceExpression => { - const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", suffix_op); - *left_leaf_ptr = &slice_expr.base; - left_leaf_ptr = &slice_expr.expr; - continue; - }, - else => unreachable, - } + *left_leaf_ptr = &suffix_op.base; + left_leaf_ptr = &suffix_op.lhs; }, State.Operand => |operand| { *left_leaf_ptr = operand; @@ -1172,6 +1147,18 @@ pub const Parser = struct { return node; } + fn createSuffixOp(self: &Parser, arena: &mem.Allocator, op: &const ast.NodeSuffixOp.SuffixOp) !&ast.NodeSuffixOp { + const node = try arena.create(ast.NodeSuffixOp); + + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = *op, + .rtoken = undefined, + }; + return node; + } + fn createIdentifier(self: &Parser, arena: &mem.Allocator, name_token: &const Token) !&ast.NodeIdentifier { const node = try arena.create(ast.NodeIdentifier); @@ -1625,6 +1612,43 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), } }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", base); + + switch (suffix_op.op) { + ast.NodeSuffixOp.SuffixOp.Call => |call_info| { + try stack.append(RenderState { .Text = ")"}); + var i = call_info.params.len; + while (i != 0) { + i -= 1; + const param_node = call_info.params.at(i); + try stack.append(RenderState { .Expression = param_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + try stack.append(RenderState { .Text = "("}); + }, + ast.NodeSuffixOp.SuffixOp.ArrayAccess => |index_expr| { + try stack.append(RenderState { .Text = "]"}); + try stack.append(RenderState { .Expression = index_expr}); + try stack.append(RenderState { .Text = "["}); + }, + ast.NodeSuffixOp.SuffixOp.Slice => |range| { + try stack.append(RenderState { .Text = "]"}); + if (range.end) |end| { + try stack.append(RenderState { .Expression = end}); + } + try stack.append(RenderState { .Text = ".."}); + try stack.append(RenderState { .Expression = range.start}); + try stack.append(RenderState { .Text = "["}); + }, + ast.NodeSuffixOp.SuffixOp.StructInitializer => @panic("TODO: StructInitializer"), + ast.NodeSuffixOp.SuffixOp.ArrayInitializer => @panic("TODO: ArrayInitializer"), + } + + try stack.append(RenderState { .Expression = suffix_op.lhs }); + }, ast.Node.Id.IntegerLiteral => { const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token)); @@ -1693,39 +1717,6 @@ pub const Parser = struct { } } }, - ast.Node.Id.Call => { - const call = @fieldParentPtr(ast.NodeCall, "base", base); - try stack.append(RenderState { .Text = ")"}); - var i = call.params.len; - while (i != 0) { - i -= 1; - const param_node = call.params.at(i); - try stack.append(RenderState { .Expression = param_node}); - if (i != 0) { - try stack.append(RenderState { .Text = ", " }); - } - } - try stack.append(RenderState { .Text = "("}); - try stack.append(RenderState { .Expression = call.callee }); - }, - ast.Node.Id.ArrayAccess => { - const arr_access = @fieldParentPtr(ast.NodeArrayAccess, "base", base); - try stack.append(RenderState { .Text = "]"}); - try stack.append(RenderState { .Expression = arr_access.index}); - try stack.append(RenderState { .Text = "["}); - try stack.append(RenderState { .Expression = arr_access.expr }); - }, - ast.Node.Id.SliceExpression => { - const slice_expr = @fieldParentPtr(ast.NodeSliceExpression, "base", base); - try stack.append(RenderState { .Text = "]"}); - if (slice_expr.end) |end| { - try stack.append(RenderState { .Expression = end}); - } - try stack.append(RenderState { .Text = ".."}); - try stack.append(RenderState { .Expression = slice_expr.start}); - try stack.append(RenderState { .Text = "["}); - try stack.append(RenderState { .Expression = slice_expr.expr}); - }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), From 0b9247fb636a47add80f7c4ec194df436629df91 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 14:20:34 +0200 Subject: [PATCH 061/132] std.zig.parser Refactor: * Slice/Array access is now not parsed in the expr contruction loop * State.ExprListItemOrEnd now takes a token id for the end token --- std/zig/ast.zig | 30 ++++++++- std/zig/parser.zig | 149 +++++++++++++++++++++++++++------------------ 2 files changed, 118 insertions(+), 61 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 380dd95ae..2ec305429 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -18,6 +18,7 @@ pub const Node = struct { InfixOp, PrefixOp, SuffixOp, + FieldInitializer, IntegerLiteral, FloatLiteral, StringLiteral, @@ -45,6 +46,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index), @@ -73,6 +75,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), @@ -101,6 +104,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), + Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), @@ -526,6 +530,30 @@ pub const NodePrefixOp = struct { } }; +pub const NodeFieldInitializer = struct { + base: Node, + dot_token: Token, + name_token: Token, + expr: &Node, + + pub fn iterate(self: &NodeFieldInitializer, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeFieldInitializer) Token { + return self.dot_token; + } + + pub fn lastToken(self: &NodeFieldInitializer) Token { + return self.expr.lastToken(); + } +}; + pub const NodeSuffixOp = struct { base: Node, lhs: &Node, @@ -537,7 +565,7 @@ pub const NodeSuffixOp = struct { ArrayAccess: &Node, Slice: SliceRange, ArrayInitializer: ArrayList(&Node), - StructInitializer: ArrayList(&Node), + StructInitializer: ArrayList(&NodeFieldInitializer), }; const CallInfo = struct { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 6d5ec54bc..43cc35fb6 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -77,6 +77,12 @@ pub const Parser = struct { ptr: &Token, }; + const ExprListState = struct { + list: &ArrayList(&ast.Node), + end: Token.Id, + ptr: &Token, + }; + const State = union(enum) { TopLevel, TopLevelExtern: ?Token, @@ -88,7 +94,7 @@ pub const Parser = struct { InfixOp: &ast.NodeInfixOp, PrefixOp: &ast.NodePrefixOp, SuffixOp: &ast.NodeSuffixOp, - SliceOrArrayAccess, + SliceOrArrayAccess: &ast.NodeSuffixOp, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, @@ -104,8 +110,8 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: &ArrayList(&ast.Node), - ExprListCommaOrEnd: &ArrayList(&ast.Node), + ExprListItemOrEnd: ExprListState, + ExprListCommaOrEnd: ExprListState, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -529,13 +535,14 @@ pub const Parser = struct { .Operand = &node.base }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.params }); try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LParen, + .ExprListItemOrEnd = ExprListState { + .list = &node.params, + .end = Token.Id.RParen, .ptr = &node.rparen_token, - }, + } }); + try stack.append(State { .ExpectToken = Token.Id.LParen, }); continue; }, Token.Id.StringLiteral => { @@ -587,6 +594,46 @@ pub const Parser = struct { } }, + State.SliceOrArrayAccess => |node| { + var token = self.getNextToken(); + + switch (token.id) { + Token.Id.Ellipsis2 => { + const start = node.op.ArrayAccess; + node.op = ast.NodeSuffixOp.SuffixOp { + .Slice = ast.NodeSuffixOp.SliceRange { + .start = start, + .end = undefined, + } + }; + try stack.append(State { .SuffixOp = node }); + try stack.append(State.AfterOperand); + + const rbracket_token = self.getNextToken(); + if (rbracket_token.id != Token.Id.RBracket) { + self.putBackToken(rbracket_token); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RBracket, + .ptr = &node.rtoken, + } + }); + try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); + } else { + node.rtoken = rbracket_token; + } + continue; + }, + Token.Id.RBracket => { + node.rtoken = token; + try stack.append(State { .SuffixOp = node }); + try stack.append(State.AfterOperand); + continue; + }, + else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) + } + }, + State.AfterOperand => { // we'll either get an infix operator (like != or ^), // or a postfix operator (like () or {}), @@ -610,7 +657,13 @@ pub const Parser = struct { }); try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); - try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params }); + try stack.append(State { + .ExprListItemOrEnd = ExprListState { + .list = &node.op.Call.params, + .end = Token.Id.RParen, + .ptr = &node.rtoken, + } + }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.LParen, @@ -620,8 +673,17 @@ pub const Parser = struct { continue; } else if (token.id == Token.Id.LBracket) { - try stack.append(State.SliceOrArrayAccess); - try stack.append(State.ExpectOperand); + const node = try arena.create(ast.NodeSuffixOp); + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = undefined, + }, + .rtoken = undefined, + }; + try stack.append(State { .SliceOrArrayAccess = node }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); continue; // TODO: Parse postfix operator @@ -637,48 +699,6 @@ pub const Parser = struct { try dest_ptr.store(expression); break; }, - State.SliceOrArrayAccess => { - var rbracket_or_ellipsis2_token = self.getNextToken(); - - switch (rbracket_or_ellipsis2_token.id) { - Token.Id.Ellipsis2 => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .Slice = ast.NodeSuffixOp.SliceRange { - .start = expression, - .end = null, - } - }); - - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - - const rbracket_token = self.getNextToken(); - if (rbracket_token.id != Token.Id.RBracket) { - self.putBackToken(rbracket_token); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.RBracket, - .ptr = &node.rtoken, - } - }); - try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); - } else { - node.rtoken = rbracket_token; - } - break; - }, - Token.Id.RBracket => { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .ArrayAccess = expression - }); - node.rtoken = token; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - break; - }, - else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) - } - }, State.InfixOp => |infix_op| { infix_op.rhs = expression; infix_op.lhs = popSuffixOp(&stack); @@ -697,26 +717,31 @@ pub const Parser = struct { } }, - State.ExprListItemOrEnd => |params| { + State.ExprListItemOrEnd => |expr_list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.RParen => continue, else => { self.putBackToken(token); - stack.append(State { .ExprListCommaOrEnd = params }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = params} }); + stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} }); }, } }, - State.ExprListCommaOrEnd => |params| { + State.ExprListCommaOrEnd => |expr_list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = params }) catch unreachable; + stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable; + }, + else => { + const IdTag = @TagType(Token.Id); + if (IdTag(expr_list_state.end) == token.id) + continue; + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id)); }, - Token.Id.RParen => continue, - else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)), } }, @@ -933,7 +958,6 @@ pub const Parser = struct { State.PrefixOp => unreachable, State.SuffixOp => unreachable, State.Operand => unreachable, - State.SliceOrArrayAccess => unreachable, } } } @@ -1649,6 +1673,11 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.FieldInitializer => { + const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); + try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); + try stack.append(RenderState { .Expression = field_init.expr }); + }, ast.Node.Id.IntegerLiteral => { const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token)); From 5c82ed2ea9360ea0931ba789a4ffb73d689b6a72 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 14:53:27 +0200 Subject: [PATCH 062/132] std.zig.parser now parses initializers... Or, it would, if it worked --- std/zig/ast.zig | 4 +- std/zig/parser.zig | 149 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 125 insertions(+), 28 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2ec305429..c4b4ef983 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -532,7 +532,7 @@ pub const NodePrefixOp = struct { pub const NodeFieldInitializer = struct { base: Node, - dot_token: Token, + period_token: Token, name_token: Token, expr: &Node, @@ -546,7 +546,7 @@ pub const NodeFieldInitializer = struct { } pub fn firstToken(self: &NodeFieldInitializer) Token { - return self.dot_token; + return self.period_token; } pub fn lastToken(self: &NodeFieldInitializer) Token { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 43cc35fb6..19ed4af4b 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -75,13 +75,16 @@ pub const Parser = struct { const ExpectTokenSave = struct { id: Token.Id, ptr: &Token, + }; - const ExprListState = struct { - list: &ArrayList(&ast.Node), - end: Token.Id, - ptr: &Token, - }; + fn ListState(comptime T: type) type { + return struct { + list: &ArrayList(T), + end: Token.Id, + ptr: &Token, + }; + } const State = union(enum) { TopLevel, @@ -110,8 +113,10 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: ExprListState, - ExprListCommaOrEnd: ExprListState, + ExprListItemOrEnd: ListState(&ast.Node), + ExprListCommaOrEnd: ListState(&ast.Node), + FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), + FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), }; /// Returns an AST tree, allocated with the parser's allocator. @@ -536,7 +541,7 @@ pub const Parser = struct { }); try stack.append(State.AfterOperand); try stack.append(State { - .ExprListItemOrEnd = ExprListState { + .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, @@ -647,8 +652,6 @@ pub const Parser = struct { continue; } else if (token.id == Token.Id.LParen) { - self.putBackToken(token); - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { .Call = ast.NodeSuffixOp.CallInfo { .params = ArrayList(&ast.Node).init(arena), @@ -658,18 +661,12 @@ pub const Parser = struct { try stack.append(State { .SuffixOp = node }); try stack.append(State.AfterOperand); try stack.append(State { - .ExprListItemOrEnd = ExprListState { + .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.op.Call.params, .end = Token.Id.RParen, .ptr = &node.rtoken, } }); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LParen, - .ptr = &node.rtoken, - }, - }); continue; } else if (token.id == Token.Id.LBracket) { @@ -686,6 +683,47 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); continue; + // TODO: This is the initializer parsing code. It doesn't work because of + // the ambiguity between function bodies and initializers: + // fn main() void {} or fn main() (void {}) + } else if (false) { //(token.id == Token.Id.LBrace) { + const next = self.getNextToken(); + + switch (next.id) { + Token.Id.Period => { + self.putBackToken(token); + + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }); + + try stack.append(State { + .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .list = &node.op.StructInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + else => { + self.putBackToken(token); + + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }); + + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.ArrayInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + } + // TODO: Parse postfix operator } else { // no postfix/infix operator after this operand. @@ -717,30 +755,89 @@ pub const Parser = struct { } }, - State.ExprListItemOrEnd => |expr_list_state| { + State.ExprListItemOrEnd => |list_state| { + var token = self.getNextToken(); + + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + stack.append(State { .ExprListCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); + }, + + State.ExprListCommaOrEnd => |list_state| { var token = self.getNextToken(); switch (token.id) { - Token.Id.RParen => continue, + Token.Id.Comma => { + stack.append(State { .ExprListItemOrEnd = list_state }) catch unreachable; + }, else => { - self.putBackToken(token); - stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} }); + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; + continue; + } + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); }, } }, - State.ExprListCommaOrEnd => |expr_list_state| { + State.FieldInitListItemOrEnd => |list_state| { + var token = self.getNextToken(); + + const IdTag = @TagType(Token.Id); + if (IdTag(list_state.end) == token.id){ + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + + const node = try arena.create(ast.NodeFieldInitializer); + *node = ast.NodeFieldInitializer { + .base = self.initNode(ast.Node.Id.FieldInitializer), + .period_token = undefined, + .name_token = undefined, + .expr = undefined, + }; + try list_state.list.append(node); + + stack.append(State { .FieldInitListCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = &node.expr} }); + try stack.append(State { .ExpectToken = Token.Id.Equal }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &node.name_token, + } + }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Period, + .ptr = &node.period_token, + } + }); + }, + + State.FieldInitListCommaOrEnd => |list_state| { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable; + stack.append(State { .FieldInitListItemOrEnd = list_state }) catch unreachable; }, else => { const IdTag = @TagType(Token.Id); - if (IdTag(expr_list_state.end) == token.id) + if (IdTag(list_state.end) == token.id) { + *list_state.ptr = token; continue; + } - return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id)); + return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); }, } }, From 9d69e94bbad0b1ff23999584f5f632cfe8db3656 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:16:32 +0200 Subject: [PATCH 063/132] std.zig.parser now parses grouped expressions * I also moved some tests down, as they fail in ways I can't fix yet --- std/zig/ast.zig | 28 +++++++++++ std/zig/parser.zig | 115 ++++++++++++++++++++++++++++----------------- 2 files changed, 99 insertions(+), 44 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index c4b4ef983..32a8a7f11 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -18,6 +18,7 @@ pub const Node = struct { InfixOp, PrefixOp, SuffixOp, + GroupedExpression, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -46,6 +47,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -75,6 +77,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -104,6 +107,7 @@ pub const Node = struct { Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), + Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -624,6 +628,30 @@ pub const NodeSuffixOp = struct { } }; +pub const NodeGroupedExpression = struct { + base: Node, + lparen: Token, + expr: &Node, + rparen: Token, + + pub fn iterate(self: &NodeGroupedExpression, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeGroupedExpression) Token { + return self.lparen; + } + + pub fn lastToken(self: &NodeGroupedExpression) Token { + return self.rparen; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 19ed4af4b..61552f2a3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -594,6 +594,27 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.LParen => { + const node = try arena.create(ast.NodeGroupedExpression); + *node = ast.NodeGroupedExpression { + .base = self.initNode(ast.Node.Id.GroupedExpression), + .lparen = token, + .expr = undefined, + .rparen = undefined, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), } @@ -1770,6 +1791,12 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.GroupedExpression => { + const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); + try stack.append(RenderState { .Text = ")"}); + try stack.append(RenderState { .Expression = grouped_expr.expr }); + try stack.append(RenderState { .Text = "("}); + }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); @@ -2240,50 +2267,6 @@ test "zig fmt: indexing" { ); } -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); -} - -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{}; - \\ !(a{}); - \\ a + b{}; - \\ (a + b){}; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; - \\} - \\ - ); -} - test "zig fmt: struct declaration" { try testCanonical( \\const S = struct { @@ -2682,6 +2665,50 @@ test "zig fmt: coroutines" { ); } +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{}; + \\ !(a{}); + \\ a + b{}; + \\ (a + b){}; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + ); +} + test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 40f35e997a2155df8141ec309e43d6dad9785965 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:17:26 +0200 Subject: [PATCH 064/132] std.zig.parser moved container initializer tests down --- std/zig/parser.zig | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 61552f2a3..bcb66a6e6 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2360,16 +2360,6 @@ test "zig fmt: union declaration" { ); } -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; - \\ - ); -} - test "zig fmt: switch" { try testCanonical( \\test "switch" { @@ -2676,6 +2666,16 @@ test "zig fmt: arrays" { ); } +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); +} + test "zig fmt: precedence" { try testCanonical( \\test "precedence" { From 4fae452684c108275e9d3003b4927c8c8d41c59b Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:33:22 +0200 Subject: [PATCH 065/132] std.zig.parser Refactored top level decl parsing * Now, the arraylist from the root node is passed through the states. * This allows us to reuse the code for enums, unions and structs --- std/zig/parser.zig | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index bcb66a6e6..99ea89bf9 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -53,6 +53,7 @@ pub const Parser = struct { } const TopLevelDeclCtx = struct { + decls: &ArrayList(&ast.Node), visib_token: ?Token, extern_token: ?Token, lib_name: ?&ast.Node, @@ -75,7 +76,6 @@ pub const Parser = struct { const ExpectTokenSave = struct { id: Token.Id, ptr: &Token, - }; fn ListState(comptime T: type) type { @@ -88,7 +88,7 @@ pub const Parser = struct { const State = union(enum) { TopLevel, - TopLevelExtern: ?Token, + TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, Expression: DestPtr, ExpectOperand, @@ -182,7 +182,14 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_pub, Token.Id.Keyword_export => { - stack.append(State { .TopLevelExtern = token }) catch unreachable; + stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }) catch unreachable; continue; }, Token.Id.Keyword_test => { @@ -208,12 +215,19 @@ pub const Parser = struct { }, else => { self.putBackToken(token); - stack.append(State { .TopLevelExtern = null }) catch unreachable; + stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = null, + .extern_token = null, + .lib_name = null, + } + }) catch unreachable; continue; }, } }, - State.TopLevelExtern => |visib_token| { + State.TopLevelExtern => |ctx| { const token = self.getNextToken(); if (token.id == Token.Id.Keyword_extern) { const lib_name_token = self.getNextToken(); @@ -229,7 +243,8 @@ pub const Parser = struct { stack.append(State { .TopLevelDecl = TopLevelDeclCtx { - .visib_token = visib_token, + .decls = ctx.decls, + .visib_token = ctx.visib_token, .extern_token = token, .lib_name = lib_name, }, @@ -237,13 +252,7 @@ pub const Parser = struct { continue; } self.putBackToken(token); - stack.append(State { - .TopLevelDecl = TopLevelDeclCtx { - .visib_token = visib_token, - .extern_token = null, - .lib_name = null, - }, - }) catch unreachable; + stack.append(State { .TopLevelDecl = ctx }) catch unreachable; continue; }, State.TopLevelDecl => |ctx| { @@ -252,7 +261,7 @@ pub const Parser = struct { Token.Id.Keyword_var, Token.Id.Keyword_const => { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts - const var_decl_node = try self.createAttachVarDecl(arena, &root_node.decls, ctx.visib_token, + const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); try stack.append(State { .VarDecl = var_decl_node }); continue; @@ -260,7 +269,7 @@ pub const Parser = struct { Token.Id.Keyword_fn => { stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts - const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); @@ -270,7 +279,7 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, &root_node.decls, fn_token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); From d602f12df8f12c3a363abec8c1976f8bc84eb2be Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 15:59:14 +0200 Subject: [PATCH 066/132] std.zig.ast Added ContainerDecl --- std/zig/ast.zig | 69 +++++++++++++++++++++++++++++++++++++++++++++- std/zig/parser.zig | 4 +++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 32a8a7f11..767a797ca 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + ContainerDecl, Identifier, FnProto, ParamDecl, @@ -40,6 +41,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), @@ -70,6 +72,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), @@ -100,6 +103,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), @@ -196,6 +200,69 @@ pub const NodeVarDecl = struct { } }; +pub const NodeContainerDecl = struct { + base: Node, + kind_token: Token, + init_arg_expr: InitArg, + kind: Kind, + decls: ArrayList(&Node), + rbrace_token: Token, + + // TODO: Different array lists for each kind. + const Kind = union(enum) { + Struct: ArrayList(&Node), + Enum: ArrayList(&Node), + Union: ArrayList(&Node), + }; + + const InitArg = union(enum) { + None, + Enum, + Type: &Node, + }; + + pub fn iterate(self: &NodeContainerDecl, index: usize) ?&Node { + var i = index; + + switch (self.init_arg_expr) { + InitArg.Type => |t| { + if (i < 1) return t; + i -= 1; + }, + InitArg.None, + InitArg.Enum => { } + } + + if (i < self.decls.len) return self.decls.at(i); + i -= self.decls.len; + + switch (self.kind) { + Kind.Struct => |fields| { + if (i < fields.len) return fields.at(i); + i -= fields.len; + }, + Kind.Enum => |tags| { + if (i < tags.len) return tags.at(i); + i -= tags.len; + }, + Kind.Union => |tags| { + if (i < tags.len) return tags.at(i); + i -= tags.len; + }, + } + + return null; + } + + pub fn firstToken(self: &NodeContainerDecl) Token { + return self.kind_token; + } + + pub fn lastToken(self: &NodeContainerDecl) Token { + return self.rbrace_token; + } +}; + pub const NodeIdentifier = struct { base: Node, name_token: Token, @@ -611,7 +678,7 @@ pub const NodeSuffixOp = struct { i -= exprs.len; }, SuffixOp.StructInitializer => |fields| { - if (i < fields.len) return fields.at(i); + if (i < fields.len) return &fields.at(i).base; i -= fields.len; }, } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 99ea89bf9..0b4e55913 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1806,6 +1806,10 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = grouped_expr.expr }); try stack.append(RenderState { .Text = "("}); }, + ast.Node.Id.ContainerDecl => { + const container_decl = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); + @panic("TODO: ContainerDecl"); + }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); From aadc14fd78c4137ffcc02e3ba67c0c60100b4d69 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 10:17:57 -0400 Subject: [PATCH 067/132] upload static linux artifacts on successful travis build --- .travis.yml | 8 ++++++++ ci/travis_linux_script | 22 +++------------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 271b6069b..df6f60a48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,11 @@ +sudo: required +services: + - docker +addons: + artifacts: + working_dir: artifacts + target_paths: + - /builds os: - linux - osx diff --git a/ci/travis_linux_script b/ci/travis_linux_script index a709539bc..55e188d55 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -9,24 +9,8 @@ echo $PATH mkdir build cd build cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -make VERBOSE=1 -make install +make -j2 install ./zig build --build-file ../build.zig test -./zig test ../test/behavior.zig --target-os windows --target-arch i386 --target-environ msvc -wine zig-cache/test.exe - -./zig test ../test/behavior.zig --target-os windows --target-arch i386 --target-environ msvc --release-fast -wine zig-cache/test.exe - -./zig test ../test/behavior.zig --target-os windows --target-arch i386 --target-environ msvc --release-safe -wine zig-cache/test.exe - -./zig test ../test/behavior.zig --target-os windows --target-arch x86_64 --target-environ msvc -wine64 zig-cache/test.exe - -#./zig test ../test/behavior.zig --target-os windows --target-arch x86_64 --target-environ msvc --release-fast -#wine64 test.exe -# -#./zig test ../test/behavior.zig --target-os windows --target-arch x86_64 --target-environ msvc --release-safe -#wine64 test.exe +mkdir $TRAVIS_BUILD_DIR/artifacts +docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR/artifacts",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From 65e4bb149e2226e3d796c6378c74d9af3af81828 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 12:04:06 -0400 Subject: [PATCH 068/132] travis artifacts: don't upload extra stuff --- ci/travis_linux_script | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index 55e188d55..b88c29c60 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -12,5 +12,6 @@ cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) make -j2 install ./zig build --build-file ../build.zig test -mkdir $TRAVIS_BUILD_DIR/artifacts -docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR/artifacts",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT +cd $TRAVIS_BUILD_DIR +git clean -fd +docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From 21b47b34d85f6b974c371b6e5b9421a87650344a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 12:59:28 -0400 Subject: [PATCH 069/132] travis: don't upload build/ folder as artifacts --- ci/travis_linux_script | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index b88c29c60..825947701 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -14,4 +14,5 @@ make -j2 install cd $TRAVIS_BUILD_DIR git clean -fd +rm -rf build docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From ec611bf8b47af0db5244af644040907bfcb63945 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 3 Apr 2018 20:00:02 +0200 Subject: [PATCH 070/132] std.zig.parser now parses regular enums, unions and struct * Still missing packed, and extern --- std/zig/ast.zig | 106 ++++++++++++++-- std/zig/parser.zig | 305 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 351 insertions(+), 60 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 767a797ca..484bc59f1 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -12,6 +12,9 @@ pub const Node = struct { Root, VarDecl, ContainerDecl, + StructField, + UnionTag, + EnumTag, Identifier, FnProto, ParamDecl, @@ -42,6 +45,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), @@ -73,6 +79,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), @@ -104,6 +113,9 @@ pub const Node = struct { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), + Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), + Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), + Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), @@ -203,16 +215,15 @@ pub const NodeVarDecl = struct { pub const NodeContainerDecl = struct { base: Node, kind_token: Token, - init_arg_expr: InitArg, kind: Kind, - decls: ArrayList(&Node), + init_arg_expr: InitArg, + fields_and_decls: ArrayList(&Node), rbrace_token: Token, - // TODO: Different array lists for each kind. - const Kind = union(enum) { - Struct: ArrayList(&Node), - Enum: ArrayList(&Node), - Union: ArrayList(&Node), + const Kind = enum { + Struct, + Enum, + Union, }; const InitArg = union(enum) { @@ -263,6 +274,87 @@ pub const NodeContainerDecl = struct { } }; +pub const NodeStructField = struct { + base: Node, + name_token: Token, + type_expr: &Node, + + pub fn iterate(self: &NodeStructField, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.type_expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeStructField) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeStructField) Token { + return self.type_expr.lastToken(); + } +}; + +pub const NodeUnionTag = struct { + base: Node, + name_token: Token, + type_expr: ?&Node, + + pub fn iterate(self: &NodeUnionTag, index: usize) ?&Node { + var i = index; + + if (self.type_expr) |type_expr| { + if (i < 1) return type_expr; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeUnionTag) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeUnionTag) Token { + if (self.type_expr) |type_expr| { + return type_expr.lastToken(); + } + + return self.name_token; + } +}; + +pub const NodeEnumTag = struct { + base: Node, + name_token: Token, + value: ?&Node, + + pub fn iterate(self: &NodeEnumTag, index: usize) ?&Node { + var i = index; + + if (self.value) |value| { + if (i < 1) return value; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeEnumTag) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeEnumTag) Token { + if (self.value) |value| { + return value.lastToken(); + } + + return self.name_token; + } +}; + pub const NodeIdentifier = struct { base: Node, name_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0b4e55913..fc00ba5f4 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -90,6 +90,7 @@ pub const Parser = struct { TopLevel, TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, + ContainerDecl: &ast.NodeContainerDecl, Expression: DestPtr, ExpectOperand, Operand: &ast.Node, @@ -117,6 +118,7 @@ pub const Parser = struct { ExprListCommaOrEnd: ListState(&ast.Node), FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), + FieldListCommaOrEnd: &ast.NodeContainerDecl, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -181,17 +183,6 @@ pub const Parser = struct { State.TopLevel => { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_pub, Token.Id.Keyword_export => { - stack.append(State { - .TopLevelExtern = TopLevelDeclCtx { - .decls = &root_node.decls, - .visib_token = token, - .extern_token = null, - .lib_name = null, - } - }) catch unreachable; - continue; - }, Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; @@ -213,16 +204,29 @@ pub const Parser = struct { root_node.eof_token = token; return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; }, + Token.Id.Keyword_pub, Token.Id.Keyword_export => { + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &root_node.decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }); + continue; + }, else => { self.putBackToken(token); - stack.append(State { + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { .TopLevelExtern = TopLevelDeclCtx { .decls = &root_node.decls, .visib_token = null, .extern_token = null, .lib_name = null, } - }) catch unreachable; + }); continue; }, } @@ -259,7 +263,6 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_var, Token.Id.Keyword_const => { - stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); @@ -267,7 +270,6 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - stack.append(State.TopLevel) catch unreachable; // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); @@ -276,7 +278,6 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - stack.append(State.TopLevel) catch unreachable; const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, @@ -336,6 +337,101 @@ pub const Parser = struct { } return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); }, + + State.ContainerDecl => |container_decl| { + const token = self.getNextToken(); + + switch (token.id) { + Token.Id.Identifier => { + switch (container_decl.kind) { + ast.NodeContainerDecl.Kind.Struct => { + const node = try arena.create(ast.NodeStructField); + *node = ast.NodeStructField { + .base = self.initNode(ast.Node.Id.StructField), + .name_token = token, + .type_expr = undefined, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); + try stack.append(State { .ExpectToken = Token.Id.Colon }); + continue; + }, + ast.NodeContainerDecl.Kind.Union => { + const node = try arena.create(ast.NodeUnionTag); + *node = ast.NodeUnionTag { + .base = self.initNode(ast.Node.Id.UnionTag), + .name_token = token, + .type_expr = null, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Colon) { + self.putBackToken(next); + continue; + } + + try stack.append(State { .Expression = DestPtr { .NullableField = &node.type_expr } }); + continue; + }, + ast.NodeContainerDecl.Kind.Enum => { + const node = try arena.create(ast.NodeEnumTag); + *node = ast.NodeEnumTag { + .base = self.initNode(ast.Node.Id.EnumTag), + .name_token = token, + .value = null, + }; + try container_decl.fields_and_decls.append(&node.base); + + try stack.append(State { .FieldListCommaOrEnd = container_decl }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Equal) { + self.putBackToken(next); + continue; + } + + try stack.append(State { .Expression = DestPtr { .NullableField = &node.value } }); + continue; + }, + } + }, + Token.Id.Keyword_pub, Token.Id.Keyword_export => { + stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &container_decl.fields_and_decls, + .visib_token = token, + .extern_token = null, + .lib_name = null, + } + }); + continue; + }, + Token.Id.RBrace => { + container_decl.rbrace_token = token; + continue; + }, + else => { + self.putBackToken(token); + stack.append(State{ .ContainerDecl = container_decl }) catch unreachable; + try stack.append(State { + .TopLevelExtern = TopLevelDeclCtx { + .decls = &container_decl.fields_and_decls, + .visib_token = null, + .extern_token = null, + .lib_name = null, + } + }); + continue; + } + } + }, + State.ExpectToken => |token_id| { _ = try self.eatToken(token_id); continue; @@ -537,6 +633,53 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { + const node = try arena.create(ast.NodeContainerDecl); + *node = ast.NodeContainerDecl { + .base = self.initNode(ast.Node.Id.ContainerDecl), + .kind_token = token, + .kind = switch (token.id) { + Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, + Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, + Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, + else => unreachable, + }, + .init_arg_expr = undefined, + .fields_and_decls = ArrayList(&ast.Node).init(arena), + .rbrace_token = undefined, + }; + + try stack.append(State { .Operand = &node.base }); + try stack.append(State.AfterOperand); + try stack.append(State { .ContainerDecl = node }); + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + + const lparen = self.getNextToken(); + if (lparen.id != Token.Id.LParen) { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; + continue; + } + + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const init_arg_token = self.getNextToken(); + switch (init_arg_token.id) { + Token.Id.Keyword_enum => { + node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; + }, + else => { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; + try stack.append(State { + .Expression = DestPtr { + .Field = &node.init_arg_expr.Type + } + }); + }, + } + continue; + }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -799,24 +942,6 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); }, - State.ExprListCommaOrEnd => |list_state| { - var token = self.getNextToken(); - switch (token.id) { - Token.Id.Comma => { - stack.append(State { .ExprListItemOrEnd = list_state }) catch unreachable; - }, - else => { - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id) { - *list_state.ptr = token; - continue; - } - - return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); - }, - } - }, - State.FieldInitListItemOrEnd => |list_state| { var token = self.getNextToken(); @@ -854,22 +979,20 @@ pub const Parser = struct { }); }, - State.FieldInitListCommaOrEnd => |list_state| { - var token = self.getNextToken(); - switch (token.id) { - Token.Id.Comma => { - stack.append(State { .FieldInitListItemOrEnd = list_state }) catch unreachable; - }, - else => { - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id) { - *list_state.ptr = token; - continue; - } + State.ExprListCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .ExprListItemOrEnd = list_state }); + continue; + }, - return self.parseError(token, "expected ',' or {}, found {}", @tagName(list_state.end), @tagName(token.id)); - }, - } + State.FieldInitListCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); + continue; + }, + + State.FieldListCommaOrEnd => |container_decl| { + try self.commaOrEnd(&stack, Token.Id.RBrace, &container_decl.rbrace_token, + State { .ContainerDecl = container_decl }); + continue; }, State.AddrOfModifiers => |addr_of_info| { @@ -1089,6 +1212,24 @@ pub const Parser = struct { } } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { + var token = self.getNextToken(); + switch (token.id) { + Token.Id.Comma => { + stack.append(state_after_comma) catch unreachable; + }, + else => { + const IdTag = @TagType(Token.Id); + if (IdTag(*end) == token.id) { + *ptr = token; + return; + } + + return self.parseError(token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); + }, + } + } + fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { var expression: &ast.Node = undefined; var left_leaf_ptr: &&ast.Node = &expression; @@ -1806,10 +1947,6 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = grouped_expr.expr }); try stack.append(RenderState { .Text = "("}); }, - ast.Node.Id.ContainerDecl => { - const container_decl = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); - @panic("TODO: ContainerDecl"); - }, ast.Node.Id.FieldInitializer => { const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base); try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token)); @@ -1851,6 +1988,46 @@ pub const Parser = struct { const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); }, + ast.Node.Id.ContainerDecl => { + const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); + try stream.print("{} {{", self.tokenizer.getTokenSlice(container_decl.kind_token)); + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + + const fields_and_decls = container_decl.fields_and_decls.toSliceConst(); + var i = fields_and_decls.len; + while (i != 0) { + i -= 1; + const node = fields_and_decls[i]; + if (i != 0) { + switch (node.id) { + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag => { + try stack.append(RenderState { .Text = "," }); + }, + else => { } + } + } + try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = fields_and_decls[i - 1]; + const prev_line_index = prev_node.lastToken().line; + const this_line_index = node.firstToken().line; + if (this_line_index - prev_line_index >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); @@ -1883,6 +2060,28 @@ pub const Parser = struct { } } }, + ast.Node.Id.StructField => { + const field = @fieldParentPtr(ast.NodeStructField, "base", base); + try stream.print("{}:", self.tokenizer.getTokenSlice(field.name_token)); + }, + ast.Node.Id.UnionTag => { + const tag = @fieldParentPtr(ast.NodeUnionTag, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.type_expr) |type_expr| { + try stream.print(": "); + try stack.append(RenderState { .Expression = type_expr}); + } + }, + ast.Node.Id.EnumTag => { + const tag = @fieldParentPtr(ast.NodeEnumTag, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.value) |value| { + try stream.print(" = "); + try stack.append(RenderState { .Expression = value}); + } + }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), From d1f8e722b5e48666b4d7d833c0cd6b36f45bd9bb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 14:23:56 -0400 Subject: [PATCH 071/132] travis: don't upload other files as artifacts --- ci/travis_linux_script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index 825947701..f80ac33c2 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -13,6 +13,6 @@ make -j2 install ./zig build --build-file ../build.zig test cd $TRAVIS_BUILD_DIR +rm .gitignore git clean -fd -rm -rf build docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT From 9dfd1a7c8a79cd5213878c56695f2fcb8aa25580 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 18:26:49 -0400 Subject: [PATCH 072/132] remove more signal handling stuff from std.os.ChildProcess 439621e44a68b436f958a84fcdb0bdac83613aea failed to remove everything. this finishes the job --- std/os/child_process.zig | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/std/os/child_process.zig b/std/os/child_process.zig index 06802e657..8bb8b2d7e 100644 --- a/std/os/child_process.zig +++ b/std/os/child_process.zig @@ -13,8 +13,6 @@ const builtin = @import("builtin"); const Os = builtin.Os; const LinkedList = std.LinkedList; -var children_nodes = LinkedList(&ChildProcess).init(); - const is_windows = builtin.os == Os.windows; pub const ChildProcess = struct { @@ -296,8 +294,6 @@ pub const ChildProcess = struct { } fn cleanupAfterWait(self: &ChildProcess, status: i32) !Term { - children_nodes.remove(&self.llnode); - defer { os.close(self.err_pipe[0]); os.close(self.err_pipe[1]); @@ -427,9 +423,6 @@ pub const ChildProcess = struct { self.llnode = LinkedList(&ChildProcess).Node.init(self); self.term = null; - // TODO make this atomic so it works even with threads - children_nodes.prepend(&self.llnode); - if (self.stdin_behavior == StdIo.Pipe) { os.close(stdin_pipe[0]); } if (self.stdout_behavior == StdIo.Pipe) { os.close(stdout_pipe[1]); } if (self.stderr_behavior == StdIo.Pipe) { os.close(stderr_pipe[1]); } @@ -773,31 +766,3 @@ fn readIntFd(fd: i32) !ErrInt { os.posixRead(fd, bytes[0..]) catch return error.SystemResources; return mem.readInt(bytes[0..], ErrInt, builtin.endian); } - -extern fn sigchld_handler(_: i32) void { - while (true) { - var status: i32 = undefined; - const pid_result = posix.waitpid(-1, &status, posix.WNOHANG); - if (pid_result == 0) { - return; - } - const err = posix.getErrno(pid_result); - if (err > 0) { - if (err == posix.ECHILD) { - return; - } - unreachable; - } - handleTerm(i32(pid_result), status); - } -} - -fn handleTerm(pid: i32, status: i32) void { - var it = children_nodes.first; - while (it) |node| : (it = node.next) { - if (node.data.pid == pid) { - node.data.handleWaitResult(status); - return; - } - } -} From 2676da61a6ce0808509872a267c8bbd307410966 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 20:33:33 -0400 Subject: [PATCH 073/132] travis: better s3 artifacts --- .travis.yml | 6 ------ ci/travis_linux_install | 2 +- ci/travis_linux_script | 19 ++++++++++++------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index df6f60a48..c5299e914 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,11 @@ sudo: required services: - docker -addons: - artifacts: - working_dir: artifacts - target_paths: - - /builds os: - linux - osx dist: trusty osx_image: xcode8.3 -sudo: required language: cpp before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ci/travis_linux_before_install; fi diff --git a/ci/travis_linux_install b/ci/travis_linux_install index 02d0d6d2e..40322c58c 100755 --- a/ci/travis_linux_install +++ b/ci/travis_linux_install @@ -4,4 +4,4 @@ set -x sudo apt-get remove -y llvm-* sudo rm -rf /usr/local/* -sudo apt-get install -y clang-6.0 libclang-6.0 libclang-6.0-dev llvm-6.0 llvm-6.0-dev liblld-6.0 liblld-6.0-dev cmake wine1.6-amd64 +sudo apt-get install -y clang-6.0 libclang-6.0 libclang-6.0-dev llvm-6.0 llvm-6.0-dev liblld-6.0 liblld-6.0-dev cmake wine1.6-amd64 s3cmd diff --git a/ci/travis_linux_script b/ci/travis_linux_script index f80ac33c2..2bb432306 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -8,11 +8,16 @@ export CXX=clang++-6.0 echo $PATH mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -make -j2 install -./zig build --build-file ../build.zig test +# cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) +# make -j2 install +# ./zig build --build-file ../build.zig test -cd $TRAVIS_BUILD_DIR -rm .gitignore -git clean -fd -docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT +if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then + mkdir $TRAVIS_BUILD_DIR/artifacts + docker run -it --mount type=bind,source="$TRAVIS_BUILD_DIR/artifacts",target=/z ziglang/static-base:llvm6-1 -j2 $TRAVIS_COMMIT + echo "access_key = $AWS_ACCESS_KEY_ID" >> ~/.s3cfg + echo "secret_key = $AWS_SECRET_ACCESS_KEY" >> ~/.s3cfg + s3cmd put -P $TRAVIS_BUILD_DIR/artifacts/* s3://ziglang.org/builds/ + touch empty + s3cmd put -P empty s3://ziglang.org/builds/zig-linux-x86_64-$TRAVIS_BRANCH.tar.xz --add-header=x-amz-website-redirect-location:/builds/$(ls $TRAVIS_BUILD_DIR/artifacts) +fi From 6050b9d8359d1ae6b808c332aa27142b0064a6be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 3 Apr 2018 21:40:36 -0400 Subject: [PATCH 074/132] travis: don't skip tests fix broken previous commit --- ci/travis_linux_script | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/travis_linux_script b/ci/travis_linux_script index 2bb432306..9b43dd20f 100755 --- a/ci/travis_linux_script +++ b/ci/travis_linux_script @@ -8,9 +8,9 @@ export CXX=clang++-6.0 echo $PATH mkdir build cd build -# cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -# make -j2 install -# ./zig build --build-file ../build.zig test +cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) +make -j2 install +./zig build --build-file ../build.zig test if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then mkdir $TRAVIS_BUILD_DIR/artifacts From abd389209b2b25ac3d3567797bff580c2ce7d9ad Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 4 Apr 2018 00:08:10 -0400 Subject: [PATCH 075/132] fix up logic for macos std.os.deleteTree --- doc/docgen.zig | 2 +- std/os/index.zig | 11 ++++++++--- std/os/test.zig | 13 ++++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 5332a62ac..56d9a0441 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -55,7 +55,7 @@ pub fn main() !void { // TODO issue #709 // disabled to pass CI tests, but obviously we want to implement this // and then remove this workaround - if (builtin.os == builtin.Os.linux) { + if (builtin.os != builtin.Os.windows) { os.deleteTree(allocator, tmp_dir_name) catch {}; } } diff --git a/std/os/index.zig b/std/os/index.zig index d8e2fd700..4b74af035 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -1050,14 +1050,14 @@ const DeleteTreeError = error { }; pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError!void { start_over: while (true) { + var got_access_denied = false; // First, try deleting the item as a file. This way we don't follow sym links. if (deleteFile(allocator, full_path)) { return; } else |err| switch (err) { error.FileNotFound => return, - - error.AccessDenied, error.IsDir => {}, + error.AccessDenied => got_access_denied = true, error.OutOfMemory, error.SymLinkLoop, @@ -1072,7 +1072,12 @@ pub fn deleteTree(allocator: &Allocator, full_path: []const u8) DeleteTreeError! } { var dir = Dir.open(allocator, full_path) catch |err| switch (err) { - error.NotDir => continue :start_over, + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; + } + continue :start_over; + }, error.OutOfMemory, error.AccessDenied, diff --git a/std/os/test.zig b/std/os/test.zig index 20d439cc4..9c718d5b6 100644 --- a/std/os/test.zig +++ b/std/os/test.zig @@ -1,18 +1,25 @@ const std = @import("../index.zig"); const os = std.os; -const debug = std.debug; +const assert = std.debug.assert; const io = std.io; const a = std.debug.global_allocator; +const builtin = @import("builtin"); + test "makePath, put some files in it, deleteTree" { + if (builtin.os == builtin.Os.windows) { + // TODO implement os.Dir for windows + // https://github.com/zig-lang/zig/issues/709 + return; + } try os.makePath(a, "os_test_tmp/b/c"); try io.writeFile(a, "os_test_tmp/b/c/file.txt", "nonsense"); try io.writeFile(a, "os_test_tmp/b/file2.txt", "blah"); try os.deleteTree(a, "os_test_tmp"); if (os.Dir.open(a, "os_test_tmp")) |dir| { - debug.assert(false); // this should not happen! + @panic("expected error"); } else |err| { - debug.assert(err == error.PathNotFound); + assert(err == error.PathNotFound); } } From 09cf82361999c1a22819467e02fc68fd22ca188e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 09:57:37 +0200 Subject: [PATCH 076/132] std.zig.parser now parses container decls --- std/zig/ast.zig | 11 +- std/zig/parser.zig | 262 +++++++++++++++++++++++++++++---------------- 2 files changed, 177 insertions(+), 96 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 484bc59f1..1d42d721d 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -214,12 +214,19 @@ pub const NodeVarDecl = struct { pub const NodeContainerDecl = struct { base: Node, - kind_token: Token, + ltoken: Token, + layout: Layout, kind: Kind, init_arg_expr: InitArg, fields_and_decls: ArrayList(&Node), rbrace_token: Token, + const Layout = enum { + Auto, + Extern, + Packed, + }; + const Kind = enum { Struct, Enum, @@ -266,7 +273,7 @@ pub const NodeContainerDecl = struct { } pub fn firstToken(self: &NodeContainerDecl) Token { - return self.kind_token; + return self.ltoken; } pub fn lastToken(self: &NodeContainerDecl) Token { diff --git a/std/zig/parser.zig b/std/zig/parser.zig index fc00ba5f4..8a4999a4e 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -59,6 +59,11 @@ pub const Parser = struct { lib_name: ?&ast.Node, }; + const ContainerExternCtx = struct { + ltoken: Token, + layout: ast.NodeContainerDecl.Layout, + }; + const DestPtr = union(enum) { Field: &&ast.Node, NullableField: &?&ast.Node, @@ -90,6 +95,7 @@ pub const Parser = struct { TopLevel, TopLevelExtern: TopLevelDeclCtx, TopLevelDecl: TopLevelDeclCtx, + ContainerExtern: ContainerExternCtx, ContainerDecl: &ast.NodeContainerDecl, Expression: DestPtr, ExpectOperand, @@ -338,6 +344,63 @@ pub const Parser = struct { return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); }, + State.ContainerExtern => |ctx| { + const token = self.getNextToken(); + + const node = try arena.create(ast.NodeContainerDecl); + *node = ast.NodeContainerDecl { + .base = self.initNode(ast.Node.Id.ContainerDecl), + .ltoken = ctx.ltoken, + .layout = ctx.layout, + .kind = switch (token.id) { + Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, + Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, + Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, + else => { + return self.parseError(token, "expected {}, {} or {}, found {}", + @tagName(Token.Id.Keyword_struct), + @tagName(Token.Id.Keyword_union), + @tagName(Token.Id.Keyword_enum), + @tagName(token.id)); + }, + }, + .init_arg_expr = undefined, + .fields_and_decls = ArrayList(&ast.Node).init(arena), + .rbrace_token = undefined, + }; + + try stack.append(State { .Operand = &node.base }); + try stack.append(State.AfterOperand); + try stack.append(State { .ContainerDecl = node }); + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + + const lparen = self.getNextToken(); + if (lparen.id != Token.Id.LParen) { + self.putBackToken(lparen); + node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; + continue; + } + + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const init_arg_token = self.getNextToken(); + switch (init_arg_token.id) { + Token.Id.Keyword_enum => { + node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; + }, + else => { + self.putBackToken(init_arg_token); + node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; + try stack.append(State { + .Expression = DestPtr { + .Field = &node.init_arg_expr.Type + } + }); + }, + } + continue; + }, + State.ContainerDecl => |container_decl| { const token = self.getNextToken(); @@ -633,52 +696,30 @@ pub const Parser = struct { try stack.append(State.AfterOperand); continue; }, + Token.Id.Keyword_packed => { + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Packed, + }, + }); + }, + Token.Id.Keyword_extern => { + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Extern, + }, + }); + }, Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { - const node = try arena.create(ast.NodeContainerDecl); - *node = ast.NodeContainerDecl { - .base = self.initNode(ast.Node.Id.ContainerDecl), - .kind_token = token, - .kind = switch (token.id) { - Token.Id.Keyword_struct => ast.NodeContainerDecl.Kind.Struct, - Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, - Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, - else => unreachable, + self.putBackToken(token); + try stack.append(State { + .ContainerExtern = ContainerExternCtx { + .ltoken = token, + .layout = ast.NodeContainerDecl.Layout.Auto, }, - .init_arg_expr = undefined, - .fields_and_decls = ArrayList(&ast.Node).init(arena), - .rbrace_token = undefined, - }; - - try stack.append(State { .Operand = &node.base }); - try stack.append(State.AfterOperand); - try stack.append(State { .ContainerDecl = node }); - try stack.append(State { .ExpectToken = Token.Id.LBrace }); - - const lparen = self.getNextToken(); - if (lparen.id != Token.Id.LParen) { - self.putBackToken(lparen); - node.init_arg_expr = ast.NodeContainerDecl.InitArg.None; - continue; - } - - try stack.append(State { .ExpectToken = Token.Id.RParen }); - - const init_arg_token = self.getNextToken(); - switch (init_arg_token.id) { - Token.Id.Keyword_enum => { - node.init_arg_expr = ast.NodeContainerDecl.InitArg.Enum; - }, - else => { - self.putBackToken(lparen); - node.init_arg_expr = ast.NodeContainerDecl.InitArg { .Type = undefined }; - try stack.append(State { - .Expression = DestPtr { - .Field = &node.init_arg_expr.Type - } - }); - }, - } - continue; + }); }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); @@ -1706,6 +1747,29 @@ pub const Parser = struct { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Expression = test_decl.name }); }, + ast.Node.Id.StructField => { + const field = @fieldParentPtr(ast.NodeStructField, "base", decl); + try stream.print("{}: ", self.tokenizer.getTokenSlice(field.name_token)); + try stack.append(RenderState { .Expression = field.type_expr}); + }, + ast.Node.Id.UnionTag => { + const tag = @fieldParentPtr(ast.NodeUnionTag, "base", decl); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.type_expr) |type_expr| { + try stream.print(": "); + try stack.append(RenderState { .Expression = type_expr}); + } + }, + ast.Node.Id.EnumTag => { + const tag = @fieldParentPtr(ast.NodeEnumTag, "base", decl); + try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); + + if (tag.value) |value| { + try stream.print(" = "); + try stack.append(RenderState { .Expression = value}); + } + }, else => unreachable, } }, @@ -1990,27 +2054,30 @@ pub const Parser = struct { }, ast.Node.Id.ContainerDecl => { const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); - try stream.print("{} {{", self.tokenizer.getTokenSlice(container_decl.kind_token)); + + switch (container_decl.layout) { + ast.NodeContainerDecl.Layout.Packed => try stream.print("packed "), + ast.NodeContainerDecl.Layout.Extern => try stream.print("extern "), + ast.NodeContainerDecl.Layout.Auto => { }, + } + + switch (container_decl.kind) { + ast.NodeContainerDecl.Kind.Struct => try stream.print("struct"), + ast.NodeContainerDecl.Kind.Enum => try stream.print("enum"), + ast.NodeContainerDecl.Kind.Union => try stream.print("union"), + } + try stack.append(RenderState { .Text = "}"}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); const fields_and_decls = container_decl.fields_and_decls.toSliceConst(); var i = fields_and_decls.len; while (i != 0) { i -= 1; const node = fields_and_decls[i]; - if (i != 0) { - switch (node.id) { - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag => { - try stack.append(RenderState { .Text = "," }); - }, - else => { } - } - } - try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState { .TopLevelDecl = node}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = blk: { @@ -2025,22 +2092,43 @@ pub const Parser = struct { break :blk "\n"; }, }); + + if (i != 0) { + const prev_node = fields_and_decls[i - 1]; + switch (prev_node.id) { + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag => { + try stack.append(RenderState { .Text = "," }); + }, + else => { } + } + } } try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "{"}); + + switch (container_decl.init_arg_expr) { + ast.NodeContainerDecl.InitArg.None => try stack.append(RenderState { .Text = " "}), + ast.NodeContainerDecl.InitArg.Enum => try stack.append(RenderState { .Text = "(enum) "}), + ast.NodeContainerDecl.InitArg.Type => |type_expr| { + try stack.append(RenderState { .Text = ") "}); + try stack.append(RenderState { .Expression = type_expr}); + try stack.append(RenderState { .Text = "("}); + }, + } }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); var i : usize = 0; - indent += 4; while (i < multiline_str_literal.tokens.len) : (i += 1) { const t = multiline_str_literal.tokens.at(i); - try stream.writeByteNTimes(' ', indent); + try stream.writeByteNTimes(' ', indent + indent_delta); try stream.print("{}", self.tokenizer.getTokenSlice(t)); } - try stream.writeByteNTimes(' ', indent); - indent -= 4; + try stream.writeByteNTimes(' ', indent + indent_delta); }, ast.Node.Id.UndefinedLiteral => { const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base); @@ -2060,31 +2148,12 @@ pub const Parser = struct { } } }, - ast.Node.Id.StructField => { - const field = @fieldParentPtr(ast.NodeStructField, "base", base); - try stream.print("{}:", self.tokenizer.getTokenSlice(field.name_token)); - }, - ast.Node.Id.UnionTag => { - const tag = @fieldParentPtr(ast.NodeUnionTag, "base", base); - try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); - - if (tag.type_expr) |type_expr| { - try stream.print(": "); - try stack.append(RenderState { .Expression = type_expr}); - } - }, - ast.Node.Id.EnumTag => { - const tag = @fieldParentPtr(ast.NodeEnumTag, "base", base); - try stream.print("{}", self.tokenizer.getTokenSlice(tag.name_token)); - - if (tag.value) |value| { - try stream.print(" = "); - try stack.append(RenderState { .Expression = value}); - } - }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, ast.Node.Id.Root, ast.Node.Id.VarDecl, ast.Node.Id.TestDecl, @@ -2489,21 +2558,21 @@ test "zig fmt: struct declaration" { \\ return *self; \\ } \\ - \\ f2: u8, + \\ f2: u8 \\}; \\ \\const Ps = packed struct { \\ a: u8, \\ b: u8, \\ - \\ c: u8, + \\ c: u8 \\}; \\ \\const Es = extern struct { \\ a: u8, \\ b: u8, \\ - \\ c: u8, + \\ c: u8 \\}; \\ ); @@ -2513,25 +2582,25 @@ test "zig fmt: enum declaration" { try testCanonical( \\const E = enum { \\ Ok, - \\ SomethingElse = 0, + \\ SomethingElse = 0 \\}; \\ \\const E2 = enum(u8) { \\ Ok, \\ SomethingElse = 255, - \\ SomethingThird, + \\ SomethingThird \\}; \\ \\const Ee = extern enum { \\ Ok, \\ SomethingElse, - \\ SomethingThird, + \\ SomethingThird \\}; \\ \\const Ep = packed enum { \\ Ok, \\ SomethingElse, - \\ SomethingThird, + \\ SomethingThird \\}; \\ ); @@ -2542,31 +2611,36 @@ test "zig fmt: union declaration" { \\const U = union { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const Ue = union(enum) { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const E = enum { \\ Int, \\ Float, - \\ Bool, + \\ None, + \\ Bool \\}; \\ \\const Ue2 = union(E) { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ \\const Eu = extern union { \\ Int: u8, \\ Float: f32, - \\ Bool: bool, + \\ None, + \\ Bool: bool \\}; \\ ); From 020724cfa0677bf42f48957d0ca7a474f6fe31e0 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 10:27:38 +0200 Subject: [PATCH 077/132] std.zig.tokenizer Tokens now don't contain a line and column field. * Instead, this information is optained by asking the tokenizer. * getTokenLocation takes a start_index, so relative loc can be optained --- std/zig/parser.zig | 23 ++++++++++------------ std/zig/tokenizer.zig | 44 +++++++++++++++---------------------------- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 8a4999a4e..f7db8fd93 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1571,12 +1571,12 @@ pub const Parser = struct { } fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { - const loc = self.tokenizer.getTokenLocation(token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args); + const loc = self.tokenizer.getTokenLocation(0, token); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); { var i: usize = 0; - while (i < token.column) : (i += 1) { + while (i < loc.column) : (i += 1) { warn(" "); } } @@ -1679,9 +1679,8 @@ pub const Parser = struct { try stack.append(RenderState { .Text = blk: { const prev_node = root_node.decls.at(i - 1); - const prev_line_index = prev_node.lastToken().line; - const this_line_index = decl.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, decl.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } break :blk "\n"; @@ -1858,10 +1857,9 @@ pub const Parser = struct { try stack.append(RenderState { .Text = blk: { if (i != 0) { - const prev_statement_node = block.statements.items[i - 1]; - const prev_line_index = prev_statement_node.lastToken().line; - const this_line_index = statement_node.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const prev_node = block.statements.items[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, statement_node.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } } @@ -2083,9 +2081,8 @@ pub const Parser = struct { .Text = blk: { if (i != 0) { const prev_node = fields_and_decls[i - 1]; - const prev_line_index = prev_node.lastToken().line; - const this_line_index = node.firstToken().line; - if (this_line_index - prev_line_index >= 2) { + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { break :blk "\n\n"; } } diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 2e4099992..342767834 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -5,8 +5,6 @@ pub const Token = struct { id: Id, start: usize, end: usize, - line: usize, - column: usize, const KeywordId = struct { bytes: []const u8, @@ -180,28 +178,34 @@ pub const Token = struct { pub const Tokenizer = struct { buffer: []const u8, index: usize, - line: usize, - column: usize, pending_invalid_token: ?Token, - pub const LineLocation = struct { + pub const Location = struct { + line: usize, + column: usize, line_start: usize, line_end: usize, }; - pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation { - var loc = LineLocation { - .line_start = 0, + pub fn getTokenLocation(self: &Tokenizer, start_index: usize, token: &const Token) Location { + var loc = Location { + .line = 0, + .column = 0, + .line_start = start_index, .line_end = self.buffer.len, }; - for (self.buffer) |c, i| { - if (i == token.start) { - loc.line_end = i; + for (self.buffer[start_index..]) |c, i| { + if (i + start_index == token.start) { + loc.line_end = i + start_index; while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {} return loc; } if (c == '\n') { + loc.line += 1; + loc.column = 0; loc.line_start = i + 1; + } else { + loc.column += 1; } } return loc; @@ -216,8 +220,6 @@ pub const Tokenizer = struct { return Tokenizer { .buffer = buffer, .index = 0, - .line = 0, - .column = 0, .pending_invalid_token = null, }; } @@ -277,8 +279,6 @@ pub const Tokenizer = struct { .id = Token.Id.Eof, .start = self.index, .end = undefined, - .line = self.line, - .column = self.column, }; while (self.index < self.buffer.len) : (self.index += 1) { const c = self.buffer[self.index]; @@ -286,12 +286,9 @@ pub const Tokenizer = struct { State.Start => switch (c) { ' ' => { result.start = self.index + 1; - result.column += 1; }, '\n' => { result.start = self.index + 1; - result.line += 1; - result.column = 0; }, 'c' => { state = State.C; @@ -977,15 +974,6 @@ pub const Tokenizer = struct { } } - for (self.buffer[start_index..self.index]) |c| { - if (c == '\n') { - self.line += 1; - self.column = 0; - } else { - self.column += 1; - } - } - if (result.id == Token.Id.Eof) { if (self.pending_invalid_token) |token| { self.pending_invalid_token = null; @@ -1009,8 +997,6 @@ pub const Tokenizer = struct { .id = Token.Id.Invalid, .start = self.index, .end = self.index + invalid_length, - .line = self.line, - .column = self.column, }; } From ca0085c46dc5acb6ea93e35192b7b52294381d75 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 10:54:48 +0200 Subject: [PATCH 078/132] std.zig.parser now parses error set declarations --- std/zig/ast.zig | 47 ++++++++++++------- std/zig/parser.zig | 111 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 137 insertions(+), 21 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1d42d721d..efb959c7f 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + ErrorSetDecl, ContainerDecl, StructField, UnionTag, @@ -44,6 +45,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), @@ -78,6 +80,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), @@ -112,6 +115,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), @@ -212,6 +216,30 @@ pub const NodeVarDecl = struct { } }; +pub const NodeErrorSetDecl = struct { + base: Node, + error_token: Token, + decls: ArrayList(&NodeIdentifier), + rbrace_token: Token, + + pub fn iterate(self: &NodeErrorSetDecl, index: usize) ?&Node { + var i = index; + + if (i < self.decls.len) return self.decls.at(i); + i -= self.decls.len; + + return null; + } + + pub fn firstToken(self: &NodeErrorSetDecl) Token { + return self.error_token; + } + + pub fn lastToken(self: &NodeErrorSetDecl) Token { + return self.rbrace_token; + } +}; + pub const NodeContainerDecl = struct { base: Node, ltoken: Token, @@ -251,23 +279,8 @@ pub const NodeContainerDecl = struct { InitArg.Enum => { } } - if (i < self.decls.len) return self.decls.at(i); - i -= self.decls.len; - - switch (self.kind) { - Kind.Struct => |fields| { - if (i < fields.len) return fields.at(i); - i -= fields.len; - }, - Kind.Enum => |tags| { - if (i < tags.len) return tags.at(i); - i -= tags.len; - }, - Kind.Union => |tags| { - if (i < tags.len) return tags.at(i); - i -= tags.len; - }, - } + if (i < self.fields_and_decls.len) return self.fields_and_decls.at(i); + i -= self.fields_and_decls.len; return null; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index f7db8fd93..8177700fb 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -685,11 +685,66 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_error => { - const node = try arena.create(ast.NodeErrorType); - *node = ast.NodeErrorType { - .base = self.initNode(ast.Node.Id.ErrorType), - .token = token, + const next = self.getNextToken(); + + if (next.id != Token.Id.LBrace) { + self.putBackToken(next); + const node = try arena.create(ast.NodeErrorType); + *node = ast.NodeErrorType { + .base = self.initNode(ast.Node.Id.ErrorType), + .token = token, + }; + try stack.append(State { + .Operand = &node.base + }); + try stack.append(State.AfterOperand); + continue; + } + + const node = try arena.create(ast.NodeErrorSetDecl); + *node = ast.NodeErrorSetDecl { + .base = self.initNode(ast.Node.Id.ErrorSetDecl), + .error_token = token, + .decls = ArrayList(&ast.NodeIdentifier).init(arena), + .rbrace_token = undefined, }; + + while (true) { + const t = self.getNextToken(); + switch (t.id) { + Token.Id.RBrace => { + node.rbrace_token = t; + break; + }, + Token.Id.Identifier => { + try node.decls.append( + try self.createIdentifier(arena, t) + ); + }, + else => { + return self.parseError(token, "expected {} or {}, found {}", + @tagName(Token.Id.RBrace), + @tagName(Token.Id.Identifier), + @tagName(token.id)); + } + } + + const t2 = self.getNextToken(); + switch (t2.id) { + Token.Id.RBrace => { + node.rbrace_token = t; + break; + }, + Token.Id.Comma => continue, + else => { + return self.parseError(token, "expected {} or {}, found {}", + @tagName(Token.Id.RBrace), + @tagName(Token.Id.Comma), + @tagName(token.id)); + } + } + } + try stack.append(State { .Operand = &node.base }); @@ -2115,6 +2170,42 @@ pub const Parser = struct { }, } }, + ast.Node.Id.ErrorSetDecl => { + const err_set_decl = @fieldParentPtr(ast.NodeErrorSetDecl, "base", base); + try stream.print("error "); + + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); + + const decls = err_set_decl.decls.toSliceConst(); + var i = decls.len; + while (i != 0) { + i -= 1; + const node = decls[i]; + try stack.append(RenderState { .Expression = &node.base}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = decls[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "{"}); + }, ast.Node.Id.MultilineStringLiteral => { const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base); try stream.print("\n"); @@ -2643,6 +2734,18 @@ test "zig fmt: union declaration" { ); } +test "zig fmt: error set declaration" { + try testCanonical( + \\const E = error { + \\ A, + \\ B, + \\ + \\ C + \\}; + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { From f68c2e0a14c7d9997db0305981ce3e5998d88a36 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Wed, 4 Apr 2018 21:32:23 +1200 Subject: [PATCH 079/132] Fix off-by-one error in all crypto functions --- std/crypto/blake2.zig | 22 ++++++++++++++++++++-- std/crypto/md5.zig | 11 ++++++++++- std/crypto/sha1.zig | 11 ++++++++++- std/crypto/sha2.zig | 22 ++++++++++++++++++++-- std/crypto/sha3.zig | 18 ++++++++++++++++++ 5 files changed, 78 insertions(+), 6 deletions(-) diff --git a/std/crypto/blake2.zig b/std/crypto/blake2.zig index ea0a68b18..99f0e629c 100644 --- a/std/crypto/blake2.zig +++ b/std/crypto/blake2.zig @@ -84,7 +84,7 @@ fn Blake2s(comptime out_len: usize) type { return struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.t += 64; d.round(b[off..off + 64], false); } @@ -229,6 +229,15 @@ test "blake2s256 streaming" { htest.assertEqual(h2, out[0..]); } +test "blake2s256 aligned final" { + var block = []u8 {0} ** Blake2s256.block_size; + var out: [Blake2s256.digest_size]u8 = undefined; + + var h = Blake2s256.init(); + h.update(block); + h.final(out[0..]); +} + ///////////////////// // Blake2b @@ -305,7 +314,7 @@ fn Blake2b(comptime out_len: usize) type { return struct { } // Full middle blocks. - while (off + 128 < b.len) : (off += 128) { + while (off + 128 <= b.len) : (off += 128) { d.t += 128; d.round(b[off..off + 128], false); } @@ -447,3 +456,12 @@ test "blake2b512 streaming" { h.final(out[0..]); htest.assertEqual(h2, out[0..]); } + +test "blake2b512 aligned final" { + var block = []u8 {0} ** Blake2b512.block_size; + var out: [Blake2b512.digest_size]u8 = undefined; + + var h = Blake2b512.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/md5.zig b/std/crypto/md5.zig index 26700cd65..705b2428a 100644 --- a/std/crypto/md5.zig +++ b/std/crypto/md5.zig @@ -59,7 +59,7 @@ pub const Md5 = struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.round(b[off..off + 64]); } @@ -253,3 +253,12 @@ test "md5 streaming" { htest.assertEqual("900150983cd24fb0d6963f7d28e17f72", out[0..]); } + +test "md5 aligned final" { + var block = []u8 {0} ** Md5.block_size; + var out: [Md5.digest_size]u8 = undefined; + + var h = Md5.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/sha1.zig b/std/crypto/sha1.zig index f0dd3c337..333597b12 100644 --- a/std/crypto/sha1.zig +++ b/std/crypto/sha1.zig @@ -60,7 +60,7 @@ pub const Sha1 = struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.round(b[off..off + 64]); } @@ -284,3 +284,12 @@ test "sha1 streaming" { h.final(out[0..]); htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); } + +test "sha1 aligned final" { + var block = []u8 {0} ** Sha1.block_size; + var out: [Sha1.digest_size]u8 = undefined; + + var h = Sha1.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/sha2.zig b/std/crypto/sha2.zig index 113bab926..b70450c0a 100644 --- a/std/crypto/sha2.zig +++ b/std/crypto/sha2.zig @@ -105,7 +105,7 @@ fn Sha2_32(comptime params: Sha2Params32) type { return struct { } // Full middle blocks. - while (off + 64 < b.len) : (off += 64) { + while (off + 64 <= b.len) : (off += 64) { d.round(b[off..off + 64]); } @@ -319,6 +319,15 @@ test "sha256 streaming" { htest.assertEqual("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", out[0..]); } +test "sha256 aligned final" { + var block = []u8 {0} ** Sha256.block_size; + var out: [Sha256.digest_size]u8 = undefined; + + var h = Sha256.init(); + h.update(block); + h.final(out[0..]); +} + ///////////////////// // Sha384 + Sha512 @@ -420,7 +429,7 @@ fn Sha2_64(comptime params: Sha2Params64) type { return struct { } // Full middle blocks. - while (off + 128 < b.len) : (off += 128) { + while (off + 128 <= b.len) : (off += 128) { d.round(b[off..off + 128]); } @@ -669,3 +678,12 @@ test "sha512 streaming" { h.final(out[0..]); htest.assertEqual(h2, out[0..]); } + +test "sha512 aligned final" { + var block = []u8 {0} ** Sha512.block_size; + var out: [Sha512.digest_size]u8 = undefined; + + var h = Sha512.init(); + h.update(block); + h.final(out[0..]); +} diff --git a/std/crypto/sha3.zig b/std/crypto/sha3.zig index 6e6a86b3d..f92f56d68 100644 --- a/std/crypto/sha3.zig +++ b/std/crypto/sha3.zig @@ -217,6 +217,15 @@ test "sha3-256 streaming" { htest.assertEqual("3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532", out[0..]); } +test "sha3-256 aligned final" { + var block = []u8 {0} ** Sha3_256.block_size; + var out: [Sha3_256.digest_size]u8 = undefined; + + var h = Sha3_256.init(); + h.update(block); + h.final(out[0..]); +} + test "sha3-384 single" { const h1 = "0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004"; htest.assertEqualHash(Sha3_384, h1 , ""); @@ -278,3 +287,12 @@ test "sha3-512 streaming" { h.final(out[0..]); htest.assertEqual(h2, out[0..]); } + +test "sha3-512 aligned final" { + var block = []u8 {0} ** Sha3_512.block_size; + var out: [Sha3_512.digest_size]u8 = undefined; + + var h = Sha3_512.init(); + h.update(block); + h.final(out[0..]); +} From 744416ce0cd23c84e33965fc8171c0d1aea0659c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 14:58:51 +0200 Subject: [PATCH 080/132] std.zig.parser should now parse operators with precedence. * This haven't been tested yet --- std/zig/parser.zig | 227 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 177 insertions(+), 50 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 8177700fb..3f7b5f4e3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1000,24 +1000,58 @@ pub const Parser = struct { var expression = popSuffixOp(&stack); while (true) { - switch (stack.pop()) { - State.Expression => |dest_ptr| { - // we're done - try dest_ptr.store(expression); + const s = stack.pop(); + if (s == State.Expression) { + const dest_ptr = s.Expression; + // we're done + try dest_ptr.store(expression); + break; + } + + var placement_ptr = &expression; + var rhs : &&ast.Node = undefined; + const node = blk: { + switch (s) { + State.InfixOp => |infix_op| { + infix_op.lhs = popSuffixOp(&stack); + infix_op.rhs = expression; + rhs = &infix_op.rhs; + break :blk &infix_op.base; + }, + State.PrefixOp => |prefix_op| { + prefix_op.rhs = expression; + rhs = &prefix_op.rhs; + break :blk &prefix_op.base; + }, + else => unreachable, + } + }; + const node_perc = precedence(node); + + while (true) { + const perc = precedence(*placement_ptr); + + if (node_perc > perc) { + *placement_ptr = node; break; - }, - State.InfixOp => |infix_op| { - infix_op.rhs = expression; - infix_op.lhs = popSuffixOp(&stack); - expression = &infix_op.base; - continue; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - expression = &prefix_op.base; - continue; - }, - else => unreachable, + } + + switch ((*placement_ptr).id) { + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", *placement_ptr); + placement_ptr = &suffix_op.lhs; + *rhs = suffix_op.lhs; + }, + ast.Node.Id.InfixOp => { + const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", *placement_ptr); + placement_ptr = &infix_op.lhs; + *rhs = infix_op.lhs; + }, + else => { + *placement_ptr = node; + break; + }, + } } } continue; @@ -1308,6 +1342,99 @@ pub const Parser = struct { } } + fn precedence(node: &ast.Node) u8 { + switch (node.id) { + ast.Node.Id.PrefixOp => { + const prefix_op = @fieldParentPtr(ast.NodePrefixOp, "base", node); + switch (prefix_op.op) { + ast.NodePrefixOp.PrefixOp.ArrayType, + ast.NodePrefixOp.PrefixOp.SliceType => return 1, + + ast.NodePrefixOp.PrefixOp.BoolNot, + ast.NodePrefixOp.PrefixOp.Negation, + ast.NodePrefixOp.PrefixOp.NegationWrap, + ast.NodePrefixOp.PrefixOp.BitNot, + ast.NodePrefixOp.PrefixOp.Deref, + ast.NodePrefixOp.PrefixOp.AddrOf, + ast.NodePrefixOp.PrefixOp.UnwrapMaybe => return 3, + + ast.NodePrefixOp.PrefixOp.Try, + ast.NodePrefixOp.PrefixOp.Return => return 255, + } + }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); + switch (suffix_op.op) { + ast.NodeSuffixOp.SuffixOp.Call, + ast.NodeSuffixOp.SuffixOp.Slice, + ast.NodeSuffixOp.SuffixOp.ArrayAccess => return 2, + + ast.NodeSuffixOp.SuffixOp.ArrayInitializer, + ast.NodeSuffixOp.SuffixOp.StructInitializer => return 5, + } + }, + ast.Node.Id.InfixOp => { + const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", node); + switch (infix_op.op) { + ast.NodeInfixOp.InfixOp.Period => return 2, + + ast.NodeInfixOp.InfixOp.ErrorUnion => return 4, + + ast.NodeInfixOp.InfixOp.Div, + ast.NodeInfixOp.InfixOp.ArrayMult, + ast.NodeInfixOp.InfixOp.Mod, + ast.NodeInfixOp.InfixOp.Mult, + ast.NodeInfixOp.InfixOp.MultWrap => return 6, + + ast.NodeInfixOp.InfixOp.Add, + ast.NodeInfixOp.InfixOp.AddWrap, + ast.NodeInfixOp.InfixOp.ArrayCat, + ast.NodeInfixOp.InfixOp.Sub, + ast.NodeInfixOp.InfixOp.SubWrap => return 7, + + ast.NodeInfixOp.InfixOp.BitShiftLeft, + ast.NodeInfixOp.InfixOp.BitShiftRight => return 8, + + ast.NodeInfixOp.InfixOp.BitAnd => return 9, + + ast.NodeInfixOp.InfixOp.BitXor => return 10, + + ast.NodeInfixOp.InfixOp.BitOr => return 11, + + ast.NodeInfixOp.InfixOp.EqualEqual, + ast.NodeInfixOp.InfixOp.BangEqual, + ast.NodeInfixOp.InfixOp.GreaterOrEqual, + ast.NodeInfixOp.InfixOp.GreaterThan, + ast.NodeInfixOp.InfixOp.LessOrEqual, + ast.NodeInfixOp.InfixOp.LessThan => return 12, + + ast.NodeInfixOp.InfixOp.BoolAnd => return 13, + + ast.NodeInfixOp.InfixOp.BoolOr => return 14, + + ast.NodeInfixOp.InfixOp.UnwrapMaybe => return 15, + + ast.NodeInfixOp.InfixOp.Assign, + ast.NodeInfixOp.InfixOp.AssignBitAnd, + ast.NodeInfixOp.InfixOp.AssignBitOr, + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, + ast.NodeInfixOp.InfixOp.AssignBitShiftRight, + ast.NodeInfixOp.InfixOp.AssignBitXor, + ast.NodeInfixOp.InfixOp.AssignDiv, + ast.NodeInfixOp.InfixOp.AssignMinus, + ast.NodeInfixOp.InfixOp.AssignMinusWrap, + ast.NodeInfixOp.InfixOp.AssignMod, + ast.NodeInfixOp.InfixOp.AssignPlus, + ast.NodeInfixOp.InfixOp.AssignPlusWrap, + ast.NodeInfixOp.InfixOp.AssignTimes, + ast.NodeInfixOp.InfixOp.AssignTimesWarp, + ast.NodeInfixOp.InfixOp.MergeErrorSets => return 16, + } + }, + else => return 0, + } + } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -2549,6 +2676,39 @@ test "zig fmt: infix operators" { ); } +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\ a = b or c; + \\ (a = b) or c; + \\} + \\ + //\\ !a{}; + //\\ !(a{}); + //\\ a + b{}; + //\\ (a + b){}; + ); +} + test "zig fmt: prefix operators" { try testCanonical( \\test "prefix operators" { @@ -3062,39 +3222,6 @@ test "zig fmt: container initializers" { ); } -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{}; - \\ !(a{}); - \\ a + b{}; - \\ (a + b){}; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; - \\} - \\ - ); -} - test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); From 8938429ea12ff2857ace5380932a7cd68d3b4ab1 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Thu, 5 Apr 2018 02:31:10 +1200 Subject: [PATCH 081/132] Add Hmac function (#890) --- CMakeLists.txt | 1 + std/crypto/hmac.zig | 81 ++++++++++++++++++++++++++++++++++++++++++++ std/crypto/index.zig | 6 ++++ 3 files changed, 88 insertions(+) create mode 100644 std/crypto/hmac.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 1acc789de..cea302cb0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -425,6 +425,7 @@ set(ZIG_STD_FILES "crypto/sha2.zig" "crypto/sha3.zig" "crypto/blake2.zig" + "crypto/hmac.zig" "cstr.zig" "debug/failing_allocator.zig" "debug/index.zig" diff --git a/std/crypto/hmac.zig b/std/crypto/hmac.zig new file mode 100644 index 000000000..2a36f15b7 --- /dev/null +++ b/std/crypto/hmac.zig @@ -0,0 +1,81 @@ +const std = @import("../index.zig"); +const crypto = std.crypto; +const debug = std.debug; +const mem = std.mem; + +pub const HmacMd5 = Hmac(crypto.Md5); +pub const HmacSha1 = Hmac(crypto.Sha1); +pub const HmacSha256 = Hmac(crypto.Sha256); + +pub fn Hmac(comptime H: type) type { + return struct { + const digest_size = H.digest_size; + + pub fn hash(output: []u8, key: []const u8, message: []const u8) void { + debug.assert(output.len >= H.digest_size); + debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption + var scratch: [H.block_size]u8 = undefined; + + // Normalize key length to block size of hash + if (key.len > H.block_size) { + H.hash(key, scratch[0..H.digest_size]); + mem.set(u8, scratch[H.digest_size..H.block_size], 0); + } else if (key.len < H.block_size) { + mem.copy(u8, scratch[0..key.len], key); + mem.set(u8, scratch[key.len..H.block_size], 0); + } else { + mem.copy(u8, scratch[0..], key); + } + + var o_key_pad: [H.block_size]u8 = undefined; + for (o_key_pad) |*b, i| { + *b = scratch[i] ^ 0x5c; + } + + var i_key_pad: [H.block_size]u8 = undefined; + for (i_key_pad) |*b, i| { + *b = scratch[i] ^ 0x36; + } + + // HMAC(k, m) = H(o_key_pad | H(i_key_pad | message)) where | is concatenation + var hmac = H.init(); + hmac.update(i_key_pad[0..]); + hmac.update(message); + hmac.final(scratch[0..H.digest_size]); + + hmac.reset(); + hmac.update(o_key_pad[0..]); + hmac.update(scratch[0..H.digest_size]); + hmac.final(output[0..H.digest_size]); + } + }; +} + +const htest = @import("test.zig"); + +test "hmac md5" { + var out: [crypto.Md5.digest_size]u8 = undefined; + HmacMd5.hash(out[0..], "", ""); + htest.assertEqual("74e6f7298a9c2d168935f58c001bad88", out[0..]); + + HmacMd5.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + htest.assertEqual("80070713463e7749b90c2dc24911e275", out[0..]); +} + +test "hmac sha1" { + var out: [crypto.Sha1.digest_size]u8 = undefined; + HmacSha1.hash(out[0..], "", ""); + htest.assertEqual("fbdb1d1b18aa6c08324b7d64b71fb76370690e1d", out[0..]); + + HmacSha1.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + htest.assertEqual("de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9", out[0..]); +} + +test "hmac sha256" { + var out: [crypto.Sha256.digest_size]u8 = undefined; + HmacSha256.hash(out[0..], "", ""); + htest.assertEqual("b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad", out[0..]); + + HmacSha256.hash(out[0..], "key", "The quick brown fox jumps over the lazy dog"); + htest.assertEqual("f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", out[0..]); +} diff --git a/std/crypto/index.zig b/std/crypto/index.zig index ee7dc2aa0..2f3902022 100644 --- a/std/crypto/index.zig +++ b/std/crypto/index.zig @@ -19,10 +19,16 @@ pub const Blake2s256 = blake2.Blake2s256; pub const Blake2b384 = blake2.Blake2b384; pub const Blake2b512 = blake2.Blake2b512; +const hmac = @import("hmac.zig"); +pub const HmacMd5 = hmac.HmacMd5; +pub const HmacSha1 = hmac.Sha1; +pub const HmacSha256 = hmac.Sha256; + test "crypto" { _ = @import("md5.zig"); _ = @import("sha1.zig"); _ = @import("sha2.zig"); _ = @import("sha3.zig"); _ = @import("blake2.zig"); + _ = @import("hmac.zig"); } From 779247ba114492a28287bc2026719091b7022e1d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Wed, 4 Apr 2018 23:36:55 +0200 Subject: [PATCH 082/132] std.zig Major Refactor * parser now parses expression like the C++ compiler does * This makes initializers work * Added control flow expression (only return is parsed) * Added catch parsing (It doesn't quite work) * The parse can now specify states as optional. * The parse will roll back on error if states are optional * This can be overriden by State.Required --- std/zig/ast.zig | 60 +- std/zig/parser.zig | 1691 ++++++++++++++++++++++++----------------- std/zig/tokenizer.zig | 10 + 3 files changed, 1077 insertions(+), 684 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index efb959c7f..4aaeef8da 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -24,6 +24,7 @@ pub const Node = struct { PrefixOp, SuffixOp, GroupedExpression, + ControlFlowExpression, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -58,6 +59,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -93,6 +95,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -128,6 +131,7 @@ pub const Node = struct { Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), + Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -531,7 +535,7 @@ pub const NodeInfixOp = struct { op: InfixOp, rhs: &Node, - const InfixOp = enum { + const InfixOp = union(enum) { Add, AddWrap, ArrayCat, @@ -558,6 +562,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, + Catch: ?&NodeIdentifier, Div, EqualEqual, ErrorUnion, @@ -651,9 +656,9 @@ pub const NodePrefixOp = struct { BitNot, BoolNot, Deref, + MaybeType, Negation, NegationWrap, - Return, ArrayType: &Node, SliceType: AddrOfInfo, Try, @@ -754,6 +759,7 @@ pub const NodeSuffixOp = struct { const CallInfo = struct { params: ArrayList(&Node), is_async: bool, + allocator: ?&Node, }; const SliceRange = struct { @@ -831,6 +837,56 @@ pub const NodeGroupedExpression = struct { } }; +pub const NodeControlFlowExpression = struct { + base: Node, + ltoken: Token, + kind: Kind, + rhs: ?&Node, + + const Kind = union(enum) { + Break: ?Token, + Continue: ?Token, + Return, + }; + + pub fn iterate(self: &NodeControlFlowExpression, index: usize) ?&Node { + var i = index; + + if (self.rhs) |rhs| { + if (i < 1) return rhs; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeControlFlowExpression) Token { + return self.ltoken; + } + + pub fn lastToken(self: &NodeControlFlowExpression) Token { + if (self.rhs) |rhs| { + return rhs.lastToken(); + } + + switch (self.kind) { + Kind.Break => |maybe_blk_token| { + if (maybe_blk_token) |blk_token| { + return blk_token; + } + }, + Kind.Continue => |maybe_blk_token| { + if (maybe_blk_token) |blk_token| { + return blk_token; + } + }, + Kind.Return => return self.ltoken, + } + + return self.ltoken; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3f7b5f4e3..1a8e7bde8 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -60,6 +60,7 @@ pub const Parser = struct { }; const ContainerExternCtx = struct { + dest_ptr: DestPtr, ltoken: Token, layout: ast.NodeContainerDecl.Layout, }; @@ -67,13 +68,18 @@ pub const Parser = struct { const DestPtr = union(enum) { Field: &&ast.Node, NullableField: &?&ast.Node, - List: &ArrayList(&ast.Node), - pub fn store(self: &const DestPtr, value: &ast.Node) !void { + pub fn store(self: &const DestPtr, value: &ast.Node) void { switch (*self) { DestPtr.Field => |ptr| *ptr = value, DestPtr.NullableField => |ptr| *ptr = value, - DestPtr.List => |list| try list.append(value), + } + } + + pub fn get(self: &const DestPtr) &ast.Node { + switch (*self) { + DestPtr.Field => |ptr| return *ptr, + DestPtr.NullableField => |ptr| return ??*ptr, } } }; @@ -97,19 +103,13 @@ pub const Parser = struct { TopLevelDecl: TopLevelDeclCtx, ContainerExtern: ContainerExternCtx, ContainerDecl: &ast.NodeContainerDecl, - Expression: DestPtr, - ExpectOperand, - Operand: &ast.Node, - AfterOperand, - InfixOp: &ast.NodeInfixOp, - PrefixOp: &ast.NodePrefixOp, - SuffixOp: &ast.NodeSuffixOp, SliceOrArrayAccess: &ast.NodeSuffixOp, AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo, - TypeExpr: DestPtr, VarDecl: &ast.NodeVarDecl, VarDeclAlign: &ast.NodeVarDecl, VarDeclEq: &ast.NodeVarDecl, + IfToken: @TagType(Token.Id), + IfTokenSave: ExpectTokenSave, ExpectToken: @TagType(Token.Id), ExpectTokenSave: ExpectTokenSave, FnProto: &ast.NodeFnProto, @@ -125,6 +125,48 @@ pub const Parser = struct { FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, + + /// A state that can be appended before any other State. If an error occures, + /// the parser will first try looking for the closest optional state. If an + /// optional state is found, the parser will revert to the state it was in + /// when the optional was added. This will polute the arena allocator with + /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. + Optional: Parser, + + /// Optional can be reverted by adding the Required state to the stack. + Required, + + Expression: DestPtr, + AssignmentExpressionBegin: DestPtr, + AssignmentExpressionEnd: DestPtr, + UnwrapExpressionBegin: DestPtr, + UnwrapExpressionEnd: DestPtr, + BoolOrExpressionBegin: DestPtr, + BoolOrExpressionEnd: DestPtr, + BoolAndExpressionBegin: DestPtr, + BoolAndExpressionEnd: DestPtr, + ComparisonExpressionBegin: DestPtr, + ComparisonExpressionEnd: DestPtr, + BinaryOrExpressionBegin: DestPtr, + BinaryOrExpressionEnd: DestPtr, + BinaryXorExpressionBegin: DestPtr, + BinaryXorExpressionEnd: DestPtr, + BinaryAndExpressionBegin: DestPtr, + BinaryAndExpressionEnd: DestPtr, + BitShiftExpressionBegin: DestPtr, + BitShiftExpressionEnd: DestPtr, + AdditionExpressionBegin: DestPtr, + AdditionExpressionEnd: DestPtr, + MultiplyExpressionBegin: DestPtr, + MultiplyExpressionEnd: DestPtr, + CurlySuffixExpressionBegin: DestPtr, + CurlySuffixExpressionEnd: DestPtr, + TypeExprBegin: DestPtr, + TypeExprEnd: DestPtr, + PrefixOpExpression: DestPtr, + SuffixOpExpressionBegin: DestPtr, + SuffixOpExpressionEnd: DestPtr, + PrimaryExpression: DestPtr, }; /// Returns an AST tree, allocated with the parser's allocator. @@ -193,12 +235,16 @@ pub const Parser = struct { stack.append(State.TopLevel) catch unreachable; const name_token = self.getNextToken(); - if (name_token.id != Token.Id.StringLiteral) - return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + if (name_token.id != Token.Id.StringLiteral) { + try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); + continue; + } const lbrace = self.getNextToken(); - if (lbrace.id != Token.Id.LBrace) - return self.parseError(token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + if (lbrace.id != Token.Id.LBrace) { + try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); + continue; + } const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); @@ -284,28 +330,35 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { - const fn_token = try self.eatToken(Token.Id.Keyword_fn); // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, fn_token, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); try stack.append(State { .FnDef = fn_proto }); try stack.append(State { .FnProto = fn_proto }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + continue; + }, + else => { + try self.parseError(&stack, token, "expected variable declaration or function, found {}", @tagName(token.id)); continue; }, - else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)), } }, State.VarDecl => |var_decl| { - var_decl.name_token = try self.eatToken(Token.Id.Identifier); stack.append(State { .VarDeclAlign = var_decl }) catch unreachable; - - const next_token = self.getNextToken(); - if (next_token.id == Token.Id.Colon) { - try stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} }); - continue; - } - - self.putBackToken(next_token); + try stack.append(State { .TypeExprBegin = DestPtr {.NullableField = &var_decl.type_node} }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &var_decl.name_token, + } + }); continue; }, State.VarDeclAlign => |var_decl| { @@ -313,9 +366,9 @@ pub const Parser = struct { const next_token = self.getNextToken(); if (next_token.id == Token.Id.Keyword_align) { - _ = try self.eatToken(Token.Id.LParen); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr{.NullableField = &var_decl.align_node} }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); continue; } @@ -341,7 +394,8 @@ pub const Parser = struct { var_decl.semicolon_token = token; continue; } - return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); + try self.parseError(&stack, token, "expected '=' or ';', found {}", @tagName(token.id)); + continue; }, State.ContainerExtern => |ctx| { @@ -357,20 +411,20 @@ pub const Parser = struct { Token.Id.Keyword_union => ast.NodeContainerDecl.Kind.Union, Token.Id.Keyword_enum => ast.NodeContainerDecl.Kind.Enum, else => { - return self.parseError(token, "expected {}, {} or {}, found {}", + try self.parseError(&stack, token, "expected {}, {} or {}, found {}", @tagName(Token.Id.Keyword_struct), @tagName(Token.Id.Keyword_union), @tagName(Token.Id.Keyword_enum), @tagName(token.id)); + continue; }, }, .init_arg_expr = undefined, .fields_and_decls = ArrayList(&ast.Node).init(arena), .rbrace_token = undefined, }; + ctx.dest_ptr.store(&node.base); - try stack.append(State { .Operand = &node.base }); - try stack.append(State.AfterOperand); try stack.append(State { .ContainerDecl = node }); try stack.append(State { .ExpectToken = Token.Id.LBrace }); @@ -496,144 +550,569 @@ pub const Parser = struct { }, State.ExpectToken => |token_id| { - _ = try self.eatToken(token_id); + _ = (try self.eatToken(&stack, token_id)) ?? continue; continue; }, State.ExpectTokenSave => |expect_token_save| { - *expect_token_save.ptr = try self.eatToken(expect_token_save.id); + *expect_token_save.ptr = (try self.eatToken(&stack, expect_token_save.id)) ?? continue; continue; }, - State.Expression => |dest_ptr| { - // save the dest_ptr for later - stack.append(state) catch unreachable; - try stack.append(State.ExpectOperand); + State.IfToken => |token_id| { + const token = self.getNextToken(); + if (@TagType(Token.Id)(token.id) != token_id) { + self.putBackToken(token); + _ = stack.pop(); + continue; + } continue; }, - State.ExpectOperand => { - // we'll either get an operand (like 1 or x), - // or a prefix operator (like ~ or return). + + State.IfTokenSave => |if_token_save| { + const token = self.getNextToken(); + if (@TagType(Token.Id)(token.id) != if_token_save.id) { + self.putBackToken(token); + _ = stack.pop(); + continue; + } + + *if_token_save.ptr = token; + continue; + }, + + State.Optional, + State.Required => { }, + + State.Expression => |dest_ptr| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_return => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Return) }); - try stack.append(State.ExpectOperand); - continue; - }, Token.Id.Keyword_try => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Try) }); - try stack.append(State.ExpectOperand); + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Try); + dest_ptr.store(&node.base); + + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; continue; }, - Token.Id.Minus => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Negation) }); - try stack.append(State.ExpectOperand); + Token.Id.Keyword_return => { + const node = try self.createControlFlowExpr(arena, token, ast.NodeControlFlowExpression.Kind.Return); + dest_ptr.store(&node.base); + + stack.append(State { .Optional = *self }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); continue; }, - Token.Id.MinusPercent => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.NegationWrap) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_break => { + @panic("TODO: break"); }, - Token.Id.Tilde => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.BitNot) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_continue => { + @panic("TODO: break"); }, - Token.Id.QuestionMarkQuestionMark => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.UnwrapMaybe) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_cancel => { + @panic("TODO: cancel"); }, - Token.Id.Bang => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.BoolNot) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_resume => { + @panic("TODO: resume"); }, - Token.Id.Asterisk => { - try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, - ast.NodePrefixOp.PrefixOp.Deref) }); - try stack.append(State.ExpectOperand); - continue; + Token.Id.Keyword_await => { + @panic("TODO: await"); }, - Token.Id.LBracket => { - const rbracket_token = self.getNextToken(); - if (rbracket_token.id == Token.Id.RBracket) { - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .SliceType = ast.NodePrefixOp.AddrOfInfo { - .align_expr = null, - .bit_offset_start_token = null, - .bit_offset_end_token = null, - .const_token = null, - .volatile_token = null, - } - }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); + Token.Id.Keyword_suspend => { + @panic("TODO: suspend"); + }, + else => { + self.putBackToken(token); + stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; + continue; + } + } + }, + + State.AssignmentExpressionBegin => |dest_ptr| { + try stack.append(State { .AssignmentExpressionEnd = dest_ptr }); + stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + continue; + }, + + State.AssignmentExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToAssignment(token.id)) |ass_id| { + const node = try self.createInfixOp(arena, token, ass_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .UnwrapExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.UnwrapExpressionBegin => |dest_ptr| { + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolOrExpressionBegin = dest_ptr }); + continue; + }, + + State.UnwrapExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_catch => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp { .Catch = null }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); + + const next = self.getNextToken(); + if (next.id != Token.Id.Pipe) { + self.putBackToken(next); continue; } - self.putBackToken(rbracket_token); - - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .ArrayType = undefined, - }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .ExpectToken = Token.Id.RBracket }); - try stack.append(State { .Expression = DestPtr { .Field = &prefix_op.op.ArrayType } }); - - }, - Token.Id.Ampersand => { - const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ - .AddrOf = ast.NodePrefixOp.AddrOfInfo { - .align_expr = null, - .bit_offset_start_token = null, - .bit_offset_end_token = null, - .const_token = null, - .volatile_token = null, + node.op.Catch = try self.createIdentifier(arena, undefined); + try stack.append(State { .ExpectToken = Token.Id.Pipe }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Identifier, + .ptr = &(??node.op.Catch).name_token } }); - try stack.append(State { .PrefixOp = prefix_op }); - try stack.append(State.ExpectOperand); - try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf }); continue; }, - Token.Id.Identifier => { - try stack.append(State { - .Operand = &(try self.createIdentifier(arena, token)).base + Token.Id.QuestionMarkQuestionMark => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.UnwrapMaybe); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BoolOrExpressionBegin => |dest_ptr| { + stack.append(State { .BoolOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolAndExpressionBegin = dest_ptr }); + continue; + }, + + State.BoolOrExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_or => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolOr); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BoolOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BoolAndExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BoolAndExpressionBegin => |dest_ptr| { + stack.append(State { .BoolAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .ComparisonExpressionBegin = dest_ptr }); + continue; + }, + + State.BoolAndExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_and => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BoolAnd); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BoolAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .ComparisonExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.ComparisonExpressionBegin => |dest_ptr| { + stack.append(State { .ComparisonExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryOrExpressionBegin = dest_ptr }); + continue; + }, + + State.ComparisonExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToComparison(token.id)) |comp_id| { + const node = try self.createInfixOp(arena, token, comp_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .ComparisonExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryOrExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.BinaryOrExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryXorExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryOrExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Pipe => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitOr); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryOrExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryXorExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BinaryXorExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryXorExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryAndExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryXorExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Caret => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitXor); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryXorExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BinaryAndExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BinaryAndExpressionBegin => |dest_ptr| { + stack.append(State { .BinaryAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BitShiftExpressionBegin = dest_ptr }); + continue; + }, + + State.BinaryAndExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Ampersand => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.BitAnd); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BinaryAndExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .BitShiftExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.BitShiftExpressionBegin => |dest_ptr| { + stack.append(State { .BitShiftExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .AdditionExpressionBegin = dest_ptr }); + continue; + }, + + State.BitShiftExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToBitShift(token.id)) |bitshift_id| { + const node = try self.createInfixOp(arena, token, bitshift_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .BitShiftExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .AdditionExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.AdditionExpressionBegin => |dest_ptr| { + stack.append(State { .AdditionExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .MultiplyExpressionBegin = dest_ptr }); + continue; + }, + + State.AdditionExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToAddition(token.id)) |add_id| { + const node = try self.createInfixOp(arena, token, add_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .AdditionExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .MultiplyExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.MultiplyExpressionBegin => |dest_ptr| { + stack.append(State { .MultiplyExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .CurlySuffixExpressionBegin = dest_ptr }); + continue; + }, + + State.MultiplyExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToMultiply(token.id)) |mult_id| { + const node = try self.createInfixOp(arena, token, mult_id); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .MultiplyExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .CurlySuffixExpressionBegin = DestPtr { .Field = &node.rhs } }); + continue; + } else { + self.putBackToken(token); + continue; + } + }, + + State.CurlySuffixExpressionBegin => |dest_ptr| { + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .TypeExprBegin = dest_ptr }); + continue; + }, + + State.CurlySuffixExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (token.id != Token.Id.LBrace) { + self.putBackToken(token); + continue; + } + + const next = self.getNextToken(); + self.putBackToken(token); + switch (next.id) { + Token.Id.Period => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .list = &node.op.StructInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } }); - try stack.append(State.AfterOperand); continue; }, + else => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .ArrayInitializer = ArrayList(&ast.Node).init(arena), + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.ArrayInitializer, + .end = Token.Id.RBrace, + .ptr = &node.rtoken, + } + }); + continue; + }, + } + }, + + State.TypeExprBegin => |dest_ptr| { + const token = self.getNextToken(); + if (token.id == Token.Id.Keyword_var) { + @panic("TODO param with type var"); + } + self.putBackToken(token); + + stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrefixOpExpression = dest_ptr }); + continue; + }, + + State.TypeExprEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Bang => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.ErrorUnion); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrefixOpExpression = DestPtr { .Field = &node.rhs } }); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.PrefixOpExpression => |dest_ptr| { + const token = self.getNextToken(); + if (tokenIdToPrefixOp(token.id)) |prefix_id| { + const node = try self.createPrefixOp(arena, token, prefix_id); + dest_ptr.store(&node.base); + + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; + if (node.op == ast.NodePrefixOp.PrefixOp.AddrOf) { + try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + } + continue; + } else { + self.putBackToken(token); + stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; + continue; + } + }, + + State.SuffixOpExpressionBegin => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_async => { + @panic("TODO: Parse async"); + }, + else => { + self.putBackToken(token); + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .PrimaryExpression = dest_ptr }); + continue; + } + } + }, + + State.SuffixOpExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.LParen => { + const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { + .Call = ast.NodeSuffixOp.CallInfo { + .params = ArrayList(&ast.Node).init(arena), + .is_async = false, // TODO: ASYNC + .allocator = null, + } + }); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.op.Call.params, + .end = Token.Id.RParen, + .ptr = &node.rtoken, + } + }); + continue; + }, + Token.Id.LBracket => { + const node = try arena.create(ast.NodeSuffixOp); + *node = ast.NodeSuffixOp { + .base = self.initNode(ast.Node.Id.SuffixOp), + .lhs = undefined, + .op = ast.NodeSuffixOp.SuffixOp { + .ArrayAccess = undefined, + }, + .rtoken = undefined, + }; + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .SliceOrArrayAccess = node }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); + continue; + }, + Token.Id.Period => { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .SuffixOpExpressionBegin = DestPtr { .Field = &node.rhs }}); + continue; + }, + else => { + self.putBackToken(token); + continue; + }, + } + }, + + State.PrimaryExpression => |dest_ptr| { + const token = self.getNextToken(); + switch (token.id) { Token.Id.IntegerLiteral => { - try stack.append(State { - .Operand = &(try self.createIntegerLiteral(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createIntegerLiteral(arena, token)).base); continue; }, Token.Id.FloatLiteral => { - try stack.append(State { - .Operand = &(try self.createFloatLiteral(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createFloatLiteral(arena, token)).base); + continue; + }, + Token.Id.StringLiteral => { + dest_ptr.store(&(try self.createStringLiteral(arena, token)).base); + continue; + }, + Token.Id.CharLiteral => { + const node = try arena.create(ast.NodeCharLiteral); + *node = ast.NodeCharLiteral { + .base = self.initNode(ast.Node.Id.CharLiteral), + .token = token, + }; + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_undefined => { - try stack.append(State { - .Operand = &(try self.createUndefined(arena, token)).base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&(try self.createUndefined(arena, token)).base); continue; }, Token.Id.Keyword_true, Token.Id.Keyword_false => { @@ -642,10 +1121,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.BoolLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_null => { @@ -654,10 +1130,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.NullLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_this => { @@ -666,10 +1139,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ThisLiteral), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, Token.Id.Keyword_unreachable => { @@ -678,12 +1148,97 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.Unreachable), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; }, + Token.Id.MultilineStringLiteralLine => { + const node = try arena.create(ast.NodeMultilineStringLiteral); + *node = ast.NodeMultilineStringLiteral { + .base = self.initNode(ast.Node.Id.MultilineStringLiteral), + .tokens = ArrayList(Token).init(arena), + }; + dest_ptr.store(&node.base); + try node.tokens.append(token); + + while (true) { + const multiline_str = self.getNextToken(); + if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { + self.putBackToken(multiline_str); + break; + } + + try node.tokens.append(multiline_str); + } + continue; + }, + Token.Id.LParen => { + const node = try arena.create(ast.NodeGroupedExpression); + *node = ast.NodeGroupedExpression { + .base = self.initNode(ast.Node.Id.GroupedExpression), + .lparen = token, + .expr = undefined, + .rparen = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.Identifier => { + dest_ptr.store(&(try self.createIdentifier(arena, token)).base); + continue; + }, + Token.Id.Builtin => { + const node = try arena.create(ast.NodeBuiltinCall); + *node = ast.NodeBuiltinCall { + .base = self.initNode(ast.Node.Id.BuiltinCall), + .builtin_token = token, + .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { + .ExprListItemOrEnd = ListState(&ast.Node) { + .list = &node.params, + .end = Token.Id.RParen, + .ptr = &node.rparen_token, + } + }); + try stack.append(State { .ExpectToken = Token.Id.LParen, }); + continue; + }, + Token.Id.LBracket => { + const rbracket_token = self.getNextToken(); + if (rbracket_token.id == Token.Id.RBracket) { + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .SliceType = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + } + }); + dest_ptr.store(&node.base); + try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + continue; + } + + self.putBackToken(rbracket_token); + + const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ + .ArrayType = undefined, + }); + dest_ptr.store(&node.base); + try stack.append(State { .ExpectToken = Token.Id.RBracket }); + try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); + + }, Token.Id.Keyword_error => { const next = self.getNextToken(); @@ -694,10 +1249,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ErrorType), .token = token, }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + dest_ptr.store(&node.base); continue; } @@ -708,6 +1260,7 @@ pub const Parser = struct { .decls = ArrayList(&ast.NodeIdentifier).init(arena), .rbrace_token = undefined, }; + dest_ptr.store(&node.base); while (true) { const t = self.getNextToken(); @@ -722,10 +1275,11 @@ pub const Parser = struct { ); }, else => { - return self.parseError(token, "expected {} or {}, found {}", + try self.parseError(&stack, token, "expected {} or {}, found {}", @tagName(Token.Id.RBrace), @tagName(Token.Id.Identifier), @tagName(token.id)); + continue; } } @@ -737,23 +1291,20 @@ pub const Parser = struct { }, Token.Id.Comma => continue, else => { - return self.parseError(token, "expected {} or {}, found {}", + try self.parseError(&stack, token, "expected {} or {}, found {}", @tagName(Token.Id.RBrace), @tagName(Token.Id.Comma), @tagName(token.id)); + continue; } } } - - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); continue; }, Token.Id.Keyword_packed => { try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Packed, }, @@ -762,6 +1313,7 @@ pub const Parser = struct { Token.Id.Keyword_extern => { try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Extern, }, @@ -771,100 +1323,25 @@ pub const Parser = struct { self.putBackToken(token); try stack.append(State { .ContainerExtern = ContainerExternCtx { + .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Auto, }, }); }, - Token.Id.Builtin => { - const node = try arena.create(ast.NodeBuiltinCall); - *node = ast.NodeBuiltinCall { - .base = self.initNode(ast.Node.Id.BuiltinCall), - .builtin_token = token, - .params = ArrayList(&ast.Node).init(arena), - .rparen_token = undefined, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.params, - .end = Token.Id.RParen, - .ptr = &node.rparen_token, - } - }); - try stack.append(State { .ExpectToken = Token.Id.LParen, }); - continue; + Token.Id.LBrace => { + @panic("TODO: Block expr"); }, - Token.Id.StringLiteral => { - const node = try self.createStringLiteral(arena, token); - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - continue; + Token.Id.Keyword_fn => { + @panic("TODO: fn proto"); }, - Token.Id.CharLiteral => { - const node = try arena.create(ast.NodeCharLiteral); - *node = ast.NodeCharLiteral { - .base = self.initNode(ast.Node.Id.CharLiteral), - .token = token, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - continue; + Token.Id.Keyword_asm => { + @panic("TODO: inline asm"); }, - Token.Id.MultilineStringLiteralLine => { - const node = try arena.create(ast.NodeMultilineStringLiteral); - *node = ast.NodeMultilineStringLiteral { - .base = self.initNode(ast.Node.Id.MultilineStringLiteral), - .tokens = ArrayList(Token).init(arena), - }; - try node.tokens.append(token); - - while (true) { - const multiline_str = self.getNextToken(); - if (multiline_str.id != Token.Id.MultilineStringLiteralLine) { - self.putBackToken(multiline_str); - break; - } - - try node.tokens.append(multiline_str); - } - - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); + else => { + try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; - }, - Token.Id.LParen => { - const node = try arena.create(ast.NodeGroupedExpression); - *node = ast.NodeGroupedExpression { - .base = self.initNode(ast.Node.Id.GroupedExpression), - .lparen = token, - .expr = undefined, - .rparen = undefined, - }; - try stack.append(State { - .Operand = &node.base - }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.RParen, - .ptr = &node.rparen, - } - }); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - continue; - }, - - else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)), + } } }, @@ -880,8 +1357,6 @@ pub const Parser = struct { .end = undefined, } }; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); const rbracket_token = self.getNextToken(); if (rbracket_token.id != Token.Id.RBracket) { @@ -900,161 +1375,12 @@ pub const Parser = struct { }, Token.Id.RBracket => { node.rtoken = token; - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); continue; }, - else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id)) - } - }, - - State.AfterOperand => { - // we'll either get an infix operator (like != or ^), - // or a postfix operator (like () or {}), - // otherwise this expression is done (like on a ; or else). - var token = self.getNextToken(); - if (tokenIdToInfixOp(token.id)) |infix_id| { - try stack.append(State { - .InfixOp = try self.createInfixOp(arena, token, infix_id) - }); - try stack.append(State.ExpectOperand); + else => { + try self.parseError(&stack, token, "expected ']' or '..', found {}", @tagName(token.id)); continue; - - } else if (token.id == Token.Id.LParen) { - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .Call = ast.NodeSuffixOp.CallInfo { - .params = ArrayList(&ast.Node).init(arena), - .is_async = false, // TODO: ASYNC - } - }); - try stack.append(State { .SuffixOp = node }); - try stack.append(State.AfterOperand); - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.op.Call.params, - .end = Token.Id.RParen, - .ptr = &node.rtoken, - } - }); - continue; - - } else if (token.id == Token.Id.LBracket) { - const node = try arena.create(ast.NodeSuffixOp); - *node = ast.NodeSuffixOp { - .base = self.initNode(ast.Node.Id.SuffixOp), - .lhs = undefined, - .op = ast.NodeSuffixOp.SuffixOp { - .ArrayAccess = undefined, - }, - .rtoken = undefined, - }; - try stack.append(State { .SliceOrArrayAccess = node }); - try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }}); - continue; - - // TODO: This is the initializer parsing code. It doesn't work because of - // the ambiguity between function bodies and initializers: - // fn main() void {} or fn main() (void {}) - } else if (false) { //(token.id == Token.Id.LBrace) { - const next = self.getNextToken(); - - switch (next.id) { - Token.Id.Period => { - self.putBackToken(token); - - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .StructInitializer = ArrayList(&ast.NodeFieldInitializer).init(arena), - }); - - try stack.append(State { - .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { - .list = &node.op.StructInitializer, - .end = Token.Id.RBrace, - .ptr = &node.rtoken, - } - }); - continue; - }, - else => { - self.putBackToken(token); - - const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { - .ArrayInitializer = ArrayList(&ast.Node).init(arena), - }); - - try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { - .list = &node.op.ArrayInitializer, - .end = Token.Id.RBrace, - .ptr = &node.rtoken, - } - }); - continue; - }, } - - // TODO: Parse postfix operator - } else { - // no postfix/infix operator after this operand. - self.putBackToken(token); - - var expression = popSuffixOp(&stack); - while (true) { - const s = stack.pop(); - if (s == State.Expression) { - const dest_ptr = s.Expression; - // we're done - try dest_ptr.store(expression); - break; - } - - var placement_ptr = &expression; - var rhs : &&ast.Node = undefined; - const node = blk: { - switch (s) { - State.InfixOp => |infix_op| { - infix_op.lhs = popSuffixOp(&stack); - infix_op.rhs = expression; - rhs = &infix_op.rhs; - break :blk &infix_op.base; - }, - State.PrefixOp => |prefix_op| { - prefix_op.rhs = expression; - rhs = &prefix_op.rhs; - break :blk &prefix_op.base; - }, - else => unreachable, - } - }; - const node_perc = precedence(node); - - while (true) { - const perc = precedence(*placement_ptr); - - if (node_perc > perc) { - *placement_ptr = node; - break; - } - - switch ((*placement_ptr).id) { - ast.Node.Id.SuffixOp => { - const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", *placement_ptr); - placement_ptr = &suffix_op.lhs; - *rhs = suffix_op.lhs; - }, - ast.Node.Id.InfixOp => { - const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", *placement_ptr); - placement_ptr = &infix_op.lhs; - *rhs = infix_op.lhs; - }, - else => { - *placement_ptr = node; - break; - }, - } - } - } - continue; } }, @@ -1069,7 +1395,7 @@ pub const Parser = struct { self.putBackToken(token); stack.append(State { .ExprListCommaOrEnd = list_state }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = list_state.list} }); + try stack.append(State { .Expression = DestPtr{ .Field = try list_state.list.addOne() } }); }, State.FieldInitListItemOrEnd => |list_state| { @@ -1130,21 +1456,30 @@ pub const Parser = struct { switch (token.id) { Token.Id.Keyword_align => { stack.append(state) catch unreachable; - if (addr_of_info.align_expr != null) return self.parseError(token, "multiple align qualifiers"); - _ = try self.eatToken(Token.Id.LParen); + if (addr_of_info.align_expr != null) { + try self.parseError(&stack, token, "multiple align qualifiers"); + continue; + } try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr{.NullableField = &addr_of_info.align_expr} }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); continue; }, Token.Id.Keyword_const => { stack.append(state) catch unreachable; - if (addr_of_info.const_token != null) return self.parseError(token, "duplicate qualifier: const"); + if (addr_of_info.const_token != null) { + try self.parseError(&stack, token, "duplicate qualifier: const"); + continue; + } addr_of_info.const_token = token; continue; }, Token.Id.Keyword_volatile => { stack.append(state) catch unreachable; - if (addr_of_info.volatile_token != null) return self.parseError(token, "duplicate qualifier: volatile"); + if (addr_of_info.volatile_token != null) { + try self.parseError(&stack, token, "duplicate qualifier: volatile"); + continue; + } addr_of_info.volatile_token = token; continue; }, @@ -1155,17 +1490,6 @@ pub const Parser = struct { } }, - State.TypeExpr => |dest_ptr| { - const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_var) { - @panic("TODO param with type var"); - } - self.putBackToken(token); - - stack.append(State { .Expression = dest_ptr }) catch unreachable; - continue; - }, - State.FnProto => |fn_proto| { stack.append(State { .FnProtoAlign = fn_proto }) catch unreachable; try stack.append(State { .ParamDecl = fn_proto }); @@ -1201,14 +1525,14 @@ pub const Parser = struct { Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; stack.append(State { - .TypeExpr = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, + .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.InferErrorSet}, }) catch unreachable; }, else => { self.putBackToken(token); fn_proto.return_type = ast.NodeFnProto.ReturnType { .Explicit = undefined }; stack.append(State { - .TypeExpr = DestPtr {.Field = &fn_proto.return_type.Explicit}, + .TypeExprBegin = DestPtr {.Field = &fn_proto.return_type.Explicit}, }) catch unreachable; }, } @@ -1251,7 +1575,7 @@ pub const Parser = struct { stack.append(State { .ParamDecl = fn_proto }) catch unreachable; try stack.append(State.ParamDeclComma); try stack.append(State { - .TypeExpr = DestPtr {.Field = ¶m_decl.type_node} + .TypeExprBegin = DestPtr {.Field = ¶m_decl.type_node} }); continue; }, @@ -1264,7 +1588,10 @@ pub const Parser = struct { continue; }, Token.Id.Comma => continue, - else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)), + else => { + try self.parseError(&stack, token, "expected ',' or ')', found {}", @tagName(token.id)); + continue; + }, } }, @@ -1278,7 +1605,10 @@ pub const Parser = struct { continue; }, Token.Id.Semicolon => continue, - else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)), + else => { + try self.parseError(&stack, token, "expected ';' or '{{', found {}", @tagName(token.id)); + continue; + }, } }, @@ -1329,112 +1659,13 @@ pub const Parser = struct { } stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.List = &block.statements} }); + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); continue; }, - - // These are data, not control flow. - State.InfixOp => unreachable, - State.PrefixOp => unreachable, - State.SuffixOp => unreachable, - State.Operand => unreachable, } } } - fn precedence(node: &ast.Node) u8 { - switch (node.id) { - ast.Node.Id.PrefixOp => { - const prefix_op = @fieldParentPtr(ast.NodePrefixOp, "base", node); - switch (prefix_op.op) { - ast.NodePrefixOp.PrefixOp.ArrayType, - ast.NodePrefixOp.PrefixOp.SliceType => return 1, - - ast.NodePrefixOp.PrefixOp.BoolNot, - ast.NodePrefixOp.PrefixOp.Negation, - ast.NodePrefixOp.PrefixOp.NegationWrap, - ast.NodePrefixOp.PrefixOp.BitNot, - ast.NodePrefixOp.PrefixOp.Deref, - ast.NodePrefixOp.PrefixOp.AddrOf, - ast.NodePrefixOp.PrefixOp.UnwrapMaybe => return 3, - - ast.NodePrefixOp.PrefixOp.Try, - ast.NodePrefixOp.PrefixOp.Return => return 255, - } - }, - ast.Node.Id.SuffixOp => { - const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); - switch (suffix_op.op) { - ast.NodeSuffixOp.SuffixOp.Call, - ast.NodeSuffixOp.SuffixOp.Slice, - ast.NodeSuffixOp.SuffixOp.ArrayAccess => return 2, - - ast.NodeSuffixOp.SuffixOp.ArrayInitializer, - ast.NodeSuffixOp.SuffixOp.StructInitializer => return 5, - } - }, - ast.Node.Id.InfixOp => { - const infix_op = @fieldParentPtr(ast.NodeInfixOp, "base", node); - switch (infix_op.op) { - ast.NodeInfixOp.InfixOp.Period => return 2, - - ast.NodeInfixOp.InfixOp.ErrorUnion => return 4, - - ast.NodeInfixOp.InfixOp.Div, - ast.NodeInfixOp.InfixOp.ArrayMult, - ast.NodeInfixOp.InfixOp.Mod, - ast.NodeInfixOp.InfixOp.Mult, - ast.NodeInfixOp.InfixOp.MultWrap => return 6, - - ast.NodeInfixOp.InfixOp.Add, - ast.NodeInfixOp.InfixOp.AddWrap, - ast.NodeInfixOp.InfixOp.ArrayCat, - ast.NodeInfixOp.InfixOp.Sub, - ast.NodeInfixOp.InfixOp.SubWrap => return 7, - - ast.NodeInfixOp.InfixOp.BitShiftLeft, - ast.NodeInfixOp.InfixOp.BitShiftRight => return 8, - - ast.NodeInfixOp.InfixOp.BitAnd => return 9, - - ast.NodeInfixOp.InfixOp.BitXor => return 10, - - ast.NodeInfixOp.InfixOp.BitOr => return 11, - - ast.NodeInfixOp.InfixOp.EqualEqual, - ast.NodeInfixOp.InfixOp.BangEqual, - ast.NodeInfixOp.InfixOp.GreaterOrEqual, - ast.NodeInfixOp.InfixOp.GreaterThan, - ast.NodeInfixOp.InfixOp.LessOrEqual, - ast.NodeInfixOp.InfixOp.LessThan => return 12, - - ast.NodeInfixOp.InfixOp.BoolAnd => return 13, - - ast.NodeInfixOp.InfixOp.BoolOr => return 14, - - ast.NodeInfixOp.InfixOp.UnwrapMaybe => return 15, - - ast.NodeInfixOp.InfixOp.Assign, - ast.NodeInfixOp.InfixOp.AssignBitAnd, - ast.NodeInfixOp.InfixOp.AssignBitOr, - ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, - ast.NodeInfixOp.InfixOp.AssignBitShiftRight, - ast.NodeInfixOp.InfixOp.AssignBitXor, - ast.NodeInfixOp.InfixOp.AssignDiv, - ast.NodeInfixOp.InfixOp.AssignMinus, - ast.NodeInfixOp.InfixOp.AssignMinusWrap, - ast.NodeInfixOp.InfixOp.AssignMod, - ast.NodeInfixOp.InfixOp.AssignPlus, - ast.NodeInfixOp.InfixOp.AssignPlusWrap, - ast.NodeInfixOp.InfixOp.AssignTimes, - ast.NodeInfixOp.InfixOp.AssignTimesWarp, - ast.NodeInfixOp.InfixOp.MergeErrorSets => return 16, - } - }, - else => return 0, - } - } - fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -1448,74 +1679,104 @@ pub const Parser = struct { return; } - return self.parseError(token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); + try self.parseError(stack, token, "expected ',' or {}, found {}", @tagName(*end), @tagName(token.id)); }, } } - fn popSuffixOp(stack: &ArrayList(State)) &ast.Node { - var expression: &ast.Node = undefined; - var left_leaf_ptr: &&ast.Node = &expression; - while (true) { - switch (stack.pop()) { - State.SuffixOp => |suffix_op| { - *left_leaf_ptr = &suffix_op.base; - left_leaf_ptr = &suffix_op.lhs; - }, - State.Operand => |operand| { - *left_leaf_ptr = operand; - break; - }, - else => unreachable, - } - } - - return expression; + fn tokenIdToAssignment(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.AmpersandEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitAnd), + Token.Id.AngleBracketAngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftLeft), + Token.Id.AngleBracketAngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitShiftRight), + Token.Id.AsteriskEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimes), + Token.Id.AsteriskPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignTimesWarp), + Token.Id.CaretEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitXor), + Token.Id.Equal => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Assign), + Token.Id.MinusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinus), + Token.Id.MinusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMinusWrap), + Token.Id.PercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignMod), + Token.Id.PipeEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignBitOr), + Token.Id.PlusEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlus), + Token.Id.PlusPercentEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignPlusWrap), + Token.Id.SlashEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AssignDiv), + else => null, + }; } - fn tokenIdToInfixOp(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + fn tokenIdToComparison(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' return switch (*id) { - Token.Id.Ampersand => ast.NodeInfixOp.InfixOp.BitAnd, - Token.Id.AmpersandEqual => ast.NodeInfixOp.InfixOp.AssignBitAnd, - Token.Id.AngleBracketAngleBracketLeft => ast.NodeInfixOp.InfixOp.BitShiftLeft, - Token.Id.AngleBracketAngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftLeft, - Token.Id.AngleBracketAngleBracketRight => ast.NodeInfixOp.InfixOp.BitShiftRight, - Token.Id.AngleBracketAngleBracketRightEqual => ast.NodeInfixOp.InfixOp.AssignBitShiftRight, - Token.Id.AngleBracketLeft => ast.NodeInfixOp.InfixOp.LessThan, - Token.Id.AngleBracketLeftEqual => ast.NodeInfixOp.InfixOp.LessOrEqual, - Token.Id.AngleBracketRight => ast.NodeInfixOp.InfixOp.GreaterThan, - Token.Id.AngleBracketRightEqual => ast.NodeInfixOp.InfixOp.GreaterOrEqual, - Token.Id.Asterisk => ast.NodeInfixOp.InfixOp.Mult, - Token.Id.AsteriskAsterisk => ast.NodeInfixOp.InfixOp.ArrayMult, - Token.Id.AsteriskEqual => ast.NodeInfixOp.InfixOp.AssignTimes, - Token.Id.AsteriskPercent => ast.NodeInfixOp.InfixOp.MultWrap, - Token.Id.AsteriskPercentEqual => ast.NodeInfixOp.InfixOp.AssignTimesWarp, - Token.Id.Bang => ast.NodeInfixOp.InfixOp.ErrorUnion, - Token.Id.BangEqual => ast.NodeInfixOp.InfixOp.BangEqual, - Token.Id.Caret => ast.NodeInfixOp.InfixOp.BitXor, - Token.Id.CaretEqual => ast.NodeInfixOp.InfixOp.AssignBitXor, - Token.Id.Equal => ast.NodeInfixOp.InfixOp.Assign, - Token.Id.EqualEqual => ast.NodeInfixOp.InfixOp.EqualEqual, - Token.Id.Keyword_and => ast.NodeInfixOp.InfixOp.BoolAnd, - Token.Id.Keyword_or => ast.NodeInfixOp.InfixOp.BoolOr, - Token.Id.Minus => ast.NodeInfixOp.InfixOp.Sub, - Token.Id.MinusEqual => ast.NodeInfixOp.InfixOp.AssignMinus, - Token.Id.MinusPercent => ast.NodeInfixOp.InfixOp.SubWrap, - Token.Id.MinusPercentEqual => ast.NodeInfixOp.InfixOp.AssignMinusWrap, - Token.Id.Percent => ast.NodeInfixOp.InfixOp.Mod, - Token.Id.PercentEqual => ast.NodeInfixOp.InfixOp.AssignMod, - Token.Id.Period => ast.NodeInfixOp.InfixOp.Period, - Token.Id.Pipe => ast.NodeInfixOp.InfixOp.BitOr, - Token.Id.PipeEqual => ast.NodeInfixOp.InfixOp.AssignBitOr, - Token.Id.PipePipe => ast.NodeInfixOp.InfixOp.MergeErrorSets, - Token.Id.Plus => ast.NodeInfixOp.InfixOp.Add, - Token.Id.PlusEqual => ast.NodeInfixOp.InfixOp.AssignPlus, - Token.Id.PlusPercent => ast.NodeInfixOp.InfixOp.AddWrap, - Token.Id.PlusPercentEqual => ast.NodeInfixOp.InfixOp.AssignPlusWrap, - Token.Id.PlusPlus => ast.NodeInfixOp.InfixOp.ArrayCat, - Token.Id.QuestionMarkQuestionMark => ast.NodeInfixOp.InfixOp.UnwrapMaybe, - Token.Id.Slash => ast.NodeInfixOp.InfixOp.Div, - Token.Id.SlashEqual => ast.NodeInfixOp.InfixOp.AssignDiv, + Token.Id.BangEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BangEqual), + Token.Id.EqualEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.EqualEqual), + Token.Id.AngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessThan), + Token.Id.AngleBracketLeftEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.LessOrEqual), + Token.Id.AngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterThan), + Token.Id.AngleBracketRightEqual => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.GreaterOrEqual), + else => null, + }; + } + + fn tokenIdToBitShift(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.AngleBracketAngleBracketLeft => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftLeft), + Token.Id.AngleBracketAngleBracketRight => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.BitShiftRight), + else => null, + }; + } + + fn tokenIdToAddition(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Minus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Sub), + Token.Id.MinusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.SubWrap), + Token.Id.Plus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Add), + Token.Id.PlusPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.AddWrap), + Token.Id.PlusPlus => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayCat), + else => null, + }; + } + + fn tokenIdToMultiply(id: &const Token.Id) ?ast.NodeInfixOp.InfixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Slash => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Div), + Token.Id.Asterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mult), + Token.Id.AsteriskAsterisk => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.ArrayMult), + Token.Id.AsteriskPercent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MultWrap), + Token.Id.Percent => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.Mod), + Token.Id.PipePipe => (ast.NodeInfixOp.InfixOp)(ast.NodeInfixOp.InfixOp.MergeErrorSets), + else => null, + }; + } + + fn tokenIdToPrefixOp(id: &const Token.Id) ?ast.NodePrefixOp.PrefixOp { + // TODO: We have to cast all cases because of this: + // error: expected type '?InfixOp', found '?@TagType(InfixOp)' + return switch (*id) { + Token.Id.Bang => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BoolNot), + Token.Id.Tilde => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.BitNot), + Token.Id.Minus => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Negation), + Token.Id.MinusPercent => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.NegationWrap), + Token.Id.Asterisk => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Deref), + Token.Id.Ampersand => ast.NodePrefixOp.PrefixOp { + .AddrOf = ast.NodePrefixOp.AddrOfInfo { + .align_expr = null, + .bit_offset_start_token = null, + .bit_offset_end_token = null, + .const_token = null, + .volatile_token = null, + }, + }, + Token.Id.QuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.MaybeType), + Token.Id.QuestionMarkQuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.UnwrapMaybe), else => null, }; } @@ -1637,6 +1898,19 @@ pub const Parser = struct { return node; } + fn createControlFlowExpr(self: &Parser, arena: &mem.Allocator, ltoken: &const Token, + kind: &const ast.NodeControlFlowExpression.Kind) !&ast.NodeControlFlowExpression + { + const node = try arena.create(ast.NodeControlFlowExpression); + *node = ast.NodeControlFlowExpression { + .base = self.initNode(ast.Node.Id.ControlFlowExpression), + .ltoken = *ltoken, + .kind = *kind, + .rhs = undefined, + }; + return node; + } + fn createInfixOp(self: &Parser, arena: &mem.Allocator, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp { const node = try arena.create(ast.NodeInfixOp); @@ -1752,39 +2026,56 @@ pub const Parser = struct { return node; } - fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { - const loc = self.tokenizer.getTokenLocation(0, token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); - warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); - { - var i: usize = 0; - while (i < loc.column) : (i += 1) { - warn(" "); + fn parseError(self: &Parser, stack: &ArrayList(State), token: &const Token, comptime fmt: []const u8, args: ...) !void { + // Before reporting an error. We pop the stack to see if our state was optional + self.revertIfOptional(stack) catch { + const loc = self.tokenizer.getTokenLocation(0, token); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); + warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); + { + var i: usize = 0; + while (i < loc.column) : (i += 1) { + warn(" "); + } } - } - { - const caret_count = token.end - token.start; - var i: usize = 0; - while (i < caret_count) : (i += 1) { - warn("~"); + { + const caret_count = token.end - token.start; + var i: usize = 0; + while (i < caret_count) : (i += 1) { + warn("~"); + } } - } - warn("\n"); - return error.ParseError; + warn("\n"); + return error.ParseError; + }; } - fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) !void { - if (token.id != id) { - return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id)); - } - } - - fn eatToken(self: &Parser, id: @TagType(Token.Id)) !Token { + fn eatToken(self: &Parser, stack: &ArrayList(State), id: @TagType(Token.Id)) !?Token { const token = self.getNextToken(); - try self.expectToken(token, id); + if (token.id != id) { + try self.parseError(stack, token, "expected {}, found {}", @tagName(id), @tagName(token.id)); + return null; + } return token; } + fn revertIfOptional(self: &Parser, stack: &ArrayList(State)) !void { + while (stack.popOrNull()) |state| { + switch (state) { + State.Optional => |revert| { + *self = state.Optional; + return; + }, + State.Required => { + return error.StateRequired; + }, + else => { } + } + } + + return error.NoOptionalStateFound; + } + fn putBackToken(self: &Parser, token: &const Token) void { self.put_back_tokens[self.put_back_count] = *token; self.put_back_count += 1; @@ -2054,51 +2345,62 @@ pub const Parser = struct { ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); - const text = switch (prefix_op_node.op) { - ast.NodeInfixOp.InfixOp.Add => " + ", - ast.NodeInfixOp.InfixOp.AddWrap => " +% ", - ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", - ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", - ast.NodeInfixOp.InfixOp.Assign => " = ", - ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", - ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", - ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", - ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", - ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", - ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", - ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", - ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", - ast.NodeInfixOp.InfixOp.AssignMod => " %= ", - ast.NodeInfixOp.InfixOp.AssignPlus => " += ", - ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", - ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", - ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", - ast.NodeInfixOp.InfixOp.BangEqual => " != ", - ast.NodeInfixOp.InfixOp.BitAnd => " & ", - ast.NodeInfixOp.InfixOp.BitOr => " | ", - ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", - ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", - ast.NodeInfixOp.InfixOp.BitXor => " ^ ", - ast.NodeInfixOp.InfixOp.BoolAnd => " and ", - ast.NodeInfixOp.InfixOp.BoolOr => " or ", - ast.NodeInfixOp.InfixOp.Div => " / ", - ast.NodeInfixOp.InfixOp.EqualEqual => " == ", - ast.NodeInfixOp.InfixOp.ErrorUnion => "!", - ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", - ast.NodeInfixOp.InfixOp.GreaterThan => " > ", - ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", - ast.NodeInfixOp.InfixOp.LessThan => " < ", - ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", - ast.NodeInfixOp.InfixOp.Mod => " % ", - ast.NodeInfixOp.InfixOp.Mult => " * ", - ast.NodeInfixOp.InfixOp.MultWrap => " *% ", - ast.NodeInfixOp.InfixOp.Period => ".", - ast.NodeInfixOp.InfixOp.Sub => " - ", - ast.NodeInfixOp.InfixOp.SubWrap => " -% ", - ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", - }; - try stack.append(RenderState { .Text = text }); + if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { + if (prefix_op_node.op.Catch) |payload| { + try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = "|" }); + } + try stack.append(RenderState { .Text = " catch " }); + } else { + const text = switch (prefix_op_node.op) { + ast.NodeInfixOp.InfixOp.Add => " + ", + ast.NodeInfixOp.InfixOp.AddWrap => " +% ", + ast.NodeInfixOp.InfixOp.ArrayCat => " ++ ", + ast.NodeInfixOp.InfixOp.ArrayMult => " ** ", + ast.NodeInfixOp.InfixOp.Assign => " = ", + ast.NodeInfixOp.InfixOp.AssignBitAnd => " &= ", + ast.NodeInfixOp.InfixOp.AssignBitOr => " |= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftLeft => " <<= ", + ast.NodeInfixOp.InfixOp.AssignBitShiftRight => " >>= ", + ast.NodeInfixOp.InfixOp.AssignBitXor => " ^= ", + ast.NodeInfixOp.InfixOp.AssignDiv => " /= ", + ast.NodeInfixOp.InfixOp.AssignMinus => " -= ", + ast.NodeInfixOp.InfixOp.AssignMinusWrap => " -%= ", + ast.NodeInfixOp.InfixOp.AssignMod => " %= ", + ast.NodeInfixOp.InfixOp.AssignPlus => " += ", + ast.NodeInfixOp.InfixOp.AssignPlusWrap => " +%= ", + ast.NodeInfixOp.InfixOp.AssignTimes => " *= ", + ast.NodeInfixOp.InfixOp.AssignTimesWarp => " *%= ", + ast.NodeInfixOp.InfixOp.BangEqual => " != ", + ast.NodeInfixOp.InfixOp.BitAnd => " & ", + ast.NodeInfixOp.InfixOp.BitOr => " | ", + ast.NodeInfixOp.InfixOp.BitShiftLeft => " << ", + ast.NodeInfixOp.InfixOp.BitShiftRight => " >> ", + ast.NodeInfixOp.InfixOp.BitXor => " ^ ", + ast.NodeInfixOp.InfixOp.BoolAnd => " and ", + ast.NodeInfixOp.InfixOp.BoolOr => " or ", + ast.NodeInfixOp.InfixOp.Div => " / ", + ast.NodeInfixOp.InfixOp.EqualEqual => " == ", + ast.NodeInfixOp.InfixOp.ErrorUnion => "!", + ast.NodeInfixOp.InfixOp.GreaterOrEqual => " >= ", + ast.NodeInfixOp.InfixOp.GreaterThan => " > ", + ast.NodeInfixOp.InfixOp.LessOrEqual => " <= ", + ast.NodeInfixOp.InfixOp.LessThan => " < ", + ast.NodeInfixOp.InfixOp.MergeErrorSets => " || ", + ast.NodeInfixOp.InfixOp.Mod => " % ", + ast.NodeInfixOp.InfixOp.Mult => " * ", + ast.NodeInfixOp.InfixOp.MultWrap => " *% ", + ast.NodeInfixOp.InfixOp.Period => ".", + ast.NodeInfixOp.InfixOp.Sub => " - ", + ast.NodeInfixOp.InfixOp.SubWrap => " -% ", + ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", + else => unreachable, + }; + + try stack.append(RenderState { .Text = text }); + } try stack.append(RenderState { .Expression = prefix_op_node.lhs }); }, ast.Node.Id.PrefixOp => { @@ -2143,9 +2445,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Deref => try stream.write("*"), ast.NodePrefixOp.PrefixOp.Negation => try stream.write("-"), ast.NodePrefixOp.PrefixOp.NegationWrap => try stream.write("-%"), - ast.NodePrefixOp.PrefixOp.Return => try stream.write("return "), ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), + ast.NodePrefixOp.PrefixOp.MaybeType => try stream.write("?"), } }, ast.Node.Id.SuffixOp => { @@ -2185,6 +2487,32 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = suffix_op.lhs }); }, + ast.Node.Id.ControlFlowExpression => { + const flow_expr = @fieldParentPtr(ast.NodeControlFlowExpression, "base", base); + switch (flow_expr.kind) { + ast.NodeControlFlowExpression.Kind.Break => |maybe_blk_token| { + try stream.print("break"); + if (maybe_blk_token) |blk_token| { + try stream.print(" :{}", self.tokenizer.getTokenSlice(blk_token)); + } + }, + ast.NodeControlFlowExpression.Kind.Continue => |maybe_blk_token| { + try stream.print("continue"); + if (maybe_blk_token) |blk_token| { + try stream.print(" :{}", self.tokenizer.getTokenSlice(blk_token)); + } + }, + ast.NodeControlFlowExpression.Kind.Return => { + try stream.print("return"); + }, + + } + + if (flow_expr.rhs) |rhs| { + try stream.print(" "); + try stack.append(RenderState { .Expression = rhs }); + } + }, ast.Node.Id.GroupedExpression => { const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); try stack.append(RenderState { .Text = ")"}); @@ -2770,7 +3098,6 @@ test "zig fmt: values" { \\ error; \\ this; \\ unreachable; - \\ suspend; \\} \\ ); @@ -2906,6 +3233,38 @@ test "zig fmt: error set declaration" { ); } +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{}; + \\} + \\ + ); +} + +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2, }; + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { @@ -3106,17 +3465,6 @@ test "zig fmt: defer" { ); } -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - test "zig fmt: comptime" { try testCanonical( \\fn a() u8 { @@ -3201,27 +3549,6 @@ test "zig fmt: coroutines" { ); } -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; - \\} - \\ - ); -} - -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; - \\ - ); -} - test "zig fmt: zig fmt" { try testCanonical(@embedFile("ast.zig")); try testCanonical(@embedFile("index.zig")); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 342767834..87597adff 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -15,8 +15,11 @@ pub const Token = struct { KeywordId{.bytes="align", .id = Id.Keyword_align}, KeywordId{.bytes="and", .id = Id.Keyword_and}, KeywordId{.bytes="asm", .id = Id.Keyword_asm}, + KeywordId{.bytes="async", .id = Id.Keyword_async}, + KeywordId{.bytes="await", .id = Id.Keyword_await}, KeywordId{.bytes="break", .id = Id.Keyword_break}, KeywordId{.bytes="catch", .id = Id.Keyword_catch}, + KeywordId{.bytes="cancel", .id = Id.Keyword_cancel}, KeywordId{.bytes="comptime", .id = Id.Keyword_comptime}, KeywordId{.bytes="const", .id = Id.Keyword_const}, KeywordId{.bytes="continue", .id = Id.Keyword_continue}, @@ -37,10 +40,12 @@ pub const Token = struct { KeywordId{.bytes="or", .id = Id.Keyword_or}, KeywordId{.bytes="packed", .id = Id.Keyword_packed}, KeywordId{.bytes="pub", .id = Id.Keyword_pub}, + KeywordId{.bytes="resume", .id = Id.Keyword_resume}, KeywordId{.bytes="return", .id = Id.Keyword_return}, KeywordId{.bytes="section", .id = Id.Keyword_section}, KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc}, KeywordId{.bytes="struct", .id = Id.Keyword_struct}, + KeywordId{.bytes="suspend", .id = Id.Keyword_suspend}, KeywordId{.bytes="switch", .id = Id.Keyword_switch}, KeywordId{.bytes="test", .id = Id.Keyword_test}, KeywordId{.bytes="this", .id = Id.Keyword_this}, @@ -134,7 +139,10 @@ pub const Token = struct { Keyword_align, Keyword_and, Keyword_asm, + Keyword_async, + Keyword_await, Keyword_break, + Keyword_cancel, Keyword_catch, Keyword_comptime, Keyword_const, @@ -156,10 +164,12 @@ pub const Token = struct { Keyword_or, Keyword_packed, Keyword_pub, + Keyword_resume, Keyword_return, Keyword_section, Keyword_stdcallcc, Keyword_struct, + Keyword_suspend, Keyword_switch, Keyword_test, Keyword_this, From 9e8519b7a2c765b427f85f0aaa456256785eceb7 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Thu, 5 Apr 2018 23:26:06 +0200 Subject: [PATCH 083/132] fix use-after-free in BufMap.set() closes #879 --- std/buf_map.zig | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/std/buf_map.zig b/std/buf_map.zig index a58df4b2d..7e2ea99f1 100644 --- a/std/buf_map.zig +++ b/std/buf_map.zig @@ -31,8 +31,8 @@ pub const BufMap = struct { if (self.hash_map.get(key)) |entry| { const value_copy = try self.copy(value); errdefer self.free(value_copy); - _ = try self.hash_map.put(key, value_copy); - self.free(entry.value); + const old_value = ??(try self.hash_map.put(key, value_copy)); + self.free(old_value); } else { const key_copy = try self.copy(key); errdefer self.free(key_copy); @@ -71,3 +71,29 @@ pub const BufMap = struct { return result; } }; + +const assert = @import("debug/index.zig").assert; +const heap = @import("heap.zig"); + +test "BufMap" { + var direct_allocator = heap.DirectAllocator.init(); + defer direct_allocator.deinit(); + + var bufmap = BufMap.init(&direct_allocator.allocator); + defer bufmap.deinit(); + + try bufmap.set("x", "1"); + assert(mem.eql(u8, ??bufmap.get("x"), "1")); + assert(1 == bufmap.count()); + + try bufmap.set("x", "2"); + assert(mem.eql(u8, ??bufmap.get("x"), "2")); + assert(1 == bufmap.count()); + + try bufmap.set("x", "3"); + assert(mem.eql(u8, ??bufmap.get("x"), "3")); + assert(1 == bufmap.count()); + + bufmap.delete("x"); + assert(0 == bufmap.count()); +} From 8980281184a5d60098f36f3db605181f08aa1caa Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Fri, 6 Apr 2018 00:10:15 +0200 Subject: [PATCH 084/132] fix llvm assert on version string with git sha LLVM's CodeViewDebug pass misparses the version string when it contains a git revision so stop doing that. This only affected Windows builds. closes #898 --- src/codegen.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index bdd28b86f..0dafcb950 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6207,7 +6207,9 @@ static void init(CodeGen *g) { g->builder = LLVMCreateBuilder(); g->dbuilder = ZigLLVMCreateDIBuilder(g->module, true); - Buf *producer = buf_sprintf("zig %s", ZIG_VERSION_STRING); + // Don't use ZIG_VERSION_STRING here, llvm misparses it when it includes + // the git revision. + Buf *producer = buf_sprintf("zig %d.%d.%d", ZIG_VERSION_MAJOR, ZIG_VERSION_MINOR, ZIG_VERSION_PATCH); const char *flags = ""; unsigned runtime_version = 0; ZigLLVMDIFile *compile_unit_file = ZigLLVMCreateFile(g->dbuilder, buf_ptr(g->root_out_name), From e45de607d6437428d82f69711a8cc8f338d019c8 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 08:56:28 +0200 Subject: [PATCH 085/132] std.zig.parser: Initializers are now parsed and fmt correctly --- std/zig/parser.zig | 127 +++++++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 1a8e7bde8..5d2a30cae 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -249,7 +249,7 @@ pub const Parser = struct { const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, token); const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); - try stack.append(State { .Block = block }); + stack.append(State { .Block = block }) catch unreachable; continue; }, Token.Id.Eof => { @@ -318,14 +318,14 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl_node = try self.createAttachVarDecl(arena, ctx.decls, ctx.visib_token, token, (?Token)(null), ctx.extern_token, ctx.lib_name); - try stack.append(State { .VarDecl = var_decl_node }); + stack.append(State { .VarDecl = var_decl_node }) catch unreachable; continue; }, Token.Id.Keyword_fn => { // TODO shouldn't need these casts const fn_proto = try self.createAttachFnProto(arena, ctx.decls, token, ctx.extern_token, ctx.lib_name, (?Token)(null), ctx.visib_token, (?Token)(null)); - try stack.append(State { .FnDef = fn_proto }); + stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); continue; }, @@ -333,7 +333,7 @@ pub const Parser = struct { // TODO shouldn't need this cast const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); - try stack.append(State { .FnDef = fn_proto }); + stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -425,7 +425,7 @@ pub const Parser = struct { }; ctx.dest_ptr.store(&node.base); - try stack.append(State { .ContainerDecl = node }); + stack.append(State { .ContainerDecl = node }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.LBrace }); const lparen = self.getNextToken(); @@ -470,7 +470,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.type_expr } }); try stack.append(State { .ExpectToken = Token.Id.Colon }); continue; @@ -484,7 +484,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; const next = self.getNextToken(); if (next.id != Token.Id.Colon) { @@ -504,7 +504,7 @@ pub const Parser = struct { }; try container_decl.fields_and_decls.append(&node.base); - try stack.append(State { .FieldListCommaOrEnd = container_decl }); + stack.append(State { .FieldListCommaOrEnd = container_decl }) catch unreachable; const next = self.getNextToken(); if (next.id != Token.Id.Equal) { @@ -629,8 +629,8 @@ pub const Parser = struct { }, State.AssignmentExpressionBegin => |dest_ptr| { - try stack.append(State { .AssignmentExpressionEnd = dest_ptr }); - stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; + stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); continue; }, @@ -926,7 +926,6 @@ pub const Parser = struct { } const next = self.getNextToken(); - self.putBackToken(token); switch (next.id) { Token.Id.Period => { const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { @@ -943,6 +942,7 @@ pub const Parser = struct { .ptr = &node.rtoken, } }); + self.putBackToken(next); continue; }, else => { @@ -960,6 +960,7 @@ pub const Parser = struct { .ptr = &node.rtoken, } }); + self.putBackToken(next); continue; }, } @@ -1180,12 +1181,12 @@ pub const Parser = struct { .rparen = undefined, }; dest_ptr.store(&node.base); - try stack.append(State { + stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RParen, .ptr = &node.rparen, } - }); + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, @@ -1202,17 +1203,18 @@ pub const Parser = struct { .rparen_token = undefined, }; dest_ptr.store(&node.base); - try stack.append(State { + stack.append(State { .ExprListItemOrEnd = ListState(&ast.Node) { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, } - }); + }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.LParen, }); continue; }, Token.Id.LBracket => { + // TODO: option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") const rbracket_token = self.getNextToken(); if (rbracket_token.id == Token.Id.RBracket) { const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ @@ -1225,7 +1227,8 @@ pub const Parser = struct { } }); dest_ptr.store(&node.base); - try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + try stack.append(State { .AddrOfModifiers = &node.op.SliceType }); continue; } @@ -1235,6 +1238,7 @@ pub const Parser = struct { .ArrayType = undefined, }); dest_ptr.store(&node.base); + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.RBracket }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); @@ -1302,32 +1306,32 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_packed => { - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Packed, }, - }); + }) catch unreachable; }, Token.Id.Keyword_extern => { - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Extern, }, - }); + }) catch unreachable; }, Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => { self.putBackToken(token); - try stack.append(State { + stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, .ltoken = token, .layout = ast.NodeContainerDecl.Layout.Auto, }, - }); + }) catch unreachable; }, Token.Id.LBrace => { @panic("TODO: Block expr"); @@ -1361,12 +1365,12 @@ pub const Parser = struct { const rbracket_token = self.getNextToken(); if (rbracket_token.id != Token.Id.RBracket) { self.putBackToken(rbracket_token); - try stack.append(State { + stack.append(State { .ExpectTokenSave = ExpectTokenSave { .id = Token.Id.RBracket, .ptr = &node.rtoken, } - }); + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } }); } else { node.rtoken = rbracket_token; @@ -1638,7 +1642,7 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), mut_token, (?Token)(comptime_token), (?Token)(null), null); - try stack.append(State { .VarDecl = var_decl }); + stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; } self.putBackToken(mut_token); @@ -1652,7 +1656,7 @@ pub const Parser = struct { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), mut_token, (?Token)(null), (?Token)(null), null); - try stack.append(State { .VarDecl = var_decl }); + stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; } self.putBackToken(mut_token); @@ -2132,6 +2136,7 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, + FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, }; @@ -2246,6 +2251,12 @@ pub const Parser = struct { } }, + RenderState.FieldInitializer => |field_init| { + try stream.print(".{}", self.tokenizer.getTokenSlice(field_init.name_token)); + try stream.print(" = "); + try stack.append(RenderState { .Expression = field_init.expr }); + }, + RenderState.VarDecl => |var_decl| { try stack.append(RenderState { .Text = ";" }); if (var_decl.init_node) |init_node| { @@ -2481,8 +2492,34 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = range.start}); try stack.append(RenderState { .Text = "["}); }, - ast.NodeSuffixOp.SuffixOp.StructInitializer => @panic("TODO: StructInitializer"), - ast.NodeSuffixOp.SuffixOp.ArrayInitializer => @panic("TODO: ArrayInitializer"), + ast.NodeSuffixOp.SuffixOp.StructInitializer => |field_inits| { + try stack.append(RenderState { .Text = " }"}); + var i = field_inits.len; + while (i != 0) { + i -= 1; + const field_init = field_inits.at(i); + try stack.append(RenderState { .FieldInitializer = field_init }); + try stack.append(RenderState { .Text = " " }); + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Text = "{"}); + }, + ast.NodeSuffixOp.SuffixOp.ArrayInitializer => |exprs| { + try stack.append(RenderState { .Text = " }"}); + var i = exprs.len; + while (i != 0) { + i -= 1; + const expr = exprs.at(i); + try stack.append(RenderState { .Expression = expr }); + try stack.append(RenderState { .Text = " " }); + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Text = "{"}); + }, } try stack.append(RenderState { .Expression = suffix_op.lhs }); @@ -3011,6 +3048,10 @@ test "zig fmt: precedence" { \\ (a!b)(); \\ !a!b; \\ !(a!b); + \\ !a{ }; + \\ !(a{ }); + \\ a + b{ }; + \\ (a + b){ }; \\ a << b + c; \\ (a << b) + c; \\ a & b << c; @@ -3030,10 +3071,6 @@ test "zig fmt: precedence" { \\ (a = b) or c; \\} \\ - //\\ !a{}; - //\\ !(a{}); - //\\ a + b{}; - //\\ (a + b){}; ); } @@ -3233,23 +3270,12 @@ test "zig fmt: error set declaration" { ); } -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - test "zig fmt: arrays" { try testCanonical( \\test "test array" { \\ const a: [2]u8 = [2]u8{ 1, 2 }; \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{}; + \\ const a: [0]u8 = []u8{ }; \\} \\ ); @@ -3260,7 +3286,18 @@ test "zig fmt: container initializers" { \\const a1 = []u8{ }; \\const a2 = []u8{ 1, 2, 3, 4 }; \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2, }; + \\const s2 = S{ .a = 1, .b = 2 }; + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} \\ ); } From f667744d44b0fc3599c85c01d3fcf3b63c4e68a6 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 09:36:11 +0200 Subject: [PATCH 086/132] std.zig.parser Fixed: * Parsing of the optional expression in contrl flow expr * Rendering of catch expressions --- std/zig/parser.zig | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5d2a30cae..dfc3b2f82 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -89,6 +89,15 @@ pub const Parser = struct { ptr: &Token, }; + const RevertState = struct { + parser: Parser, + tokenizer: Tokenizer, + + // We expect, that if something is optional, then there is a field, + // that needs to be set to null, when we revert. + ptr: &?&ast.Node, + }; + fn ListState(comptime T: type) type { return struct { list: &ArrayList(T), @@ -131,7 +140,7 @@ pub const Parser = struct { /// optional state is found, the parser will revert to the state it was in /// when the optional was added. This will polute the arena allocator with /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. - Optional: Parser, + Optional: RevertState, /// Optional can be reverted by adding the Required state to the stack. Required, @@ -598,7 +607,13 @@ pub const Parser = struct { const node = try self.createControlFlowExpr(arena, token, ast.NodeControlFlowExpression.Kind.Return); dest_ptr.store(&node.base); - stack.append(State { .Optional = *self }) catch unreachable; + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); continue; }, @@ -673,7 +688,7 @@ pub const Parser = struct { continue; } - node.op.Catch = try self.createIdentifier(arena, undefined); + node.op.Catch = try self.createIdentifier(arena, Token(undefined)); try stack.append(State { .ExpectToken = Token.Id.Pipe }); try stack.append(State { .ExpectTokenSave = ExpectTokenSave { @@ -1910,7 +1925,7 @@ pub const Parser = struct { .base = self.initNode(ast.Node.Id.ControlFlowExpression), .ltoken = *ltoken, .kind = *kind, - .rhs = undefined, + .rhs = null, }; return node; } @@ -2067,7 +2082,9 @@ pub const Parser = struct { while (stack.popOrNull()) |state| { switch (state) { State.Optional => |revert| { - *self = state.Optional; + *self = revert.parser; + *self.tokenizer = revert.tokenizer; + *revert.ptr = null; return; }, State.Required => { @@ -2359,7 +2376,7 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { - try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Text = "| " }); try stack.append(RenderState { .Expression = &payload.base }); try stack.append(RenderState { .Text = "|" }); } @@ -2956,6 +2973,10 @@ test "zig fmt: return" { \\ return 0; \\} \\ + \\fn bar() void { + \\ return; + \\} + \\ ); } From c34ce2cbc6ca6f623e2e6e1d444c2b51e43dc493 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Fri, 6 Apr 2018 23:10:54 +1200 Subject: [PATCH 087/132] Add common hash/checksum functions - SipHash64, SipHash128 - Crc32 (fast + small variants) - Adler32 - Fnv1a (32, 64 and 128 bit variants) --- CMakeLists.txt | 5 + std/hash/adler.zig | 112 +++++++++++++++ std/hash/crc.zig | 180 ++++++++++++++++++++++++ std/hash/fnv.zig | 60 ++++++++ std/hash/index.zig | 22 +++ std/hash/siphash.zig | 320 +++++++++++++++++++++++++++++++++++++++++++ std/index.zig | 2 + 7 files changed, 701 insertions(+) create mode 100644 std/hash/adler.zig create mode 100644 std/hash/crc.zig create mode 100644 std/hash/fnv.zig create mode 100644 std/hash/index.zig create mode 100644 std/hash/siphash.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index cea302cb0..2bb9bf517 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -438,6 +438,11 @@ set(ZIG_STD_FILES "fmt/errol/lookup.zig" "fmt/index.zig" "hash_map.zig" + "hash/index.zig" + "hash/adler.zig" + "hash/crc.zig" + "hash/fnv.zig" + "hash/siphash.zig" "heap.zig" "index.zig" "io.zig" diff --git a/std/hash/adler.zig b/std/hash/adler.zig new file mode 100644 index 000000000..c77a5aaf5 --- /dev/null +++ b/std/hash/adler.zig @@ -0,0 +1,112 @@ +// Adler32 checksum. +// +// https://tools.ietf.org/html/rfc1950#section-9 +// https://github.com/madler/zlib/blob/master/adler32.c + +const std = @import("../index.zig"); +const debug = std.debug; + +pub const Adler32 = struct { + const base = 65521; + const nmax = 5552; + + adler: u32, + + pub fn init() Adler32 { + return Adler32 { + .adler = 1, + }; + } + + // This fast variant is taken from zlib. It reduces the required modulos and unrolls longer + // buffer inputs and should be much quicker. + pub fn update(self: &Adler32, input: []const u8) void { + var s1 = self.adler & 0xffff; + var s2 = (self.adler >> 16) & 0xffff; + + if (input.len == 1) { + s1 +%= input[0]; + if (s1 >= base) { + s1 -= base; + } + s2 +%= s1; + if (s2 >= base) { + s2 -= base; + } + } + else if (input.len < 16) { + for (input) |b| { + s1 +%= b; + s2 +%= s1; + } + if (s1 >= base) { + s1 -= base; + } + + s2 %= base; + } + else { + var i: usize = 0; + while (i + nmax <= input.len) : (i += nmax) { + const n = nmax / 16; // note: 16 | nmax + + var rounds: usize = 0; + while (rounds < n) : (rounds += 1) { + comptime var j: usize = 0; + inline while (j < 16) : (j += 1) { + s1 +%= input[i + n * j]; + s2 +%= s1; + } + } + } + + if (i < input.len) { + while (i + 16 <= input.len) : (i += 16) { + comptime var j: usize = 0; + inline while (j < 16) : (j += 1) { + s1 +%= input[i + j]; + s2 +%= s1; + } + } + while (i < input.len) : (i += 1) { + s1 +%= input[i]; + s2 +%= s1; + } + + s1 %= base; + s2 %= base; + } + } + + self.adler = s1 | (s2 << 16); + } + + pub fn final(self: &Adler32) u32 { + return self.adler; + } + + pub fn hash(input: []const u8) u32 { + var c = Adler32.init(); + c.update(input); + return c.final(); + } +}; + +test "adler32 sanity" { + debug.assert(Adler32.hash("a") == 0x620062); + debug.assert(Adler32.hash("example") == 0xbc002ed); +} + +test "adler32 long" { + const long1 = []u8 {1} ** 1024; + debug.assert(Adler32.hash(long1[0..]) == 0x06780401); + + const long2 = []u8 {1} ** 1025; + debug.assert(Adler32.hash(long2[0..]) == 0x0a7a0402); +} + +test "adler32 very long" { + const long = []u8 {1} ** 5553; + debug.assert(Adler32.hash(long[0..]) == 0x707f15b2); +} + diff --git a/std/hash/crc.zig b/std/hash/crc.zig new file mode 100644 index 000000000..f88069ce3 --- /dev/null +++ b/std/hash/crc.zig @@ -0,0 +1,180 @@ +// There are two implementations of CRC32 implemented with the following key characteristics: +// +// - Crc32WithPoly uses 8Kb of tables but is ~10x faster than the small method. +// +// - Crc32SmallWithPoly uses only 64 bytes of memory but is slower. Be aware that this is +// still moderately fast just slow relative to the slicing approach. + +const std = @import("../index.zig"); +const debug = std.debug; + +pub const Polynomial = struct { + const IEEE = 0xedb88320; + const Castagnoli = 0x82f63b78; + const Koopman = 0xeb31d82e; +}; + +// IEEE is by far the most common CRC and so is aliased by default. +pub const Crc32 = Crc32WithPoly(Polynomial.IEEE); + +// slicing-by-8 crc32 implementation. +pub fn Crc32WithPoly(comptime poly: u32) type { + return struct { + const Self = this; + const lookup_tables = comptime block: { + @setEvalBranchQuota(20000); + var tables: [8][256]u32 = undefined; + + for (tables[0]) |*e, i| { + var crc = u32(i); + var j: usize = 0; while (j < 8) : (j += 1) { + if (crc & 1 == 1) { + crc = (crc >> 1) ^ poly; + } else { + crc = (crc >> 1); + } + } + *e = crc; + } + + var i: usize = 0; + while (i < 256) : (i += 1) { + var crc = tables[0][i]; + var j: usize = 1; while (j < 8) : (j += 1) { + const index = @truncate(u8, crc); + crc = tables[0][index] ^ (crc >> 8); + tables[j][i] = crc; + } + } + + break :block tables; + }; + + crc: u32, + + pub fn init() Self { + return Self { + .crc = 0xffffffff, + }; + } + + pub fn update(self: &Self, input: []const u8) void { + var i: usize = 0; + while (i + 8 <= input.len) : (i += 8) { + const p = input[i..i+8]; + + // Unrolling this way gives ~50Mb/s increase + self.crc ^= (u32(p[0]) << 0); + self.crc ^= (u32(p[1]) << 8); + self.crc ^= (u32(p[2]) << 16); + self.crc ^= (u32(p[3]) << 24); + + self.crc = + lookup_tables[0][p[7]] ^ + lookup_tables[1][p[6]] ^ + lookup_tables[2][p[5]] ^ + lookup_tables[3][p[4]] ^ + lookup_tables[4][@truncate(u8, self.crc >> 24)] ^ + lookup_tables[5][@truncate(u8, self.crc >> 16)] ^ + lookup_tables[6][@truncate(u8, self.crc >> 8)] ^ + lookup_tables[7][@truncate(u8, self.crc >> 0)]; + } + + while (i < input.len) : (i += 1) { + const index = @truncate(u8, self.crc) ^ input[i]; + self.crc = (self.crc >> 8) ^ lookup_tables[0][index]; + } + } + + pub fn final(self: &Self) u32 { + return ~self.crc; + } + + pub fn hash(input: []const u8) u32 { + var c = Self.init(); + c.update(input); + return c.final(); + } + }; +} + +test "crc32 ieee" { + const Crc32Ieee = Crc32WithPoly(Polynomial.IEEE); + + debug.assert(Crc32Ieee.hash("") == 0x00000000); + debug.assert(Crc32Ieee.hash("a") == 0xe8b7be43); + debug.assert(Crc32Ieee.hash("abc") == 0x352441c2); +} + +test "crc32 castagnoli" { + const Crc32Castagnoli = Crc32WithPoly(Polynomial.Castagnoli); + + debug.assert(Crc32Castagnoli.hash("") == 0x00000000); + debug.assert(Crc32Castagnoli.hash("a") == 0xc1d04330); + debug.assert(Crc32Castagnoli.hash("abc") == 0x364b3fb7); +} + +// half-byte lookup table implementation. +pub fn Crc32SmallWithPoly(comptime poly: u32) type { + return struct { + const Self = this; + const lookup_table = comptime block: { + var table: [16]u32 = undefined; + + for (table) |*e, i| { + var crc = u32(i * 16); + var j: usize = 0; while (j < 8) : (j += 1) { + if (crc & 1 == 1) { + crc = (crc >> 1) ^ poly; + } else { + crc = (crc >> 1); + } + } + *e = crc; + } + + break :block table; + }; + + crc: u32, + + pub fn init() Self { + return Self { + .crc = 0xffffffff, + }; + } + + pub fn update(self: &Self, input: []const u8) void { + for (input) |b| { + self.crc = lookup_table[@truncate(u4, self.crc ^ (b >> 0))] ^ (self.crc >> 4); + self.crc = lookup_table[@truncate(u4, self.crc ^ (b >> 4))] ^ (self.crc >> 4); + } + } + + pub fn final(self: &Self) u32 { + return ~self.crc; + } + + pub fn hash(input: []const u8) u32 { + var c = Self.init(); + c.update(input); + return c.final(); + } + }; +} + +test "small crc32 ieee" { + const Crc32Ieee = Crc32SmallWithPoly(Polynomial.IEEE); + + debug.assert(Crc32Ieee.hash("") == 0x00000000); + debug.assert(Crc32Ieee.hash("a") == 0xe8b7be43); + debug.assert(Crc32Ieee.hash("abc") == 0x352441c2); +} + +test "small crc32 castagnoli" { + const Crc32Castagnoli = Crc32SmallWithPoly(Polynomial.Castagnoli); + + debug.assert(Crc32Castagnoli.hash("") == 0x00000000); + debug.assert(Crc32Castagnoli.hash("a") == 0xc1d04330); + debug.assert(Crc32Castagnoli.hash("abc") == 0x364b3fb7); +} diff --git a/std/hash/fnv.zig b/std/hash/fnv.zig new file mode 100644 index 000000000..88b965b76 --- /dev/null +++ b/std/hash/fnv.zig @@ -0,0 +1,60 @@ +// FNV1a - Fowler-Noll-Vo hash function +// +// FNV1a is a fast, non-cryptographic hash function with fairly good distribution properties. +// +// https://tools.ietf.org/html/draft-eastlake-fnv-14 + +const std = @import("../index.zig"); +const debug = std.debug; + +pub const Fnv1a_32 = Fnv1a(u32, 0x01000193 , 0x811c9dc5); +pub const Fnv1a_64 = Fnv1a(u64, 0x100000001b3, 0xcbf29ce484222325); +pub const Fnv1a_128 = Fnv1a(u128, 0x1000000000000000000013b, 0x6c62272e07bb014262b821756295c58d); + +fn Fnv1a(comptime T: type, comptime prime: T, comptime offset: T) type { + return struct { + const Self = this; + + value: T, + + pub fn init() Self { + return Self { + .value = offset, + }; + } + + pub fn update(self: &Self, input: []const u8) void { + for (input) |b| { + self.value ^= b; + self.value *%= prime; + } + } + + pub fn final(self: &Self) T { + return self.value; + } + + pub fn hash(input: []const u8) T { + var c = Self.init(); + c.update(input); + return c.final(); + } + }; +} + +test "fnv1a-32" { + debug.assert(Fnv1a_32.hash("") == 0x811c9dc5); + debug.assert(Fnv1a_32.hash("a") == 0xe40c292c); + debug.assert(Fnv1a_32.hash("foobar") == 0xbf9cf968); +} + +test "fnv1a-64" { + debug.assert(Fnv1a_64.hash("") == 0xcbf29ce484222325); + debug.assert(Fnv1a_64.hash("a") == 0xaf63dc4c8601ec8c); + debug.assert(Fnv1a_64.hash("foobar") == 0x85944171f73967e8); +} + +test "fnv1a-128" { + debug.assert(Fnv1a_128.hash("") == 0x6c62272e07bb014262b821756295c58d); + debug.assert(Fnv1a_128.hash("a") == 0xd228cb696f1a8caf78912b704e4a8964); +} diff --git a/std/hash/index.zig b/std/hash/index.zig new file mode 100644 index 000000000..8cce35f3c --- /dev/null +++ b/std/hash/index.zig @@ -0,0 +1,22 @@ +const adler = @import("adler.zig"); +pub const Adler32 = adler.Adler32; + +// pub for polynomials + generic crc32 construction +pub const crc = @import("crc.zig"); +pub const Crc32 = crc.Crc32; + +const fnv = @import("fnv.zig"); +pub const Fnv1a_32 = fnv.Fnv1a_32; +pub const Fnv1a_64 = fnv.Fnv1a_64; +pub const Fnv1a_128 = fnv.Fnv1a_128; + +const siphash = @import("siphash.zig"); +pub const SipHash64 = siphash.SipHash64; +pub const SipHash128 = siphash.SipHash128; + +test "hash" { + _ = @import("adler.zig"); + _ = @import("crc.zig"); + _ = @import("fnv.zig"); + _ = @import("siphash.zig"); +} diff --git a/std/hash/siphash.zig b/std/hash/siphash.zig new file mode 100644 index 000000000..301c35cf0 --- /dev/null +++ b/std/hash/siphash.zig @@ -0,0 +1,320 @@ +// Siphash +// +// SipHash is a moderately fast, non-cryptographic keyed hash function designed for resistance +// against hash flooding DoS attacks. +// +// https://131002.net/siphash/ + +const std = @import("../index.zig"); +const debug = std.debug; +const math = std.math; +const mem = std.mem; + +const Endian = @import("builtin").Endian; + +pub fn SipHash64(comptime c_rounds: usize, comptime d_rounds: usize) type { + return SipHash(u64, c_rounds, d_rounds); +} + +pub fn SipHash128(comptime c_rounds: usize, comptime d_rounds: usize) type { + return SipHash(u128, c_rounds, d_rounds); +} + +fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) type { + debug.assert(T == u64 or T == u128); + debug.assert(c_rounds > 0 and d_rounds > 0); + + return struct { + const Self = this; + const digest_size = 64; + const block_size = 64; + + v0: u64, + v1: u64, + v2: u64, + v3: u64, + + // streaming cache + buf: [8]u8, + buf_len: usize, + msg_len: u8, + + pub fn init(key: []const u8) Self { + debug.assert(key.len >= 16); + + const k0 = mem.readInt(key[0..8], u64, Endian.Little); + const k1 = mem.readInt(key[8..16], u64, Endian.Little); + + var d = Self { + .v0 = k0 ^ 0x736f6d6570736575, + .v1 = k1 ^ 0x646f72616e646f6d, + .v2 = k0 ^ 0x6c7967656e657261, + .v3 = k1 ^ 0x7465646279746573, + + .buf = undefined, + .buf_len = 0, + .msg_len = 0, + }; + + if (T == u128) { + d.v1 ^= 0xee; + } + + return d; + } + + pub fn update(d: &Self, b: []const u8) void { + var off: usize = 0; + + // Partial from previous. + if (d.buf_len != 0 and d.buf_len + b.len > 8) { + off += 8 - d.buf_len; + mem.copy(u8, d.buf[d.buf_len..], b[0..off]); + d.round(d.buf[0..]); + d.buf_len = 0; + } + + // Full middle blocks. + while (off + 8 <= b.len) : (off += 8) { + d.round(b[off..off + 8]); + } + + // Remainder for next pass. + mem.copy(u8, d.buf[d.buf_len..], b[off..]); + d.buf_len += u8(b[off..].len); + d.msg_len +%= @truncate(u8, b.len); + } + + pub fn final(d: &Self) T { + // Padding + mem.set(u8, d.buf[d.buf_len..], 0); + d.buf[7] = d.msg_len; + d.round(d.buf[0..]); + + if (T == u128) { + d.v2 ^= 0xee; + } else { + d.v2 ^= 0xff; + } + + comptime var i: usize = 0; + inline while (i < d_rounds) : (i += 1) { + @inlineCall(sipRound, d); + } + + const b1 = d.v0 ^ d.v1 ^ d.v2 ^ d.v3; + if (T == u64) { + return b1; + } + + d.v1 ^= 0xdd; + + comptime var j: usize = 0; + inline while (j < d_rounds) : (j += 1) { + @inlineCall(sipRound, d); + } + + const b2 = d.v0 ^ d.v1 ^ d.v2 ^ d.v3; + return (u128(b2) << 64) | b1; + } + + fn round(d: &Self, b: []const u8) void { + debug.assert(b.len == 8); + + const m = mem.readInt(b[0..], u64, Endian.Little); + d.v3 ^= m; + + comptime var i: usize = 0; + inline while (i < c_rounds) : (i += 1) { + @inlineCall(sipRound, d); + } + + d.v0 ^= m; + } + + fn sipRound(d: &Self) void { + d.v0 +%= d.v1; + d.v1 = math.rotl(u64, d.v1, u64(13)); + d.v1 ^= d.v0; + d.v0 = math.rotl(u64, d.v0, u64(32)); + d.v2 +%= d.v3; + d.v3 = math.rotl(u64, d.v3, u64(16)); + d.v3 ^= d.v2; + d.v0 +%= d.v3; + d.v3 = math.rotl(u64, d.v3, u64(21)); + d.v3 ^= d.v0; + d.v2 +%= d.v1; + d.v1 = math.rotl(u64, d.v1, u64(17)); + d.v1 ^= d.v2; + d.v2 = math.rotl(u64, d.v2, u64(32)); + } + + pub fn hash(key: []const u8, input: []const u8) T { + var c = Self.init(key); + c.update(input); + return c.final(); + } + }; +} + +// Test vectors from reference implementation. +// https://github.com/veorq/SipHash/blob/master/vectors.h +const test_key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"; + +test "siphash64-2-4 sanity" { + const vectors = [][]const u8 { + "\x31\x0e\x0e\xdd\x47\xdb\x6f\x72", // "" + "\xfd\x67\xdc\x93\xc5\x39\xf8\x74", // "\x00" + "\x5a\x4f\xa9\xd9\x09\x80\x6c\x0d", // "\x00\x01" ... etc + "\x2d\x7e\xfb\xd7\x96\x66\x67\x85", + "\xb7\x87\x71\x27\xe0\x94\x27\xcf", + "\x8d\xa6\x99\xcd\x64\x55\x76\x18", + "\xce\xe3\xfe\x58\x6e\x46\xc9\xcb", + "\x37\xd1\x01\x8b\xf5\x00\x02\xab", + "\x62\x24\x93\x9a\x79\xf5\xf5\x93", + "\xb0\xe4\xa9\x0b\xdf\x82\x00\x9e", + "\xf3\xb9\xdd\x94\xc5\xbb\x5d\x7a", + "\xa7\xad\x6b\x22\x46\x2f\xb3\xf4", + "\xfb\xe5\x0e\x86\xbc\x8f\x1e\x75", + "\x90\x3d\x84\xc0\x27\x56\xea\x14", + "\xee\xf2\x7a\x8e\x90\xca\x23\xf7", + "\xe5\x45\xbe\x49\x61\xca\x29\xa1", + "\xdb\x9b\xc2\x57\x7f\xcc\x2a\x3f", + "\x94\x47\xbe\x2c\xf5\xe9\x9a\x69", + "\x9c\xd3\x8d\x96\xf0\xb3\xc1\x4b", + "\xbd\x61\x79\xa7\x1d\xc9\x6d\xbb", + "\x98\xee\xa2\x1a\xf2\x5c\xd6\xbe", + "\xc7\x67\x3b\x2e\xb0\xcb\xf2\xd0", + "\x88\x3e\xa3\xe3\x95\x67\x53\x93", + "\xc8\xce\x5c\xcd\x8c\x03\x0c\xa8", + "\x94\xaf\x49\xf6\xc6\x50\xad\xb8", + "\xea\xb8\x85\x8a\xde\x92\xe1\xbc", + "\xf3\x15\xbb\x5b\xb8\x35\xd8\x17", + "\xad\xcf\x6b\x07\x63\x61\x2e\x2f", + "\xa5\xc9\x1d\xa7\xac\xaa\x4d\xde", + "\x71\x65\x95\x87\x66\x50\xa2\xa6", + "\x28\xef\x49\x5c\x53\xa3\x87\xad", + "\x42\xc3\x41\xd8\xfa\x92\xd8\x32", + "\xce\x7c\xf2\x72\x2f\x51\x27\x71", + "\xe3\x78\x59\xf9\x46\x23\xf3\xa7", + "\x38\x12\x05\xbb\x1a\xb0\xe0\x12", + "\xae\x97\xa1\x0f\xd4\x34\xe0\x15", + "\xb4\xa3\x15\x08\xbe\xff\x4d\x31", + "\x81\x39\x62\x29\xf0\x90\x79\x02", + "\x4d\x0c\xf4\x9e\xe5\xd4\xdc\xca", + "\x5c\x73\x33\x6a\x76\xd8\xbf\x9a", + "\xd0\xa7\x04\x53\x6b\xa9\x3e\x0e", + "\x92\x59\x58\xfc\xd6\x42\x0c\xad", + "\xa9\x15\xc2\x9b\xc8\x06\x73\x18", + "\x95\x2b\x79\xf3\xbc\x0a\xa6\xd4", + "\xf2\x1d\xf2\xe4\x1d\x45\x35\xf9", + "\x87\x57\x75\x19\x04\x8f\x53\xa9", + "\x10\xa5\x6c\xf5\xdf\xcd\x9a\xdb", + "\xeb\x75\x09\x5c\xcd\x98\x6c\xd0", + "\x51\xa9\xcb\x9e\xcb\xa3\x12\xe6", + "\x96\xaf\xad\xfc\x2c\xe6\x66\xc7", + "\x72\xfe\x52\x97\x5a\x43\x64\xee", + "\x5a\x16\x45\xb2\x76\xd5\x92\xa1", + "\xb2\x74\xcb\x8e\xbf\x87\x87\x0a", + "\x6f\x9b\xb4\x20\x3d\xe7\xb3\x81", + "\xea\xec\xb2\xa3\x0b\x22\xa8\x7f", + "\x99\x24\xa4\x3c\xc1\x31\x57\x24", + "\xbd\x83\x8d\x3a\xaf\xbf\x8d\xb7", + "\x0b\x1a\x2a\x32\x65\xd5\x1a\xea", + "\x13\x50\x79\xa3\x23\x1c\xe6\x60", + "\x93\x2b\x28\x46\xe4\xd7\x06\x66", + "\xe1\x91\x5f\x5c\xb1\xec\xa4\x6c", + "\xf3\x25\x96\x5c\xa1\x6d\x62\x9f", + "\x57\x5f\xf2\x8e\x60\x38\x1b\xe5", + "\x72\x45\x06\xeb\x4c\x32\x8a\x95", + }; + + const siphash = SipHash64(2, 4); + + var buffer: [64]u8 = undefined; + for (vectors) |vector, i| { + buffer[i] = u8(i); + + const expected = mem.readInt(vector, u64, Endian.Little); + debug.assert(siphash.hash(test_key, buffer[0..i]) == expected); + } +} + +test "siphash128-2-4 sanity" { + const vectors = [][]const u8 { + "\xa3\x81\x7f\x04\xba\x25\xa8\xe6\x6d\xf6\x72\x14\xc7\x55\x02\x93", + "\xda\x87\xc1\xd8\x6b\x99\xaf\x44\x34\x76\x59\x11\x9b\x22\xfc\x45", + "\x81\x77\x22\x8d\xa4\xa4\x5d\xc7\xfc\xa3\x8b\xde\xf6\x0a\xff\xe4", + "\x9c\x70\xb6\x0c\x52\x67\xa9\x4e\x5f\x33\xb6\xb0\x29\x85\xed\x51", + "\xf8\x81\x64\xc1\x2d\x9c\x8f\xaf\x7d\x0f\x6e\x7c\x7b\xcd\x55\x79", + "\x13\x68\x87\x59\x80\x77\x6f\x88\x54\x52\x7a\x07\x69\x0e\x96\x27", + "\x14\xee\xca\x33\x8b\x20\x86\x13\x48\x5e\xa0\x30\x8f\xd7\xa1\x5e", + "\xa1\xf1\xeb\xbe\xd8\xdb\xc1\x53\xc0\xb8\x4a\xa6\x1f\xf0\x82\x39", + "\x3b\x62\xa9\xba\x62\x58\xf5\x61\x0f\x83\xe2\x64\xf3\x14\x97\xb4", + "\x26\x44\x99\x06\x0a\xd9\xba\xab\xc4\x7f\x8b\x02\xbb\x6d\x71\xed", + "\x00\x11\x0d\xc3\x78\x14\x69\x56\xc9\x54\x47\xd3\xf3\xd0\xfb\xba", + "\x01\x51\xc5\x68\x38\x6b\x66\x77\xa2\xb4\xdc\x6f\x81\xe5\xdc\x18", + "\xd6\x26\xb2\x66\x90\x5e\xf3\x58\x82\x63\x4d\xf6\x85\x32\xc1\x25", + "\x98\x69\xe2\x47\xe9\xc0\x8b\x10\xd0\x29\x93\x4f\xc4\xb9\x52\xf7", + "\x31\xfc\xef\xac\x66\xd7\xde\x9c\x7e\xc7\x48\x5f\xe4\x49\x49\x02", + "\x54\x93\xe9\x99\x33\xb0\xa8\x11\x7e\x08\xec\x0f\x97\xcf\xc3\xd9", + "\x6e\xe2\xa4\xca\x67\xb0\x54\xbb\xfd\x33\x15\xbf\x85\x23\x05\x77", + "\x47\x3d\x06\xe8\x73\x8d\xb8\x98\x54\xc0\x66\xc4\x7a\xe4\x77\x40", + "\xa4\x26\xe5\xe4\x23\xbf\x48\x85\x29\x4d\xa4\x81\xfe\xae\xf7\x23", + "\x78\x01\x77\x31\xcf\x65\xfa\xb0\x74\xd5\x20\x89\x52\x51\x2e\xb1", + "\x9e\x25\xfc\x83\x3f\x22\x90\x73\x3e\x93\x44\xa5\xe8\x38\x39\xeb", + "\x56\x8e\x49\x5a\xbe\x52\x5a\x21\x8a\x22\x14\xcd\x3e\x07\x1d\x12", + "\x4a\x29\xb5\x45\x52\xd1\x6b\x9a\x46\x9c\x10\x52\x8e\xff\x0a\xae", + "\xc9\xd1\x84\xdd\xd5\xa9\xf5\xe0\xcf\x8c\xe2\x9a\x9a\xbf\x69\x1c", + "\x2d\xb4\x79\xae\x78\xbd\x50\xd8\x88\x2a\x8a\x17\x8a\x61\x32\xad", + "\x8e\xce\x5f\x04\x2d\x5e\x44\x7b\x50\x51\xb9\xea\xcb\x8d\x8f\x6f", + "\x9c\x0b\x53\xb4\xb3\xc3\x07\xe8\x7e\xae\xe0\x86\x78\x14\x1f\x66", + "\xab\xf2\x48\xaf\x69\xa6\xea\xe4\xbf\xd3\xeb\x2f\x12\x9e\xeb\x94", + "\x06\x64\xda\x16\x68\x57\x4b\x88\xb9\x35\xf3\x02\x73\x58\xae\xf4", + "\xaa\x4b\x9d\xc4\xbf\x33\x7d\xe9\x0c\xd4\xfd\x3c\x46\x7c\x6a\xb7", + "\xea\x5c\x7f\x47\x1f\xaf\x6b\xde\x2b\x1a\xd7\xd4\x68\x6d\x22\x87", + "\x29\x39\xb0\x18\x32\x23\xfa\xfc\x17\x23\xde\x4f\x52\xc4\x3d\x35", + "\x7c\x39\x56\xca\x5e\xea\xfc\x3e\x36\x3e\x9d\x55\x65\x46\xeb\x68", + "\x77\xc6\x07\x71\x46\xf0\x1c\x32\xb6\xb6\x9d\x5f\x4e\xa9\xff\xcf", + "\x37\xa6\x98\x6c\xb8\x84\x7e\xdf\x09\x25\xf0\xf1\x30\x9b\x54\xde", + "\xa7\x05\xf0\xe6\x9d\xa9\xa8\xf9\x07\x24\x1a\x2e\x92\x3c\x8c\xc8", + "\x3d\xc4\x7d\x1f\x29\xc4\x48\x46\x1e\x9e\x76\xed\x90\x4f\x67\x11", + "\x0d\x62\xbf\x01\xe6\xfc\x0e\x1a\x0d\x3c\x47\x51\xc5\xd3\x69\x2b", + "\x8c\x03\x46\x8b\xca\x7c\x66\x9e\xe4\xfd\x5e\x08\x4b\xbe\xe7\xb5", + "\x52\x8a\x5b\xb9\x3b\xaf\x2c\x9c\x44\x73\xcc\xe5\xd0\xd2\x2b\xd9", + "\xdf\x6a\x30\x1e\x95\xc9\x5d\xad\x97\xae\x0c\xc8\xc6\x91\x3b\xd8", + "\x80\x11\x89\x90\x2c\x85\x7f\x39\xe7\x35\x91\x28\x5e\x70\xb6\xdb", + "\xe6\x17\x34\x6a\xc9\xc2\x31\xbb\x36\x50\xae\x34\xcc\xca\x0c\x5b", + "\x27\xd9\x34\x37\xef\xb7\x21\xaa\x40\x18\x21\xdc\xec\x5a\xdf\x89", + "\x89\x23\x7d\x9d\xed\x9c\x5e\x78\xd8\xb1\xc9\xb1\x66\xcc\x73\x42", + "\x4a\x6d\x80\x91\xbf\x5e\x7d\x65\x11\x89\xfa\x94\xa2\x50\xb1\x4c", + "\x0e\x33\xf9\x60\x55\xe7\xae\x89\x3f\xfc\x0e\x3d\xcf\x49\x29\x02", + "\xe6\x1c\x43\x2b\x72\x0b\x19\xd1\x8e\xc8\xd8\x4b\xdc\x63\x15\x1b", + "\xf7\xe5\xae\xf5\x49\xf7\x82\xcf\x37\x90\x55\xa6\x08\x26\x9b\x16", + "\x43\x8d\x03\x0f\xd0\xb7\xa5\x4f\xa8\x37\xf2\xad\x20\x1a\x64\x03", + "\xa5\x90\xd3\xee\x4f\xbf\x04\xe3\x24\x7e\x0d\x27\xf2\x86\x42\x3f", + "\x5f\xe2\xc1\xa1\x72\xfe\x93\xc4\xb1\x5c\xd3\x7c\xae\xf9\xf5\x38", + "\x2c\x97\x32\x5c\xbd\x06\xb3\x6e\xb2\x13\x3d\xd0\x8b\x3a\x01\x7c", + "\x92\xc8\x14\x22\x7a\x6b\xca\x94\x9f\xf0\x65\x9f\x00\x2a\xd3\x9e", + "\xdc\xe8\x50\x11\x0b\xd8\x32\x8c\xfb\xd5\x08\x41\xd6\x91\x1d\x87", + "\x67\xf1\x49\x84\xc7\xda\x79\x12\x48\xe3\x2b\xb5\x92\x25\x83\xda", + "\x19\x38\xf2\xcf\x72\xd5\x4e\xe9\x7e\x94\x16\x6f\xa9\x1d\x2a\x36", + "\x74\x48\x1e\x96\x46\xed\x49\xfe\x0f\x62\x24\x30\x16\x04\x69\x8e", + "\x57\xfc\xa5\xde\x98\xa9\xd6\xd8\x00\x64\x38\xd0\x58\x3d\x8a\x1d", + "\x9f\xec\xde\x1c\xef\xdc\x1c\xbe\xd4\x76\x36\x74\xd9\x57\x53\x59", + "\xe3\x04\x0c\x00\xeb\x28\xf1\x53\x66\xca\x73\xcb\xd8\x72\xe7\x40", + "\x76\x97\x00\x9a\x6a\x83\x1d\xfe\xcc\xa9\x1c\x59\x93\x67\x0f\x7a", + "\x58\x53\x54\x23\x21\xf5\x67\xa0\x05\xd5\x47\xa4\xf0\x47\x59\xbd", + "\x51\x50\xd1\x77\x2f\x50\x83\x4a\x50\x3e\x06\x9a\x97\x3f\xbd\x7c", + }; + + const siphash = SipHash128(2, 4); + + var buffer: [64]u8 = undefined; + for (vectors) |vector, i| { + buffer[i] = u8(i); + + const expected = mem.readInt(vector, u128, Endian.Little); + debug.assert(siphash.hash(test_key, buffer[0..i]) == expected); + } +} diff --git a/std/index.zig b/std/index.zig index 4bc1444ac..f2af70b28 100644 --- a/std/index.zig +++ b/std/index.zig @@ -19,6 +19,7 @@ pub const elf = @import("elf.zig"); pub const empty_import = @import("empty.zig"); pub const endian = @import("endian.zig"); pub const fmt = @import("fmt/index.zig"); +pub const hash = @import("hash/index.zig"); pub const heap = @import("heap.zig"); pub const io = @import("io.zig"); pub const macho = @import("macho.zig"); @@ -51,6 +52,7 @@ test "std" { _ = @import("empty.zig"); _ = @import("endian.zig"); _ = @import("fmt/index.zig"); + _ = @import("hash/index.zig"); _ = @import("io.zig"); _ = @import("macho.zig"); _ = @import("math/index.zig"); From 820de1716b83ee99ed94461996ca592a302eccae Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Fri, 6 Apr 2018 15:37:49 +0200 Subject: [PATCH 088/132] std.zig.parser now parses labeled blocks. * There is also some code for switch range parsing --- std/zig/ast.zig | 88 +++++++++++++++++++++++-- std/zig/parser.zig | 161 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 215 insertions(+), 34 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 4aaeef8da..bc066a992 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,8 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Switch, + SwitchCase, InfixOp, PrefixOp, SuffixOp, @@ -55,6 +57,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -91,6 +95,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -127,6 +133,8 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), + Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -506,9 +514,10 @@ pub const NodeParamDecl = struct { pub const NodeBlock = struct { base: Node, - begin_token: Token, - end_token: Token, + label: ?Token, + lbrace: Token, statements: ArrayList(&Node), + rbrace: Token, pub fn iterate(self: &NodeBlock, index: usize) ?&Node { var i = index; @@ -520,11 +529,80 @@ pub const NodeBlock = struct { } pub fn firstToken(self: &NodeBlock) Token { - return self.begin_token; + if (self.label) |label| { + return label; + } + + return self.lbrace; } pub fn lastToken(self: &NodeBlock) Token { - return self.end_token; + return self.rbrace; + } +}; + +pub const NodeSwitch = struct { + base: Node, + switch_token: Token, + expr: &Node, + cases: ArrayList(&NodeSwitchCase), + rbrace: Token, + + pub fn iterate(self: &NodeSwitch, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + if (i < self.cases.len) return self.cases.at(i); + i -= self.cases.len; + + return null; + } + + pub fn firstToken(self: &NodeSwitch) Token { + return self.switch_token; + } + + pub fn lastToken(self: &NodeSwitch) Token { + return self.rbrace; + } +}; + +pub const NodeSwitchCase = struct { + base: Node, + items: ArrayList(&Node), + capture: ?Capture, + expr: &Node, + + const Capture = struct { + symbol: &NodeIdentifier, + is_ptr: bool, + }; + + pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { + var i = index; + + if (i < self.items.len) return self.items.at(i); + i -= self.items.len; + + if (self.capture) |capture| { + if (i < 1) return &capture.base; + i -= 1; + } + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeSwitchCase) Token { + return self.items.at(0).firstToken(); + } + + pub fn lastToken(self: &NodeSwitchCase) Token { + return self.expr.lastToken(); } }; @@ -575,6 +653,7 @@ pub const NodeInfixOp = struct { Mult, MultWrap, Period, + Range, Sub, SubWrap, UnwrapMaybe, @@ -625,6 +704,7 @@ pub const NodeInfixOp = struct { InfixOp.Mult, InfixOp.MultWrap, InfixOp.Period, + InfixOp.Range, InfixOp.Sub, InfixOp.SubWrap, InfixOp.UnwrapMaybe => {}, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index dfc3b2f82..3337968d6 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -146,6 +146,8 @@ pub const Parser = struct { Required, Expression: DestPtr, + RangeExpressionBegin: DestPtr, + RangeExpressionEnd: DestPtr, AssignmentExpressionBegin: DestPtr, AssignmentExpressionEnd: DestPtr, UnwrapExpressionBegin: DestPtr, @@ -256,7 +258,7 @@ pub const Parser = struct { } const name = try self.createStringLiteral(arena, name_token); - const block = try self.createBlock(arena, token); + const block = try self.createBlock(arena, (?Token)(null), token); const test_decl = try self.createAttachTestDecl(arena, &root_node.decls, token, &name.base, block); stack.append(State { .Block = block }) catch unreachable; continue; @@ -643,6 +645,27 @@ pub const Parser = struct { } }, + State.RangeExpressionBegin => |dest_ptr| { + stack.append(State { .RangeExpressionEnd = dest_ptr }) catch unreachable; + try stack.append(State { .Expression = dest_ptr }); + continue; + }, + + State.RangeExpressionEnd => |dest_ptr| { + const token = self.getNextToken(); + if (token.id == Token.Id.Ellipsis3) { + const node = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Range); + node.lhs = dest_ptr.get(); + dest_ptr.store(&node.base); + + stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + continue; + } else { + self.putBackToken(token); + continue; + } + }, + State.AssignmentExpressionBegin => |dest_ptr| { stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); @@ -1205,10 +1228,6 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); continue; }, - Token.Id.Identifier => { - dest_ptr.store(&(try self.createIdentifier(arena, token)).base); - continue; - }, Token.Id.Builtin => { const node = try arena.create(ast.NodeBuiltinCall); *node = ast.NodeBuiltinCall { @@ -1348,8 +1367,32 @@ pub const Parser = struct { }, }) catch unreachable; }, + Token.Id.Identifier => { + const next = self.getNextToken(); + if (next.id != Token.Id.Colon) { + self.putBackToken(next); + dest_ptr.store(&(try self.createIdentifier(arena, token)).base); + continue; + } + + const block = try self.createBlock(arena, (?Token)(token), Token(undefined)); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LBrace, + .ptr = &block.lbrace, + } + }); + continue; + }, Token.Id.LBrace => { - @panic("TODO: Block expr"); + const block = try self.createBlock(arena, (?Token)(null), token); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; }, Token.Id.Keyword_fn => { @panic("TODO: fn proto"); @@ -1618,7 +1661,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch(token.id) { Token.Id.LBrace => { - const block = try self.createBlock(arena, token); + const block = try self.createBlock(arena, (?Token)(null), token); fn_proto.body_node = &block.base; stack.append(State { .Block = block }) catch unreachable; continue; @@ -1635,7 +1678,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.RBrace => { - block.end_token = token; + block.rbrace = token; continue; }, else => { @@ -1648,38 +1691,64 @@ pub const Parser = struct { }, State.Statement => |block| { - { - // Look for comptime var, comptime const - const comptime_token = self.getNextToken(); - if (comptime_token.id == Token.Id.Keyword_comptime) { + const next = self.getNextToken(); + switch (next.id) { + Token.Id.Keyword_comptime => { const mut_token = self.getNextToken(); if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { // TODO shouldn't need these casts const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(comptime_token), (?Token)(null), null); + mut_token, (?Token)(next), (?Token)(null), null); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; + } else { + self.putBackToken(mut_token); + @panic("TODO: comptime block"); } - self.putBackToken(mut_token); - } - self.putBackToken(comptime_token); - } - { - // Look for const, var - const mut_token = self.getNextToken(); - if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) { - // TODO shouldn't need these casts + }, + Token.Id.Keyword_var, Token.Id.Keyword_const => { const var_decl = try self.createAttachVarDecl(arena, &block.statements, (?Token)(null), - mut_token, (?Token)(null), (?Token)(null), null); + next, (?Token)(null), (?Token)(null), null); stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; + }, + Token.Id.Identifier => { + const maybe_colon = self.getNextToken(); + if (maybe_colon.id != Token.Id.Colon) { + self.putBackToken(maybe_colon); + self.putBackToken(next); + stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + continue; + } + + const inner_block = try self.createBlock(arena, (?Token)(next), Token(undefined)); + try block.statements.append(&inner_block.base); + + stack.append(State { .Block = inner_block }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LBrace, + .ptr = &inner_block.lbrace, + } + }); + continue; + }, + Token.Id.LBrace => { + const inner_block = try self.createBlock(arena, (?Token)(null), next); + try block.statements.append(&inner_block.base); + + stack.append(State { .Block = inner_block }) catch unreachable; + continue; + }, + else => { + self.putBackToken(next); + stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + continue; } - self.putBackToken(mut_token); } - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); - continue; }, } } @@ -1905,14 +1974,15 @@ pub const Parser = struct { return node; } - fn createBlock(self: &Parser, arena: &mem.Allocator, begin_token: &const Token) !&ast.NodeBlock { + fn createBlock(self: &Parser, arena: &mem.Allocator, label: &const ?Token, lbrace: &const Token) !&ast.NodeBlock { const node = try arena.create(ast.NodeBlock); *node = ast.NodeBlock { .base = self.initNode(ast.Node.Id.Block), - .begin_token = *begin_token, - .end_token = undefined, + .label = *label, + .lbrace = *lbrace, .statements = ArrayList(&ast.Node).init(arena), + .rbrace = undefined, }; return node; } @@ -2340,6 +2410,10 @@ pub const Parser = struct { }, ast.Node.Id.Block => { const block = @fieldParentPtr(ast.NodeBlock, "base", base); + if (block.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + if (block.statements.len == 0) { try stream.write("{}"); } else { @@ -2747,6 +2821,8 @@ pub const Parser = struct { }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), + ast.Node.Id.Switch => @panic("TODO switch"), + ast.Node.Id.SwitchCase => @panic("TODO switch case"), ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -2791,6 +2867,9 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, + ast.Node.Id.Block => { + try stack.append(RenderState { .Expression = base}); + }, else => { try stack.append(RenderState { .Text = ";"}); try stack.append(RenderState { .Expression = base}); @@ -3323,6 +3402,28 @@ test "zig fmt: catch" { ); } +test "zig fmt: blocks" { + try testCanonical( + \\test "blocks" { + \\ { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ blk: { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ const r = blk: { + \\ const a = 0; + \\ const b = 0; + \\ }; + \\} + \\ + ); +} + test "zig fmt: switch" { try testCanonical( \\test "switch" { From bdff5bfa3e9f6ab490771b54109cb200b180b4da Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sat, 7 Apr 2018 01:38:38 +0200 Subject: [PATCH 089/132] std.zig.parser now parses switch --- std/zig/ast.zig | 21 ++++ std/zig/parser.zig | 260 ++++++++++++++++++++++++++++++++++++------ std/zig/tokenizer.zig | 6 + 3 files changed, 251 insertions(+), 36 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index bc066a992..ca384efed 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -22,6 +22,7 @@ pub const Node = struct { Block, Switch, SwitchCase, + SwitchElse, InfixOp, PrefixOp, SuffixOp, @@ -59,6 +60,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -97,6 +99,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -135,6 +138,7 @@ pub const Node = struct { Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -606,6 +610,23 @@ pub const NodeSwitchCase = struct { } }; +pub const NodeSwitchElse = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeSwitchElse, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeSwitchElse) Token { + return self.token; + } + + pub fn lastToken(self: &NodeSwitchElse) Token { + return self.token; + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 3337968d6..09f782750 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -98,10 +98,15 @@ pub const Parser = struct { ptr: &?&ast.Node, }; - fn ListState(comptime T: type) type { + const ExprListCtx = struct { + list: &ArrayList(&ast.Node), + end: Token.Id, + ptr: &Token, + }; + + fn ListSave(comptime T: type) type { return struct { list: &ArrayList(T), - end: Token.Id, ptr: &Token, }; } @@ -129,11 +134,16 @@ pub const Parser = struct { FnDef: &ast.NodeFnProto, Block: &ast.NodeBlock, Statement: &ast.NodeBlock, - ExprListItemOrEnd: ListState(&ast.Node), - ExprListCommaOrEnd: ListState(&ast.Node), - FieldInitListItemOrEnd: ListState(&ast.NodeFieldInitializer), - FieldInitListCommaOrEnd: ListState(&ast.NodeFieldInitializer), + ExprListItemOrEnd: ExprListCtx, + ExprListCommaOrEnd: ExprListCtx, + FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), + FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, + SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), + SwitchCaseCapture: &?ast.NodeSwitchCase.Capture, + SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), + SwitchCaseItem: &ArrayList(&ast.Node), + SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), /// A state that can be appended before any other State. If an error occures, /// the parser will first try looking for the closest optional state. If an @@ -245,17 +255,8 @@ pub const Parser = struct { Token.Id.Keyword_test => { stack.append(State.TopLevel) catch unreachable; - const name_token = self.getNextToken(); - if (name_token.id != Token.Id.StringLiteral) { - try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.StringLiteral), @tagName(name_token.id)); - continue; - } - - const lbrace = self.getNextToken(); - if (lbrace.id != Token.Id.LBrace) { - try self.parseError(&stack, token, "expected {}, found {}", @tagName(Token.Id.LBrace), @tagName(name_token.id)); - continue; - } + const name_token = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + const lbrace = (try self.eatToken(&stack, Token.Id.LBrace)) ?? continue; const name = try self.createStringLiteral(arena, name_token); const block = try self.createBlock(arena, (?Token)(null), token); @@ -974,9 +975,8 @@ pub const Parser = struct { stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .FieldInitListItemOrEnd = ListState(&ast.NodeFieldInitializer) { + .FieldInitListItemOrEnd = ListSave(&ast.NodeFieldInitializer) { .list = &node.op.StructInitializer, - .end = Token.Id.RBrace, .ptr = &node.rtoken, } }); @@ -992,7 +992,7 @@ pub const Parser = struct { stack.append(State { .CurlySuffixExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.op.ArrayInitializer, .end = Token.Id.RBrace, .ptr = &node.rtoken, @@ -1084,7 +1084,7 @@ pub const Parser = struct { stack.append(State { .SuffixOpExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.op.Call.params, .end = Token.Id.RParen, .ptr = &node.rtoken, @@ -1238,7 +1238,7 @@ pub const Parser = struct { }; dest_ptr.store(&node.base); stack.append(State { - .ExprListItemOrEnd = ListState(&ast.Node) { + .ExprListItemOrEnd = ExprListCtx { .list = &node.params, .end = Token.Id.RParen, .ptr = &node.rparen_token, @@ -1400,6 +1400,43 @@ pub const Parser = struct { Token.Id.Keyword_asm => { @panic("TODO: inline asm"); }, + Token.Id.Keyword_if => { + @panic("TODO: inline if"); + }, + Token.Id.Keyword_while => { + @panic("TODO: inline while"); + }, + Token.Id.Keyword_for => { + @panic("TODO: inline for"); + }, + Token.Id.Keyword_switch => { + const node = try arena.create(ast.NodeSwitch); + *node = ast.NodeSwitch { + .base = self.initNode(ast.Node.Id.Switch), + .switch_token = token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { + .list = &node.cases, + .ptr = &node.rbrace, + }, + }) catch unreachable; + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + Token.Id.Keyword_comptime => { + @panic("TODO: inline comptime"); + }, + Token.Id.Keyword_suspend => { + @panic("TODO: inline suspend"); + }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -1463,8 +1500,7 @@ pub const Parser = struct { State.FieldInitListItemOrEnd => |list_state| { var token = self.getNextToken(); - const IdTag = @TagType(Token.Id); - if (IdTag(list_state.end) == token.id){ + if (token.id == Token.Id.RBrace){ *list_state.ptr = token; continue; } @@ -1497,13 +1533,82 @@ pub const Parser = struct { }); }, + State.SwitchCaseOrEnd => |list_state| { + var token = self.getNextToken(); + + if (token.id == Token.Id.RBrace){ + *list_state.ptr = token; + continue; + } + + self.putBackToken(token); + + const node = try arena.create(ast.NodeSwitchCase); + *node = ast.NodeSwitchCase { + .base = self.initNode(ast.Node.Id.SwitchCase), + .items = ArrayList(&ast.Node).init(arena), + .capture = null, + .expr = undefined, + }; + try list_state.list.append(node); + stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; + try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); + try stack.append(State { .SwitchCaseCapture = &node.capture }); + + const maybe_else = self.getNextToken(); + if (maybe_else.id == Token.Id.Keyword_else) { + const else_node = try arena.create(ast.NodeSwitchElse); + *else_node = ast.NodeSwitchElse { + .base = self.initNode(ast.Node.Id.SwitchElse), + .token = maybe_else, + }; + try node.items.append(&else_node.base); + try stack.append(State { .ExpectToken = Token.Id.EqualAngleBracketRight }); + continue; + } else { + self.putBackToken(maybe_else); + try stack.append(State { .SwitchCaseItem = &node.items }); + continue; + } + }, + + State.SwitchCaseCapture => |capture| { + const token = self.getNextToken(); + if (token.id != Token.Id.Pipe) { + self.putBackToken(token); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + *capture = ast.NodeSwitchCase.Capture { + .symbol = try self.createIdentifier(arena, ident), + .is_ptr = is_ptr + }; + }, + + State.SwitchCaseItem => |case_items| { + stack.append(State { .SwitchCaseItemCommaOrEnd = case_items }) catch unreachable; + try stack.append(State { .RangeExpressionBegin = DestPtr{ .Field = try case_items.addOne() } }); + }, + State.ExprListCommaOrEnd => |list_state| { try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .ExprListItemOrEnd = list_state }); continue; }, State.FieldInitListCommaOrEnd => |list_state| { - try self.commaOrEnd(&stack, list_state.end, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); + try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .FieldInitListItemOrEnd = list_state }); continue; }, @@ -1513,6 +1618,16 @@ pub const Parser = struct { continue; }, + State.SwitchCaseCommaOrEnd => |list_state| { + try self.commaOrEnd(&stack, Token.Id.RBrace, list_state.ptr, State { .SwitchCaseOrEnd = list_state }); + continue; + }, + + State.SwitchCaseItemCommaOrEnd => |case_items| { + try self.commaOrEnd(&stack, Token.Id.EqualAngleBracketRight, null, State { .SwitchCaseItem = case_items }); + continue; + }, + State.AddrOfModifiers => |addr_of_info| { var token = self.getNextToken(); switch (token.id) { @@ -1741,6 +1856,11 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, + Token.Id.Keyword_switch => { + self.putBackToken(next); + stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; + continue; + }, else => { self.putBackToken(next); stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; @@ -1754,7 +1874,7 @@ pub const Parser = struct { } } - fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, ptr: &Token, state_after_comma: &const State) !void { + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { Token.Id.Comma => { @@ -1763,7 +1883,9 @@ pub const Parser = struct { else => { const IdTag = @TagType(Token.Id); if (IdTag(*end) == token.id) { - *ptr = token; + if (maybe_ptr) |ptr| { + *ptr = token; + } return; } @@ -2498,7 +2620,8 @@ pub const Parser = struct { ast.NodeInfixOp.InfixOp.Sub => " - ", ast.NodeInfixOp.InfixOp.SubWrap => " -% ", ast.NodeInfixOp.InfixOp.UnwrapMaybe => " ?? ", - else => unreachable, + ast.NodeInfixOp.InfixOp.Range => " ... ", + ast.NodeInfixOp.InfixOp.Catch => unreachable, }; try stack.append(RenderState { .Text = text }); @@ -2821,8 +2944,73 @@ pub const Parser = struct { }, ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), - ast.Node.Id.Switch => @panic("TODO switch"), - ast.Node.Id.SwitchCase => @panic("TODO switch case"), + ast.Node.Id.Switch => { + const switch_node = @fieldParentPtr(ast.NodeSwitch, "base", base); + try stream.print("{} (", self.tokenizer.getTokenSlice(switch_node.switch_token)); + + try stack.append(RenderState { .Text = "}"}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Text = "\n"}); + + const cases = switch_node.cases.toSliceConst(); + var i = cases.len; + while (i != 0) { + i -= 1; + const node = cases[i]; + try stack.append(RenderState { .Expression = &node.base}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_node = cases[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); + + if (i != 0) { + try stack.append(RenderState { .Text = "," }); + } + } + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = ") {"}); + try stack.append(RenderState { .Expression = switch_node.expr }); + }, + ast.Node.Id.SwitchCase => { + const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base); + + try stack.append(RenderState { .Expression = switch_case.expr }); + if (switch_case.capture) |capture| { + try stack.append(RenderState { .Text = "| "}); + try stack.append(RenderState { .Expression = &capture.symbol.base }); + + if (capture.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + try stack.append(RenderState { .Text = "|"}); + } + try stack.append(RenderState { .Text = " => "}); + + const items = switch_case.items.toSliceConst(); + var i = items.len; + while (i != 0) { + i -= 1; + try stack.append(RenderState { .Expression = items[i] }); + + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + }, + ast.Node.Id.SwitchElse => { + const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -2867,7 +3055,7 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, - ast.Node.Id.Block => { + ast.Node.Id.Block, ast.Node.Id.Switch => { try stack.append(RenderState { .Expression = base}); }, else => { @@ -3436,24 +3624,24 @@ test "zig fmt: switch" { \\ else => { \\ const a = 1; \\ const b = a; - \\ }, + \\ } \\ } \\ \\ const res = switch (0) { \\ 0 => 0, \\ 1 => 2, - \\ else => 4, + \\ else => 4 \\ }; \\ \\ const Union = union(enum) { \\ Int: i64, - \\ Float: f64, + \\ Float: f64 \\ }; \\ - \\ const u = Union { .Int = 0 }; + \\ const u = Union{ .Int = 0 }; \\ switch (u) { \\ Union.Int => |int| {}, - \\ Union.Float => |*float| unreachable, + \\ Union.Float => |*float| unreachable \\ } \\} \\ diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 87597adff..422aa2062 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -86,6 +86,7 @@ pub const Token = struct { PipeEqual, Equal, EqualEqual, + EqualAngleBracketRight, BangEqual, LParen, RParen, @@ -688,6 +689,11 @@ pub const Tokenizer = struct { self.index += 1; break; }, + '>' => { + result.id = Token.Id.EqualAngleBracketRight; + self.index += 1; + break; + }, else => { result.id = Token.Id.Equal; break; From e4d0b46c0c07f3151b4959c5e2e0d4c304981ead Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Sun, 8 Apr 2018 17:05:08 +0200 Subject: [PATCH 090/132] std.zig.parser WIP generalizing parsing of payloads * Note, it doesn't work :) --- std/zig/ast.zig | 108 ++++++++++++++++++++++++++++++++++++++++----- std/zig/parser.zig | 104 ++++++++++++++++++++++++++----------------- 2 files changed, 161 insertions(+), 51 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index ca384efed..0d060eb68 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,9 +20,11 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Payload, Switch, SwitchCase, SwitchElse, + While, InfixOp, PrefixOp, SuffixOp, @@ -58,9 +60,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), + Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -97,9 +101,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -136,9 +142,11 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), - Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), + Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), + Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -545,6 +553,31 @@ pub const NodeBlock = struct { } }; +pub const NodePayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + symbol: &NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodePayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbol.base; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodePayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodePayload) Token { + return self.rpipe; + } +}; + pub const NodeSwitch = struct { base: Node, switch_token: Token, @@ -576,22 +609,17 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - capture: ?Capture, + payload: ?&Node, expr: &Node, - const Capture = struct { - symbol: &NodeIdentifier, - is_ptr: bool, - }; - pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { var i = index; if (i < self.items.len) return self.items.at(i); i -= self.items.len; - if (self.capture) |capture| { - if (i < 1) return &capture.base; + if (self.payload) |payload| { + if (i < 1) return payload; i -= 1; } @@ -627,6 +655,66 @@ pub const NodeSwitchElse = struct { } }; +pub const NodeWhile = struct { + base: Node, + while_token: Token, + condition: &Node, + payload: ?&NodePayload, + continue_expr: ?&Node, + body: &Node, + @"else": ?Else, + + const Else = struct { + capture: ?&NodeIdentifier, + body: &Node, + }; + + + pub fn iterate(self: &NodeWhile, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.condition; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (self.continue_expr) |continue_expr| { + if (i < 1) return continue_expr; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (@"else".capture) |capture| { + if (i < 1) return &capture.base; + i -= 1; + } + + if (i < 1) return @"else".body; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeWhile) Token { + return self.while_token; + } + + pub fn lastToken(self: &NodeWhile) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, @@ -661,7 +749,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodeIdentifier, + Catch: ?&Node, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 09f782750..0d8b8a324 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -104,6 +104,11 @@ pub const Parser = struct { ptr: &Token, }; + const ElseCtx = struct { + payload: ?DestPtr, + body: DestPtr, + }; + fn ListSave(comptime T: type) type { return struct { list: &ArrayList(T), @@ -140,7 +145,7 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - SwitchCaseCapture: &?ast.NodeSwitchCase.Capture, + Payload: DestPtr, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -635,9 +640,6 @@ pub const Parser = struct { Token.Id.Keyword_await => { @panic("TODO: await"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: suspend"); - }, else => { self.putBackToken(token); stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; @@ -705,21 +707,14 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - - const next = self.getNextToken(); - if (next.id != Token.Id.Pipe) { - self.putBackToken(next); - continue; - } - - node.op.Catch = try self.createIdentifier(arena, Token(undefined)); - try stack.append(State { .ExpectToken = Token.Id.Pipe }); try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Identifier, - .ptr = &(??node.op.Catch).name_token + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.op.Catch, } }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1404,12 +1399,25 @@ pub const Parser = struct { @panic("TODO: inline if"); }, Token.Id.Keyword_while => { + const node = try arena.create(ast.NodeWhile); + *node = ast.NodeWhile { + .base = self.initNode(ast.Node.Id.While), + .while_token = token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + @panic("TODO: inline while"); }, Token.Id.Keyword_for => { @panic("TODO: inline for"); }, Token.Id.Keyword_switch => { + @breakpoint(); const node = try arena.create(ast.NodeSwitch); *node = ast.NodeSwitch { .base = self.initNode(ast.Node.Id.Switch), @@ -1434,9 +1442,6 @@ pub const Parser = struct { Token.Id.Keyword_comptime => { @panic("TODO: inline comptime"); }, - Token.Id.Keyword_suspend => { - @panic("TODO: inline suspend"); - }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -1547,13 +1552,21 @@ pub const Parser = struct { *node = ast.NodeSwitchCase { .base = self.initNode(ast.Node.Id.SwitchCase), .items = ArrayList(&ast.Node).init(arena), - .capture = null, + .payload = null, .expr = undefined, }; try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .SwitchCaseCapture = &node.capture }); + try stack.append(State { + .Optional = RevertState { + .tokenizer = *self.tokenizer, + .parser = *self, + .ptr = &node.payload, + } + }); + try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } }); + try stack.append(State.Required); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1572,12 +1585,8 @@ pub const Parser = struct { } }, - State.SwitchCaseCapture => |capture| { - const token = self.getNextToken(); - if (token.id != Token.Id.Pipe) { - self.putBackToken(token); - continue; - } + State.Payload => |dest_ptr| { + const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; const is_ptr = blk: { const asterik = self.getNextToken(); @@ -1590,11 +1599,16 @@ pub const Parser = struct { }; const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - *capture = ast.NodeSwitchCase.Capture { + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), + .lpipe = lpipe, + .is_ptr = is_ptr, .symbol = try self.createIdentifier(arena, ident), - .is_ptr = is_ptr + .rpipe = rpipe }; + dest_ptr.store(&node.base); }, State.SwitchCaseItem => |case_items| { @@ -1856,6 +1870,8 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, + Token.Id.Keyword_suspend, Token.Id.Keyword_if, + Token.Id.Keyword_while, Token.Id.Keyword_for, Token.Id.Keyword_switch => { self.putBackToken(next); stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; @@ -2572,9 +2588,8 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { - try stack.append(RenderState { .Text = "| " }); - try stack.append(RenderState { .Expression = &payload.base }); - try stack.append(RenderState { .Text = "|" }); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " catch " }); } else { @@ -2764,6 +2779,17 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, + ast.Node.Id.Payload => { + const payload = @fieldParentPtr(ast.NodePayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + try stack.append(RenderState { .Expression = &payload.symbol.base }); + + if (payload.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + + try stack.append(RenderState { .Text = "|"}); + }, ast.Node.Id.GroupedExpression => { const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base); try stack.append(RenderState { .Text = ")"}); @@ -2985,14 +3011,9 @@ pub const Parser = struct { const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base); try stack.append(RenderState { .Expression = switch_case.expr }); - if (switch_case.capture) |capture| { - try stack.append(RenderState { .Text = "| "}); - try stack.append(RenderState { .Expression = &capture.symbol.base }); - - if (capture.is_ptr) { - try stack.append(RenderState { .Text = "*"}); - } - try stack.append(RenderState { .Text = "|"}); + if (switch_case.payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = payload }); } try stack.append(RenderState { .Text = " => "}); @@ -3011,6 +3032,7 @@ pub const Parser = struct { const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); }, + ast.Node.Id.While => @panic("TODO: Render while"), ast.Node.Id.StructField, ast.Node.Id.UnionTag, From d26905c102f45382a5aa4bf59deda0ccc8c6e50f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 29 Mar 2018 00:24:04 -0400 Subject: [PATCH 091/132] error return traces for the early return case it would work but LLVM is not correctly spilling the addresses. See #821 --- src/all_types.hpp | 19 ++++ src/analyze.cpp | 22 +++- src/codegen.cpp | 272 +++++++++++++++++++++++++++++++++++++++------- src/ir.cpp | 146 ++++++++++++++++--------- src/ir_print.cpp | 24 +++- 5 files changed, 391 insertions(+), 92 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 6951230aa..6893f60fb 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -1656,6 +1656,8 @@ struct CodeGen { LLVMValueRef coro_save_fn_val; LLVMValueRef coro_promise_fn_val; LLVMValueRef coro_alloc_helper_fn_val; + LLVMValueRef merge_err_ret_traces_fn_val; + LLVMValueRef add_error_return_trace_addr_fn_val; bool error_during_imports; const char **clang_argv; @@ -2054,6 +2056,7 @@ enum IrInstructionId { IrInstructionIdAwaitBookkeeping, IrInstructionIdSaveErrRetAddr, IrInstructionIdAddImplicitReturnType, + IrInstructionIdMergeErrRetTraces, }; struct IrInstruction { @@ -2892,6 +2895,11 @@ struct IrInstructionExport { struct IrInstructionErrorReturnTrace { IrInstruction base; + + enum Nullable { + Null, + NonNull, + } nullable; }; struct IrInstructionErrorUnion { @@ -3024,6 +3032,13 @@ struct IrInstructionAddImplicitReturnType { IrInstruction *value; }; +struct IrInstructionMergeErrRetTraces { + IrInstruction base; + + IrInstruction *coro_promise_ptr; + TypeStructField *resolved_field; +}; + static const size_t slice_ptr_index = 0; static const size_t slice_len_index = 1; @@ -3033,11 +3048,15 @@ static const size_t maybe_null_index = 1; static const size_t err_union_err_index = 0; static const size_t err_union_payload_index = 1; +// TODO call graph analysis to find out what this number needs to be for every function +static const size_t stack_trace_ptr_count = 30; + #define ASYNC_ALLOC_FIELD_NAME "allocFn" #define ASYNC_FREE_FIELD_NAME "freeFn" #define AWAITER_HANDLE_FIELD_NAME "awaiter_handle" #define RESULT_FIELD_NAME "result" #define RESULT_PTR_FIELD_NAME "result_ptr" +#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr" enum FloatMode { diff --git a/src/analyze.cpp b/src/analyze.cpp index 291e7e764..16788c5e6 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -468,10 +468,26 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) TypeTableEntry *awaiter_handle_type = get_maybe_type(g, g->builtin_types.entry_promise); TypeTableEntry *result_ptr_type = get_pointer_to_type(g, return_type, false); - const char *field_names[] = {AWAITER_HANDLE_FIELD_NAME, RESULT_FIELD_NAME, RESULT_PTR_FIELD_NAME}; - TypeTableEntry *field_types[] = {awaiter_handle_type, return_type, result_ptr_type}; + + ZigList field_names = {}; + field_names.append(AWAITER_HANDLE_FIELD_NAME); + field_names.append(RESULT_FIELD_NAME); + field_names.append(RESULT_PTR_FIELD_NAME); + if (g->have_err_ret_tracing) { + field_names.append(ERR_RET_TRACE_PTR_FIELD_NAME); + } + + ZigList field_types = {}; + field_types.append(awaiter_handle_type); + field_types.append(return_type); + field_types.append(result_ptr_type); + if (g->have_err_ret_tracing) { + field_types.append(get_ptr_to_stack_trace_type(g)); + } + + assert(field_names.length == field_types.length); Buf *name = buf_sprintf("AsyncFramePromise(%s)", buf_ptr(&return_type->name)); - TypeTableEntry *entry = get_struct_type(g, buf_ptr(name), field_names, field_types, 3); + TypeTableEntry *entry = get_struct_type(g, buf_ptr(name), field_names.items, field_types.items, field_names.length); return_type->promise_frame_parent = entry; return entry; diff --git a/src/codegen.cpp b/src/codegen.cpp index 0dafcb950..fa8e069e2 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1114,6 +1114,207 @@ static LLVMValueRef get_return_address_fn_val(CodeGen *g) { return g->return_address_fn_val; } +static LLVMValueRef get_add_error_return_trace_addr_fn(CodeGen *g) { + if (g->add_error_return_trace_addr_fn_val != nullptr) + return g->add_error_return_trace_addr_fn_val; + + LLVMTypeRef arg_types[] = { + get_ptr_to_stack_trace_type(g)->type_ref, + g->builtin_types.entry_usize->type_ref, + }; + LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), arg_types, 2, false); + + Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_add_err_ret_trace_addr"), false); + LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); + addLLVMFnAttr(fn_val, "alwaysinline"); + LLVMSetLinkage(fn_val, LLVMInternalLinkage); + LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); + addLLVMFnAttr(fn_val, "nounwind"); + add_uwtable_attr(g, fn_val); + addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); + if (g->build_mode == BuildModeDebug) { + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); + } + + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); + LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); + LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); + LLVMPositionBuilderAtEnd(g->builder, entry_block); + ZigLLVMClearCurrentDebugLocation(g->builder); + + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->type_ref; + + // stack_trace.instruction_addresses[stack_trace.index % stack_trace.instruction_addresses.len] = return_address; + + LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0); + LLVMValueRef address_value = LLVMGetParam(fn_val, 1); + + size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; + LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)index_field_index, ""); + size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; + LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)addresses_field_index, ""); + + TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); + size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; + LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); + + LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, ""); + LLVMValueRef index_val = gen_load_untyped(g, index_field_ptr, 0, false, ""); + LLVMValueRef modded_val = LLVMBuildURem(g->builder, index_val, len_value, ""); + LLVMValueRef address_indices[] = { + modded_val, + }; + + LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, ""); + LLVMValueRef address_slot = LLVMBuildInBoundsGEP(g->builder, ptr_value, address_indices, 1, ""); + + gen_store_untyped(g, address_value, address_slot, 0, false); + + // stack_trace.index += 1; + LLVMValueRef index_plus_one_val = LLVMBuildNUWAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), ""); + gen_store_untyped(g, index_plus_one_val, index_field_ptr, 0, false); + + // return; + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, prev_block); + LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); + + g->add_error_return_trace_addr_fn_val = fn_val; + return fn_val; +} + +static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) { + if (g->merge_err_ret_traces_fn_val) + return g->merge_err_ret_traces_fn_val; + + assert(g->stack_trace_type != nullptr); + + LLVMTypeRef param_types[] = { + get_ptr_to_stack_trace_type(g)->type_ref, + get_ptr_to_stack_trace_type(g)->type_ref, + }; + LLVMTypeRef fn_type_ref = LLVMFunctionType(LLVMVoidType(), param_types, 2, false); + + Buf *fn_name = get_mangled_name(g, buf_create_from_str("__zig_merge_error_return_traces"), false); + LLVMValueRef fn_val = LLVMAddFunction(g->module, buf_ptr(fn_name), fn_type_ref); + LLVMSetLinkage(fn_val, LLVMInternalLinkage); + LLVMSetFunctionCallConv(fn_val, get_llvm_cc(g, CallingConventionUnspecified)); + addLLVMFnAttr(fn_val, "nounwind"); + add_uwtable_attr(g, fn_val); + addLLVMArgAttr(fn_val, (unsigned)0, "nonnull"); + addLLVMArgAttr(fn_val, (unsigned)0, "noalias"); + addLLVMArgAttr(fn_val, (unsigned)0, "writeonly"); + addLLVMArgAttr(fn_val, (unsigned)1, "nonnull"); + addLLVMArgAttr(fn_val, (unsigned)1, "noalias"); + addLLVMArgAttr(fn_val, (unsigned)1, "readonly"); + if (g->build_mode == BuildModeDebug) { + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim", "true"); + ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); + } + + // this is above the ZigLLVMClearCurrentDebugLocation + LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g); + + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); + LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); + LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); + LLVMPositionBuilderAtEnd(g->builder, entry_block); + ZigLLVMClearCurrentDebugLocation(g->builder); + + // var frame_index: usize = undefined; + // var frames_left: usize = undefined; + // if (src_stack_trace.index < src_stack_trace.instruction_addresses.len) { + // frame_index = 0; + // frames_left = src_stack_trace.index; + // if (frames_left == 0) return; + // } else { + // frame_index = (src_stack_trace.index + 1) % src_stack_trace.instruction_addresses.len; + // frames_left = src_stack_trace.instruction_addresses.len; + // } + // while (true) { + // __zig_add_err_ret_trace_addr(dest_stack_trace, src_stack_trace.instruction_addresses[frame_index]); + // frames_left -= 1; + // if (frames_left == 0) return; + // frame_index = (frame_index + 1) % src_stack_trace.instruction_addresses.len; + // } + LLVMBasicBlockRef return_block = LLVMAppendBasicBlock(fn_val, "Return"); + + LLVMValueRef frame_index_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->type_ref, "frame_index"); + LLVMValueRef frames_left_ptr = LLVMBuildAlloca(g->builder, g->builtin_types.entry_usize->type_ref, "frames_left"); + + LLVMValueRef dest_stack_trace_ptr = LLVMGetParam(fn_val, 0); + LLVMValueRef src_stack_trace_ptr = LLVMGetParam(fn_val, 1); + + size_t src_index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; + size_t src_addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; + LLVMValueRef src_index_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, + (unsigned)src_index_field_index, ""); + LLVMValueRef src_addresses_field_ptr = LLVMBuildStructGEP(g->builder, src_stack_trace_ptr, + (unsigned)src_addresses_field_index, ""); + TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef src_ptr_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)ptr_field_index, ""); + size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; + LLVMValueRef src_len_field_ptr = LLVMBuildStructGEP(g->builder, src_addresses_field_ptr, (unsigned)len_field_index, ""); + LLVMValueRef src_index_val = LLVMBuildLoad(g->builder, src_index_field_ptr, ""); + LLVMValueRef src_ptr_val = LLVMBuildLoad(g->builder, src_ptr_field_ptr, ""); + LLVMValueRef src_len_val = LLVMBuildLoad(g->builder, src_len_field_ptr, ""); + LLVMValueRef no_wrap_bit = LLVMBuildICmp(g->builder, LLVMIntULT, src_index_val, src_len_val, ""); + LLVMBasicBlockRef no_wrap_block = LLVMAppendBasicBlock(fn_val, "NoWrap"); + LLVMBasicBlockRef yes_wrap_block = LLVMAppendBasicBlock(fn_val, "YesWrap"); + LLVMBasicBlockRef loop_block = LLVMAppendBasicBlock(fn_val, "Loop"); + LLVMBuildCondBr(g->builder, no_wrap_bit, no_wrap_block, yes_wrap_block); + + LLVMPositionBuilderAtEnd(g->builder, no_wrap_block); + LLVMValueRef usize_zero = LLVMConstNull(g->builtin_types.entry_usize->type_ref); + LLVMBuildStore(g->builder, usize_zero, frame_index_ptr); + LLVMBuildStore(g->builder, src_index_val, frames_left_ptr); + LLVMValueRef frames_left_eq_zero_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, src_index_val, usize_zero, ""); + LLVMBuildCondBr(g->builder, frames_left_eq_zero_bit, return_block, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, yes_wrap_block); + LLVMValueRef usize_one = LLVMConstInt(g->builtin_types.entry_usize->type_ref, 1, false); + LLVMValueRef plus_one = LLVMBuildNUWAdd(g->builder, src_index_val, usize_one, ""); + LLVMValueRef mod_len = LLVMBuildURem(g->builder, plus_one, src_len_val, ""); + LLVMBuildStore(g->builder, mod_len, frame_index_ptr); + LLVMBuildStore(g->builder, src_len_val, frames_left_ptr); + LLVMBuildBr(g->builder, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, loop_block); + LLVMValueRef ptr_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); + LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, ""); + LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, ""); + LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val}; + LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); + LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, ""); + LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, ""); + LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, ""); + LLVMBasicBlockRef continue_block = LLVMAppendBasicBlock(fn_val, "Continue"); + LLVMBuildCondBr(g->builder, done_bit, return_block, continue_block); + + LLVMPositionBuilderAtEnd(g->builder, return_block); + LLVMBuildRetVoid(g->builder); + + LLVMPositionBuilderAtEnd(g->builder, continue_block); + LLVMBuildStore(g->builder, new_frames_left, frames_left_ptr); + LLVMValueRef prev_index = LLVMBuildLoad(g->builder, frame_index_ptr, ""); + LLVMValueRef index_plus_one = LLVMBuildNUWAdd(g->builder, prev_index, usize_one, ""); + LLVMValueRef index_mod_len = LLVMBuildURem(g->builder, index_plus_one, src_len_val, ""); + LLVMBuildStore(g->builder, index_mod_len, frame_index_ptr); + LLVMBuildBr(g->builder, loop_block); + + LLVMPositionBuilderAtEnd(g->builder, prev_block); + LLVMSetCurrentDebugLocation(g->builder, prev_debug_location); + + g->merge_err_ret_traces_fn_val = fn_val; + return fn_val; + +} + static LLVMValueRef get_return_err_fn(CodeGen *g) { if (g->return_err_fn != nullptr) return g->return_err_fn; @@ -1140,50 +1341,24 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) { ZigLLVMAddFunctionAttr(fn_val, "no-frame-pointer-elim-non-leaf", nullptr); } + // this is above the ZigLLVMClearCurrentDebugLocation + LLVMValueRef add_error_return_trace_addr_fn_val = get_add_error_return_trace_addr_fn(g); + LLVMBasicBlockRef entry_block = LLVMAppendBasicBlock(fn_val, "Entry"); LLVMBasicBlockRef prev_block = LLVMGetInsertBlock(g->builder); LLVMValueRef prev_debug_location = LLVMGetCurrentDebugLocation(g->builder); LLVMPositionBuilderAtEnd(g->builder, entry_block); ZigLLVMClearCurrentDebugLocation(g->builder); - LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->type_ref; - - // stack_trace.instruction_addresses[stack_trace.index % stack_trace.instruction_addresses.len] = return_address; - LLVMValueRef err_ret_trace_ptr = LLVMGetParam(fn_val, 0); - size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; - LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)index_field_index, ""); - size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; - LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, err_ret_trace_ptr, (unsigned)addresses_field_index, ""); - - TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; - size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; - LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); - size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; - LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - - LLVMValueRef len_value = gen_load_untyped(g, len_field_ptr, 0, false, ""); - LLVMValueRef index_val = gen_load_untyped(g, index_field_ptr, 0, false, ""); - LLVMValueRef modded_val = LLVMBuildURem(g->builder, index_val, len_value, ""); - LLVMValueRef address_indices[] = { - modded_val, - }; - - LLVMValueRef ptr_value = gen_load_untyped(g, ptr_field_ptr, 0, false, ""); - LLVMValueRef address_slot = LLVMBuildInBoundsGEP(g->builder, ptr_value, address_indices, 1, ""); + LLVMTypeRef usize_type_ref = g->builtin_types.entry_usize->type_ref; LLVMValueRef zero = LLVMConstNull(g->builtin_types.entry_i32->type_ref); LLVMValueRef return_address_ptr = LLVMBuildCall(g->builder, get_return_address_fn_val(g), &zero, 1, ""); LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, ""); - LLVMValueRef address_value = LLVMBuildPtrToInt(g->builder, return_address, usize_type_ref, ""); - gen_store_untyped(g, address_value, address_slot, 0, false); - - // stack_trace.index += 1; - LLVMValueRef index_plus_one_val = LLVMBuildAdd(g->builder, index_val, LLVMConstInt(usize_type_ref, 1, false), ""); - gen_store_untyped(g, index_plus_one_val, index_field_ptr, 0, false); - - // return; + LLVMValueRef args[] = { err_ret_trace_ptr, return_address }; + LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); LLVMBuildRetVoid(g->builder); LLVMPositionBuilderAtEnd(g->builder, prev_block); @@ -1641,7 +1816,6 @@ static LLVMValueRef ir_render_save_err_ret_addr(CodeGen *g, IrExecutable *execut }; LLVMValueRef call_instruction = ZigLLVMBuildCall(g->builder, return_err_fn, args, 1, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); - LLVMSetTailCall(call_instruction, true); return call_instruction; } @@ -4204,6 +4378,22 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutable *executable, return LLVMBuildIntToPtr(g->builder, uncasted_result, operand_type->type_ref, ""); } +static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *executable, + IrInstructionMergeErrRetTraces *instruction) +{ + assert(g->have_err_ret_tracing); + + LLVMValueRef coro_promise_ptr = ir_llvm_value(g, instruction->coro_promise_ptr); + TypeStructField *field = instruction->resolved_field; + LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, coro_promise_ptr, field->gen_index, ""); + LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, ptr_field_ptr, ""); + LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); + + LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; + LLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, ""); + return nullptr; +} + static void set_debug_location(CodeGen *g, IrInstruction *instruction) { AstNode *source_node = instruction->source_node; Scope *scope = instruction->scope; @@ -4421,6 +4611,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_atomic_rmw(g, executable, (IrInstructionAtomicRmw *)instruction); case IrInstructionIdSaveErrRetAddr: return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction); + case IrInstructionIdMergeErrRetTraces: + return ir_render_merge_err_ret_traces(g, executable, (IrInstructionMergeErrRetTraces *)instruction); } zig_unreachable(); } @@ -5313,12 +5505,12 @@ static void do_code_gen(CodeGen *g) { bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && (is_async || !have_err_ret_trace_arg); - if (have_err_ret_trace_stack) { - // TODO call graph analysis to find out what this number needs to be for every function - static const size_t stack_trace_ptr_count = 30; - + bool have_exactly_one_err_ret_value = !have_err_ret_trace_stack && g->have_err_ret_tracing && is_async && + type_can_fail(fn_table_entry->type_entry->data.fn.fn_type_id.return_type); + if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { TypeTableEntry *usize = g->builtin_types.entry_usize; - TypeTableEntry *array_type = get_array_type(g, usize, stack_trace_ptr_count); + uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; + TypeTableEntry *array_type = get_array_type(g, usize, ret_addr_count); LLVMValueRef err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); @@ -5341,7 +5533,7 @@ static void do_code_gen(CodeGen *g) { size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); + gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } else { g->cur_err_ret_trace_val_stack = nullptr; } @@ -5943,6 +6135,8 @@ static void define_builtin_compile_vars(CodeGen *g) { os_path_join(g->cache_dir, buf_create_from_str(builtin_zig_basename), builtin_zig_path); Buf *contents = buf_alloc(); + // Modifications to this struct must be coordinated with code that does anything with + // g->stack_trace_type. There are hard-coded references to the field indexes. buf_append_str(contents, "pub const StackTrace = struct {\n" " index: usize,\n" diff --git a/src/ir.cpp b/src/ir.cpp index 4fe6769f7..c771adca4 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -725,6 +725,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionAddImplicitRetur return IrInstructionIdAddImplicitReturnType; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionMergeErrRetTraces *) { + return IrInstructionIdMergeErrRetTraces; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -972,6 +976,12 @@ static IrInstruction *ir_build_const_promise_init(IrBuilder *irb, Scope *scope, const_instruction->base.value.data.x_struct.fields[1].special = ConstValSpecialUndef; const_instruction->base.value.data.x_struct.fields[2].type = struct_type->data.structure.fields[2].type_entry; const_instruction->base.value.data.x_struct.fields[2].special = ConstValSpecialUndef; + if (irb->codegen->have_err_ret_tracing) { + assert(struct_type->data.structure.src_field_count == 4); + + const_instruction->base.value.data.x_struct.fields[3].type = struct_type->data.structure.fields[3].type_entry; + const_instruction->base.value.data.x_struct.fields[3].special = ConstValSpecialUndef; + } return &const_instruction->base; } @@ -2495,8 +2505,9 @@ static IrInstruction *ir_build_arg_type(IrBuilder *irb, Scope *scope, AstNode *s return &instruction->base; } -static IrInstruction *ir_build_error_return_trace(IrBuilder *irb, Scope *scope, AstNode *source_node) { +static IrInstruction *ir_build_error_return_trace(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstructionErrorReturnTrace::Nullable nullable) { IrInstructionErrorReturnTrace *instruction = ir_build_instruction(irb, scope, source_node); + instruction->nullable = nullable; return &instruction->base; } @@ -2717,6 +2728,18 @@ static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *s return &instruction->base; } +static IrInstruction *ir_build_merge_err_ret_traces(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *coro_promise_ptr, TypeStructField *resolved_field) +{ + IrInstructionMergeErrRetTraces *instruction = ir_build_instruction(irb, scope, source_node); + instruction->coro_promise_ptr = coro_promise_ptr; + instruction->resolved_field = resolved_field; + + ir_ref_instruction(coro_promise_ptr, irb->current_basic_block); + + return &instruction->base; +} + static void ir_count_defers(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, size_t *results) { results[ReturnKindUnconditional] = 0; results[ReturnKindError] = 0; @@ -2822,34 +2845,6 @@ static IrInstruction *ir_gen_async_return(IrBuilder *irb, Scope *scope, AstNode // the above blocks are rendered by ir_gen after the rest of codegen } -static bool exec_have_err_ret_trace(CodeGen *g, IrExecutable *exec) { - if (!g->have_err_ret_tracing) - return false; - FnTableEntry *fn_entry = exec_fn_entry(exec); - if (fn_entry == nullptr) - return false; - if (exec->is_inline) - return false; - return type_can_fail(fn_entry->type_entry->data.fn.fn_type_id.return_type); -} - -static void ir_gen_save_err_ret_addr(IrBuilder *irb, Scope *scope, AstNode *node) { - if (!exec_have_err_ret_trace(irb->codegen, irb->exec)) - return; - - bool is_async = exec_is_async(irb->exec); - - if (is_async) { - //IrInstruction *err_ret_addr_ptr = ir_build_load_ptr(irb, scope, node, irb->exec->coro_err_ret_addr_ptr); - //IrInstruction *return_address_ptr = ir_build_instr_addr(irb, scope, node); - //IrInstruction *return_address_usize = ir_build_ptr_to_int(irb, scope, node, return_address_ptr); - //ir_build_store_ptr(irb, scope, node, err_ret_addr_ptr, return_address_usize); - return; - } - - ir_build_save_err_ret_addr(irb, scope, node); -} - static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, LVal lval) { assert(node->type == NodeTypeReturnExpr); @@ -2895,8 +2890,9 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, IrInstruction *is_err = ir_build_test_err(irb, scope, node, return_value); + bool should_inline = ir_should_inline(irb->exec, scope); IrInstruction *is_comptime; - if (ir_should_inline(irb->exec, scope)) { + if (should_inline) { is_comptime = ir_build_const_bool(irb, scope, node, true); } else { is_comptime = ir_build_test_comptime(irb, scope, node, is_err); @@ -2909,7 +2905,9 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, if (have_err_defers) { ir_gen_defers_for_block(irb, scope, outer_scope, true); } - ir_gen_save_err_ret_addr(irb, scope, node); + if (irb->codegen->have_err_ret_tracing && !should_inline) { + ir_build_save_err_ret_addr(irb, scope, node); + } ir_build_br(irb, scope, node, ret_stmt_block, is_comptime); ir_set_cursor_at_end_and_append_block(irb, ok_block); @@ -2938,7 +2936,8 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, IrBasicBlock *return_block = ir_create_basic_block(irb, scope, "ErrRetReturn"); IrBasicBlock *continue_block = ir_create_basic_block(irb, scope, "ErrRetContinue"); IrInstruction *is_comptime; - if (ir_should_inline(irb->exec, scope)) { + bool should_inline = ir_should_inline(irb->exec, scope); + if (should_inline) { is_comptime = ir_build_const_bool(irb, scope, node, true); } else { is_comptime = ir_build_test_comptime(irb, scope, node, is_err_val); @@ -2948,7 +2947,9 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_set_cursor_at_end_and_append_block(irb, return_block); ir_gen_defers_for_block(irb, scope, outer_scope, true); IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); - ir_gen_save_err_ret_addr(irb, scope, node); + if (irb->codegen->have_err_ret_tracing && !should_inline) { + ir_build_save_err_ret_addr(irb, scope, node); + } ir_gen_async_return(irb, scope, node, err_val, false); ir_set_cursor_at_end_and_append_block(irb, continue_block); @@ -4242,7 +4243,7 @@ static IrInstruction *ir_gen_builtin_fn_call(IrBuilder *irb, Scope *scope, AstNo } case BuiltinFnIdErrorReturnTrace: { - return ir_build_error_return_trace(irb, scope, node); + return ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::Null); } case BuiltinFnIdAtomicRmw: { @@ -6148,10 +6149,13 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast IrInstruction *is_non_null = ir_build_test_nonnull(irb, parent_scope, node, maybe_await_handle); IrBasicBlock *yes_suspend_block = ir_create_basic_block(irb, parent_scope, "YesSuspend"); IrBasicBlock *no_suspend_block = ir_create_basic_block(irb, parent_scope, "NoSuspend"); - IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "Merge"); + IrBasicBlock *merge_block = ir_create_basic_block(irb, parent_scope, "MergeSuspend"); ir_build_cond_br(irb, parent_scope, node, is_non_null, no_suspend_block, yes_suspend_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); + if (irb->codegen->have_err_ret_tracing) { + ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, nullptr); + } Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_field_name); IrInstruction *no_suspend_result = ir_build_load_ptr(irb, parent_scope, node, promise_result_ptr); @@ -6460,13 +6464,19 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec irb->exec->coro_handle = ir_build_coro_begin(irb, coro_scope, node, coro_id, coro_mem_ptr); Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); - irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, + irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, awaiter_handle_field_name); Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); - irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_field_name); + irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); - irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, coro_scope, node, coro_promise_ptr, result_ptr_field_name); - ir_build_store_ptr(irb, coro_scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); + irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name); + ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); + if (irb->codegen->have_err_ret_tracing) { + IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull); + Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); + IrInstruction *coro_err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); + ir_build_store_ptr(irb, scope, node, coro_err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); + } irb->exec->coro_early_final = ir_create_basic_block(irb, scope, "CoroEarlyFinal"); @@ -11579,18 +11589,25 @@ static bool exec_has_err_ret_trace(CodeGen *g, IrExecutable *exec) { static TypeTableEntry *ir_analyze_instruction_error_return_trace(IrAnalyze *ira, IrInstructionErrorReturnTrace *instruction) { - TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(ira->codegen); - TypeTableEntry *nullable_type = get_maybe_type(ira->codegen, ptr_to_stack_trace_type); - if (!exec_has_err_ret_trace(ira->codegen, ira->new_irb.exec)) { - ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); - out_val->data.x_maybe = nullptr; + if (instruction->nullable == IrInstructionErrorReturnTrace::Null) { + TypeTableEntry *ptr_to_stack_trace_type = get_ptr_to_stack_trace_type(ira->codegen); + TypeTableEntry *nullable_type = get_maybe_type(ira->codegen, ptr_to_stack_trace_type); + if (!exec_has_err_ret_trace(ira->codegen, ira->new_irb.exec)) { + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->data.x_maybe = nullptr; + return nullable_type; + } + IrInstruction *new_instruction = ir_build_error_return_trace(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, instruction->nullable); + ir_link_new_instruction(new_instruction, &instruction->base); return nullable_type; + } else { + assert(ira->codegen->have_err_ret_tracing); + IrInstruction *new_instruction = ir_build_error_return_trace(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, instruction->nullable); + ir_link_new_instruction(new_instruction, &instruction->base); + return get_ptr_to_stack_trace_type(ira->codegen); } - - IrInstruction *new_instruction = ir_build_error_return_trace(&ira->new_irb, instruction->base.scope, - instruction->base.source_node); - ir_link_new_instruction(new_instruction, &instruction->base); - return nullable_type; } static TypeTableEntry *ir_analyze_instruction_error_union(IrAnalyze *ira, @@ -17904,6 +17921,34 @@ static TypeTableEntry *ir_analyze_instruction_await_bookkeeping(IrAnalyze *ira, return out_val->type; } +static TypeTableEntry *ir_analyze_instruction_merge_err_ret_traces(IrAnalyze *ira, + IrInstructionMergeErrRetTraces *instruction) +{ + IrInstruction *coro_promise_ptr = instruction->coro_promise_ptr->other; + if (type_is_invalid(coro_promise_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; + + assert(coro_promise_ptr->value.type->id == TypeTableEntryIdPointer); + TypeTableEntry *promise_frame_type = coro_promise_ptr->value.type->data.pointer.child_type; + assert(promise_frame_type->id == TypeTableEntryIdStruct); + TypeTableEntry *promise_result_type = promise_frame_type->data.structure.fields[1].type_entry; + + if (!type_can_fail(promise_result_type)) { + ConstExprValue *out_val = ir_build_const_from(ira, &instruction->base); + out_val->type = ira->codegen->builtin_types.entry_void; + return out_val->type; + } + + TypeStructField *field = find_struct_type_field(promise_frame_type, buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME)); + assert(field != nullptr); + + IrInstruction *result = ir_build_merge_err_ret_traces(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, coro_promise_ptr, field); + ir_link_new_instruction(result, &instruction->base); + result->value.type = ira->codegen->builtin_types.entry_void; + return result->value.type; +} + static TypeTableEntry *ir_analyze_instruction_save_err_ret_addr(IrAnalyze *ira, IrInstructionSaveErrRetAddr *instruction) { IrInstruction *result = ir_build_save_err_ret_addr(&ira->new_irb, instruction->base.scope, instruction->base.source_node); @@ -18155,6 +18200,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_save_err_ret_addr(ira, (IrInstructionSaveErrRetAddr *)instruction); case IrInstructionIdAddImplicitReturnType: return ir_analyze_instruction_add_implicit_return_type(ira, (IrInstructionAddImplicitReturnType *)instruction); + case IrInstructionIdMergeErrRetTraces: + return ir_analyze_instruction_merge_err_ret_traces(ira, (IrInstructionMergeErrRetTraces *)instruction); } zig_unreachable(); } @@ -18282,6 +18329,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdAwaitBookkeeping: case IrInstructionIdSaveErrRetAddr: case IrInstructionIdAddImplicitReturnType: + case IrInstructionIdMergeErrRetTraces: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index b14d49a4c..432d287bb 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1024,7 +1024,16 @@ static void ir_print_export(IrPrint *irp, IrInstructionExport *instruction) { } static void ir_print_error_return_trace(IrPrint *irp, IrInstructionErrorReturnTrace *instruction) { - fprintf(irp->f, "@errorReturnTrace()"); + fprintf(irp->f, "@errorReturnTrace("); + switch (instruction->nullable) { + case IrInstructionErrorReturnTrace::Null: + fprintf(irp->f, "Null"); + break; + case IrInstructionErrorReturnTrace::NonNull: + fprintf(irp->f, "NonNull"); + break; + } + fprintf(irp->f, ")"); } static void ir_print_error_union(IrPrint *irp, IrInstructionErrorUnion *instruction) { @@ -1179,6 +1188,16 @@ static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImpl fprintf(irp->f, ")"); } +static void ir_print_merge_err_ret_traces(IrPrint *irp, IrInstructionMergeErrRetTraces *instruction) { + fprintf(irp->f, "@mergeErrRetTraces("); + ir_print_other_instruction(irp, instruction->coro_promise_ptr); + fprintf(irp->f, ","); + if (instruction->resolved_field != nullptr) { + fprintf(irp->f, "field '%s'", buf_ptr(instruction->resolved_field->name)); + } + fprintf(irp->f, ")"); +} + static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { ir_print_prefix(irp, instruction); switch (instruction->id) { @@ -1559,6 +1578,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdAddImplicitReturnType: ir_print_add_implicit_return_type(irp, (IrInstructionAddImplicitReturnType *)instruction); break; + case IrInstructionIdMergeErrRetTraces: + ir_print_merge_err_ret_traces(irp, (IrInstructionMergeErrRetTraces *)instruction); + break; } fprintf(irp->f, "\n"); } From e4083b7391fd829e3060d24e10d13c8d52d889b0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 29 Mar 2018 01:24:07 -0400 Subject: [PATCH 092/132] codegen: fix not putting llvm allocas together --- src/codegen.cpp | 53 ++++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index fa8e069e2..88331e302 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5502,38 +5502,19 @@ static void do_code_gen(CodeGen *g) { g->cur_err_ret_trace_val_arg = nullptr; } + // error return tracing setup bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && (is_async || !have_err_ret_trace_arg); bool have_exactly_one_err_ret_value = !have_err_ret_trace_stack && g->have_err_ret_tracing && is_async && type_can_fail(fn_table_entry->type_entry->data.fn.fn_type_id.return_type); + LLVMValueRef err_ret_array_val = nullptr; if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { - TypeTableEntry *usize = g->builtin_types.entry_usize; uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; - TypeTableEntry *array_type = get_array_type(g, usize, ret_addr_count); - LLVMValueRef err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", + TypeTableEntry *array_type = get_array_type(g, g->builtin_types.entry_usize, ret_addr_count); + err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); - size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; - LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); - gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false); - - size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; - LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, ""); - - TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; - size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; - LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); - LLVMValueRef zero = LLVMConstNull(usize->type_ref); - LLVMValueRef indices[] = {zero, zero}; - LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val, - indices, 2, ""); - gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr, - get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false)); - - size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; - LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } else { g->cur_err_ret_trace_val_stack = nullptr; } @@ -5628,6 +5609,32 @@ static void do_code_gen(CodeGen *g) { } } + // finishing error return trace setup. we have to do this after all the allocas. + if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { + uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; + TypeTableEntry *usize = g->builtin_types.entry_usize; + size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; + LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); + gen_store_untyped(g, LLVMConstNull(usize->type_ref), index_field_ptr, 0, false); + + size_t addresses_field_index = g->stack_trace_type->data.structure.fields[1].gen_index; + LLVMValueRef addresses_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)addresses_field_index, ""); + + TypeTableEntry *slice_type = g->stack_trace_type->data.structure.fields[1].type_entry; + size_t ptr_field_index = slice_type->data.structure.fields[slice_ptr_index].gen_index; + LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)ptr_field_index, ""); + LLVMValueRef zero = LLVMConstNull(usize->type_ref); + LLVMValueRef indices[] = {zero, zero}; + LLVMValueRef err_ret_array_val_elem0_ptr = LLVMBuildInBoundsGEP(g->builder, err_ret_array_val, + indices, 2, ""); + TypeTableEntry *ptr_ptr_usize_type = get_pointer_to_type(g, get_pointer_to_type(g, usize, false), false); + gen_store(g, err_ret_array_val_elem0_ptr, ptr_field_ptr, ptr_ptr_usize_type); + + size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; + LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); + gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); + } + FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id; // create debug variable declarations for parameters From ada441157f4a388950946e7f4db65c273f23c063 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 16:04:21 -0400 Subject: [PATCH 093/132] put the error return addresses in the coro frame --- src/all_types.hpp | 12 +++++- src/analyze.cpp | 7 +++- src/codegen.cpp | 33 +++++++-------- src/ir.cpp | 101 +++++++++++++++++++++++++++++----------------- src/ir_print.cpp | 13 ++++-- 5 files changed, 105 insertions(+), 61 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 6893f60fb..52b8ede82 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2057,6 +2057,7 @@ enum IrInstructionId { IrInstructionIdSaveErrRetAddr, IrInstructionIdAddImplicitReturnType, IrInstructionIdMergeErrRetTraces, + IrInstructionIdMarkErrRetTracePtr, }; struct IrInstruction { @@ -3036,7 +3037,13 @@ struct IrInstructionMergeErrRetTraces { IrInstruction base; IrInstruction *coro_promise_ptr; - TypeStructField *resolved_field; + IrInstruction *err_ret_trace_ptr; +}; + +struct IrInstructionMarkErrRetTracePtr { + IrInstruction base; + + IrInstruction *err_ret_trace_ptr; }; static const size_t slice_ptr_index = 0; @@ -3056,7 +3063,8 @@ static const size_t stack_trace_ptr_count = 30; #define AWAITER_HANDLE_FIELD_NAME "awaiter_handle" #define RESULT_FIELD_NAME "result" #define RESULT_PTR_FIELD_NAME "result_ptr" -#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr" +#define RETURN_ADDRESSES_FIELD_NAME "return_addresses" +#define ERR_RET_TRACE_FIELD_NAME "err_ret_trace" enum FloatMode { diff --git a/src/analyze.cpp b/src/analyze.cpp index 16788c5e6..ae2a1a1b1 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -474,7 +474,8 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_names.append(RESULT_FIELD_NAME); field_names.append(RESULT_PTR_FIELD_NAME); if (g->have_err_ret_tracing) { - field_names.append(ERR_RET_TRACE_PTR_FIELD_NAME); + field_names.append(ERR_RET_TRACE_FIELD_NAME); + field_names.append(RETURN_ADDRESSES_FIELD_NAME); } ZigList field_types = {}; @@ -482,7 +483,9 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_types.append(return_type); field_types.append(result_ptr_type); if (g->have_err_ret_tracing) { - field_types.append(get_ptr_to_stack_trace_type(g)); + get_ptr_to_stack_trace_type(g); + field_types.append(g->stack_trace_type); + field_types.append(get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count)); } assert(field_names.length == field_types.length); diff --git a/src/codegen.cpp b/src/codegen.cpp index 88331e302..914e9c709 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4383,10 +4383,7 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe { assert(g->have_err_ret_tracing); - LLVMValueRef coro_promise_ptr = ir_llvm_value(g, instruction->coro_promise_ptr); - TypeStructField *field = instruction->resolved_field; - LLVMValueRef ptr_field_ptr = LLVMBuildStructGEP(g->builder, coro_promise_ptr, field->gen_index, ""); - LLVMValueRef src_trace_ptr = LLVMBuildLoad(g->builder, ptr_field_ptr, ""); + LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->err_ret_trace_ptr); LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; @@ -4394,6 +4391,14 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe return nullptr; } +static LLVMValueRef ir_render_mark_err_ret_trace_ptr(CodeGen *g, IrExecutable *executable, + IrInstructionMarkErrRetTracePtr *instruction) +{ + assert(g->have_err_ret_tracing); + g->cur_err_ret_trace_val_stack = ir_llvm_value(g, instruction->err_ret_trace_ptr); + return nullptr; +} + static void set_debug_location(CodeGen *g, IrInstruction *instruction) { AstNode *source_node = instruction->source_node; Scope *scope = instruction->scope; @@ -4613,6 +4618,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutable *executable, return ir_render_save_err_ret_addr(g, executable, (IrInstructionSaveErrRetAddr *)instruction); case IrInstructionIdMergeErrRetTraces: return ir_render_merge_err_ret_traces(g, executable, (IrInstructionMergeErrRetTraces *)instruction); + case IrInstructionIdMarkErrRetTracePtr: + return ir_render_mark_err_ret_trace_ptr(g, executable, (IrInstructionMarkErrRetTracePtr *)instruction); } zig_unreachable(); } @@ -5504,16 +5511,11 @@ static void do_code_gen(CodeGen *g) { // error return tracing setup bool is_async = fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync; - bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && - (is_async || !have_err_ret_trace_arg); - bool have_exactly_one_err_ret_value = !have_err_ret_trace_stack && g->have_err_ret_tracing && is_async && - type_can_fail(fn_table_entry->type_entry->data.fn.fn_type_id.return_type); + bool have_err_ret_trace_stack = g->have_err_ret_tracing && fn_table_entry->calls_or_awaits_errorable_fn && !is_async && !have_err_ret_trace_arg; LLVMValueRef err_ret_array_val = nullptr; - if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { - uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; - TypeTableEntry *array_type = get_array_type(g, g->builtin_types.entry_usize, ret_addr_count); - err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", - get_abi_alignment(g, array_type)); + if (have_err_ret_trace_stack) { + TypeTableEntry *array_type = get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count); + err_ret_array_val = build_alloca(g, array_type, "error_return_trace_addresses", get_abi_alignment(g, array_type)); g->cur_err_ret_trace_val_stack = build_alloca(g, g->stack_trace_type, "error_return_trace", get_abi_alignment(g, g->stack_trace_type)); } else { g->cur_err_ret_trace_val_stack = nullptr; @@ -5610,8 +5612,7 @@ static void do_code_gen(CodeGen *g) { } // finishing error return trace setup. we have to do this after all the allocas. - if (have_err_ret_trace_stack || have_exactly_one_err_ret_value) { - uint32_t ret_addr_count = have_exactly_one_err_ret_value ? 1 : stack_trace_ptr_count; + if (have_err_ret_trace_stack) { TypeTableEntry *usize = g->builtin_types.entry_usize; size_t index_field_index = g->stack_trace_type->data.structure.fields[0].gen_index; LLVMValueRef index_field_ptr = LLVMBuildStructGEP(g->builder, g->cur_err_ret_trace_val_stack, (unsigned)index_field_index, ""); @@ -5632,7 +5633,7 @@ static void do_code_gen(CodeGen *g) { size_t len_field_index = slice_type->data.structure.fields[slice_len_index].gen_index; LLVMValueRef len_field_ptr = LLVMBuildStructGEP(g->builder, addresses_field_ptr, (unsigned)len_field_index, ""); - gen_store(g, LLVMConstInt(usize->type_ref, ret_addr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); + gen_store(g, LLVMConstInt(usize->type_ref, stack_trace_ptr_count, false), len_field_ptr, get_pointer_to_type(g, usize, false)); } FnTypeId *fn_type_id = &fn_table_entry->type_entry->data.fn.fn_type_id; diff --git a/src/ir.cpp b/src/ir.cpp index c771adca4..3f8744ecb 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -729,6 +729,10 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionMergeErrRetTrace return IrInstructionIdMergeErrRetTraces; } +static constexpr IrInstructionId ir_instruction_id(IrInstructionMarkErrRetTracePtr *) { + return IrInstructionIdMarkErrRetTracePtr; +} + template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { T *special_instruction = allocate(1); @@ -960,31 +964,6 @@ static IrInstruction *ir_build_const_c_str_lit(IrBuilder *irb, Scope *scope, Ast return &const_instruction->base; } -static IrInstruction *ir_build_const_promise_init(IrBuilder *irb, Scope *scope, AstNode *source_node, - TypeTableEntry *return_type) -{ - TypeTableEntry *struct_type = get_promise_frame_type(irb->codegen, return_type); - - IrInstructionConst *const_instruction = ir_build_instruction(irb, scope, source_node); - const_instruction->base.value.type = struct_type; - const_instruction->base.value.special = ConstValSpecialStatic; - const_instruction->base.value.data.x_struct.fields = allocate(struct_type->data.structure.src_field_count); - const_instruction->base.value.data.x_struct.fields[0].type = struct_type->data.structure.fields[0].type_entry; - const_instruction->base.value.data.x_struct.fields[0].special = ConstValSpecialStatic; - const_instruction->base.value.data.x_struct.fields[0].data.x_maybe = nullptr; - const_instruction->base.value.data.x_struct.fields[1].type = return_type; - const_instruction->base.value.data.x_struct.fields[1].special = ConstValSpecialUndef; - const_instruction->base.value.data.x_struct.fields[2].type = struct_type->data.structure.fields[2].type_entry; - const_instruction->base.value.data.x_struct.fields[2].special = ConstValSpecialUndef; - if (irb->codegen->have_err_ret_tracing) { - assert(struct_type->data.structure.src_field_count == 4); - - const_instruction->base.value.data.x_struct.fields[3].type = struct_type->data.structure.fields[3].type_entry; - const_instruction->base.value.data.x_struct.fields[3].special = ConstValSpecialUndef; - } - return &const_instruction->base; -} - static IrInstruction *ir_build_bin_op(IrBuilder *irb, Scope *scope, AstNode *source_node, IrBinOp op_id, IrInstruction *op1, IrInstruction *op2, bool safety_check_on) { @@ -2729,13 +2708,23 @@ static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *s } static IrInstruction *ir_build_merge_err_ret_traces(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction *coro_promise_ptr, TypeStructField *resolved_field) + IrInstruction *coro_promise_ptr, IrInstruction *err_ret_trace_ptr) { IrInstructionMergeErrRetTraces *instruction = ir_build_instruction(irb, scope, source_node); instruction->coro_promise_ptr = coro_promise_ptr; - instruction->resolved_field = resolved_field; + instruction->err_ret_trace_ptr = err_ret_trace_ptr; ir_ref_instruction(coro_promise_ptr, irb->current_basic_block); + ir_ref_instruction(err_ret_trace_ptr, irb->current_basic_block); + + return &instruction->base; +} + +static IrInstruction *ir_build_mark_err_ret_trace_ptr(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *err_ret_trace_ptr) { + IrInstructionMarkErrRetTracePtr *instruction = ir_build_instruction(irb, scope, source_node); + instruction->err_ret_trace_ptr = err_ret_trace_ptr; + + ir_ref_instruction(err_ret_trace_ptr, irb->current_basic_block); return &instruction->base; } @@ -6154,7 +6143,9 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); if (irb->codegen->have_err_ret_tracing) { - ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, nullptr); + Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); + IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_field_name); + ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_ptr); } Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_field_name); @@ -6421,8 +6412,11 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec VariableTableEntry *promise_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); return_type = fn_entry->type_entry->data.fn.fn_type_id.return_type; - IrInstruction *promise_init = ir_build_const_promise_init(irb, coro_scope, node, return_type); - ir_build_var_decl(irb, coro_scope, node, promise_var, nullptr, nullptr, promise_init); + IrInstruction *undef = ir_build_const_undefined(irb, coro_scope, node); + TypeTableEntry *coro_frame_type = get_promise_frame_type(irb->codegen, return_type); + IrInstruction *coro_frame_type_value = ir_build_const_type(irb, coro_scope, node, coro_frame_type); + // TODO mark this var decl as "no safety" e.g. disable initializing the undef value to 0xaa + ir_build_var_decl(irb, coro_scope, node, promise_var, coro_frame_type_value, nullptr, undef); IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); VariableTableEntry *await_handle_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); @@ -6456,7 +6450,6 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec ir_set_cursor_at_end_and_append_block(irb, alloc_err_block); // we can return undefined here, because the caller passes a pointer to the error struct field // in the error union result, and we populate it in case of allocation failure. - IrInstruction *undef = ir_build_const_undefined(irb, coro_scope, node); ir_build_return(irb, coro_scope, node, undef); ir_set_cursor_at_end_and_append_block(irb, alloc_ok_block); @@ -6466,16 +6459,32 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); irb->exec->coro_awaiter_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, awaiter_handle_field_name); + ir_build_store_ptr(irb, scope, node, irb->exec->coro_awaiter_field_ptr, null_value); Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); irb->exec->coro_result_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_field_name); result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); irb->exec->coro_result_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, result_ptr_field_name); ir_build_store_ptr(irb, scope, node, irb->exec->coro_result_ptr_field_ptr, irb->exec->coro_result_field_ptr); if (irb->codegen->have_err_ret_tracing) { - IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, scope, node, IrInstructionErrorReturnTrace::NonNull); - Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); - IrInstruction *coro_err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); - ir_build_store_ptr(irb, scope, node, coro_err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); + // initialize the error return trace + Buf *return_addresses_field_name = buf_create_from_str(RETURN_ADDRESSES_FIELD_NAME); + IrInstruction *return_addresses_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, return_addresses_field_name); + + Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); + IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name); + ir_build_mark_err_ret_trace_ptr(irb, scope, node, err_ret_trace_ptr); + + // coordinate with builtin.zig + Buf *index_name = buf_create_from_str("index"); + IrInstruction *index_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, index_name); + IrInstruction *zero = ir_build_const_usize(irb, scope, node, 0); + ir_build_store_ptr(irb, scope, node, index_ptr, zero); + + Buf *instruction_addresses_name = buf_create_from_str("instruction_addresses"); + IrInstruction *addrs_slice_ptr = ir_build_field_ptr(irb, scope, node, err_ret_trace_ptr, instruction_addresses_name); + + IrInstruction *slice_value = ir_build_slice(irb, scope, node, return_addresses_ptr, zero, nullptr, false); + ir_build_store_ptr(irb, scope, node, addrs_slice_ptr, slice_value); } @@ -17939,11 +17948,12 @@ static TypeTableEntry *ir_analyze_instruction_merge_err_ret_traces(IrAnalyze *ir return out_val->type; } - TypeStructField *field = find_struct_type_field(promise_frame_type, buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME)); - assert(field != nullptr); + IrInstruction *err_ret_trace_ptr = instruction->err_ret_trace_ptr->other; + if (type_is_invalid(err_ret_trace_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; IrInstruction *result = ir_build_merge_err_ret_traces(&ira->new_irb, instruction->base.scope, - instruction->base.source_node, coro_promise_ptr, field); + instruction->base.source_node, coro_promise_ptr, err_ret_trace_ptr); ir_link_new_instruction(result, &instruction->base); result->value.type = ira->codegen->builtin_types.entry_void; return result->value.type; @@ -17957,6 +17967,18 @@ static TypeTableEntry *ir_analyze_instruction_save_err_ret_addr(IrAnalyze *ira, return result->value.type; } +static TypeTableEntry *ir_analyze_instruction_mark_err_ret_trace_ptr(IrAnalyze *ira, IrInstructionMarkErrRetTracePtr *instruction) { + IrInstruction *err_ret_trace_ptr = instruction->err_ret_trace_ptr->other; + if (type_is_invalid(err_ret_trace_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; + + IrInstruction *result = ir_build_mark_err_ret_trace_ptr(&ira->new_irb, instruction->base.scope, + instruction->base.source_node, err_ret_trace_ptr); + ir_link_new_instruction(result, &instruction->base); + result->value.type = ira->codegen->builtin_types.entry_void; + return result->value.type; +} + static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstruction *instruction) { switch (instruction->id) { case IrInstructionIdInvalid: @@ -18202,6 +18224,8 @@ static TypeTableEntry *ir_analyze_instruction_nocast(IrAnalyze *ira, IrInstructi return ir_analyze_instruction_add_implicit_return_type(ira, (IrInstructionAddImplicitReturnType *)instruction); case IrInstructionIdMergeErrRetTraces: return ir_analyze_instruction_merge_err_ret_traces(ira, (IrInstructionMergeErrRetTraces *)instruction); + case IrInstructionIdMarkErrRetTracePtr: + return ir_analyze_instruction_mark_err_ret_trace_ptr(ira, (IrInstructionMarkErrRetTracePtr *)instruction); } zig_unreachable(); } @@ -18330,6 +18354,7 @@ bool ir_has_side_effects(IrInstruction *instruction) { case IrInstructionIdSaveErrRetAddr: case IrInstructionIdAddImplicitReturnType: case IrInstructionIdMergeErrRetTraces: + case IrInstructionIdMarkErrRetTracePtr: return true; case IrInstructionIdPhi: diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 432d287bb..20dfb10b8 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1192,9 +1192,13 @@ static void ir_print_merge_err_ret_traces(IrPrint *irp, IrInstructionMergeErrRet fprintf(irp->f, "@mergeErrRetTraces("); ir_print_other_instruction(irp, instruction->coro_promise_ptr); fprintf(irp->f, ","); - if (instruction->resolved_field != nullptr) { - fprintf(irp->f, "field '%s'", buf_ptr(instruction->resolved_field->name)); - } + ir_print_other_instruction(irp, instruction->err_ret_trace_ptr); + fprintf(irp->f, ")"); +} + +static void ir_print_mark_err_ret_trace_ptr(IrPrint *irp, IrInstructionMarkErrRetTracePtr *instruction) { + fprintf(irp->f, "@markErrRetTracePtr("); + ir_print_other_instruction(irp, instruction->err_ret_trace_ptr); fprintf(irp->f, ")"); } @@ -1581,6 +1585,9 @@ static void ir_print_instruction(IrPrint *irp, IrInstruction *instruction) { case IrInstructionIdMergeErrRetTraces: ir_print_merge_err_ret_traces(irp, (IrInstructionMergeErrRetTraces *)instruction); break; + case IrInstructionIdMarkErrRetTracePtr: + ir_print_mark_err_ret_trace_ptr(irp, (IrInstructionMarkErrRetTracePtr *)instruction); + break; } fprintf(irp->f, "\n"); } From 9e98ea552dcf03a4a05a920c8f027d09130dd688 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 16:40:59 -0400 Subject: [PATCH 094/132] fix calling convention at callsite of zig-generated fns --- src/codegen.cpp | 6 +++--- test/cases/coroutines.zig | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 914e9c709..34eda6dd9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1289,7 +1289,7 @@ static LLVMValueRef get_merge_err_ret_traces_fn_val(CodeGen *g) { LLVMValueRef addr_ptr = LLVMBuildInBoundsGEP(g->builder, src_ptr_val, &ptr_index, 1, ""); LLVMValueRef this_addr_val = LLVMBuildLoad(g->builder, addr_ptr, ""); LLVMValueRef args[] = {dest_stack_trace_ptr, this_addr_val}; - LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); + ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, ""); LLVMValueRef prev_frames_left = LLVMBuildLoad(g->builder, frames_left_ptr, ""); LLVMValueRef new_frames_left = LLVMBuildNUWSub(g->builder, prev_frames_left, usize_one, ""); LLVMValueRef done_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, new_frames_left, usize_zero, ""); @@ -1358,7 +1358,7 @@ static LLVMValueRef get_return_err_fn(CodeGen *g) { LLVMValueRef return_address = LLVMBuildPtrToInt(g->builder, return_address_ptr, usize_type_ref, ""); LLVMValueRef args[] = { err_ret_trace_ptr, return_address }; - LLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, ""); + ZigLLVMBuildCall(g->builder, add_error_return_trace_addr_fn_val, args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAlways, ""); LLVMBuildRetVoid(g->builder); LLVMPositionBuilderAtEnd(g->builder, prev_block); @@ -4387,7 +4387,7 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; - LLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, ""); + ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); return nullptr; } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 922c1a7e5..553732373 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const builtin = @import("builtin"); const assert = std.debug.assert; var x: i32 = 1; @@ -189,3 +190,30 @@ async fn failing() !void { suspend; return error.Fail; } + +test "error return trace across suspend points" { + const p = nonFailing(); + resume p; + const p2 = try async printTrace(p); + cancel p2; +} + +fn nonFailing() promise->error!void { + return async suspendThenFail() catch unreachable; +} + +async fn suspendThenFail() error!void { + suspend; + return error.Fail; +} + +async fn printTrace(p: promise->error!void) void { + (await p) catch |e| { + std.debug.assert(e == error.Fail); + if (@errorReturnTrace()) |trace| { + assert(trace.index == 1); + } else if (builtin.mode != builtin.Mode.ReleaseFast) { + @panic("expected return trace"); + } + }; +} From ee1a4f4c1d888d1485d8bb13ee0fa756bf729b08 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 17:44:29 -0400 Subject: [PATCH 095/132] error return traces work with async return case --- src/all_types.hpp | 12 +++++++---- src/analyze.cpp | 3 ++- src/codegen.cpp | 4 ++-- src/ir.cpp | 43 ++++++++++++++++++++++++++++++--------- src/ir_print.cpp | 4 +++- test/cases/coroutines.zig | 9 +++++++- 6 files changed, 56 insertions(+), 19 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index 52b8ede82..25d4f70e2 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -3037,7 +3037,8 @@ struct IrInstructionMergeErrRetTraces { IrInstruction base; IrInstruction *coro_promise_ptr; - IrInstruction *err_ret_trace_ptr; + IrInstruction *src_err_ret_trace_ptr; + IrInstruction *dest_err_ret_trace_ptr; }; struct IrInstructionMarkErrRetTracePtr { @@ -3058,13 +3059,16 @@ static const size_t err_union_payload_index = 1; // TODO call graph analysis to find out what this number needs to be for every function static const size_t stack_trace_ptr_count = 30; +// these belong to the async function +#define RETURN_ADDRESSES_FIELD_NAME "return_addresses" +#define ERR_RET_TRACE_FIELD_NAME "err_ret_trace" +#define RESULT_FIELD_NAME "result" #define ASYNC_ALLOC_FIELD_NAME "allocFn" #define ASYNC_FREE_FIELD_NAME "freeFn" #define AWAITER_HANDLE_FIELD_NAME "awaiter_handle" -#define RESULT_FIELD_NAME "result" +// these point to data belonging to the awaiter +#define ERR_RET_TRACE_PTR_FIELD_NAME "err_ret_trace_ptr" #define RESULT_PTR_FIELD_NAME "result_ptr" -#define RETURN_ADDRESSES_FIELD_NAME "return_addresses" -#define ERR_RET_TRACE_FIELD_NAME "err_ret_trace" enum FloatMode { diff --git a/src/analyze.cpp b/src/analyze.cpp index ae2a1a1b1..3db49a11c 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -474,6 +474,7 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_names.append(RESULT_FIELD_NAME); field_names.append(RESULT_PTR_FIELD_NAME); if (g->have_err_ret_tracing) { + field_names.append(ERR_RET_TRACE_PTR_FIELD_NAME); field_names.append(ERR_RET_TRACE_FIELD_NAME); field_names.append(RETURN_ADDRESSES_FIELD_NAME); } @@ -483,7 +484,7 @@ TypeTableEntry *get_promise_frame_type(CodeGen *g, TypeTableEntry *return_type) field_types.append(return_type); field_types.append(result_ptr_type); if (g->have_err_ret_tracing) { - get_ptr_to_stack_trace_type(g); + field_types.append(get_ptr_to_stack_trace_type(g)); field_types.append(g->stack_trace_type); field_types.append(get_array_type(g, g->builtin_types.entry_usize, stack_trace_ptr_count)); } diff --git a/src/codegen.cpp b/src/codegen.cpp index 34eda6dd9..be83f6834 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4383,8 +4383,8 @@ static LLVMValueRef ir_render_merge_err_ret_traces(CodeGen *g, IrExecutable *exe { assert(g->have_err_ret_tracing); - LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->err_ret_trace_ptr); - LLVMValueRef dest_trace_ptr = get_cur_err_ret_trace_val(g, instruction->base.scope); + LLVMValueRef src_trace_ptr = ir_llvm_value(g, instruction->src_err_ret_trace_ptr); + LLVMValueRef dest_trace_ptr = ir_llvm_value(g, instruction->dest_err_ret_trace_ptr); LLVMValueRef args[] = { dest_trace_ptr, src_trace_ptr }; ZigLLVMBuildCall(g->builder, get_merge_err_ret_traces_fn_val(g), args, 2, get_llvm_cc(g, CallingConventionUnspecified), ZigLLVM_FnInlineAuto, ""); diff --git a/src/ir.cpp b/src/ir.cpp index 3f8744ecb..4ab8b130c 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2708,14 +2708,16 @@ static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *s } static IrInstruction *ir_build_merge_err_ret_traces(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction *coro_promise_ptr, IrInstruction *err_ret_trace_ptr) + IrInstruction *coro_promise_ptr, IrInstruction *src_err_ret_trace_ptr, IrInstruction *dest_err_ret_trace_ptr) { IrInstructionMergeErrRetTraces *instruction = ir_build_instruction(irb, scope, source_node); instruction->coro_promise_ptr = coro_promise_ptr; - instruction->err_ret_trace_ptr = err_ret_trace_ptr; + instruction->src_err_ret_trace_ptr = src_err_ret_trace_ptr; + instruction->dest_err_ret_trace_ptr = dest_err_ret_trace_ptr; ir_ref_instruction(coro_promise_ptr, irb->current_basic_block); - ir_ref_instruction(err_ret_trace_ptr, irb->current_basic_block); + ir_ref_instruction(src_err_ret_trace_ptr, irb->current_basic_block); + ir_ref_instruction(dest_err_ret_trace_ptr, irb->current_basic_block); return &instruction->base; } @@ -6115,6 +6117,13 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast Buf *result_ptr_field_name = buf_create_from_str(RESULT_PTR_FIELD_NAME); IrInstruction *result_ptr_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_ptr_field_name); + if (irb->codegen->have_err_ret_tracing) { + IrInstruction *err_ret_trace_ptr = ir_build_error_return_trace(irb, parent_scope, node, IrInstructionErrorReturnTrace::NonNull); + Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); + IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); + ir_build_store_ptr(irb, parent_scope, node, err_ret_trace_ptr_field_ptr, err_ret_trace_ptr); + } + Buf *awaiter_handle_field_name = buf_create_from_str(AWAITER_HANDLE_FIELD_NAME); IrInstruction *awaiter_field_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, awaiter_handle_field_name); @@ -6144,8 +6153,9 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast ir_set_cursor_at_end_and_append_block(irb, no_suspend_block); if (irb->codegen->have_err_ret_tracing) { Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); - IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_field_name); - ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_ptr); + IrInstruction *src_err_ret_trace_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, err_ret_trace_field_name); + IrInstruction *dest_err_ret_trace_ptr = ir_build_error_return_trace(irb, parent_scope, node, IrInstructionErrorReturnTrace::NonNull); + ir_build_merge_err_ret_traces(irb, parent_scope, node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr); } Buf *result_field_name = buf_create_from_str(RESULT_FIELD_NAME); IrInstruction *promise_result_ptr = ir_build_field_ptr(irb, parent_scope, node, coro_promise_ptr, result_field_name); @@ -6402,6 +6412,8 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *coro_id; IrInstruction *u8_ptr_type; IrInstruction *const_bool_false; + IrInstruction *coro_promise_ptr; + IrInstruction *err_ret_trace_ptr; TypeTableEntry *return_type; Buf *result_ptr_field_name; VariableTableEntry *coro_size_var; @@ -6417,7 +6429,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *coro_frame_type_value = ir_build_const_type(irb, coro_scope, node, coro_frame_type); // TODO mark this var decl as "no safety" e.g. disable initializing the undef value to 0xaa ir_build_var_decl(irb, coro_scope, node, promise_var, coro_frame_type_value, nullptr, undef); - IrInstruction *coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); + coro_promise_ptr = ir_build_var_ptr(irb, coro_scope, node, promise_var, false, false); VariableTableEntry *await_handle_var = ir_create_var(irb, node, coro_scope, nullptr, false, false, true, const_bool_false); IrInstruction *null_value = ir_build_const_null(irb, coro_scope, node); @@ -6471,7 +6483,7 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *return_addresses_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, return_addresses_field_name); Buf *err_ret_trace_field_name = buf_create_from_str(ERR_RET_TRACE_FIELD_NAME); - IrInstruction *err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name); + err_ret_trace_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_field_name); ir_build_mark_err_ret_trace_ptr(irb, scope, node, err_ret_trace_ptr); // coordinate with builtin.zig @@ -6536,6 +6548,12 @@ bool ir_gen(CodeGen *codegen, AstNode *node, Scope *scope, IrExecutable *ir_exec IrInstruction *size_of_ret_val = ir_build_size_of(irb, scope, node, return_type_inst); ir_build_memcpy(irb, scope, node, result_ptr_as_u8_ptr, return_value_ptr_as_u8_ptr, size_of_ret_val); } + if (irb->codegen->have_err_ret_tracing) { + Buf *err_ret_trace_ptr_field_name = buf_create_from_str(ERR_RET_TRACE_PTR_FIELD_NAME); + IrInstruction *err_ret_trace_ptr_field_ptr = ir_build_field_ptr(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr_field_name); + IrInstruction *dest_err_ret_trace_ptr = ir_build_load_ptr(irb, scope, node, err_ret_trace_ptr_field_ptr); + ir_build_merge_err_ret_traces(irb, scope, node, coro_promise_ptr, err_ret_trace_ptr, dest_err_ret_trace_ptr); + } ir_build_br(irb, scope, node, check_free_block, const_bool_false); ir_set_cursor_at_end_and_append_block(irb, irb->exec->coro_final_cleanup_block); @@ -13098,6 +13116,7 @@ static IrInstruction *ir_analyze_container_member_access_inner(IrAnalyze *ira, { if (!is_slice(bare_struct_type)) { ScopeDecls *container_scope = get_container_scope(bare_struct_type); + assert(container_scope != nullptr); auto entry = container_scope->decl_table.maybe_get(field_name); Tld *tld = entry ? entry->value : nullptr; if (tld && tld->id == TldIdFn) { @@ -17948,12 +17967,16 @@ static TypeTableEntry *ir_analyze_instruction_merge_err_ret_traces(IrAnalyze *ir return out_val->type; } - IrInstruction *err_ret_trace_ptr = instruction->err_ret_trace_ptr->other; - if (type_is_invalid(err_ret_trace_ptr->value.type)) + IrInstruction *src_err_ret_trace_ptr = instruction->src_err_ret_trace_ptr->other; + if (type_is_invalid(src_err_ret_trace_ptr->value.type)) + return ira->codegen->builtin_types.entry_invalid; + + IrInstruction *dest_err_ret_trace_ptr = instruction->dest_err_ret_trace_ptr->other; + if (type_is_invalid(dest_err_ret_trace_ptr->value.type)) return ira->codegen->builtin_types.entry_invalid; IrInstruction *result = ir_build_merge_err_ret_traces(&ira->new_irb, instruction->base.scope, - instruction->base.source_node, coro_promise_ptr, err_ret_trace_ptr); + instruction->base.source_node, coro_promise_ptr, src_err_ret_trace_ptr, dest_err_ret_trace_ptr); ir_link_new_instruction(result, &instruction->base); result->value.type = ira->codegen->builtin_types.entry_void; return result->value.type; diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 20dfb10b8..99f79ff75 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1192,7 +1192,9 @@ static void ir_print_merge_err_ret_traces(IrPrint *irp, IrInstructionMergeErrRet fprintf(irp->f, "@mergeErrRetTraces("); ir_print_other_instruction(irp, instruction->coro_promise_ptr); fprintf(irp->f, ","); - ir_print_other_instruction(irp, instruction->err_ret_trace_ptr); + ir_print_other_instruction(irp, instruction->src_err_ret_trace_ptr); + fprintf(irp->f, ","); + ir_print_other_instruction(irp, instruction->dest_err_ret_trace_ptr); fprintf(irp->f, ")"); } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 553732373..6d28b98c9 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -191,13 +191,20 @@ async fn failing() !void { return error.Fail; } -test "error return trace across suspend points" { +test "error return trace across suspend points - early return" { const p = nonFailing(); resume p; const p2 = try async printTrace(p); cancel p2; } +test "error return trace across suspend points - async return" { + const p = nonFailing(); + const p2 = try async printTrace(p); + resume p; + cancel p2; +} + fn nonFailing() promise->error!void { return async suspendThenFail() catch unreachable; } From eae355d77168d655273afa298dc921c40a36bd0b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Apr 2018 19:14:49 -0400 Subject: [PATCH 096/132] add docs for packed enum --- doc/langref.html.in | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 7f837186b..ac597b321 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -1947,8 +1947,24 @@ const Foo = extern enum { A, B, C }; export fn entry(foo: Foo) void { } {#code_end#} {#header_close#} -

TODO packed enum

- {#see_also|@memberName|@memberCount|@tagName#} + {#header_open|packed enum#} +

By default, the size of enums is not guaranteed.

+

packed enum causes the size of the enum to be the same as the size of the integer tag type + of the enum:

+ {#code_begin|test#} +const std = @import("std"); + +test "packed enum" { + const Number = packed enum(u8) { + One, + Two, + Three, + }; + std.debug.assert(@sizeOf(Number) == @sizeOf(u8)); +} + {#code_end#} + {#header_close#} + {#see_also|@memberName|@memberCount|@tagName|@sizeOf#} {#header_close#} {#header_open|union#} {#code_begin|test|union#} From 292d0cbdadd453874af6e2065638a88d4dda8a10 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 18:03:02 -0400 Subject: [PATCH 097/132] add docs for union methods --- doc/langref.html.in | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index ac597b321..856d62f14 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -2033,7 +2033,27 @@ test "union variant switch" { assert(mem.eql(u8, what_is_it, "this is a number")); } -// TODO union methods +// Unions can have methods just like structs and enums: + +const Variant = union(enum) { + Int: i32, + Bool: bool, + + fn truthy(self: &const Variant) bool { + return switch (*self) { + Variant.Int => |x_int| x_int != 0, + Variant.Bool => |x_bool| x_bool, + }; + } +}; + +test "union method" { + var v1 = Variant { .Int = 1 }; + var v2 = Variant { .Bool = false }; + + assert(v1.truthy()); + assert(!v2.truthy()); +} const Small = union { From 0d22a00f6fde79f851a7d19c2096c07f541ed0be Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 7 Mar 2018 03:55:52 -0500 Subject: [PATCH 098/132] *WIP* async/await TCP server --- CMakeLists.txt | 2 - src/all_types.hpp | 12 +- src/analyze.cpp | 1 - src/ast_render.cpp | 3 - src/ir.cpp | 21 +- src/parser.cpp | 4 +- std/endian.zig | 25 -- std/event.zig | 202 +++++++++++++++ std/fmt/index.zig | 2 +- std/index.zig | 4 +- std/linked_list.zig | 1 + std/mem.zig | 26 ++ std/net.zig | 280 +++++++++------------ std/os/index.zig | 373 ++++++++++++++++++++++++++-- std/os/linux/i386.zig | 505 -------------------------------------- std/os/linux/index.zig | 216 +++++++++++----- test/cases/coroutines.zig | 15 ++ 17 files changed, 888 insertions(+), 804 deletions(-) delete mode 100644 std/endian.zig create mode 100644 std/event.zig delete mode 100644 std/os/linux/i386.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bb9bf517..c6f169d63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -432,7 +432,6 @@ set(ZIG_STD_FILES "dwarf.zig" "elf.zig" "empty.zig" - "endian.zig" "fmt/errol/enum3.zig" "fmt/errol/index.zig" "fmt/errol/lookup.zig" @@ -503,7 +502,6 @@ set(ZIG_STD_FILES "os/get_user_id.zig" "os/index.zig" "os/linux/errno.zig" - "os/linux/i386.zig" "os/linux/index.zig" "os/linux/x86_64.zig" "os/path.zig" diff --git a/src/all_types.hpp b/src/all_types.hpp index 25d4f70e2..d434ea187 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -359,7 +359,6 @@ enum NodeType { NodeTypeRoot, NodeTypeFnProto, NodeTypeFnDef, - NodeTypeFnDecl, NodeTypeParamDecl, NodeTypeBlock, NodeTypeGroupedExpr, @@ -453,10 +452,6 @@ struct AstNodeFnDef { AstNode *body; }; -struct AstNodeFnDecl { - AstNode *fn_proto; -}; - struct AstNodeParamDecl { Buf *name; AstNode *type; @@ -713,10 +708,6 @@ struct AstNodeSwitchRange { AstNode *end; }; -struct AstNodeLabel { - Buf *name; -}; - struct AstNodeCompTime { AstNode *expr; }; @@ -892,7 +883,6 @@ struct AstNode { union { AstNodeRoot root; AstNodeFnDef fn_def; - AstNodeFnDecl fn_decl; AstNodeFnProto fn_proto; AstNodeParamDecl param_decl; AstNodeBlock block; @@ -917,7 +907,6 @@ struct AstNode { AstNodeSwitchExpr switch_expr; AstNodeSwitchProng switch_prong; AstNodeSwitchRange switch_range; - AstNodeLabel label; AstNodeCompTime comptime_expr; AstNodeAsmExpr asm_expr; AstNodeFieldAccessExpr field_access_expr; @@ -2702,6 +2691,7 @@ struct IrInstructionFnProto { IrInstruction **param_types; IrInstruction *align_value; + IrInstruction *async_allocator_type_value; IrInstruction *return_type; IrInstruction *async_allocator_type_value; bool is_var_args; diff --git a/src/analyze.cpp b/src/analyze.cpp index 3db49a11c..c73e6b39e 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3236,7 +3236,6 @@ void scan_decls(CodeGen *g, ScopeDecls *decls_scope, AstNode *node) { break; case NodeTypeContainerDecl: case NodeTypeParamDecl: - case NodeTypeFnDecl: case NodeTypeReturnExpr: case NodeTypeDefer: case NodeTypeBlock: diff --git a/src/ast_render.cpp b/src/ast_render.cpp index 7b5fc03ea..2c3e1fc87 100644 --- a/src/ast_render.cpp +++ b/src/ast_render.cpp @@ -148,8 +148,6 @@ static const char *node_type_str(NodeType node_type) { return "Root"; case NodeTypeFnDef: return "FnDef"; - case NodeTypeFnDecl: - return "FnDecl"; case NodeTypeFnProto: return "FnProto"; case NodeTypeParamDecl: @@ -1098,7 +1096,6 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) { } break; } - case NodeTypeFnDecl: case NodeTypeParamDecl: case NodeTypeTestDecl: case NodeTypeStructField: diff --git a/src/ir.cpp b/src/ir.cpp index 4ab8b130c..a80318357 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2153,12 +2153,12 @@ static IrInstruction *ir_build_unwrap_err_payload_from(IrBuilder *irb, IrInstruc } static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, - IrInstruction *async_allocator_type_value, bool is_var_args) + IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, IrInstruction *async_allocator_type_value, bool is_var_args) { IrInstructionFnProto *instruction = ir_build_instruction(irb, scope, source_node); instruction->param_types = param_types; instruction->align_value = align_value; + instruction->async_allocator_type_value = async_allocator_type_value; instruction->return_type = return_type; instruction->async_allocator_type_value = async_allocator_type_value; instruction->is_var_args = is_var_args; @@ -6041,6 +6041,13 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } + IrInstruction *async_allocator_type_value = nullptr; + if (node->data.fn_proto.async_allocator_type != nullptr) { + async_allocator_type_value = ir_gen_node(irb, node->data.fn_proto.async_allocator_type, parent_scope); + if (async_allocator_type_value == irb->codegen->invalid_instruction) + return irb->codegen->invalid_instruction; + } + IrInstruction *return_type; if (node->data.fn_proto.return_var_token == nullptr) { if (node->data.fn_proto.return_type == nullptr) { @@ -6061,8 +6068,7 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } - return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, - async_allocator_type_value, is_var_args); + return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, async_allocator_type_value, is_var_args); } static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *parent_scope, AstNode *node) { @@ -6273,7 +6279,6 @@ static IrInstruction *ir_gen_node_raw(IrBuilder *irb, AstNode *node, Scope *scop case NodeTypeSwitchRange: case NodeTypeStructField: case NodeTypeFnDef: - case NodeTypeFnDecl: case NodeTypeTestDecl: zig_unreachable(); case NodeTypeBlock: @@ -16741,6 +16746,12 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc return ira->codegen->builtin_types.entry_invalid; } + if (instruction->async_allocator_type_value != nullptr) { + fn_type_id.async_allocator_type = ir_resolve_type(ira, instruction->async_allocator_type_value->other); + if (type_is_invalid(fn_type_id.async_allocator_type)) + return ira->codegen->builtin_types.entry_invalid; + } + IrInstruction *return_type_value = instruction->return_type->other; fn_type_id.return_type = ir_resolve_type(ira, return_type_value); if (type_is_invalid(fn_type_id.return_type)) diff --git a/src/parser.cpp b/src/parser.cpp index d6faf4c98..b54a17362 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1037,6 +1037,7 @@ static AstNode *ast_parse_suffix_op_expr(ParseContext *pc, size_t *token_index, Token *async_token = &pc->tokens->at(*token_index); if (async_token->id == TokenIdKeywordAsync) { + size_t token_index_of_async = *token_index; *token_index += 1; AstNode *allocator_expr_node = nullptr; @@ -2923,9 +2924,6 @@ void ast_visit_node_children(AstNode *node, void (*visit)(AstNode **, void *cont visit_field(&node->data.fn_def.fn_proto, visit, context); visit_field(&node->data.fn_def.body, visit, context); break; - case NodeTypeFnDecl: - visit_field(&node->data.fn_decl.fn_proto, visit, context); - break; case NodeTypeParamDecl: visit_field(&node->data.param_decl.type, visit, context); break; diff --git a/std/endian.zig b/std/endian.zig deleted file mode 100644 index 121505d24..000000000 --- a/std/endian.zig +++ /dev/null @@ -1,25 +0,0 @@ -const mem = @import("mem.zig"); -const builtin = @import("builtin"); - -pub fn swapIfLe(comptime T: type, x: T) T { - return swapIf(builtin.Endian.Little, T, x); -} - -pub fn swapIfBe(comptime T: type, x: T) T { - return swapIf(builtin.Endian.Big, T, x); -} - -pub fn swapIf(endian: builtin.Endian, comptime T: type, x: T) T { - return if (builtin.endian == endian) swap(T, x) else x; -} - -pub fn swap(comptime T: type, x: T) T { - var buf: [@sizeOf(T)]u8 = undefined; - mem.writeInt(buf[0..], x, builtin.Endian.Little); - return mem.readInt(buf, T, builtin.Endian.Big); -} - -test "swap" { - const debug = @import("debug/index.zig"); - debug.assert(swap(u32, 0xDEADBEEF) == 0xEFBEADDE); -} diff --git a/std/event.zig b/std/event.zig new file mode 100644 index 000000000..07fc64293 --- /dev/null +++ b/std/event.zig @@ -0,0 +1,202 @@ +const std = @import("index.zig"); +const assert = std.debug.assert; +const event = this; +const mem = std.mem; +const posix = std.os.posix; + +pub const TcpServer = struct { + handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File) void, + + loop: &Loop, + sockfd: i32, + accept_coro: ?promise, + + waiting_for_emfile_node: PromiseNode, + + const PromiseNode = std.LinkedList(promise).Node; + + pub fn init(loop: &Loop) !TcpServer { + const sockfd = try std.os.posixSocket(posix.AF_INET, + posix.SOCK_STREAM|posix.SOCK_CLOEXEC|posix.SOCK_NONBLOCK, + posix.PROTO_tcp); + errdefer std.os.close(sockfd); + + // TODO can't initialize handler coroutine here because we need well defined copy elision + return TcpServer { + .loop = loop, + .sockfd = sockfd, + .accept_coro = null, + .handleRequestFn = undefined, + .waiting_for_emfile_node = undefined, + }; + } + + pub fn listen(self: &TcpServer, address: &const std.net.Address, + handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File)void) !void + { + self.handleRequestFn = handleRequestFn; + + try std.os.posixBind(self.sockfd, &address.sockaddr); + try std.os.posixListen(self.sockfd, posix.SOMAXCONN); + + self.accept_coro = try async(self.loop.allocator) (TcpServer.handler)(self); // TODO #817 + errdefer cancel ??self.accept_coro; + + try self.loop.addFd(self.sockfd, ??self.accept_coro); + errdefer self.loop.removeFd(self.sockfd); + + } + + pub fn deinit(self: &TcpServer) void { + self.loop.removeFd(self.sockfd); + if (self.accept_coro) |accept_coro| cancel accept_coro; + std.os.close(self.sockfd); + } + + pub async fn handler(self: &TcpServer) void { + while (true) { + var accepted_addr: std.net.Address = undefined; + if (std.os.posixAccept(self.sockfd, &accepted_addr.sockaddr, + posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| + { + var socket = std.os.File.openHandle(accepted_fd); + // TODO #817 + _ = async(self.loop.allocator) (self.handleRequestFn)(self, accepted_addr, + socket) catch |err| switch (err) + { + error.OutOfMemory => { + socket.close(); + continue; + }, + }; + } else |err| switch (err) { + error.WouldBlock => { + suspend; // we will get resumed by epoll_wait in the event loop + continue; + }, + error.ProcessFdQuotaExceeded => { + errdefer std.os.emfile_promise_queue.remove(&self.waiting_for_emfile_node); + suspend |p| { + self.waiting_for_emfile_node = PromiseNode.init(p); + std.os.emfile_promise_queue.append(&self.waiting_for_emfile_node); + } + continue; + }, + error.ConnectionAborted, + error.FileDescriptorClosed => continue, + + error.PageFault => unreachable, + error.InvalidSyscall => unreachable, + error.FileDescriptorNotASocket => unreachable, + error.OperationNotSupported => unreachable, + + error.SystemFdQuotaExceeded, + error.SystemResources, + error.ProtocolFailure, + error.BlockedByFirewall, + error.Unexpected => { + @panic("TODO handle this error"); + }, + } + } + } +}; + +pub const Loop = struct { + allocator: &mem.Allocator, + epollfd: i32, + keep_running: bool, + + fn init(allocator: &mem.Allocator) !Loop { + const epollfd = try std.os.linuxEpollCreate(std.os.linux.EPOLL_CLOEXEC); + return Loop { + .keep_running = true, + .allocator = allocator, + .epollfd = epollfd, + }; + } + + pub fn addFd(self: &Loop, fd: i32, prom: promise) !void { + var ev = std.os.linux.epoll_event { + .events = std.os.linux.EPOLLIN|std.os.linux.EPOLLET, + .data = std.os.linux.epoll_data { + .ptr = @ptrToInt(prom), + }, + }; + try std.os.linuxEpollCtl(self.epollfd, std.os.linux.EPOLL_CTL_ADD, fd, &ev); + } + + pub fn removeFd(self: &Loop, fd: i32) void { + std.os.linuxEpollCtl(self.epollfd, std.os.linux.EPOLL_CTL_DEL, fd, undefined) catch {}; + } + + async fn waitFd(self: &Loop, fd: i32) !void { + defer self.removeFd(fd); + suspend |p| { + try self.addFd(fd, p); + } + } + + pub fn stop(self: &Loop) void { + // TODO make atomic + self.keep_running = false; + // TODO activate an fd in the epoll set + } + + pub fn run(self: &Loop) void { + while (self.keep_running) { + var events: [16]std.os.linux.epoll_event = undefined; + const count = std.os.linuxEpollWait(self.epollfd, events[0..], -1); + for (events[0..count]) |ev| { + const p = @intToPtr(promise, ev.data.ptr); + resume p; + } + } + } +}; + +test "listen on a port, send bytes, receive bytes" { + const MyServer = struct { + tcp_server: TcpServer, + + const Self = this; + + async(&mem.Allocator) fn handler(tcp_server: &TcpServer, _addr: &const std.net.Address, + _socket: &const std.os.File) void + { + const self = @fieldParentPtr(Self, "tcp_server", tcp_server); + var socket = *_socket; // TODO https://github.com/zig-lang/zig/issues/733 + defer socket.close(); + const next_handler = async errorableHandler(self, _addr, socket) catch |err| switch (err) { + error.OutOfMemory => return, + }; + (await next_handler) catch |err| switch (err) { + + }; + suspend |p| { cancel p; } + } + + async fn errorableHandler(self: &Self, _addr: &const std.net.Address, + _socket: &const std.os.File) !void + { + const addr = *_addr; // TODO https://github.com/zig-lang/zig/issues/733 + var socket = *_socket; // TODO https://github.com/zig-lang/zig/issues/733 + + var adapter = std.io.FileOutStream.init(&socket); + var stream = &adapter.stream; + try stream.print("hello from server\n") catch unreachable; + } + }; + + const ip4addr = std.net.parseIp4("127.0.0.1") catch unreachable; + const addr = std.net.Address.initIp4(ip4addr, 0); + + var loop = try Loop.init(std.debug.global_allocator); + var server = MyServer { + .tcp_server = try TcpServer.init(&loop), + }; + defer server.tcp_server.deinit(); + try server.tcp_server.listen(addr, MyServer.handler); + + loop.run(); +} diff --git a/std/fmt/index.zig b/std/fmt/index.zig index bd5b5710e..cfdd70e95 100644 --- a/std/fmt/index.zig +++ b/std/fmt/index.zig @@ -465,7 +465,7 @@ pub fn parseUnsigned(comptime T: type, buf: []const u8, radix: u8) ParseUnsigned return x; } -fn charToDigit(c: u8, radix: u8) (error{InvalidCharacter}!u8) { +pub fn charToDigit(c: u8, radix: u8) (error{InvalidCharacter}!u8) { const value = switch (c) { '0' ... '9' => c - '0', 'A' ... 'Z' => c - 'A' + 10, diff --git a/std/index.zig b/std/index.zig index f2af70b28..f8ec787a0 100644 --- a/std/index.zig +++ b/std/index.zig @@ -17,7 +17,7 @@ pub const debug = @import("debug/index.zig"); pub const dwarf = @import("dwarf.zig"); pub const elf = @import("elf.zig"); pub const empty_import = @import("empty.zig"); -pub const endian = @import("endian.zig"); +pub const event = @import("event.zig"); pub const fmt = @import("fmt/index.zig"); pub const hash = @import("hash/index.zig"); pub const heap = @import("heap.zig"); @@ -50,13 +50,13 @@ test "std" { _ = @import("dwarf.zig"); _ = @import("elf.zig"); _ = @import("empty.zig"); - _ = @import("endian.zig"); _ = @import("fmt/index.zig"); _ = @import("hash/index.zig"); _ = @import("io.zig"); _ = @import("macho.zig"); _ = @import("math/index.zig"); _ = @import("mem.zig"); + _ = @import("net.zig"); _ = @import("heap.zig"); _ = @import("net.zig"); _ = @import("os/index.zig"); diff --git a/std/linked_list.zig b/std/linked_list.zig index c916a5313..45595f3ef 100644 --- a/std/linked_list.zig +++ b/std/linked_list.zig @@ -161,6 +161,7 @@ fn BaseLinkedList(comptime T: type, comptime ParentType: type, comptime field_na } list.len -= 1; + assert(list.len == 0 or (list.first != null and list.last != null)); } /// Remove and return the last node in the list. diff --git a/std/mem.zig b/std/mem.zig index 97cb35ae6..8a59d6251 100644 --- a/std/mem.zig +++ b/std/mem.zig @@ -3,6 +3,7 @@ const debug = std.debug; const assert = debug.assert; const math = std.math; const builtin = @import("builtin"); +const mem = this; pub const Allocator = struct { const Error = error {OutOfMemory}; @@ -550,3 +551,28 @@ test "std.mem.rotate" { assert(eql(i32, arr, []i32{ 1, 2, 4, 5, 3 })); } + +// TODO: When https://github.com/zig-lang/zig/issues/649 is solved these can be done by +// endian-casting the pointer and then dereferencing + +pub fn endianSwapIfLe(comptime T: type, x: T) T { + return endianSwapIf(builtin.Endian.Little, T, x); +} + +pub fn endianSwapIfBe(comptime T: type, x: T) T { + return endianSwapIf(builtin.Endian.Big, T, x); +} + +pub fn endianSwapIf(endian: builtin.Endian, comptime T: type, x: T) T { + return if (builtin.endian == endian) endianSwap(T, x) else x; +} + +pub fn endianSwap(comptime T: type, x: T) T { + var buf: [@sizeOf(T)]u8 = undefined; + mem.writeInt(buf[0..], x, builtin.Endian.Little); + return mem.readInt(buf, T, builtin.Endian.Big); +} + +test "std.mem.endianSwap" { + assert(endianSwap(u32, 0xDEADBEEF) == 0xEFBEADDE); +} diff --git a/std/net.zig b/std/net.zig index 1140b6449..595baae9d 100644 --- a/std/net.zig +++ b/std/net.zig @@ -1,143 +1,103 @@ const std = @import("index.zig"); -const linux = std.os.linux; const assert = std.debug.assert; -const endian = std.endian; +const net = this; +const posix = std.os.posix; +const mem = std.mem; -// TODO don't trust this file, it bit rotted. start over +pub const Address = struct { + sockaddr: posix.sockaddr, -const Connection = struct { - socket_fd: i32, - - pub fn send(c: Connection, buf: []const u8) !usize { - const send_ret = linux.sendto(c.socket_fd, buf.ptr, buf.len, 0, null, 0); - const send_err = linux.getErrno(send_ret); - switch (send_err) { - 0 => return send_ret, - linux.EINVAL => unreachable, - linux.EFAULT => unreachable, - linux.ECONNRESET => return error.ConnectionReset, - linux.EINTR => return error.SigInterrupt, - // TODO there are more possible errors - else => return error.Unexpected, - } + pub fn initIp4(ip4: u32, port: u16) Address { + return Address { + .sockaddr = posix.sockaddr { + .in = posix.sockaddr_in { + .family = posix.AF_INET, + .port = std.mem.endianSwapIfLe(u16, port), + .addr = ip4, + .zero = []u8{0} ** 8, + }, + }, + }; } - pub fn recv(c: Connection, buf: []u8) ![]u8 { - const recv_ret = linux.recvfrom(c.socket_fd, buf.ptr, buf.len, 0, null, null); - const recv_err = linux.getErrno(recv_ret); - switch (recv_err) { - 0 => return buf[0..recv_ret], - linux.EINVAL => unreachable, - linux.EFAULT => unreachable, - linux.ENOTSOCK => return error.NotSocket, - linux.EINTR => return error.SigInterrupt, - linux.ENOMEM => return error.OutOfMemory, - linux.ECONNREFUSED => return error.ConnectionRefused, - linux.EBADF => return error.BadFd, - // TODO more error values - else => return error.Unexpected, - } + pub fn initIp6(ip6: &const Ip6Addr, port: u16) Address { + return Address { + .family = posix.AF_INET6, + .sockaddr = posix.sockaddr { + .in6 = posix.sockaddr_in6 { + .family = posix.AF_INET6, + .port = std.mem.endianSwapIfLe(u16, port), + .flowinfo = 0, + .addr = ip6.addr, + .scope_id = ip6.scope_id, + }, + }, + }; } - pub fn close(c: Connection) !void { - switch (linux.getErrno(linux.close(c.socket_fd))) { - 0 => return, - linux.EBADF => unreachable, - linux.EINTR => return error.SigInterrupt, - linux.EIO => return error.Io, - else => return error.Unexpected, + pub fn format(self: &const Address, out_stream: var) !void { + switch (self.sockaddr.in.family) { + posix.AF_INET => { + const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in.port); + const bytes = ([]const u8)((&self.sockaddr.in.addr)[0..1]); + try out_stream.print("{}.{}.{}.{}:{}", bytes[0], bytes[1], bytes[2], bytes[3], native_endian_port); + }, + posix.AF_INET6 => { + const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in6.port); + try out_stream.print("[TODO render ip6 address]:{}", native_endian_port); + }, + else => try out_stream.write("(unrecognized address family)"), } } }; -const Address = struct { - family: u16, +pub fn parseIp4(buf: []const u8) !u32 { + var result: u32 = undefined; + const out_ptr = ([]u8)((&result)[0..1]); + + var x: u8 = 0; + var index: u8 = 0; + var saw_any_digits = false; + for (buf) |c| { + if (c == '.') { + if (!saw_any_digits) { + return error.InvalidCharacter; + } + if (index == 3) { + return error.InvalidEnd; + } + out_ptr[index] = x; + index += 1; + x = 0; + saw_any_digits = false; + } else if (c >= '0' and c <= '9') { + saw_any_digits = true; + const digit = c - '0'; + if (@mulWithOverflow(u8, x, 10, &x)) { + return error.Overflow; + } + if (@addWithOverflow(u8, x, digit, &x)) { + return error.Overflow; + } + } else { + return error.InvalidCharacter; + } + } + if (index == 3 and saw_any_digits) { + out_ptr[index] = x; + return result; + } + + return error.Incomplete; +} + +pub const Ip6Addr = struct { scope_id: u32, addr: [16]u8, - sort_key: i32, }; -pub fn lookup(hostname: []const u8, out_addrs: []Address) ![]Address { - if (hostname.len == 0) { - - unreachable; // TODO - } - - unreachable; // TODO -} - -pub fn connectAddr(addr: &Address, port: u16) !Connection { - const socket_ret = linux.socket(addr.family, linux.SOCK_STREAM, linux.PROTO_tcp); - const socket_err = linux.getErrno(socket_ret); - if (socket_err > 0) { - // TODO figure out possible errors from socket() - return error.Unexpected; - } - const socket_fd = i32(socket_ret); - - const connect_ret = if (addr.family == linux.AF_INET) x: { - var os_addr: linux.sockaddr_in = undefined; - os_addr.family = addr.family; - os_addr.port = endian.swapIfLe(u16, port); - @memcpy((&u8)(&os_addr.addr), &addr.addr[0], 4); - @memset(&os_addr.zero[0], 0, @sizeOf(@typeOf(os_addr.zero))); - break :x linux.connect(socket_fd, (&linux.sockaddr)(&os_addr), @sizeOf(linux.sockaddr_in)); - } else if (addr.family == linux.AF_INET6) x: { - var os_addr: linux.sockaddr_in6 = undefined; - os_addr.family = addr.family; - os_addr.port = endian.swapIfLe(u16, port); - os_addr.flowinfo = 0; - os_addr.scope_id = addr.scope_id; - @memcpy(&os_addr.addr[0], &addr.addr[0], 16); - break :x linux.connect(socket_fd, (&linux.sockaddr)(&os_addr), @sizeOf(linux.sockaddr_in6)); - } else { - unreachable; - }; - const connect_err = linux.getErrno(connect_ret); - if (connect_err > 0) { - switch (connect_err) { - linux.ETIMEDOUT => return error.TimedOut, - else => { - // TODO figure out possible errors from connect() - return error.Unexpected; - }, - } - } - - return Connection { - .socket_fd = socket_fd, - }; -} - -pub fn connect(hostname: []const u8, port: u16) !Connection { - var addrs_buf: [1]Address = undefined; - const addrs_slice = try lookup(hostname, addrs_buf[0..]); - const main_addr = &addrs_slice[0]; - - return connectAddr(main_addr, port); -} - -pub fn parseIpLiteral(buf: []const u8) !Address { - - return error.InvalidIpLiteral; -} - -fn hexDigit(c: u8) u8 { - // TODO use switch with range - if ('0' <= c and c <= '9') { - return c - '0'; - } else if ('A' <= c and c <= 'Z') { - return c - 'A' + 10; - } else if ('a' <= c and c <= 'z') { - return c - 'a' + 10; - } else { - return @maxValue(u8); - } -} - -fn parseIp6(buf: []const u8) !Address { - var result: Address = undefined; - result.family = linux.AF_INET6; +pub fn parseIp6(buf: []const u8) !Ip6Addr { + var result: Ip6Addr = undefined; result.scope_id = 0; const ip_slice = result.addr[0..]; @@ -156,14 +116,14 @@ fn parseIp6(buf: []const u8) !Address { return error.Overflow; } } else { - return error.InvalidChar; + return error.InvalidCharacter; } } else if (c == ':') { if (!saw_any_digits) { - return error.InvalidChar; + return error.InvalidCharacter; } if (index == 14) { - return error.JunkAtEnd; + return error.InvalidEnd; } ip_slice[index] = @truncate(u8, x >> 8); index += 1; @@ -174,7 +134,7 @@ fn parseIp6(buf: []const u8) !Address { saw_any_digits = false; } else if (c == '%') { if (!saw_any_digits) { - return error.InvalidChar; + return error.InvalidCharacter; } if (index == 14) { ip_slice[index] = @truncate(u8, x >> 8); @@ -185,10 +145,7 @@ fn parseIp6(buf: []const u8) !Address { scope_id = true; saw_any_digits = false; } else { - const digit = hexDigit(c); - if (digit == @maxValue(u8)) { - return error.InvalidChar; - } + const digit = try std.fmt.charToDigit(c, 16); if (@mulWithOverflow(u16, x, 16, &x)) { return error.Overflow; } @@ -216,42 +173,27 @@ fn parseIp6(buf: []const u8) !Address { return error.Incomplete; } -fn parseIp4(buf: []const u8) !u32 { - var result: u32 = undefined; - const out_ptr = ([]u8)((&result)[0..1]); +test "std.net.parseIp4" { + assert((try parseIp4("127.0.0.1")) == std.mem.endianSwapIfLe(u32, 0x7f000001)); - var x: u8 = 0; - var index: u8 = 0; - var saw_any_digits = false; - for (buf) |c| { - if (c == '.') { - if (!saw_any_digits) { - return error.InvalidChar; - } - if (index == 3) { - return error.JunkAtEnd; - } - out_ptr[index] = x; - index += 1; - x = 0; - saw_any_digits = false; - } else if (c >= '0' and c <= '9') { - saw_any_digits = true; - const digit = c - '0'; - if (@mulWithOverflow(u8, x, 10, &x)) { - return error.Overflow; - } - if (@addWithOverflow(u8, x, digit, &x)) { - return error.Overflow; - } - } else { - return error.InvalidChar; - } - } - if (index == 3 and saw_any_digits) { - out_ptr[index] = x; - return result; - } - - return error.Incomplete; + testParseIp4Fail("256.0.0.1", error.Overflow); + testParseIp4Fail("x.0.0.1", error.InvalidCharacter); + testParseIp4Fail("127.0.0.1.1", error.InvalidEnd); + testParseIp4Fail("127.0.0.", error.Incomplete); + testParseIp4Fail("100..0.1", error.InvalidCharacter); +} + +fn testParseIp4Fail(buf: []const u8, expected_err: error) void { + if (parseIp4(buf)) |_| { + @panic("expected error"); + } else |e| { + assert(e == expected_err); + } +} + +test "std.net.parseIp6" { + const addr = try parseIp6("FF01:0:0:0:0:0:0:FB"); + assert(addr.addr[0] == 0xff); + assert(addr.addr[1] == 0x01); + assert(addr.addr[2] == 0x00); } diff --git a/std/os/index.zig b/std/os/index.zig index 4b74af035..6f9db7edc 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -4,6 +4,19 @@ const Os = builtin.Os; const is_windows = builtin.os == Os.windows; const os = this; +test "std.os" { + _ = @import("child_process.zig"); + _ = @import("darwin.zig"); + _ = @import("darwin_errno.zig"); + _ = @import("get_user_id.zig"); + _ = @import("linux/errno.zig"); + _ = @import("linux/index.zig"); + _ = @import("linux/x86_64.zig"); + _ = @import("path.zig"); + _ = @import("test.zig"); + _ = @import("windows/index.zig"); +} + pub const windows = @import("windows/index.zig"); pub const darwin = @import("darwin.zig"); pub const linux = @import("linux/index.zig"); @@ -14,6 +27,7 @@ pub const posix = switch(builtin.os) { Os.zen => zen, else => @compileError("Unsupported OS"), }; +pub const net = @import("net.zig"); pub const ChildProcess = @import("child_process.zig").ChildProcess; pub const path = @import("path.zig"); @@ -173,6 +187,13 @@ pub fn exit(status: u8) noreturn { } } +/// When a file descriptor is closed on linux, it pops the first +/// node from this queue and resumes it. +/// Async functions which get the EMFILE error code can suspend, +/// putting their coroutine handle into this list. +/// TODO make this an atomic linked list +pub var emfile_promise_queue = std.LinkedList(promise).init(); + /// Closes the file handle. Keeps trying if it gets interrupted by a signal. pub fn close(handle: FileHandle) void { if (is_windows) { @@ -180,10 +201,12 @@ pub fn close(handle: FileHandle) void { } else { while (true) { const err = posix.getErrno(posix.close(handle)); - if (err == posix.EINTR) { - continue; - } else { - return; + switch (err) { + posix.EINTR => continue, + else => { + if (emfile_promise_queue.popFirst()) |p| resume p.data; + return; + }, } } } @@ -1753,27 +1776,16 @@ fn testWindowsCmdLine(input_cmd_line: &const u8, expected_args: []const []const assert(it.next(debug.global_allocator) == null); } -test "std.os" { - _ = @import("child_process.zig"); - _ = @import("darwin_errno.zig"); - _ = @import("darwin.zig"); - _ = @import("get_user_id.zig"); - _ = @import("linux/errno.zig"); - //_ = @import("linux_i386.zig"); - _ = @import("linux/x86_64.zig"); - _ = @import("linux/index.zig"); - _ = @import("path.zig"); - _ = @import("windows/index.zig"); - _ = @import("test.zig"); -} - - // TODO make this a build variable that you can set const unexpected_error_tracing = false; +const UnexpectedError = error { + /// The Operating System returned an undocumented error code. + Unexpected, +}; /// Call this when you made a syscall or something that sets errno /// and you get an unexpected error. -pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { +pub fn unexpectedErrorPosix(errno: usize) UnexpectedError { if (unexpected_error_tracing) { debug.warn("unexpected errno: {}\n", errno); debug.dumpCurrentStackTrace(null); @@ -1783,7 +1795,7 @@ pub fn unexpectedErrorPosix(errno: usize) (error{Unexpected}) { /// Call this when you made a windows DLL call or something that does SetLastError /// and you get an unexpected error. -pub fn unexpectedErrorWindows(err: windows.DWORD) (error{Unexpected}) { +pub fn unexpectedErrorWindows(err: windows.DWORD) UnexpectedError { if (unexpected_error_tracing) { debug.warn("unexpected GetLastError(): {}\n", err); debug.dumpCurrentStackTrace(null); @@ -1898,3 +1910,322 @@ pub fn isTty(handle: FileHandle) bool { } } } + +pub const PosixSocketError = error { + /// Permission to create a socket of the specified type and/or + /// pro‐tocol is denied. + PermissionDenied, + + /// The implementation does not support the specified address family. + AddressFamilyNotSupported, + + /// Unknown protocol, or protocol family not available. + ProtocolFamilyNotAvailable, + + /// The per-process limit on the number of open file descriptors has been reached. + ProcessFdQuotaExceeded, + + /// The system-wide limit on the total number of open files has been reached. + SystemFdQuotaExceeded, + + /// Insufficient memory is available. The socket cannot be created until sufficient + /// resources are freed. + SystemResources, + + /// The protocol type or the specified protocol is not supported within this domain. + ProtocolNotSupported, +}; + +pub fn posixSocket(domain: u32, socket_type: u32, protocol: u32) !i32 { + const rc = posix.socket(domain, socket_type, protocol); + const err = posix.getErrno(rc); + switch (err) { + 0 => return i32(rc), + posix.EACCES => return PosixSocketError.PermissionDenied, + posix.EAFNOSUPPORT => return PosixSocketError.AddressFamilyNotSupported, + posix.EINVAL => return PosixSocketError.ProtocolFamilyNotAvailable, + posix.EMFILE => return PosixSocketError.ProcessFdQuotaExceeded, + posix.ENFILE => return PosixSocketError.SystemFdQuotaExceeded, + posix.ENOBUFS, posix.ENOMEM => return PosixSocketError.SystemResources, + posix.EPROTONOSUPPORT => return PosixSocketError.ProtocolNotSupported, + else => return unexpectedErrorPosix(err), + } +} + +pub const PosixBindError = error { + /// The address is protected, and the user is not the superuser. + /// For UNIX domain sockets: Search permission is denied on a component + /// of the path prefix. + AccessDenied, + + /// The given address is already in use, or in the case of Internet domain sockets, + /// The port number was specified as zero in the socket + /// address structure, but, upon attempting to bind to an ephemeral port, it was + /// determined that all port numbers in the ephemeral port range are currently in + /// use. See the discussion of /proc/sys/net/ipv4/ip_local_port_range ip(7). + AddressInUse, + + /// sockfd is not a valid file descriptor. + InvalidFileDescriptor, + + /// The socket is already bound to an address, or addrlen is wrong, or addr is not + /// a valid address for this socket's domain. + InvalidSocketOrAddress, + + /// The file descriptor sockfd does not refer to a socket. + FileDescriptorNotASocket, + + /// A nonexistent interface was requested or the requested address was not local. + AddressNotAvailable, + + /// addr points outside the user's accessible address space. + PageFault, + + /// Too many symbolic links were encountered in resolving addr. + SymLinkLoop, + + /// addr is too long. + NameTooLong, + + /// A component in the directory prefix of the socket pathname does not exist. + FileNotFound, + + /// Insufficient kernel memory was available. + SystemResources, + + /// A component of the path prefix is not a directory. + NotDir, + + /// The socket inode would reside on a read-only filesystem. + ReadOnlyFileSystem, + + Unexpected, +}; + +/// addr is `&const T` where T is one of the sockaddr +pub fn posixBind(fd: i32, addr: &const posix.sockaddr) PosixBindError!void { + const rc = posix.bind(fd, addr, @sizeOf(posix.sockaddr)); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + posix.EACCES => return PosixBindError.AccessDenied, + posix.EADDRINUSE => return PosixBindError.AddressInUse, + posix.EBADF => return PosixBindError.InvalidFileDescriptor, + posix.EINVAL => return PosixBindError.InvalidSocketOrAddress, + posix.ENOTSOCK => return PosixBindError.FileDescriptorNotASocket, + posix.EADDRNOTAVAIL => return PosixBindError.AddressNotAvailable, + posix.EFAULT => return PosixBindError.PageFault, + posix.ELOOP => return PosixBindError.SymLinkLoop, + posix.ENAMETOOLONG => return PosixBindError.NameTooLong, + posix.ENOENT => return PosixBindError.FileNotFound, + posix.ENOMEM => return PosixBindError.SystemResources, + posix.ENOTDIR => return PosixBindError.NotDir, + posix.EROFS => return PosixBindError.ReadOnlyFileSystem, + else => return unexpectedErrorPosix(err), + } +} + +const PosixListenError = error { + /// Another socket is already listening on the same port. + /// For Internet domain sockets, the socket referred to by sockfd had not previously + /// been bound to an address and, upon attempting to bind it to an ephemeral port, it + /// was determined that all port numbers in the ephemeral port range are currently in + /// use. See the discussion of /proc/sys/net/ipv4/ip_local_port_range in ip(7). + AddressInUse, + + /// The argument sockfd is not a valid file descriptor. + InvalidFileDescriptor, + + /// The file descriptor sockfd does not refer to a socket. + FileDescriptorNotASocket, + + /// The socket is not of a type that supports the listen() operation. + OperationNotSupported, + + Unexpected, +}; + +pub fn posixListen(sockfd: i32, backlog: u32) PosixListenError!void { + const rc = posix.listen(sockfd, backlog); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + posix.EADDRINUSE => return PosixListenError.AddressInUse, + posix.EBADF => return PosixListenError.InvalidFileDescriptor, + posix.ENOTSOCK => return PosixListenError.FileDescriptorNotASocket, + posix.EOPNOTSUPP => return PosixListenError.OperationNotSupported, + else => return unexpectedErrorPosix(err), + } +} + +pub const PosixAcceptError = error { + /// The socket is marked nonblocking and no connections are present to be accepted. + WouldBlock, + + /// sockfd is not an open file descriptor. + FileDescriptorClosed, + + ConnectionAborted, + + /// The addr argument is not in a writable part of the user address space. + PageFault, + + /// Socket is not listening for connections, or addrlen is invalid (e.g., is negative), + /// or invalid value in flags. + InvalidSyscall, + + /// The per-process limit on the number of open file descriptors has been reached. + ProcessFdQuotaExceeded, + + /// The system-wide limit on the total number of open files has been reached. + SystemFdQuotaExceeded, + + /// Not enough free memory. This often means that the memory allocation is limited + /// by the socket buffer limits, not by the system memory. + SystemResources, + + /// The file descriptor sockfd does not refer to a socket. + FileDescriptorNotASocket, + + /// The referenced socket is not of type SOCK_STREAM. + OperationNotSupported, + + ProtocolFailure, + + /// Firewall rules forbid connection. + BlockedByFirewall, + + Unexpected, +}; + +pub fn posixAccept(fd: i32, addr: &posix.sockaddr, flags: u32) PosixAcceptError!i32 { + while (true) { + var sockaddr_size = u32(@sizeOf(posix.sockaddr)); + const rc = posix.accept4(fd, addr, &sockaddr_size, flags); + const err = posix.getErrno(rc); + switch (err) { + 0 => return i32(rc), + posix.EINTR => continue, + else => return unexpectedErrorPosix(err), + + posix.EAGAIN => return PosixAcceptError.WouldBlock, + posix.EBADF => return PosixAcceptError.FileDescriptorClosed, + posix.ECONNABORTED => return PosixAcceptError.ConnectionAborted, + posix.EFAULT => return PosixAcceptError.PageFault, + posix.EINVAL => return PosixAcceptError.InvalidSyscall, + posix.EMFILE => return PosixAcceptError.ProcessFdQuotaExceeded, + posix.ENFILE => return PosixAcceptError.SystemFdQuotaExceeded, + posix.ENOBUFS, posix.ENOMEM => return PosixAcceptError.SystemResources, + posix.ENOTSOCK => return PosixAcceptError.FileDescriptorNotASocket, + posix.EOPNOTSUPP => return PosixAcceptError.OperationNotSupported, + posix.EPROTO => return PosixAcceptError.ProtocolFailure, + posix.EPERM => return PosixAcceptError.BlockedByFirewall, + } + } +} + +pub const LinuxEpollCreateError = error { + /// Invalid value specified in flags. + InvalidSyscall, + + /// The per-user limit on the number of epoll instances imposed by + /// /proc/sys/fs/epoll/max_user_instances was encountered. See epoll(7) for further + /// details. + /// Or, The per-process limit on the number of open file descriptors has been reached. + ProcessFdQuotaExceeded, + + /// The system-wide limit on the total number of open files has been reached. + SystemFdQuotaExceeded, + + /// There was insufficient memory to create the kernel object. + SystemResources, + + Unexpected, +}; + +pub fn linuxEpollCreate(flags: u32) LinuxEpollCreateError!i32 { + const rc = posix.epoll_create1(flags); + const err = posix.getErrno(rc); + switch (err) { + 0 => return i32(rc), + else => return unexpectedErrorPosix(err), + + posix.EINVAL => return LinuxEpollCreateError.InvalidSyscall, + posix.EMFILE => return LinuxEpollCreateError.ProcessFdQuotaExceeded, + posix.ENFILE => return LinuxEpollCreateError.SystemFdQuotaExceeded, + posix.ENOMEM => return LinuxEpollCreateError.SystemResources, + } +} + +pub const LinuxEpollCtlError = error { + /// epfd or fd is not a valid file descriptor. + InvalidFileDescriptor, + + /// op was EPOLL_CTL_ADD, and the supplied file descriptor fd is already registered + /// with this epoll instance. + FileDescriptorAlreadyPresentInSet, + + /// epfd is not an epoll file descriptor, or fd is the same as epfd, or the requested + /// operation op is not supported by this interface, or + /// An invalid event type was specified along with EPOLLEXCLUSIVE in events, or + /// op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE, or + /// op was EPOLL_CTL_MOD and the EPOLLEXCLUSIVE flag has previously been applied to + /// this epfd, fd pair, or + /// EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance. + InvalidSyscall, + + /// fd refers to an epoll instance and this EPOLL_CTL_ADD operation would result in a + /// circular loop of epoll instances monitoring one another. + OperationCausesCircularLoop, + + /// op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll + /// instance. + FileDescriptorNotRegistered, + + /// There was insufficient memory to handle the requested op control operation. + SystemResources, + + /// The limit imposed by /proc/sys/fs/epoll/max_user_watches was encountered while + /// trying to register (EPOLL_CTL_ADD) a new file descriptor on an epoll instance. + /// See epoll(7) for further details. + UserResourceLimitReached, + + /// The target file fd does not support epoll. This error can occur if fd refers to, + /// for example, a regular file or a directory. + FileDescriptorIncompatibleWithEpoll, + + Unexpected, +}; + +pub fn linuxEpollCtl(epfd: i32, op: u32, fd: i32, event: &linux.epoll_event) LinuxEpollCtlError!void { + const rc = posix.epoll_ctl(epfd, op, fd, event); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + else => return unexpectedErrorPosix(err), + + posix.EBADF => return LinuxEpollCtlError.InvalidFileDescriptor, + posix.EEXIST => return LinuxEpollCtlError.FileDescriptorAlreadyPresentInSet, + posix.EINVAL => return LinuxEpollCtlError.InvalidSyscall, + posix.ELOOP => return LinuxEpollCtlError.OperationCausesCircularLoop, + posix.ENOENT => return LinuxEpollCtlError.FileDescriptorNotRegistered, + posix.ENOMEM => return LinuxEpollCtlError.SystemResources, + posix.ENOSPC => return LinuxEpollCtlError.UserResourceLimitReached, + posix.EPERM => return LinuxEpollCtlError.FileDescriptorIncompatibleWithEpoll, + } +} + +pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usize { + while (true) { + const rc = posix.epoll_wait(epfd, &events[0], u32(events.len), timeout); + const err = posix.getErrno(rc); + switch (err) { + 0 => return rc, + posix.EINTR => continue, + posix.EBADF => unreachable, + posix.EFAULT => unreachable, + posix.EINVAL => unreachable, + else => unreachable, + } + } +} diff --git a/std/os/linux/i386.zig b/std/os/linux/i386.zig deleted file mode 100644 index 7450ad34f..000000000 --- a/std/os/linux/i386.zig +++ /dev/null @@ -1,505 +0,0 @@ -const std = @import("../../index.zig"); -const linux = std.os.linux; -const socklen_t = linux.socklen_t; -const iovec = linux.iovec; - -pub const SYS_restart_syscall = 0; -pub const SYS_exit = 1; -pub const SYS_fork = 2; -pub const SYS_read = 3; -pub const SYS_write = 4; -pub const SYS_open = 5; -pub const SYS_close = 6; -pub const SYS_waitpid = 7; -pub const SYS_creat = 8; -pub const SYS_link = 9; -pub const SYS_unlink = 10; -pub const SYS_execve = 11; -pub const SYS_chdir = 12; -pub const SYS_time = 13; -pub const SYS_mknod = 14; -pub const SYS_chmod = 15; -pub const SYS_lchown = 16; -pub const SYS_break = 17; -pub const SYS_oldstat = 18; -pub const SYS_lseek = 19; -pub const SYS_getpid = 20; -pub const SYS_mount = 21; -pub const SYS_umount = 22; -pub const SYS_setuid = 23; -pub const SYS_getuid = 24; -pub const SYS_stime = 25; -pub const SYS_ptrace = 26; -pub const SYS_alarm = 27; -pub const SYS_oldfstat = 28; -pub const SYS_pause = 29; -pub const SYS_utime = 30; -pub const SYS_stty = 31; -pub const SYS_gtty = 32; -pub const SYS_access = 33; -pub const SYS_nice = 34; -pub const SYS_ftime = 35; -pub const SYS_sync = 36; -pub const SYS_kill = 37; -pub const SYS_rename = 38; -pub const SYS_mkdir = 39; -pub const SYS_rmdir = 40; -pub const SYS_dup = 41; -pub const SYS_pipe = 42; -pub const SYS_times = 43; -pub const SYS_prof = 44; -pub const SYS_brk = 45; -pub const SYS_setgid = 46; -pub const SYS_getgid = 47; -pub const SYS_signal = 48; -pub const SYS_geteuid = 49; -pub const SYS_getegid = 50; -pub const SYS_acct = 51; -pub const SYS_umount2 = 52; -pub const SYS_lock = 53; -pub const SYS_ioctl = 54; -pub const SYS_fcntl = 55; -pub const SYS_mpx = 56; -pub const SYS_setpgid = 57; -pub const SYS_ulimit = 58; -pub const SYS_oldolduname = 59; -pub const SYS_umask = 60; -pub const SYS_chroot = 61; -pub const SYS_ustat = 62; -pub const SYS_dup2 = 63; -pub const SYS_getppid = 64; -pub const SYS_getpgrp = 65; -pub const SYS_setsid = 66; -pub const SYS_sigaction = 67; -pub const SYS_sgetmask = 68; -pub const SYS_ssetmask = 69; -pub const SYS_setreuid = 70; -pub const SYS_setregid = 71; -pub const SYS_sigsuspend = 72; -pub const SYS_sigpending = 73; -pub const SYS_sethostname = 74; -pub const SYS_setrlimit = 75; -pub const SYS_getrlimit = 76; -pub const SYS_getrusage = 77; -pub const SYS_gettimeofday = 78; -pub const SYS_settimeofday = 79; -pub const SYS_getgroups = 80; -pub const SYS_setgroups = 81; -pub const SYS_select = 82; -pub const SYS_symlink = 83; -pub const SYS_oldlstat = 84; -pub const SYS_readlink = 85; -pub const SYS_uselib = 86; -pub const SYS_swapon = 87; -pub const SYS_reboot = 88; -pub const SYS_readdir = 89; -pub const SYS_mmap = 90; -pub const SYS_munmap = 91; -pub const SYS_truncate = 92; -pub const SYS_ftruncate = 93; -pub const SYS_fchmod = 94; -pub const SYS_fchown = 95; -pub const SYS_getpriority = 96; -pub const SYS_setpriority = 97; -pub const SYS_profil = 98; -pub const SYS_statfs = 99; -pub const SYS_fstatfs = 100; -pub const SYS_ioperm = 101; -pub const SYS_socketcall = 102; -pub const SYS_syslog = 103; -pub const SYS_setitimer = 104; -pub const SYS_getitimer = 105; -pub const SYS_stat = 106; -pub const SYS_lstat = 107; -pub const SYS_fstat = 108; -pub const SYS_olduname = 109; -pub const SYS_iopl = 110; -pub const SYS_vhangup = 111; -pub const SYS_idle = 112; -pub const SYS_vm86old = 113; -pub const SYS_wait4 = 114; -pub const SYS_swapoff = 115; -pub const SYS_sysinfo = 116; -pub const SYS_ipc = 117; -pub const SYS_fsync = 118; -pub const SYS_sigreturn = 119; -pub const SYS_clone = 120; -pub const SYS_setdomainname = 121; -pub const SYS_uname = 122; -pub const SYS_modify_ldt = 123; -pub const SYS_adjtimex = 124; -pub const SYS_mprotect = 125; -pub const SYS_sigprocmask = 126; -pub const SYS_create_module = 127; -pub const SYS_init_module = 128; -pub const SYS_delete_module = 129; -pub const SYS_get_kernel_syms = 130; -pub const SYS_quotactl = 131; -pub const SYS_getpgid = 132; -pub const SYS_fchdir = 133; -pub const SYS_bdflush = 134; -pub const SYS_sysfs = 135; -pub const SYS_personality = 136; -pub const SYS_afs_syscall = 137; -pub const SYS_setfsuid = 138; -pub const SYS_setfsgid = 139; -pub const SYS__llseek = 140; -pub const SYS_getdents = 141; -pub const SYS__newselect = 142; -pub const SYS_flock = 143; -pub const SYS_msync = 144; -pub const SYS_readv = 145; -pub const SYS_writev = 146; -pub const SYS_getsid = 147; -pub const SYS_fdatasync = 148; -pub const SYS__sysctl = 149; -pub const SYS_mlock = 150; -pub const SYS_munlock = 151; -pub const SYS_mlockall = 152; -pub const SYS_munlockall = 153; -pub const SYS_sched_setparam = 154; -pub const SYS_sched_getparam = 155; -pub const SYS_sched_setscheduler = 156; -pub const SYS_sched_getscheduler = 157; -pub const SYS_sched_yield = 158; -pub const SYS_sched_get_priority_max = 159; -pub const SYS_sched_get_priority_min = 160; -pub const SYS_sched_rr_get_interval = 161; -pub const SYS_nanosleep = 162; -pub const SYS_mremap = 163; -pub const SYS_setresuid = 164; -pub const SYS_getresuid = 165; -pub const SYS_vm86 = 166; -pub const SYS_query_module = 167; -pub const SYS_poll = 168; -pub const SYS_nfsservctl = 169; -pub const SYS_setresgid = 170; -pub const SYS_getresgid = 171; -pub const SYS_prctl = 172; -pub const SYS_rt_sigreturn = 173; -pub const SYS_rt_sigaction = 174; -pub const SYS_rt_sigprocmask = 175; -pub const SYS_rt_sigpending = 176; -pub const SYS_rt_sigtimedwait = 177; -pub const SYS_rt_sigqueueinfo = 178; -pub const SYS_rt_sigsuspend = 179; -pub const SYS_pread64 = 180; -pub const SYS_pwrite64 = 181; -pub const SYS_chown = 182; -pub const SYS_getcwd = 183; -pub const SYS_capget = 184; -pub const SYS_capset = 185; -pub const SYS_sigaltstack = 186; -pub const SYS_sendfile = 187; -pub const SYS_getpmsg = 188; -pub const SYS_putpmsg = 189; -pub const SYS_vfork = 190; -pub const SYS_ugetrlimit = 191; -pub const SYS_mmap2 = 192; -pub const SYS_truncate64 = 193; -pub const SYS_ftruncate64 = 194; -pub const SYS_stat64 = 195; -pub const SYS_lstat64 = 196; -pub const SYS_fstat64 = 197; -pub const SYS_lchown32 = 198; -pub const SYS_getuid32 = 199; -pub const SYS_getgid32 = 200; -pub const SYS_geteuid32 = 201; -pub const SYS_getegid32 = 202; -pub const SYS_setreuid32 = 203; -pub const SYS_setregid32 = 204; -pub const SYS_getgroups32 = 205; -pub const SYS_setgroups32 = 206; -pub const SYS_fchown32 = 207; -pub const SYS_setresuid32 = 208; -pub const SYS_getresuid32 = 209; -pub const SYS_setresgid32 = 210; -pub const SYS_getresgid32 = 211; -pub const SYS_chown32 = 212; -pub const SYS_setuid32 = 213; -pub const SYS_setgid32 = 214; -pub const SYS_setfsuid32 = 215; -pub const SYS_setfsgid32 = 216; -pub const SYS_pivot_root = 217; -pub const SYS_mincore = 218; -pub const SYS_madvise = 219; -pub const SYS_madvise1 = 219; -pub const SYS_getdents64 = 220; -pub const SYS_fcntl64 = 221; -pub const SYS_gettid = 224; -pub const SYS_readahead = 225; -pub const SYS_setxattr = 226; -pub const SYS_lsetxattr = 227; -pub const SYS_fsetxattr = 228; -pub const SYS_getxattr = 229; -pub const SYS_lgetxattr = 230; -pub const SYS_fgetxattr = 231; -pub const SYS_listxattr = 232; -pub const SYS_llistxattr = 233; -pub const SYS_flistxattr = 234; -pub const SYS_removexattr = 235; -pub const SYS_lremovexattr = 236; -pub const SYS_fremovexattr = 237; -pub const SYS_tkill = 238; -pub const SYS_sendfile64 = 239; -pub const SYS_futex = 240; -pub const SYS_sched_setaffinity = 241; -pub const SYS_sched_getaffinity = 242; -pub const SYS_set_thread_area = 243; -pub const SYS_get_thread_area = 244; -pub const SYS_io_setup = 245; -pub const SYS_io_destroy = 246; -pub const SYS_io_getevents = 247; -pub const SYS_io_submit = 248; -pub const SYS_io_cancel = 249; -pub const SYS_fadvise64 = 250; -pub const SYS_exit_group = 252; -pub const SYS_lookup_dcookie = 253; -pub const SYS_epoll_create = 254; -pub const SYS_epoll_ctl = 255; -pub const SYS_epoll_wait = 256; -pub const SYS_remap_file_pages = 257; -pub const SYS_set_tid_address = 258; -pub const SYS_timer_create = 259; -pub const SYS_timer_settime = SYS_timer_create+1; -pub const SYS_timer_gettime = SYS_timer_create+2; -pub const SYS_timer_getoverrun = SYS_timer_create+3; -pub const SYS_timer_delete = SYS_timer_create+4; -pub const SYS_clock_settime = SYS_timer_create+5; -pub const SYS_clock_gettime = SYS_timer_create+6; -pub const SYS_clock_getres = SYS_timer_create+7; -pub const SYS_clock_nanosleep = SYS_timer_create+8; -pub const SYS_statfs64 = 268; -pub const SYS_fstatfs64 = 269; -pub const SYS_tgkill = 270; -pub const SYS_utimes = 271; -pub const SYS_fadvise64_64 = 272; -pub const SYS_vserver = 273; -pub const SYS_mbind = 274; -pub const SYS_get_mempolicy = 275; -pub const SYS_set_mempolicy = 276; -pub const SYS_mq_open = 277; -pub const SYS_mq_unlink = SYS_mq_open+1; -pub const SYS_mq_timedsend = SYS_mq_open+2; -pub const SYS_mq_timedreceive = SYS_mq_open+3; -pub const SYS_mq_notify = SYS_mq_open+4; -pub const SYS_mq_getsetattr = SYS_mq_open+5; -pub const SYS_kexec_load = 283; -pub const SYS_waitid = 284; -pub const SYS_add_key = 286; -pub const SYS_request_key = 287; -pub const SYS_keyctl = 288; -pub const SYS_ioprio_set = 289; -pub const SYS_ioprio_get = 290; -pub const SYS_inotify_init = 291; -pub const SYS_inotify_add_watch = 292; -pub const SYS_inotify_rm_watch = 293; -pub const SYS_migrate_pages = 294; -pub const SYS_openat = 295; -pub const SYS_mkdirat = 296; -pub const SYS_mknodat = 297; -pub const SYS_fchownat = 298; -pub const SYS_futimesat = 299; -pub const SYS_fstatat64 = 300; -pub const SYS_unlinkat = 301; -pub const SYS_renameat = 302; -pub const SYS_linkat = 303; -pub const SYS_symlinkat = 304; -pub const SYS_readlinkat = 305; -pub const SYS_fchmodat = 306; -pub const SYS_faccessat = 307; -pub const SYS_pselect6 = 308; -pub const SYS_ppoll = 309; -pub const SYS_unshare = 310; -pub const SYS_set_robust_list = 311; -pub const SYS_get_robust_list = 312; -pub const SYS_splice = 313; -pub const SYS_sync_file_range = 314; -pub const SYS_tee = 315; -pub const SYS_vmsplice = 316; -pub const SYS_move_pages = 317; -pub const SYS_getcpu = 318; -pub const SYS_epoll_pwait = 319; -pub const SYS_utimensat = 320; -pub const SYS_signalfd = 321; -pub const SYS_timerfd_create = 322; -pub const SYS_eventfd = 323; -pub const SYS_fallocate = 324; -pub const SYS_timerfd_settime = 325; -pub const SYS_timerfd_gettime = 326; -pub const SYS_signalfd4 = 327; -pub const SYS_eventfd2 = 328; -pub const SYS_epoll_create1 = 329; -pub const SYS_dup3 = 330; -pub const SYS_pipe2 = 331; -pub const SYS_inotify_init1 = 332; -pub const SYS_preadv = 333; -pub const SYS_pwritev = 334; -pub const SYS_rt_tgsigqueueinfo = 335; -pub const SYS_perf_event_open = 336; -pub const SYS_recvmmsg = 337; -pub const SYS_fanotify_init = 338; -pub const SYS_fanotify_mark = 339; -pub const SYS_prlimit64 = 340; -pub const SYS_name_to_handle_at = 341; -pub const SYS_open_by_handle_at = 342; -pub const SYS_clock_adjtime = 343; -pub const SYS_syncfs = 344; -pub const SYS_sendmmsg = 345; -pub const SYS_setns = 346; -pub const SYS_process_vm_readv = 347; -pub const SYS_process_vm_writev = 348; -pub const SYS_kcmp = 349; -pub const SYS_finit_module = 350; -pub const SYS_sched_setattr = 351; -pub const SYS_sched_getattr = 352; -pub const SYS_renameat2 = 353; -pub const SYS_seccomp = 354; -pub const SYS_getrandom = 355; -pub const SYS_memfd_create = 356; -pub const SYS_bpf = 357; -pub const SYS_execveat = 358; -pub const SYS_socket = 359; -pub const SYS_socketpair = 360; -pub const SYS_bind = 361; -pub const SYS_connect = 362; -pub const SYS_listen = 363; -pub const SYS_accept4 = 364; -pub const SYS_getsockopt = 365; -pub const SYS_setsockopt = 366; -pub const SYS_getsockname = 367; -pub const SYS_getpeername = 368; -pub const SYS_sendto = 369; -pub const SYS_sendmsg = 370; -pub const SYS_recvfrom = 371; -pub const SYS_recvmsg = 372; -pub const SYS_shutdown = 373; -pub const SYS_userfaultfd = 374; -pub const SYS_membarrier = 375; -pub const SYS_mlock2 = 376; - - -pub const O_CREAT = 0o100; -pub const O_EXCL = 0o200; -pub const O_NOCTTY = 0o400; -pub const O_TRUNC = 0o1000; -pub const O_APPEND = 0o2000; -pub const O_NONBLOCK = 0o4000; -pub const O_DSYNC = 0o10000; -pub const O_SYNC = 0o4010000; -pub const O_RSYNC = 0o4010000; -pub const O_DIRECTORY = 0o200000; -pub const O_NOFOLLOW = 0o400000; -pub const O_CLOEXEC = 0o2000000; - -pub const O_ASYNC = 0o20000; -pub const O_DIRECT = 0o40000; -pub const O_LARGEFILE = 0o100000; -pub const O_NOATIME = 0o1000000; -pub const O_PATH = 0o10000000; -pub const O_TMPFILE = 0o20200000; -pub const O_NDELAY = O_NONBLOCK; - -pub const F_DUPFD = 0; -pub const F_GETFD = 1; -pub const F_SETFD = 2; -pub const F_GETFL = 3; -pub const F_SETFL = 4; - -pub const F_SETOWN = 8; -pub const F_GETOWN = 9; -pub const F_SETSIG = 10; -pub const F_GETSIG = 11; - -pub const F_GETLK = 12; -pub const F_SETLK = 13; -pub const F_SETLKW = 14; - -pub const F_SETOWN_EX = 15; -pub const F_GETOWN_EX = 16; - -pub const F_GETOWNER_UIDS = 17; - -pub inline fn syscall0(number: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number)); -} - -pub inline fn syscall1(number: usize, arg1: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1)); -} - -pub inline fn syscall2(number: usize, arg1: usize, arg2: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2)); -} - -pub inline fn syscall3(number: usize, arg1: usize, arg2: usize, arg3: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3)); -} - -pub inline fn syscall4(number: usize, arg1: usize, arg2: usize, arg3: usize, arg4: usize) usize { - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3), - [arg4] "{esi}" (arg4)); -} - -pub inline fn syscall5(number: usize, arg1: usize, arg2: usize, arg3: usize, - arg4: usize, arg5: usize) usize -{ - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3), - [arg4] "{esi}" (arg4), - [arg5] "{edi}" (arg5)); -} - -pub inline fn syscall6(number: usize, arg1: usize, arg2: usize, arg3: usize, - arg4: usize, arg5: usize, arg6: usize) usize -{ - return asm volatile ("int $0x80" - : [ret] "={eax}" (-> usize) - : [number] "{eax}" (number), - [arg1] "{ebx}" (arg1), - [arg2] "{ecx}" (arg2), - [arg3] "{edx}" (arg3), - [arg4] "{esi}" (arg4), - [arg5] "{edi}" (arg5), - [arg6] "{ebp}" (arg6)); -} - -pub nakedcc fn restore() void { - asm volatile ( - \\popl %%eax - \\movl $119, %%eax - \\int $0x80 - : - : - : "rcx", "r11"); -} - -pub nakedcc fn restore_rt() void { - asm volatile ("int $0x80" - : - : [number] "{eax}" (usize(SYS_rt_sigreturn)) - : "rcx", "r11"); -} diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 8fd8bcbe7..602ff66e7 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -101,17 +101,6 @@ pub const SIG_BLOCK = 0; pub const SIG_UNBLOCK = 1; pub const SIG_SETMASK = 2; -pub const SOCK_STREAM = 1; -pub const SOCK_DGRAM = 2; -pub const SOCK_RAW = 3; -pub const SOCK_RDM = 4; -pub const SOCK_SEQPACKET = 5; -pub const SOCK_DCCP = 6; -pub const SOCK_PACKET = 10; -pub const SOCK_CLOEXEC = 0o2000000; -pub const SOCK_NONBLOCK = 0o4000; - - pub const PROTO_ip = 0o000; pub const PROTO_icmp = 0o001; pub const PROTO_igmp = 0o002; @@ -149,6 +138,20 @@ pub const PROTO_encap = 0o142; pub const PROTO_pim = 0o147; pub const PROTO_raw = 0o377; +pub const SHUT_RD = 0; +pub const SHUT_WR = 1; +pub const SHUT_RDWR = 2; + +pub const SOCK_STREAM = 1; +pub const SOCK_DGRAM = 2; +pub const SOCK_RAW = 3; +pub const SOCK_RDM = 4; +pub const SOCK_SEQPACKET = 5; +pub const SOCK_DCCP = 6; +pub const SOCK_PACKET = 10; +pub const SOCK_CLOEXEC = 0o2000000; +pub const SOCK_NONBLOCK = 0o4000; + pub const PF_UNSPEC = 0; pub const PF_LOCAL = 1; pub const PF_UNIX = PF_LOCAL; @@ -193,7 +196,10 @@ pub const PF_CAIF = 37; pub const PF_ALG = 38; pub const PF_NFC = 39; pub const PF_VSOCK = 40; -pub const PF_MAX = 41; +pub const PF_KCM = 41; +pub const PF_QIPCRTR = 42; +pub const PF_SMC = 43; +pub const PF_MAX = 44; pub const AF_UNSPEC = PF_UNSPEC; pub const AF_LOCAL = PF_LOCAL; @@ -239,8 +245,137 @@ pub const AF_CAIF = PF_CAIF; pub const AF_ALG = PF_ALG; pub const AF_NFC = PF_NFC; pub const AF_VSOCK = PF_VSOCK; +pub const AF_KCM = PF_KCM; +pub const AF_QIPCRTR = PF_QIPCRTR; +pub const AF_SMC = PF_SMC; pub const AF_MAX = PF_MAX; +pub const SO_DEBUG = 1; +pub const SO_REUSEADDR = 2; +pub const SO_TYPE = 3; +pub const SO_ERROR = 4; +pub const SO_DONTROUTE = 5; +pub const SO_BROADCAST = 6; +pub const SO_SNDBUF = 7; +pub const SO_RCVBUF = 8; +pub const SO_KEEPALIVE = 9; +pub const SO_OOBINLINE = 10; +pub const SO_NO_CHECK = 11; +pub const SO_PRIORITY = 12; +pub const SO_LINGER = 13; +pub const SO_BSDCOMPAT = 14; +pub const SO_REUSEPORT = 15; +pub const SO_PASSCRED = 16; +pub const SO_PEERCRED = 17; +pub const SO_RCVLOWAT = 18; +pub const SO_SNDLOWAT = 19; +pub const SO_RCVTIMEO = 20; +pub const SO_SNDTIMEO = 21; +pub const SO_ACCEPTCONN = 30; +pub const SO_SNDBUFFORCE = 32; +pub const SO_RCVBUFFORCE = 33; +pub const SO_PROTOCOL = 38; +pub const SO_DOMAIN = 39; + +pub const SO_SECURITY_AUTHENTICATION = 22; +pub const SO_SECURITY_ENCRYPTION_TRANSPORT = 23; +pub const SO_SECURITY_ENCRYPTION_NETWORK = 24; + +pub const SO_BINDTODEVICE = 25; + +pub const SO_ATTACH_FILTER = 26; +pub const SO_DETACH_FILTER = 27; +pub const SO_GET_FILTER = SO_ATTACH_FILTER; + +pub const SO_PEERNAME = 28; +pub const SO_TIMESTAMP = 29; +pub const SCM_TIMESTAMP = SO_TIMESTAMP; + +pub const SO_PEERSEC = 31; +pub const SO_PASSSEC = 34; +pub const SO_TIMESTAMPNS = 35; +pub const SCM_TIMESTAMPNS = SO_TIMESTAMPNS; +pub const SO_MARK = 36; +pub const SO_TIMESTAMPING = 37; +pub const SCM_TIMESTAMPING = SO_TIMESTAMPING; +pub const SO_RXQ_OVFL = 40; +pub const SO_WIFI_STATUS = 41; +pub const SCM_WIFI_STATUS = SO_WIFI_STATUS; +pub const SO_PEEK_OFF = 42; +pub const SO_NOFCS = 43; +pub const SO_LOCK_FILTER = 44; +pub const SO_SELECT_ERR_QUEUE = 45; +pub const SO_BUSY_POLL = 46; +pub const SO_MAX_PACING_RATE = 47; +pub const SO_BPF_EXTENSIONS = 48; +pub const SO_INCOMING_CPU = 49; +pub const SO_ATTACH_BPF = 50; +pub const SO_DETACH_BPF = SO_DETACH_FILTER; +pub const SO_ATTACH_REUSEPORT_CBPF = 51; +pub const SO_ATTACH_REUSEPORT_EBPF = 52; +pub const SO_CNX_ADVICE = 53; +pub const SCM_TIMESTAMPING_OPT_STATS = 54; +pub const SO_MEMINFO = 55; +pub const SO_INCOMING_NAPI_ID = 56; +pub const SO_COOKIE = 57; +pub const SCM_TIMESTAMPING_PKTINFO = 58; +pub const SO_PEERGROUPS = 59; +pub const SO_ZEROCOPY = 60; + +pub const SOL_SOCKET = 1; + +pub const SOL_IP = 0; +pub const SOL_IPV6 = 41; +pub const SOL_ICMPV6 = 58; + +pub const SOL_RAW = 255; +pub const SOL_DECNET = 261; +pub const SOL_X25 = 262; +pub const SOL_PACKET = 263; +pub const SOL_ATM = 264; +pub const SOL_AAL = 265; +pub const SOL_IRDA = 266; +pub const SOL_NETBEUI = 267; +pub const SOL_LLC = 268; +pub const SOL_DCCP = 269; +pub const SOL_NETLINK = 270; +pub const SOL_TIPC = 271; +pub const SOL_RXRPC = 272; +pub const SOL_PPPOL2TP = 273; +pub const SOL_BLUETOOTH = 274; +pub const SOL_PNPIPE = 275; +pub const SOL_RDS = 276; +pub const SOL_IUCV = 277; +pub const SOL_CAIF = 278; +pub const SOL_ALG = 279; +pub const SOL_NFC = 280; +pub const SOL_KCM = 281; +pub const SOL_TLS = 282; + +pub const SOMAXCONN = 128; + +pub const MSG_OOB = 0x0001; +pub const MSG_PEEK = 0x0002; +pub const MSG_DONTROUTE = 0x0004; +pub const MSG_CTRUNC = 0x0008; +pub const MSG_PROXY = 0x0010; +pub const MSG_TRUNC = 0x0020; +pub const MSG_DONTWAIT = 0x0040; +pub const MSG_EOR = 0x0080; +pub const MSG_WAITALL = 0x0100; +pub const MSG_FIN = 0x0200; +pub const MSG_SYN = 0x0400; +pub const MSG_CONFIRM = 0x0800; +pub const MSG_RST = 0x1000; +pub const MSG_ERRQUEUE = 0x2000; +pub const MSG_NOSIGNAL = 0x4000; +pub const MSG_MORE = 0x8000; +pub const MSG_WAITFORONE = 0x10000; +pub const MSG_BATCH = 0x40000; +pub const MSG_ZEROCOPY = 0x4000000; +pub const MSG_FASTOPEN = 0x20000000; +pub const MSG_CMSG_CLOEXEC = 0x40000000; + pub const DT_UNKNOWN = 0; pub const DT_FIFO = 1; pub const DT_CHR = 2; @@ -599,30 +734,27 @@ pub fn sigismember(set: &const sigset_t, sig: u6) bool { return ((*set)[usize(s) / usize.bit_count] & (usize(1) << (s & (usize.bit_count - 1)))) != 0; } - +pub const in_port_t = u16; pub const sa_family_t = u16; pub const socklen_t = u32; -pub const in_addr = u32; -pub const in6_addr = [16]u8; -pub const sockaddr = extern struct { - family: sa_family_t, - port: u16, - data: [12]u8, +pub const sockaddr = extern union { + in: sockaddr_in, + in6: sockaddr_in6, }; pub const sockaddr_in = extern struct { family: sa_family_t, - port: u16, - addr: in_addr, + port: in_port_t, + addr: u32, zero: [8]u8, }; pub const sockaddr_in6 = extern struct { family: sa_family_t, - port: u16, + port: in_port_t, flowinfo: u32, - addr: in6_addr, + addr: [16]u8, scope_id: u32, }; @@ -639,8 +771,8 @@ pub fn getpeername(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t) us return syscall3(SYS_getpeername, usize(fd), @ptrToInt(addr), @ptrToInt(len)); } -pub fn socket(domain: i32, socket_type: i32, protocol: i32) usize { - return syscall3(SYS_socket, usize(domain), usize(socket_type), usize(protocol)); +pub fn socket(domain: u32, socket_type: u32, protocol: u32) usize { + return syscall3(SYS_socket, domain, socket_type, protocol); } pub fn setsockopt(fd: i32, level: i32, optname: i32, optval: &const u8, optlen: socklen_t) usize { @@ -677,8 +809,8 @@ pub fn bind(fd: i32, addr: &const sockaddr, len: socklen_t) usize { return syscall3(SYS_bind, usize(fd), @ptrToInt(addr), usize(len)); } -pub fn listen(fd: i32, backlog: i32) usize { - return syscall2(SYS_listen, usize(fd), usize(backlog)); +pub fn listen(fd: i32, backlog: u32) usize { + return syscall2(SYS_listen, usize(fd), backlog); } pub fn sendto(fd: i32, buf: &const u8, len: usize, flags: u32, addr: ?&const sockaddr, alen: socklen_t) usize { @@ -697,34 +829,6 @@ pub fn accept4(fd: i32, noalias addr: &sockaddr, noalias len: &socklen_t, flags: return syscall4(SYS_accept4, usize(fd), @ptrToInt(addr), @ptrToInt(len), flags); } -// error NameTooLong; -// error SystemResources; -// error Io; -// -// pub fn if_nametoindex(name: []u8) !u32 { -// var ifr: ifreq = undefined; -// -// if (name.len >= ifr.ifr_name.len) { -// return error.NameTooLong; -// } -// -// const socket_ret = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0); -// const socket_err = getErrno(socket_ret); -// if (socket_err > 0) { -// return error.SystemResources; -// } -// const socket_fd = i32(socket_ret); -// @memcpy(&ifr.ifr_name[0], &name[0], name.len); -// ifr.ifr_name[name.len] = 0; -// const ioctl_ret = ioctl(socket_fd, SIOCGIFINDEX, &ifr); -// close(socket_fd); -// const ioctl_err = getErrno(ioctl_ret); -// if (ioctl_err > 0) { -// return error.Io; -// } -// return ifr.ifr_ifindex; -// } - pub fn fstat(fd: i32, stat_buf: &Stat) usize { return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } @@ -749,7 +853,7 @@ pub fn epoll_create1(flags: usize) usize { return syscall1(SYS_epoll_create1, flags); } -pub fn epoll_ctl(epoll_fd: i32, op: i32, fd: i32, ev: &epoll_event) usize { +pub fn epoll_ctl(epoll_fd: i32, op: u32, fd: i32, ev: &epoll_event) usize { return syscall4(SYS_epoll_ctl, usize(epoll_fd), usize(op), usize(fd), @ptrToInt(ev)); } diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index 6d28b98c9..c4149d2f5 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -133,6 +133,7 @@ fn early_seq(c: u8) void { early_points[early_seq_index] = c; early_seq_index += 1; } +<<<<<<< HEAD test "coro allocation failure" { var failing_allocator = std.debug.FailingAllocator.init(std.debug.global_allocator, 0); @@ -224,3 +225,17 @@ async fn printTrace(p: promise->error!void) void { } }; } + +test "coroutine in a struct field" { + const Foo = struct { + bar: async fn() void, + }; + var foo = Foo { + .bar = simpleAsyncFn2, + }; + cancel try async foo.bar(); +} + +async fn simpleAsyncFn2() void { + suspend; +} From b85ef656caa4d6845996d60a091332e9113d17e2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 8 Mar 2018 10:07:21 -0500 Subject: [PATCH 099/132] running into the llvm corosplit error again --- src/ir.cpp | 11 +++++++++++ src/ir_print.cpp | 6 ++++++ std/event.zig | 8 ++++---- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/ir.cpp b/src/ir.cpp index a80318357..5fb4a61ef 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2647,6 +2647,17 @@ static IrInstruction *ir_build_coro_alloc_helper(IrBuilder *irb, Scope *scope, A return &instruction->base; } +static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *scope, AstNode *source_node, + IrInstruction *value) +{ + IrInstructionAddImplicitReturnType *instruction = ir_build_instruction(irb, scope, source_node); + instruction->value = value; + + ir_ref_instruction(value, irb->current_basic_block); + + return &instruction->base; +} + static IrInstruction *ir_build_atomic_rmw(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *operand_type, IrInstruction *ptr, IrInstruction *op, IrInstruction *operand, IrInstruction *ordering, AtomicRmwOp resolved_op, AtomicOrder resolved_ordering) diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 99f79ff75..33fa3ce13 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1146,6 +1146,12 @@ static void ir_print_coro_alloc_helper(IrPrint *irp, IrInstructionCoroAllocHelpe fprintf(irp->f, ")"); } +static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImplicitReturnType *instruction) { + fprintf(irp->f, "@addImplicitReturnType("); + ir_print_other_instruction(irp, instruction->value); + fprintf(irp->f, ")"); +} + static void ir_print_atomic_rmw(IrPrint *irp, IrInstructionAtomicRmw *instruction) { fprintf(irp->f, "@atomicRmw("); if (instruction->operand_type != nullptr) { diff --git a/std/event.zig b/std/event.zig index 07fc64293..7ede8e20d 100644 --- a/std/event.zig +++ b/std/event.zig @@ -168,10 +168,10 @@ test "listen on a port, send bytes, receive bytes" { var socket = *_socket; // TODO https://github.com/zig-lang/zig/issues/733 defer socket.close(); const next_handler = async errorableHandler(self, _addr, socket) catch |err| switch (err) { - error.OutOfMemory => return, + error.OutOfMemory => @panic("unable to handle connection: out of memory"), }; - (await next_handler) catch |err| switch (err) { - + (await next_handler) catch |err| { + std.debug.panic("unable to handle connection: {}\n", err); }; suspend |p| { cancel p; } } @@ -184,7 +184,7 @@ test "listen on a port, send bytes, receive bytes" { var adapter = std.io.FileOutStream.init(&socket); var stream = &adapter.stream; - try stream.print("hello from server\n") catch unreachable; + try stream.print("hello from server\n"); } }; From 8f4ad95777bb0fd14c095bf626dfc2928a7dddca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 8 Mar 2018 22:57:12 -0500 Subject: [PATCH 100/132] update what std tests to run --- std/index.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/index.zig b/std/index.zig index f8ec787a0..7084c5518 100644 --- a/std/index.zig +++ b/std/index.zig @@ -50,6 +50,7 @@ test "std" { _ = @import("dwarf.zig"); _ = @import("elf.zig"); _ = @import("empty.zig"); + //TODO_ = @import("event.zig"); _ = @import("fmt/index.zig"); _ = @import("hash/index.zig"); _ = @import("io.zig"); @@ -58,7 +59,6 @@ test "std" { _ = @import("mem.zig"); _ = @import("net.zig"); _ = @import("heap.zig"); - _ = @import("net.zig"); _ = @import("os/index.zig"); _ = @import("rand/index.zig"); _ = @import("sort.zig"); From acd8f6ef184c031f672fa6d6a690d831344de4b4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 18:49:20 -0400 Subject: [PATCH 101/132] fixups from rebase --- src/all_types.hpp | 1 - src/ir.cpp | 31 ++++--------------------------- src/ir_print.cpp | 6 ------ src/parser.cpp | 1 - test/cases/coroutines.zig | 1 - 5 files changed, 4 insertions(+), 36 deletions(-) diff --git a/src/all_types.hpp b/src/all_types.hpp index d434ea187..d27a5c7a1 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -2691,7 +2691,6 @@ struct IrInstructionFnProto { IrInstruction **param_types; IrInstruction *align_value; - IrInstruction *async_allocator_type_value; IrInstruction *return_type; IrInstruction *async_allocator_type_value; bool is_var_args; diff --git a/src/ir.cpp b/src/ir.cpp index 5fb4a61ef..5348e8ba1 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2153,12 +2153,12 @@ static IrInstruction *ir_build_unwrap_err_payload_from(IrBuilder *irb, IrInstruc } static IrInstruction *ir_build_fn_proto(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, IrInstruction *async_allocator_type_value, bool is_var_args) + IrInstruction **param_types, IrInstruction *align_value, IrInstruction *return_type, + IrInstruction *async_allocator_type_value, bool is_var_args) { IrInstructionFnProto *instruction = ir_build_instruction(irb, scope, source_node); instruction->param_types = param_types; instruction->align_value = align_value; - instruction->async_allocator_type_value = async_allocator_type_value; instruction->return_type = return_type; instruction->async_allocator_type_value = async_allocator_type_value; instruction->is_var_args = is_var_args; @@ -2647,17 +2647,6 @@ static IrInstruction *ir_build_coro_alloc_helper(IrBuilder *irb, Scope *scope, A return &instruction->base; } -static IrInstruction *ir_build_add_implicit_return_type(IrBuilder *irb, Scope *scope, AstNode *source_node, - IrInstruction *value) -{ - IrInstructionAddImplicitReturnType *instruction = ir_build_instruction(irb, scope, source_node); - instruction->value = value; - - ir_ref_instruction(value, irb->current_basic_block); - - return &instruction->base; -} - static IrInstruction *ir_build_atomic_rmw(IrBuilder *irb, Scope *scope, AstNode *source_node, IrInstruction *operand_type, IrInstruction *ptr, IrInstruction *op, IrInstruction *operand, IrInstruction *ordering, AtomicRmwOp resolved_op, AtomicOrder resolved_ordering) @@ -6052,13 +6041,6 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } - IrInstruction *async_allocator_type_value = nullptr; - if (node->data.fn_proto.async_allocator_type != nullptr) { - async_allocator_type_value = ir_gen_node(irb, node->data.fn_proto.async_allocator_type, parent_scope); - if (async_allocator_type_value == irb->codegen->invalid_instruction) - return irb->codegen->invalid_instruction; - } - IrInstruction *return_type; if (node->data.fn_proto.return_var_token == nullptr) { if (node->data.fn_proto.return_type == nullptr) { @@ -6079,7 +6061,8 @@ static IrInstruction *ir_gen_fn_proto(IrBuilder *irb, Scope *parent_scope, AstNo return irb->codegen->invalid_instruction; } - return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, async_allocator_type_value, is_var_args); + return ir_build_fn_proto(irb, parent_scope, node, param_types, align_value, return_type, + async_allocator_type_value, is_var_args); } static IrInstruction *ir_gen_cancel(IrBuilder *irb, Scope *parent_scope, AstNode *node) { @@ -16757,12 +16740,6 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc return ira->codegen->builtin_types.entry_invalid; } - if (instruction->async_allocator_type_value != nullptr) { - fn_type_id.async_allocator_type = ir_resolve_type(ira, instruction->async_allocator_type_value->other); - if (type_is_invalid(fn_type_id.async_allocator_type)) - return ira->codegen->builtin_types.entry_invalid; - } - IrInstruction *return_type_value = instruction->return_type->other; fn_type_id.return_type = ir_resolve_type(ira, return_type_value); if (type_is_invalid(fn_type_id.return_type)) diff --git a/src/ir_print.cpp b/src/ir_print.cpp index 33fa3ce13..99f79ff75 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -1146,12 +1146,6 @@ static void ir_print_coro_alloc_helper(IrPrint *irp, IrInstructionCoroAllocHelpe fprintf(irp->f, ")"); } -static void ir_print_add_implicit_return_type(IrPrint *irp, IrInstructionAddImplicitReturnType *instruction) { - fprintf(irp->f, "@addImplicitReturnType("); - ir_print_other_instruction(irp, instruction->value); - fprintf(irp->f, ")"); -} - static void ir_print_atomic_rmw(IrPrint *irp, IrInstructionAtomicRmw *instruction) { fprintf(irp->f, "@atomicRmw("); if (instruction->operand_type != nullptr) { diff --git a/src/parser.cpp b/src/parser.cpp index b54a17362..2bd94033c 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1037,7 +1037,6 @@ static AstNode *ast_parse_suffix_op_expr(ParseContext *pc, size_t *token_index, Token *async_token = &pc->tokens->at(*token_index); if (async_token->id == TokenIdKeywordAsync) { - size_t token_index_of_async = *token_index; *token_index += 1; AstNode *allocator_expr_node = nullptr; diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index c4149d2f5..fbd8f0860 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -133,7 +133,6 @@ fn early_seq(c: u8) void { early_points[early_seq_index] = c; early_seq_index += 1; } -<<<<<<< HEAD test "coro allocation failure" { var failing_allocator = std.debug.FailingAllocator.init(std.debug.global_allocator, 0); From cbda0fa78c37dd84821061309469c85a2281174c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 8 Apr 2018 20:08:40 -0400 Subject: [PATCH 102/132] basic tcp server working when used with netcat --- std/event.zig | 22 ++++++++++++++-------- std/net.zig | 6 ++++++ std/os/index.zig | 24 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/std/event.zig b/std/event.zig index 7ede8e20d..2ea5d0386 100644 --- a/std/event.zig +++ b/std/event.zig @@ -5,11 +5,12 @@ const mem = std.mem; const posix = std.os.posix; pub const TcpServer = struct { - handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File) void, + handleRequestFn: async<&mem.Allocator> fn (&TcpServer, &const std.net.Address, &const std.os.File) void, loop: &Loop, sockfd: i32, accept_coro: ?promise, + listen_address: std.net.Address, waiting_for_emfile_node: PromiseNode, @@ -28,18 +29,20 @@ pub const TcpServer = struct { .accept_coro = null, .handleRequestFn = undefined, .waiting_for_emfile_node = undefined, + .listen_address = undefined, }; } pub fn listen(self: &TcpServer, address: &const std.net.Address, - handleRequestFn: async(&mem.Allocator) fn (&TcpServer, &const std.net.Address, &const std.os.File)void) !void + handleRequestFn: async<&mem.Allocator> fn (&TcpServer, &const std.net.Address, &const std.os.File)void) !void { self.handleRequestFn = handleRequestFn; try std.os.posixBind(self.sockfd, &address.sockaddr); try std.os.posixListen(self.sockfd, posix.SOMAXCONN); + self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(self.sockfd)); - self.accept_coro = try async(self.loop.allocator) (TcpServer.handler)(self); // TODO #817 + self.accept_coro = try async TcpServer.handler(self); errdefer cancel ??self.accept_coro; try self.loop.addFd(self.sockfd, ??self.accept_coro); @@ -60,10 +63,7 @@ pub const TcpServer = struct { posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| { var socket = std.os.File.openHandle(accepted_fd); - // TODO #817 - _ = async(self.loop.allocator) (self.handleRequestFn)(self, accepted_addr, - socket) catch |err| switch (err) - { + _ = async self.handleRequestFn(self, accepted_addr, socket) catch |err| switch (err) { error.OutOfMemory => { socket.close(); continue; @@ -161,7 +161,7 @@ test "listen on a port, send bytes, receive bytes" { const Self = this; - async(&mem.Allocator) fn handler(tcp_server: &TcpServer, _addr: &const std.net.Address, + async<&mem.Allocator> fn handler(tcp_server: &TcpServer, _addr: &const std.net.Address, _socket: &const std.os.File) void { const self = @fieldParentPtr(Self, "tcp_server", tcp_server); @@ -198,5 +198,11 @@ test "listen on a port, send bytes, receive bytes" { defer server.tcp_server.deinit(); try server.tcp_server.listen(addr, MyServer.handler); + var stderr_file = try std.io.getStdErr(); + var stderr_stream = &std.io.FileOutStream.init(&stderr_file).stream; + try stderr_stream.print("\nlistening at "); + try server.tcp_server.listen_address.format(stderr_stream); + try stderr_stream.print("\n"); + loop.run(); } diff --git a/std/net.zig b/std/net.zig index 595baae9d..3dddffda9 100644 --- a/std/net.zig +++ b/std/net.zig @@ -35,6 +35,12 @@ pub const Address = struct { }; } + pub fn initPosix(addr: &const posix.sockaddr) Address { + return Address { + .sockaddr = *addr, + }; + } + pub fn format(self: &const Address, out_stream: var) !void { switch (self.sockaddr.in.family) { posix.AF_INET => { diff --git a/std/os/index.zig b/std/os/index.zig index 6f9db7edc..e3ab34b35 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -2229,3 +2229,27 @@ pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usiz } } } + +pub const PosixGetSockNameError = error { + /// Insufficient resources were available in the system to perform the operation. + SystemResources, + + Unexpected, +}; + +pub fn posixGetSockName(sockfd: i32) PosixGetSockNameError!posix.sockaddr { + var addr: posix.sockaddr = undefined; + var addrlen: posix.socklen_t = @sizeOf(posix.sockaddr); + const rc = posix.getsockname(sockfd, &addr, &addrlen); + const err = posix.getErrno(rc); + switch (err) { + 0 => return addr, + else => return unexpectedErrorPosix(err), + + posix.EBADF => unreachable, + posix.EFAULT => unreachable, + posix.EINVAL => unreachable, + posix.ENOTSOCK => unreachable, + posix.ENOBUFS => return PosixGetSockNameError.SystemResources, + } +} From e85a10e9f5f6b17736babd321da8dceb72ea17af Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Apr 2018 00:52:45 -0400 Subject: [PATCH 103/132] async tcp server proof of concept --- CMakeLists.txt | 1 + src/codegen.cpp | 3 + src/ir.cpp | 33 ++++++---- std/c/darwin.zig | 8 +++ std/event.zig | 45 ++++++++++--- std/index.zig | 2 +- std/net.zig | 27 +++++--- std/os/darwin.zig | 3 + std/os/index.zig | 133 +++++++++++++++++++++++++++++++++++++- std/os/linux/index.zig | 12 ++-- test/cases/coroutines.zig | 14 ---- 11 files changed, 231 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6f169d63..b5171d726 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -432,6 +432,7 @@ set(ZIG_STD_FILES "dwarf.zig" "elf.zig" "empty.zig" + "event.zig" "fmt/errol/enum3.zig" "fmt/errol/index.zig" "fmt/errol/lookup.zig" diff --git a/src/codegen.cpp b/src/codegen.cpp index be83f6834..2aca14352 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -408,6 +408,9 @@ static uint32_t get_err_ret_trace_arg_index(CodeGen *g, FnTableEntry *fn_table_e if (!g->have_err_ret_tracing) { return UINT32_MAX; } + if (fn_table_entry->type_entry->data.fn.fn_type_id.cc == CallingConventionAsync) { + return 0; + } TypeTableEntry *fn_type = fn_table_entry->type_entry; if (!fn_type_can_fail(&fn_type->data.fn.fn_type_id)) { return UINT32_MAX; diff --git a/src/ir.cpp b/src/ir.cpp index 5348e8ba1..0b072cc69 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -2755,9 +2755,10 @@ static IrInstruction *ir_mark_gen(IrInstruction *instruction) { static bool ir_gen_defers_for_block(IrBuilder *irb, Scope *inner_scope, Scope *outer_scope, bool gen_error_defers) { Scope *scope = inner_scope; + bool is_noreturn = false; while (scope != outer_scope) { if (!scope) - return false; + return is_noreturn; if (scope->id == ScopeIdDefer) { AstNode *defer_node = scope->source_node; @@ -2770,14 +2771,18 @@ static bool ir_gen_defers_for_block(IrBuilder *irb, Scope *inner_scope, Scope *o Scope *defer_expr_scope = defer_node->data.defer.expr_scope; IrInstruction *defer_expr_value = ir_gen_node(irb, defer_expr_node, defer_expr_scope); if (defer_expr_value != irb->codegen->invalid_instruction) { - ir_mark_gen(ir_build_check_statement_is_void(irb, defer_expr_scope, defer_expr_node, defer_expr_value)); + if (defer_expr_value->value.type != nullptr && defer_expr_value->value.type->id == TypeTableEntryIdUnreachable) { + is_noreturn = true; + } else { + ir_mark_gen(ir_build_check_statement_is_void(irb, defer_expr_scope, defer_expr_node, defer_expr_value)); + } } } } scope = scope->parent; } - return true; + return is_noreturn; } static void ir_set_cursor_at_end(IrBuilder *irb, IrBasicBlock *basic_block) { @@ -2936,12 +2941,13 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_mark_gen(ir_build_cond_br(irb, scope, node, is_err_val, return_block, continue_block, is_comptime)); ir_set_cursor_at_end_and_append_block(irb, return_block); - ir_gen_defers_for_block(irb, scope, outer_scope, true); - IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); - if (irb->codegen->have_err_ret_tracing && !should_inline) { - ir_build_save_err_ret_addr(irb, scope, node); + if (!ir_gen_defers_for_block(irb, scope, outer_scope, true)) { + IrInstruction *err_val = ir_build_unwrap_err_code(irb, scope, node, err_union_ptr); + if (irb->codegen->have_err_ret_tracing && !should_inline) { + ir_build_save_err_ret_addr(irb, scope, node); + } + ir_gen_async_return(irb, scope, node, err_val, false); } - ir_gen_async_return(irb, scope, node, err_val, false); ir_set_cursor_at_end_and_append_block(irb, continue_block); IrInstruction *unwrapped_ptr = ir_build_unwrap_err_payload(irb, scope, node, err_union_ptr, false); @@ -5695,7 +5701,7 @@ static IrInstruction *ir_gen_continue(IrBuilder *irb, Scope *continue_scope, Ast IrBasicBlock *dest_block = loop_scope->continue_block; ir_gen_defers_for_block(irb, continue_scope, dest_block->scope, false); - return ir_build_br(irb, continue_scope, node, dest_block, is_comptime); + return ir_mark_gen(ir_build_br(irb, continue_scope, node, dest_block, is_comptime)); } static IrInstruction *ir_gen_error_type(IrBuilder *irb, Scope *scope, AstNode *node) { @@ -6178,7 +6184,7 @@ static IrInstruction *ir_gen_await_expr(IrBuilder *irb, Scope *parent_scope, Ast ir_set_cursor_at_end_and_append_block(irb, cleanup_block); ir_gen_defers_for_block(irb, parent_scope, outer_scope, true); - ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false); + ir_mark_gen(ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); ir_set_cursor_at_end_and_append_block(irb, resume_block); IrInstruction *yes_suspend_result = ir_build_load_ptr(irb, parent_scope, node, my_result_var_ptr); @@ -6254,7 +6260,7 @@ static IrInstruction *ir_gen_suspend(IrBuilder *irb, Scope *parent_scope, AstNod ir_set_cursor_at_end_and_append_block(irb, cleanup_block); ir_gen_defers_for_block(irb, parent_scope, outer_scope, true); - ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false); + ir_mark_gen(ir_build_br(irb, parent_scope, node, irb->exec->coro_final_cleanup_block, const_bool_false)); ir_set_cursor_at_end_and_append_block(irb, resume_block); return ir_build_const_void(irb, parent_scope, node); @@ -16746,6 +16752,11 @@ static TypeTableEntry *ir_analyze_instruction_fn_proto(IrAnalyze *ira, IrInstruc return ira->codegen->builtin_types.entry_invalid; if (fn_type_id.cc == CallingConventionAsync) { + if (instruction->async_allocator_type_value == nullptr) { + ir_add_error(ira, &instruction->base, + buf_sprintf("async fn proto missing allocator type")); + return ira->codegen->builtin_types.entry_invalid; + } IrInstruction *async_allocator_type_value = instruction->async_allocator_type_value->other; fn_type_id.async_allocator_type = ir_resolve_type(ira, async_allocator_type_value); if (type_is_invalid(fn_type_id.async_allocator_type)) diff --git a/std/c/darwin.zig b/std/c/darwin.zig index aa49dfa3d..feb689cdc 100644 --- a/std/c/darwin.zig +++ b/std/c/darwin.zig @@ -55,3 +55,11 @@ pub const dirent = extern struct { d_type: u8, d_name: u8, // field address is address of first byte of name }; + +pub const sockaddr = extern struct { + sa_len: u8, + sa_family: sa_family_t, + sa_data: [14]u8, +}; + +pub const sa_family_t = u8; diff --git a/std/event.zig b/std/event.zig index 2ea5d0386..bdad7fcc1 100644 --- a/std/event.zig +++ b/std/event.zig @@ -1,4 +1,5 @@ const std = @import("index.zig"); +const builtin = @import("builtin"); const assert = std.debug.assert; const event = this; const mem = std.mem; @@ -38,7 +39,7 @@ pub const TcpServer = struct { { self.handleRequestFn = handleRequestFn; - try std.os.posixBind(self.sockfd, &address.sockaddr); + try std.os.posixBind(self.sockfd, &address.os_addr); try std.os.posixListen(self.sockfd, posix.SOMAXCONN); self.listen_address = std.net.Address.initPosix(try std.os.posixGetSockName(self.sockfd)); @@ -59,7 +60,7 @@ pub const TcpServer = struct { pub async fn handler(self: &TcpServer) void { while (true) { var accepted_addr: std.net.Address = undefined; - if (std.os.posixAccept(self.sockfd, &accepted_addr.sockaddr, + if (std.os.posixAccept(self.sockfd, &accepted_addr.os_addr, posix.SOCK_NONBLOCK | posix.SOCK_CLOEXEC)) |accepted_fd| { var socket = std.os.File.openHandle(accepted_fd); @@ -118,7 +119,7 @@ pub const Loop = struct { pub fn addFd(self: &Loop, fd: i32, prom: promise) !void { var ev = std.os.linux.epoll_event { - .events = std.os.linux.EPOLLIN|std.os.linux.EPOLLET, + .events = std.os.linux.EPOLLIN|std.os.linux.EPOLLOUT|std.os.linux.EPOLLET, .data = std.os.linux.epoll_data { .ptr = @ptrToInt(prom), }, @@ -155,7 +156,24 @@ pub const Loop = struct { } }; +pub async fn connect(loop: &Loop, _address: &const std.net.Address) !std.os.File { + var address = *_address; // TODO https://github.com/zig-lang/zig/issues/733 + + const sockfd = try std.os.posixSocket(posix.AF_INET, posix.SOCK_STREAM|posix.SOCK_CLOEXEC|posix.SOCK_NONBLOCK, posix.PROTO_tcp); + errdefer std.os.close(sockfd); + + try std.os.posixConnectAsync(sockfd, &address.os_addr); + try await try async loop.waitFd(sockfd); + try std.os.posixGetSockOptConnectError(sockfd); + + return std.os.File.openHandle(sockfd); +} + test "listen on a port, send bytes, receive bytes" { + if (builtin.os != builtin.Os.linux) { + // TODO build abstractions for other operating systems + return; + } const MyServer = struct { tcp_server: TcpServer, @@ -198,11 +216,20 @@ test "listen on a port, send bytes, receive bytes" { defer server.tcp_server.deinit(); try server.tcp_server.listen(addr, MyServer.handler); - var stderr_file = try std.io.getStdErr(); - var stderr_stream = &std.io.FileOutStream.init(&stderr_file).stream; - try stderr_stream.print("\nlistening at "); - try server.tcp_server.listen_address.format(stderr_stream); - try stderr_stream.print("\n"); - + const p = try async doAsyncTest(&loop, server.tcp_server.listen_address); + defer cancel p; loop.run(); } + +async fn doAsyncTest(loop: &Loop, address: &const std.net.Address) void { + errdefer @panic("test failure"); + + var socket_file = try await try async event.connect(loop, address); + defer socket_file.close(); + + var buf: [512]u8 = undefined; + const amt_read = try socket_file.read(buf[0..]); + const msg = buf[0..amt_read]; + assert(mem.eql(u8, msg, "hello from server\n")); + loop.stop(); +} diff --git a/std/index.zig b/std/index.zig index 7084c5518..07c4360aa 100644 --- a/std/index.zig +++ b/std/index.zig @@ -50,7 +50,7 @@ test "std" { _ = @import("dwarf.zig"); _ = @import("elf.zig"); _ = @import("empty.zig"); - //TODO_ = @import("event.zig"); + _ = @import("event.zig"); _ = @import("fmt/index.zig"); _ = @import("hash/index.zig"); _ = @import("io.zig"); diff --git a/std/net.zig b/std/net.zig index 3dddffda9..8e1b8d97b 100644 --- a/std/net.zig +++ b/std/net.zig @@ -1,15 +1,26 @@ const std = @import("index.zig"); +const builtin = @import("builtin"); const assert = std.debug.assert; const net = this; const posix = std.os.posix; const mem = std.mem; +pub const TmpWinAddr = struct { + family: u8, + data: [14]u8, +}; + +pub const OsAddress = switch (builtin.os) { + builtin.Os.windows => TmpWinAddr, + else => posix.sockaddr, +}; + pub const Address = struct { - sockaddr: posix.sockaddr, + os_addr: OsAddress, pub fn initIp4(ip4: u32, port: u16) Address { return Address { - .sockaddr = posix.sockaddr { + .os_addr = posix.sockaddr { .in = posix.sockaddr_in { .family = posix.AF_INET, .port = std.mem.endianSwapIfLe(u16, port), @@ -23,7 +34,7 @@ pub const Address = struct { pub fn initIp6(ip6: &const Ip6Addr, port: u16) Address { return Address { .family = posix.AF_INET6, - .sockaddr = posix.sockaddr { + .os_addr = posix.sockaddr { .in6 = posix.sockaddr_in6 { .family = posix.AF_INET6, .port = std.mem.endianSwapIfLe(u16, port), @@ -37,19 +48,19 @@ pub const Address = struct { pub fn initPosix(addr: &const posix.sockaddr) Address { return Address { - .sockaddr = *addr, + .os_addr = *addr, }; } pub fn format(self: &const Address, out_stream: var) !void { - switch (self.sockaddr.in.family) { + switch (self.os_addr.in.family) { posix.AF_INET => { - const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in.port); - const bytes = ([]const u8)((&self.sockaddr.in.addr)[0..1]); + const native_endian_port = std.mem.endianSwapIfLe(u16, self.os_addr.in.port); + const bytes = ([]const u8)((&self.os_addr.in.addr)[0..1]); try out_stream.print("{}.{}.{}.{}:{}", bytes[0], bytes[1], bytes[2], bytes[3], native_endian_port); }, posix.AF_INET6 => { - const native_endian_port = std.mem.endianSwapIfLe(u16, self.sockaddr.in6.port); + const native_endian_port = std.mem.endianSwapIfLe(u16, self.os_addr.in6.port); try out_stream.print("[TODO render ip6 address]:{}", native_endian_port); }, else => try out_stream.write("(unrecognized address family)"), diff --git a/std/os/darwin.zig b/std/os/darwin.zig index f8b1fbed3..40da55315 100644 --- a/std/os/darwin.zig +++ b/std/os/darwin.zig @@ -301,6 +301,9 @@ pub const timespec = c.timespec; pub const Stat = c.Stat; pub const dirent = c.dirent; +pub const sa_family_t = c.sa_family_t; +pub const sockaddr = c.sockaddr; + /// Renamed from `sigaction` to `Sigaction` to avoid conflict with the syscall. pub const Sigaction = struct { handler: extern fn(i32)void, diff --git a/std/os/index.zig b/std/os/index.zig index e3ab34b35..b6caed6f5 100644 --- a/std/os/index.zig +++ b/std/os/index.zig @@ -2217,7 +2217,7 @@ pub fn linuxEpollCtl(epfd: i32, op: u32, fd: i32, event: &linux.epoll_event) Lin pub fn linuxEpollWait(epfd: i32, events: []linux.epoll_event, timeout: i32) usize { while (true) { - const rc = posix.epoll_wait(epfd, &events[0], u32(events.len), timeout); + const rc = posix.epoll_wait(epfd, events.ptr, u32(events.len), timeout); const err = posix.getErrno(rc); switch (err) { 0 => return rc, @@ -2253,3 +2253,134 @@ pub fn posixGetSockName(sockfd: i32) PosixGetSockNameError!posix.sockaddr { posix.ENOBUFS => return PosixGetSockNameError.SystemResources, } } + +pub const PosixConnectError = error { + /// For UNIX domain sockets, which are identified by pathname: Write permission is denied on the socket + /// file, or search permission is denied for one of the directories in the path prefix. + /// or + /// The user tried to connect to a broadcast address without having the socket broadcast flag enabled or + /// the connection request failed because of a local firewall rule. + PermissionDenied, + + /// Local address is already in use. + AddressInUse, + + /// (Internet domain sockets) The socket referred to by sockfd had not previously been bound to an + /// address and, upon attempting to bind it to an ephemeral port, it was determined that all port numbers + /// in the ephemeral port range are currently in use. See the discussion of + /// /proc/sys/net/ipv4/ip_local_port_range in ip(7). + AddressNotAvailable, + + /// The passed address didn't have the correct address family in its sa_family field. + AddressFamilyNotSupported, + + /// Insufficient entries in the routing cache. + SystemResources, + + /// A connect() on a stream socket found no one listening on the remote address. + ConnectionRefused, + + /// Network is unreachable. + NetworkUnreachable, + + /// Timeout while attempting connection. The server may be too busy to accept new connections. Note + /// that for IP sockets the timeout may be very long when syncookies are enabled on the server. + ConnectionTimedOut, + + Unexpected, +}; + +pub fn posixConnect(sockfd: i32, sockaddr: &const posix.sockaddr) PosixConnectError!void { + while (true) { + const rc = posix.connect(sockfd, sockaddr, @sizeOf(posix.sockaddr)); + const err = posix.getErrno(rc); + switch (err) { + 0 => return, + else => return unexpectedErrorPosix(err), + + posix.EACCES => return PosixConnectError.PermissionDenied, + posix.EPERM => return PosixConnectError.PermissionDenied, + posix.EADDRINUSE => return PosixConnectError.AddressInUse, + posix.EADDRNOTAVAIL => return PosixConnectError.AddressNotAvailable, + posix.EAFNOSUPPORT => return PosixConnectError.AddressFamilyNotSupported, + posix.EAGAIN => return PosixConnectError.SystemResources, + posix.EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. + posix.EBADF => unreachable, // sockfd is not a valid open file descriptor. + posix.ECONNREFUSED => return PosixConnectError.ConnectionRefused, + posix.EFAULT => unreachable, // The socket structure address is outside the user's address space. + posix.EINPROGRESS => unreachable, // The socket is nonblocking and the connection cannot be completed immediately. + posix.EINTR => continue, + posix.EISCONN => unreachable, // The socket is already connected. + posix.ENETUNREACH => return PosixConnectError.NetworkUnreachable, + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + posix.EPROTOTYPE => unreachable, // The socket type does not support the requested communications protocol. + posix.ETIMEDOUT => return PosixConnectError.ConnectionTimedOut, + } + } +} + +/// Same as posixConnect except it is for blocking socket file descriptors. +/// It expects to receive EINPROGRESS. +pub fn posixConnectAsync(sockfd: i32, sockaddr: &const posix.sockaddr) PosixConnectError!void { + while (true) { + const rc = posix.connect(sockfd, sockaddr, @sizeOf(posix.sockaddr)); + const err = posix.getErrno(rc); + switch (err) { + 0, posix.EINPROGRESS => return, + else => return unexpectedErrorPosix(err), + + posix.EACCES => return PosixConnectError.PermissionDenied, + posix.EPERM => return PosixConnectError.PermissionDenied, + posix.EADDRINUSE => return PosixConnectError.AddressInUse, + posix.EADDRNOTAVAIL => return PosixConnectError.AddressNotAvailable, + posix.EAFNOSUPPORT => return PosixConnectError.AddressFamilyNotSupported, + posix.EAGAIN => return PosixConnectError.SystemResources, + posix.EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. + posix.EBADF => unreachable, // sockfd is not a valid open file descriptor. + posix.ECONNREFUSED => return PosixConnectError.ConnectionRefused, + posix.EFAULT => unreachable, // The socket structure address is outside the user's address space. + posix.EINTR => continue, + posix.EISCONN => unreachable, // The socket is already connected. + posix.ENETUNREACH => return PosixConnectError.NetworkUnreachable, + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + posix.EPROTOTYPE => unreachable, // The socket type does not support the requested communications protocol. + posix.ETIMEDOUT => return PosixConnectError.ConnectionTimedOut, + } + } +} + +pub fn posixGetSockOptConnectError(sockfd: i32) PosixConnectError!void { + var err_code: i32 = undefined; + var size: u32 = @sizeOf(i32); + const rc = posix.getsockopt(sockfd, posix.SOL_SOCKET, posix.SO_ERROR, @ptrCast(&u8, &err_code), &size); + assert(size == 4); + const err = posix.getErrno(rc); + switch (err) { + 0 => switch (err_code) { + 0 => return, + else => return unexpectedErrorPosix(err), + + posix.EACCES => return PosixConnectError.PermissionDenied, + posix.EPERM => return PosixConnectError.PermissionDenied, + posix.EADDRINUSE => return PosixConnectError.AddressInUse, + posix.EADDRNOTAVAIL => return PosixConnectError.AddressNotAvailable, + posix.EAFNOSUPPORT => return PosixConnectError.AddressFamilyNotSupported, + posix.EAGAIN => return PosixConnectError.SystemResources, + posix.EALREADY => unreachable, // The socket is nonblocking and a previous connection attempt has not yet been completed. + posix.EBADF => unreachable, // sockfd is not a valid open file descriptor. + posix.ECONNREFUSED => return PosixConnectError.ConnectionRefused, + posix.EFAULT => unreachable, // The socket structure address is outside the user's address space. + posix.EISCONN => unreachable, // The socket is already connected. + posix.ENETUNREACH => return PosixConnectError.NetworkUnreachable, + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + posix.EPROTOTYPE => unreachable, // The socket type does not support the requested communications protocol. + posix.ETIMEDOUT => return PosixConnectError.ConnectionTimedOut, + }, + else => return unexpectedErrorPosix(err), + posix.EBADF => unreachable, // The argument sockfd is not a valid file descriptor. + posix.EFAULT => unreachable, // The address pointed to by optval or optlen is not in a valid part of the process address space. + posix.EINVAL => unreachable, + posix.ENOPROTOOPT => unreachable, // The option is unknown at the level indicated. + posix.ENOTSOCK => unreachable, // The file descriptor sockfd does not refer to a socket. + } +} diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 602ff66e7..7f27fc83d 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -775,12 +775,12 @@ pub fn socket(domain: u32, socket_type: u32, protocol: u32) usize { return syscall3(SYS_socket, domain, socket_type, protocol); } -pub fn setsockopt(fd: i32, level: i32, optname: i32, optval: &const u8, optlen: socklen_t) usize { - return syscall5(SYS_setsockopt, usize(fd), usize(level), usize(optname), usize(optval), @ptrToInt(optlen)); +pub fn setsockopt(fd: i32, level: u32, optname: u32, optval: &const u8, optlen: socklen_t) usize { + return syscall5(SYS_setsockopt, usize(fd), level, optname, usize(optval), @ptrToInt(optlen)); } -pub fn getsockopt(fd: i32, level: i32, optname: i32, noalias optval: &u8, noalias optlen: &socklen_t) usize { - return syscall5(SYS_getsockopt, usize(fd), usize(level), usize(optname), @ptrToInt(optval), @ptrToInt(optlen)); +pub fn getsockopt(fd: i32, level: u32, optname: u32, noalias optval: &u8, noalias optlen: &socklen_t) usize { + return syscall5(SYS_getsockopt, usize(fd), level, optname, @ptrToInt(optval), @ptrToInt(optlen)); } pub fn sendmsg(fd: i32, msg: &const msghdr, flags: u32) usize { @@ -833,14 +833,14 @@ pub fn fstat(fd: i32, stat_buf: &Stat) usize { return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } -pub const epoll_data = extern union { +pub const epoll_data = packed union { ptr: usize, fd: i32, @"u32": u32, @"u64": u64, }; -pub const epoll_event = extern struct { +pub const epoll_event = packed struct { events: u32, data: epoll_data, }; diff --git a/test/cases/coroutines.zig b/test/cases/coroutines.zig index fbd8f0860..6d28b98c9 100644 --- a/test/cases/coroutines.zig +++ b/test/cases/coroutines.zig @@ -224,17 +224,3 @@ async fn printTrace(p: promise->error!void) void { } }; } - -test "coroutine in a struct field" { - const Foo = struct { - bar: async fn() void, - }; - var foo = Foo { - .bar = simpleAsyncFn2, - }; - cancel try async foo.bar(); -} - -async fn simpleAsyncFn2() void { - suspend; -} From e260c8ca632fb2569f99d182dd6d1daea2b6df63 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:11:18 +0200 Subject: [PATCH 104/132] std.zig.parser now parses while loops and labeled break and continue --- std/zig/ast.zig | 62 ++++-- std/zig/parser.zig | 541 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 468 insertions(+), 135 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 0d060eb68..22b14fe36 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { ParamDecl, Block, Payload, + Else, Switch, SwitchCase, SwitchElse, @@ -61,6 +62,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), @@ -102,6 +104,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), @@ -143,6 +146,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), @@ -578,6 +582,35 @@ pub const NodePayload = struct { } }; +pub const NodeElse = struct { + base: Node, + else_token: Token, + payload: ?&NodePayload, + body: &Node, + + pub fn iterate(self: &NodeElse, index: usize) ?&Node { + var i = index; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeElse) Token { + return self.else_token; + } + + pub fn lastToken(self: &NodeElse) Token { + return self.body.lastToken(); + } +}; + pub const NodeSwitch = struct { base: Node, switch_token: Token, @@ -609,7 +642,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&Node, + payload: ?&NodePayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -657,18 +690,14 @@ pub const NodeSwitchElse = struct { pub const NodeWhile = struct { base: Node, + label: ?Token, + inline_token: ?Token, while_token: Token, condition: &Node, payload: ?&NodePayload, continue_expr: ?&Node, body: &Node, - @"else": ?Else, - - const Else = struct { - capture: ?&NodeIdentifier, - body: &Node, - }; - + @"else": ?&NodeElse, pub fn iterate(self: &NodeWhile, index: usize) ?&Node { var i = index; @@ -690,12 +719,7 @@ pub const NodeWhile = struct { i -= 1; if (self.@"else") |@"else"| { - if (@"else".capture) |capture| { - if (i < 1) return &capture.base; - i -= 1; - } - - if (i < 1) return @"else".body; + if (i < 1) return &@"else".base; i -= 1; } @@ -703,6 +727,14 @@ pub const NodeWhile = struct { } pub fn firstToken(self: &NodeWhile) Token { + if (self.label) |label| { + return label; + } + + if (self.inline_token) |inline_token| { + return inline_token; + } + return self.while_token; } @@ -749,7 +781,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&Node, + Catch: ?&NodePayload, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0d8b8a324..7c8fbb4e5 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -116,6 +116,24 @@ pub const Parser = struct { }; } + const LabelCtx = struct { + label: ?Token, + dest_ptr: DestPtr, + }; + + const InlineCtx = struct { + label: ?Token, + inline_token: ?Token, + dest_ptr: DestPtr, + }; + + const LoopCtx = struct { + label: ?Token, + inline_token: ?Token, + loop_token: Token, + dest_ptr: DestPtr, + }; + const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, @@ -137,15 +155,22 @@ pub const Parser = struct { ParamDecl: &ast.NodeFnProto, ParamDeclComma, FnDef: &ast.NodeFnProto, + LabeledExpression: LabelCtx, + Inline: InlineCtx, + While: LoopCtx, + For: LoopCtx, Block: &ast.NodeBlock, + Else: &?&ast.NodeElse, + WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, + Semicolon: &const &const ast.Node, ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - Payload: DestPtr, + Payload: &?&ast.NodePayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -157,9 +182,6 @@ pub const Parser = struct { /// "leaked" nodes. TODO: Figure out if it's nessesary to handle leaked nodes. Optional: RevertState, - /// Optional can be reverted by adding the Required state to the stack. - Required, - Expression: DestPtr, RangeExpressionBegin: DestPtr, RangeExpressionEnd: DestPtr, @@ -598,8 +620,7 @@ pub const Parser = struct { continue; }, - State.Optional, - State.Required => { }, + State.Optional => { }, State.Expression => |dest_ptr| { const token = self.getNextToken(); @@ -626,10 +647,51 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_break => { - @panic("TODO: break"); + const label = blk: { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + break :blk null; + } + + break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + }; + + const node = try self.createControlFlowExpr(arena, token, + ast.NodeControlFlowExpression.Kind { + .Break = label, + } + ); + dest_ptr.store(&node.base); + + stack.append(State { + .Optional = RevertState { + .parser = *self, + .tokenizer = *self.tokenizer, + .ptr = &node.rhs, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = &node.rhs } }); + continue; }, Token.Id.Keyword_continue => { - @panic("TODO: break"); + const label = blk: { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + break :blk null; + } + + break :blk (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + }; + + const node = try self.createControlFlowExpr(arena, token, + ast.NodeControlFlowExpression.Kind { + .Continue = label, + } + ); + dest_ptr.store(&node.base); + continue; }, Token.Id.Keyword_cancel => { @panic("TODO: cancel"); @@ -707,14 +769,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { - .Optional = RevertState { - .tokenizer = *self.tokenizer, - .parser = *self, - .ptr = &node.op.Catch, - } - }); - try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } }); + try stack.append(State { .Payload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1370,16 +1425,12 @@ pub const Parser = struct { continue; } - const block = try self.createBlock(arena, (?Token)(token), Token(undefined)); - dest_ptr.store(&block.base); - - stack.append(State { .Block = block }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LBrace, - .ptr = &block.lbrace, + stack.append(State { + .LabeledExpression = LabelCtx { + .label = token, + .dest_ptr = dest_ptr } - }); + }) catch unreachable; continue; }, Token.Id.LBrace => { @@ -1398,26 +1449,31 @@ pub const Parser = struct { Token.Id.Keyword_if => { @panic("TODO: inline if"); }, + Token.Id.Keyword_inline => { + stack.append(State { + .Inline = InlineCtx { + .label = null, + .inline_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, Token.Id.Keyword_while => { - const node = try arena.create(ast.NodeWhile); - *node = ast.NodeWhile { - .base = self.initNode(ast.Node.Id.While), - .while_token = token, - .condition = undefined, - .payload = null, - .continue_expr = null, - .body = undefined, - .@"else" = null, - }; - dest_ptr.store(&node.base); - - @panic("TODO: inline while"); + stack.append(State { + .While = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; }, Token.Id.Keyword_for => { @panic("TODO: inline for"); }, Token.Id.Keyword_switch => { - @breakpoint(); const node = try arena.create(ast.NodeSwitch); *node = ast.NodeSwitch { .base = self.initNode(ast.Node.Id.Switch), @@ -1558,15 +1614,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { - .Optional = RevertState { - .tokenizer = *self.tokenizer, - .parser = *self, - .ptr = &node.payload, - } - }); - try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } }); - try stack.append(State.Required); + try stack.append(State { .Payload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1585,32 +1633,6 @@ pub const Parser = struct { } }, - State.Payload => |dest_ptr| { - const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - - const is_ptr = blk: { - const asterik = self.getNextToken(); - if (asterik.id == Token.Id.Asterisk) { - break :blk true; - } else { - self.putBackToken(asterik); - break :blk false; - } - }; - - const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; - const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePayload); - *node = ast.NodePayload { - .base = self.initNode(ast.Node.Id.Payload), - .lpipe = lpipe, - .is_ptr = is_ptr, - .symbol = try self.createIdentifier(arena, ident), - .rpipe = rpipe - }; - dest_ptr.store(&node.base); - }, - State.SwitchCaseItem => |case_items| { stack.append(State { .SwitchCaseItemCommaOrEnd = case_items }) catch unreachable; try stack.append(State { .RangeExpressionBegin = DestPtr{ .Field = try case_items.addOne() } }); @@ -1642,6 +1664,68 @@ pub const Parser = struct { continue; }, + State.Else => |dest| { + const else_token = self.getNextToken(); + if (else_token.id != Token.Id.Keyword_else) { + self.putBackToken(else_token); + continue; + } + + const node = try arena.create(ast.NodeElse); + *node = ast.NodeElse { + .base = self.initNode(ast.Node.Id.Else), + .else_token = else_token, + .payload = null, + .body = undefined, + }; + *dest = node; + + stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + }, + + State.WhileContinueExpr => |dest| { + const colon = self.getNextToken(); + if (colon.id != Token.Id.Colon) { + self.putBackToken(colon); + continue; + } + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); + }, + + State.Payload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), + .lpipe = lpipe, + .is_ptr = is_ptr, + .symbol = try self.createIdentifier(arena, ident), + .rpipe = rpipe + }; + *dest = node; + }, + State.AddrOfModifiers => |addr_of_info| { var token = self.getNextToken(); switch (token.id) { @@ -1803,6 +1887,114 @@ pub const Parser = struct { } }, + State.LabeledExpression => |ctx| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.LBrace => { + const block = try self.createBlock(arena, (?Token)(ctx.label), token); + ctx.dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; + }, + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .label = ctx.label, + .inline_token = null, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .label = ctx.label, + .inline_token = null, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_inline => { + stack.append(State { + .Inline = InlineCtx { + .label = ctx.label, + .inline_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + else => { + try self.parseError(&stack, token, "expected 'while', 'for', 'inline' or '{{', found {}", @tagName(token.id)); + continue; + }, + } + }, + + State.Inline => |ctx| { + const token = self.getNextToken(); + switch (token.id) { + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .inline_token = ctx.inline_token, + .label = ctx.label, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .inline_token = ctx.inline_token, + .label = ctx.label, + .loop_token = token, + .dest_ptr = ctx.dest_ptr, + } + }) catch unreachable; + continue; + }, + else => { + try self.parseError(&stack, token, "expected 'while' or 'for', found {}", @tagName(token.id)); + continue; + }, + } + }, + + State.While => |ctx| { + const node = try arena.create(ast.NodeWhile); + *node = ast.NodeWhile { + .base = self.initNode(ast.Node.Id.While), + .label = ctx.label, + .inline_token = ctx.inline_token, + .while_token = ctx.loop_token, + .condition = undefined, + .payload = null, + .continue_expr = null, + .body = undefined, + .@"else" = null, + }; + ctx.dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .WhileContinueExpr = &node.continue_expr }); + try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + + State.For => |ctx| { + }, + State.Block => |block| { const token = self.getNextToken(); switch (token.id) { @@ -1841,28 +2033,6 @@ pub const Parser = struct { stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; }, - Token.Id.Identifier => { - const maybe_colon = self.getNextToken(); - if (maybe_colon.id != Token.Id.Colon) { - self.putBackToken(maybe_colon); - self.putBackToken(next); - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); - continue; - } - - const inner_block = try self.createBlock(arena, (?Token)(next), Token(undefined)); - try block.statements.append(&inner_block.base); - - stack.append(State { .Block = inner_block }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.LBrace, - .ptr = &inner_block.lbrace, - } - }); - continue; - }, Token.Id.LBrace => { const inner_block = try self.createBlock(arena, (?Token)(null), next); try block.statements.append(&inner_block.base); @@ -1870,22 +2040,53 @@ pub const Parser = struct { stack.append(State { .Block = inner_block }) catch unreachable; continue; }, - Token.Id.Keyword_suspend, Token.Id.Keyword_if, - Token.Id.Keyword_while, Token.Id.Keyword_for, - Token.Id.Keyword_switch => { - self.putBackToken(next); - stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable; - continue; - }, else => { self.putBackToken(next); - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }); + const statememt = try block.statements.addOne(); + stack.append(State { .Semicolon = statememt }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = statememt } }); continue; } } }, + + State.Semicolon => |node_ptr| { + const node = *node_ptr; + switch (node.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => continue, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", node); + if (while_node.@"else") |@"else"| { + stack.append(State { .Semicolon = &@"else".base }) catch unreachable; + continue; + } + + stack.append(State { .Semicolon = &while_node.body }) catch unreachable; + continue; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", node); + stack.append(State { .Semicolon = &else_node.body }) catch unreachable; + continue; + }, + else => { + _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; + } + } + } } } } @@ -2295,9 +2496,6 @@ pub const Parser = struct { *revert.ptr = null; return; }, - State.Required => { - return error.StateRequired; - }, else => { } } } @@ -2361,6 +2559,7 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, + Semicolon: &ast.Node, FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, @@ -2589,7 +2788,7 @@ pub const Parser = struct { if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) { if (prefix_op_node.op.Catch) |payload| { try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = payload }); + try stack.append(RenderState { .Expression = &payload.base }); } try stack.append(RenderState { .Text = " catch " }); } else { @@ -3013,7 +3212,7 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = switch_case.expr }); if (switch_case.payload) |payload| { try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = payload }); + try stack.append(RenderState { .Expression = &payload.base }); } try stack.append(RenderState { .Text = " => "}); @@ -3032,7 +3231,74 @@ pub const Parser = struct { const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token)); }, - ast.Node.Id.While => @panic("TODO: Render while"), + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); + + if (else_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = else_node.body }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = else_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (else_node.payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &payload.base }); + } + }, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); + if (while_node.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + + if (while_node.inline_token) |inline_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(inline_token)); + } + + try stream.print("{} ", self.tokenizer.getTokenSlice(while_node.while_token)); + + if (while_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (while_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Text = "\n" }); + } + } + + if (while_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = while_node.body }); + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = while_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (while_node.continue_expr) |continue_expr| { + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = continue_expr }); + try stack.append(RenderState { .Text = ": (" }); + try stack.append(RenderState { .Text = " " }); + } + + if (while_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = while_node.condition }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -3077,13 +3343,45 @@ pub const Parser = struct { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base); try stack.append(RenderState { .VarDecl = var_decl}); }, - ast.Node.Id.Block, ast.Node.Id.Switch => { - try stack.append(RenderState { .Expression = base}); + else => { + try stack.append(RenderState { .Semicolon = base }); + try stack.append(RenderState { .Expression = base }); + }, + } + }, + RenderState.Semicolon => |base| { + switch (base.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => {}, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); + if (while_node.@"else") |@"else"| { + stack.append(RenderState { .Semicolon = &@"else".base }) catch unreachable; + continue; + } + + stack.append(RenderState { .Semicolon = while_node.body }) catch unreachable; + continue; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", base); + stack.append(RenderState { .Semicolon = else_node.body }) catch unreachable; + continue; }, else => { - try stack.append(RenderState { .Text = ";"}); - try stack.append(RenderState { .Expression = base}); - }, + try stack.append(RenderState { .Text = ";" }); + } } }, RenderState.Indent => |new_indent| indent = new_indent, @@ -3690,8 +3988,11 @@ test "zig fmt: while" { \\ continue; \\ \\ i = 0; - \\ var j usize = 0; - \\ while (i < 10) : ({ i += 1; j += 1; }) { + \\ var j: usize = 0; + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) { \\ continue; \\ } \\ @@ -3711,7 +4012,7 @@ test "zig fmt: while" { \\ break 7; \\ } else { \\ unreachable; - \\ } + \\ }; \\ \\ var a: error!u8 = 0; \\ while (a) |v| { @@ -3721,7 +4022,7 @@ test "zig fmt: while" { \\ } \\ \\ comptime var k: usize = 0; - \\ inline while (i < 10) (i += 1) + \\ inline while (i < 10) : (i += 1) \\ j += 2; \\} \\ From e24409ebe0f50be9e01810a5f61bb4c09db57d28 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:17:57 +0200 Subject: [PATCH 105/132] std.zig.parser unified code for rendering and parsing semicolon in statements --- std/zig/parser.zig | 109 +++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 69 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7c8fbb4e5..0fd757a01 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2053,44 +2053,49 @@ pub const Parser = struct { State.Semicolon => |node_ptr| { const node = *node_ptr; - switch (node.id) { - ast.Node.Id.Root, - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag, - ast.Node.Id.ParamDecl, - ast.Node.Id.Block, - ast.Node.Id.Payload, - ast.Node.Id.Switch, - ast.Node.Id.SwitchCase, - ast.Node.Id.SwitchElse, - ast.Node.Id.FieldInitializer, - ast.Node.Id.LineComment, - ast.Node.Id.TestDecl => continue, - ast.Node.Id.While => { - const while_node = @fieldParentPtr(ast.NodeWhile, "base", node); - if (while_node.@"else") |@"else"| { - stack.append(State { .Semicolon = &@"else".base }) catch unreachable; - continue; - } - - stack.append(State { .Semicolon = &while_node.body }) catch unreachable; - continue; - }, - ast.Node.Id.Else => { - const else_node = @fieldParentPtr(ast.NodeElse, "base", node); - stack.append(State { .Semicolon = &else_node.body }) catch unreachable; - continue; - }, - else => { - _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; - } + if (requireSemiColon(node)) { + _ = (try self.eatToken(&stack, Token.Id.Semicolon)) ?? continue; } } } } } + fn requireSemiColon(node: &const ast.Node) bool { + var n = node; + while (true) { + switch (n.id) { + ast.Node.Id.Root, + ast.Node.Id.StructField, + ast.Node.Id.UnionTag, + ast.Node.Id.EnumTag, + ast.Node.Id.ParamDecl, + ast.Node.Id.Block, + ast.Node.Id.Payload, + ast.Node.Id.Switch, + ast.Node.Id.SwitchCase, + ast.Node.Id.SwitchElse, + ast.Node.Id.FieldInitializer, + ast.Node.Id.LineComment, + ast.Node.Id.TestDecl => return false, + ast.Node.Id.While => { + const while_node = @fieldParentPtr(ast.NodeWhile, "base", n); + if (while_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = while_node.body; + }, + ast.Node.Id.Else => { + const else_node = @fieldParentPtr(ast.NodeElse, "base", n); + n = else_node.body; + }, + else => return true, + } + } + } + fn commaOrEnd(self: &Parser, stack: &ArrayList(State), end: &const Token.Id, maybe_ptr: ?&Token, state_after_comma: &const State) !void { var token = self.getNextToken(); switch (token.id) { @@ -2559,7 +2564,6 @@ pub const Parser = struct { Expression: &ast.Node, VarDecl: &ast.NodeVarDecl, Statement: &ast.Node, - Semicolon: &ast.Node, FieldInitializer: &ast.NodeFieldInitializer, PrintIndent, Indent: usize, @@ -3344,46 +3348,13 @@ pub const Parser = struct { try stack.append(RenderState { .VarDecl = var_decl}); }, else => { - try stack.append(RenderState { .Semicolon = base }); + if (requireSemiColon(base)) { + try stack.append(RenderState { .Text = ";" }); + } try stack.append(RenderState { .Expression = base }); }, } }, - RenderState.Semicolon => |base| { - switch (base.id) { - ast.Node.Id.Root, - ast.Node.Id.StructField, - ast.Node.Id.UnionTag, - ast.Node.Id.EnumTag, - ast.Node.Id.ParamDecl, - ast.Node.Id.Block, - ast.Node.Id.Payload, - ast.Node.Id.Switch, - ast.Node.Id.SwitchCase, - ast.Node.Id.SwitchElse, - ast.Node.Id.FieldInitializer, - ast.Node.Id.LineComment, - ast.Node.Id.TestDecl => {}, - ast.Node.Id.While => { - const while_node = @fieldParentPtr(ast.NodeWhile, "base", base); - if (while_node.@"else") |@"else"| { - stack.append(RenderState { .Semicolon = &@"else".base }) catch unreachable; - continue; - } - - stack.append(RenderState { .Semicolon = while_node.body }) catch unreachable; - continue; - }, - ast.Node.Id.Else => { - const else_node = @fieldParentPtr(ast.NodeElse, "base", base); - stack.append(RenderState { .Semicolon = else_node.body }) catch unreachable; - continue; - }, - else => { - try stack.append(RenderState { .Text = ";" }); - } - } - }, RenderState.Indent => |new_indent| indent = new_indent, RenderState.PrintIndent => try stream.writeByteNTimes(' ', indent), } From 7dd55a8007c540415d3e704e490609dd86bea924 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 11:48:25 +0200 Subject: [PATCH 106/132] std.zig.parser now parses for loops --- std/zig/ast.zig | 150 +++++++++++++++++++++++++++++---- std/zig/parser.zig | 202 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 319 insertions(+), 33 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 22b14fe36..c97e33ba1 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,12 +20,15 @@ pub const Node = struct { FnProto, ParamDecl, Block, - Payload, + ErrorPayload, + ValuePayload, + ValueIndexPayload, Else, Switch, SwitchCase, SwitchElse, While, + For, InfixOp, PrefixOp, SuffixOp, @@ -61,12 +64,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), + Id.For => @fieldParentPtr(NodeFor, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -103,12 +109,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), + Id.For => @fieldParentPtr(NodeFor, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -145,12 +154,15 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), - Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), + Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), + Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), + Id.For => @fieldParentPtr(NodeFor, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -557,27 +569,82 @@ pub const NodeBlock = struct { } }; -pub const NodePayload = struct { +pub const NodeErrorPayload = struct { base: Node, lpipe: Token, - is_ptr: bool, - symbol: &NodeIdentifier, + error_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodePayload, index: usize) ?&Node { + pub fn iterate(self: &NodeErrorPayload, index: usize) ?&Node { var i = index; - if (i < 1) return &self.symbol.base; + if (i < 1) return &self.error_symbol.base; i -= 1; return null; } - pub fn firstToken(self: &NodePayload) Token { + pub fn firstToken(self: &NodeErrorPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodePayload) Token { + pub fn lastToken(self: &NodeErrorPayload) Token { + return self.rpipe; + } +}; + +pub const NodeValuePayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + value_symbol: &NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodeValuePayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.value_symbol.base; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeValuePayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodeValuePayload) Token { + return self.rpipe; + } +}; + +pub const NodeValueIndexPayload = struct { + base: Node, + lpipe: Token, + is_ptr: bool, + value_symbol: &NodeIdentifier, + index_symbol: ?&NodeIdentifier, + rpipe: Token, + + pub fn iterate(self: &NodeValueIndexPayload, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.value_symbol.base; + i -= 1; + + if (self.index_symbol) |index_symbol| { + if (i < 1) return &index_symbol.base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeValueIndexPayload) Token { + return self.lpipe; + } + + pub fn lastToken(self: &NodeValueIndexPayload) Token { return self.rpipe; } }; @@ -585,7 +652,7 @@ pub const NodePayload = struct { pub const NodeElse = struct { base: Node, else_token: Token, - payload: ?&NodePayload, + payload: ?&NodeErrorPayload, body: &Node, pub fn iterate(self: &NodeElse, index: usize) ?&Node { @@ -642,7 +709,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&NodePayload, + payload: ?&NodeValuePayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -694,7 +761,7 @@ pub const NodeWhile = struct { inline_token: ?Token, while_token: Token, condition: &Node, - payload: ?&NodePayload, + payload: ?&NodeValuePayload, continue_expr: ?&Node, body: &Node, @"else": ?&NodeElse, @@ -747,6 +814,59 @@ pub const NodeWhile = struct { } }; +pub const NodeFor = struct { + base: Node, + label: ?Token, + inline_token: ?Token, + for_token: Token, + array_expr: &Node, + payload: ?&NodeValueIndexPayload, + body: &Node, + @"else": ?&NodeElse, + + pub fn iterate(self: &NodeFor, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.array_expr; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (i < 1) return &@"else".base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeFor) Token { + if (self.label) |label| { + return label; + } + + if (self.inline_token) |inline_token| { + return inline_token; + } + + return self.for_token; + } + + pub fn lastToken(self: &NodeFor) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, @@ -781,7 +901,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodePayload, + Catch: ?&NodeErrorPayload, Div, EqualEqual, ErrorUnion, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0fd757a01..2b2afaeab 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -170,7 +170,9 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - Payload: &?&ast.NodePayload, + ErrorPayload: &?&ast.NodeErrorPayload, + ValuePayload: &?&ast.NodeValuePayload, + ValueIndexPayload: &?&ast.NodeValueIndexPayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -769,7 +771,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { .Payload = &node.op.Catch }); + try stack.append(State { .ErrorPayload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1471,7 +1473,15 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_for => { - @panic("TODO: inline for"); + stack.append(State { + .For = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; }, Token.Id.Keyword_switch => { const node = try arena.create(ast.NodeSwitch); @@ -1614,7 +1624,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ValuePayload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1681,7 +1691,7 @@ pub const Parser = struct { *dest = node; stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ErrorPayload = &node.payload }); }, State.WhileContinueExpr => |dest| { @@ -1696,7 +1706,26 @@ pub const Parser = struct { try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); }, - State.Payload => |dest| { + State.ErrorPayload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodeErrorPayload); + *node = ast.NodeErrorPayload { + .base = self.initNode(ast.Node.Id.ErrorPayload), + .lpipe = lpipe, + .error_symbol = try self.createIdentifier(arena, error_symbol), + .rpipe = rpipe + }; + *dest = node; + }, + + State.ValuePayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1713,14 +1742,56 @@ pub const Parser = struct { } }; - const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodePayload); - *node = ast.NodePayload { - .base = self.initNode(ast.Node.Id.Payload), + const node = try arena.create(ast.NodeValuePayload); + *node = ast.NodeValuePayload { + .base = self.initNode(ast.Node.Id.ValuePayload), .lpipe = lpipe, .is_ptr = is_ptr, - .symbol = try self.createIdentifier(arena, ident), + .value_symbol = try self.createIdentifier(arena, value_symbol), + .rpipe = rpipe + }; + *dest = node; + }, + + State.ValueIndexPayload => |dest| { + const lpipe = self.getNextToken(); + if (lpipe.id != Token.Id.Pipe) { + self.putBackToken(lpipe); + continue; + } + + const is_ptr = blk: { + const asterik = self.getNextToken(); + if (asterik.id == Token.Id.Asterisk) { + break :blk true; + } else { + self.putBackToken(asterik); + break :blk false; + } + }; + + const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + const index_symbol = blk: { + const comma = self.getNextToken(); + if (comma.id != Token.Id.Comma) { + self.putBackToken(comma); + break :blk null; + } + + const symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + break :blk try self.createIdentifier(arena, symbol); + }; + + const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; + const node = try arena.create(ast.NodeValueIndexPayload); + *node = ast.NodeValueIndexPayload { + .base = self.initNode(ast.Node.Id.ValueIndexPayload), + .lpipe = lpipe, + .is_ptr = is_ptr, + .value_symbol = try self.createIdentifier(arena, value_symbol), + .index_symbol = index_symbol, .rpipe = rpipe }; *dest = node; @@ -1986,13 +2057,32 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .WhileContinueExpr = &node.continue_expr }); - try stack.append(State { .Payload = &node.payload }); + try stack.append(State { .ValuePayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); }, State.For => |ctx| { + const node = try arena.create(ast.NodeFor); + *node = ast.NodeFor { + .base = self.initNode(ast.Node.Id.For), + .label = ctx.label, + .inline_token = ctx.inline_token, + .for_token = ctx.loop_token, + .array_expr = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + ctx.dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .ValueIndexPayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.array_expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); }, State.Block => |block| { @@ -2071,7 +2161,9 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.ParamDecl, ast.Node.Id.Block, - ast.Node.Id.Payload, + ast.Node.Id.ErrorPayload, + ast.Node.Id.ValuePayload, + ast.Node.Id.ValueIndexPayload, ast.Node.Id.Switch, ast.Node.Id.SwitchCase, ast.Node.Id.SwitchElse, @@ -2087,6 +2179,15 @@ pub const Parser = struct { n = while_node.body; }, + ast.Node.Id.For => { + const for_node = @fieldParentPtr(ast.NodeFor, "base", n); + if (for_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = for_node.body; + }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; @@ -2982,10 +3083,33 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, - ast.Node.Id.Payload => { - const payload = @fieldParentPtr(ast.NodePayload, "base", base); + ast.Node.Id.ErrorPayload => { + const payload = @fieldParentPtr(ast.NodeErrorPayload, "base", base); try stack.append(RenderState { .Text = "|"}); - try stack.append(RenderState { .Expression = &payload.symbol.base }); + try stack.append(RenderState { .Expression = &payload.error_symbol.base }); + try stack.append(RenderState { .Text = "|"}); + }, + ast.Node.Id.ValuePayload => { + const payload = @fieldParentPtr(ast.NodeValuePayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + try stack.append(RenderState { .Expression = &payload.value_symbol.base }); + + if (payload.is_ptr) { + try stack.append(RenderState { .Text = "*"}); + } + + try stack.append(RenderState { .Text = "|"}); + }, + ast.Node.Id.ValueIndexPayload => { + const payload = @fieldParentPtr(ast.NodeValueIndexPayload, "base", base); + try stack.append(RenderState { .Text = "|"}); + + if (payload.index_symbol) |index_symbol| { + try stack.append(RenderState { .Expression = &index_symbol.base }); + try stack.append(RenderState { .Text = ", "}); + } + + try stack.append(RenderState { .Expression = &payload.value_symbol.base }); if (payload.is_ptr) { try stack.append(RenderState { .Text = "*"}); @@ -3303,6 +3427,48 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = while_node.condition }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.For => { + const for_node = @fieldParentPtr(ast.NodeFor, "base", base); + if (for_node.label) |label| { + try stream.print("{}: ", self.tokenizer.getTokenSlice(label)); + } + + if (for_node.inline_token) |inline_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(inline_token)); + } + + try stream.print("{} ", self.tokenizer.getTokenSlice(for_node.for_token)); + + if (for_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (for_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Text = "\n" }); + } + } + + if (for_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Expression = for_node.body }); + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = for_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } + + if (for_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = for_node.array_expr }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4022,10 +4188,10 @@ test "zig fmt: for" { \\ continue; \\ \\ const res = for (a) |v, i| { - \\ breal v; + \\ break v; \\ } else { \\ unreachable; - \\ } + \\ }; \\ \\ var num: usize = 0; \\ inline for (a) |v, i| { From c19f5a2356b441e3dde95e85e029fec6ef4f836d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 12:51:18 +0200 Subject: [PATCH 107/132] std.zig.parser now parses if statements --- std/zig/ast.zig | 47 +++++++++++++++++++++ std/zig/parser.zig | 101 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 137 insertions(+), 11 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index c97e33ba1..9e077a356 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -29,6 +29,7 @@ pub const Node = struct { SwitchElse, While, For, + If, InfixOp, PrefixOp, SuffixOp, @@ -73,6 +74,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index), Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index), Id.For => @fieldParentPtr(NodeFor, "base", base).iterate(index), + Id.If => @fieldParentPtr(NodeIf, "base", base).iterate(index), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), @@ -118,6 +120,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(), Id.For => @fieldParentPtr(NodeFor, "base", base).firstToken(), + Id.If => @fieldParentPtr(NodeIf, "base", base).firstToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), @@ -163,6 +166,7 @@ pub const Node = struct { Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(), Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(), Id.For => @fieldParentPtr(NodeFor, "base", base).lastToken(), + Id.If => @fieldParentPtr(NodeIf, "base", base).lastToken(), Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), @@ -867,6 +871,49 @@ pub const NodeFor = struct { } }; +pub const NodeIf = struct { + base: Node, + if_token: Token, + condition: &Node, + payload: ?&NodeValuePayload, + body: &Node, + @"else": ?&NodeElse, + + pub fn iterate(self: &NodeIf, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.condition; + i -= 1; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (i < 1) return self.body; + i -= 1; + + if (self.@"else") |@"else"| { + if (i < 1) return &@"else".base; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeIf) Token { + return self.if_token; + } + + pub fn lastToken(self: &NodeIf) Token { + if (self.@"else") |@"else"| { + return @"else".body.lastToken(); + } + + return self.body.lastToken(); + } +}; + pub const NodeInfixOp = struct { base: Node, op_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 2b2afaeab..09383a313 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1449,7 +1449,24 @@ pub const Parser = struct { @panic("TODO: inline asm"); }, Token.Id.Keyword_if => { - @panic("TODO: inline if"); + const node = try arena.create(ast.NodeIf); + *node = ast.NodeIf { + .base = self.initNode(ast.Node.Id.If), + .if_token = token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + continue; }, Token.Id.Keyword_inline => { stack.append(State { @@ -2188,6 +2205,15 @@ pub const Parser = struct { n = for_node.body; }, + ast.Node.Id.If => { + const if_node = @fieldParentPtr(ast.NodeIf, "base", n); + if (if_node.@"else") |@"else"| { + n = @"else".base; + continue; + } + + n = if_node.body; + }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; @@ -3363,14 +3389,19 @@ pub const Parser = struct { const else_node = @fieldParentPtr(ast.NodeElse, "base", base); try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); - if (else_node.body.id == ast.Node.Id.Block) { - try stack.append(RenderState { .Expression = else_node.body }); - } else { - try stack.append(RenderState { .Indent = indent }); - try stack.append(RenderState { .Expression = else_node.body }); - try stack.append(RenderState.PrintIndent); - try stack.append(RenderState { .Indent = indent + indent_delta }); - try stack.append(RenderState { .Text = "\n" }); + switch (else_node.body.id) { + ast.Node.Id.Block, ast.Node.Id.If, + ast.Node.Id.For, ast.Node.Id.While, + ast.Node.Id.Switch => { + try stack.append(RenderState { .Expression = else_node.body }); + }, + else => { + try stack.append(RenderState { .Indent = indent }); + try stack.append(RenderState { .Expression = else_node.body }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + } } if (else_node.payload) |payload| { @@ -3396,6 +3427,7 @@ pub const Parser = struct { if (while_node.body.id == ast.Node.Id.Block) { try stack.append(RenderState { .Text = " " }); } else { + try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = "\n" }); } } @@ -3445,6 +3477,7 @@ pub const Parser = struct { if (for_node.body.id == ast.Node.Id.Block) { try stack.append(RenderState { .Text = " " }); } else { + try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Text = "\n" }); } } @@ -3469,6 +3502,53 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = for_node.array_expr }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.If => { + const if_node = @fieldParentPtr(ast.NodeIf, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(if_node.if_token)); + + switch (if_node.body.id) { + ast.Node.Id.Block, ast.Node.Id.If, + ast.Node.Id.For, ast.Node.Id.While, + ast.Node.Id.Switch => { + if (if_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = &@"else".base }); + + if (if_node.body.id == ast.Node.Id.Block) { + try stack.append(RenderState { .Text = " " }); + } else { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Text = "\n" }); + } + } + }, + else => { + if (if_node.@"else") |@"else"| { + try stack.append(RenderState { .Expression = @"else".body }); + + if (@"else".payload) |payload| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &payload.base }); + } + + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(@"else".else_token) }); + try stack.append(RenderState { .Text = " " }); + } + } + } + + try stack.append(RenderState { .Expression = if_node.body }); + try stack.append(RenderState { .Text = " " }); + + if (if_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = ")" }); + try stack.append(RenderState { .Expression = if_node.condition }); + try stack.append(RenderState { .Text = "(" }); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4210,8 +4290,7 @@ test "zig fmt: if" { \\ unreachable; \\ } \\ - \\ if (10 < 0) - \\ unreachable; + \\ if (10 < 0) unreachable; \\ \\ if (10 < 0) { \\ unreachable; From d04346d2ac04c079c1722db691d1536c3d717735 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 13:07:46 +0200 Subject: [PATCH 108/132] ast.zig.parser now parses defer statements --- std/zig/ast.zig | 33 +++++++++++++++++++++++++++++++++ std/zig/parser.zig | 27 +++++++++++++++++++++++++++ std/zig/tokenizer.zig | 2 ++ 3 files changed, 62 insertions(+) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 9e077a356..29eaa56be 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -20,6 +20,7 @@ pub const Node = struct { FnProto, ParamDecl, Block, + Defer, ErrorPayload, ValuePayload, ValueIndexPayload, @@ -65,6 +66,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), @@ -111,6 +113,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), @@ -157,6 +160,7 @@ pub const Node = struct { Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), @@ -573,6 +577,35 @@ pub const NodeBlock = struct { } }; +pub const NodeDefer = struct { + base: Node, + defer_token: Token, + kind: Kind, + expr: &Node, + + const Kind = enum { + Error, + Unconditional, + }; + + pub fn iterate(self: &NodeDefer, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeDefer) Token { + return self.defer_token; + } + + pub fn lastToken(self: &NodeDefer) Token { + return self.expr.lastToken(); + } +}; + pub const NodeErrorPayload = struct { base: Node, lpipe: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 09383a313..b783af3ea 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -2140,6 +2140,24 @@ pub const Parser = struct { stack.append(State { .VarDecl = var_decl }) catch unreachable; continue; }, + Token.Id.Keyword_defer, Token.Id.Keyword_errdefer => { + const node = try arena.create(ast.NodeDefer); + *node = ast.NodeDefer { + .base = self.initNode(ast.Node.Id.Defer), + .defer_token = next, + .kind = switch (next.id) { + Token.Id.Keyword_defer => ast.NodeDefer.Kind.Unconditional, + Token.Id.Keyword_errdefer => ast.NodeDefer.Kind.Error, + else => unreachable, + }, + .expr = undefined, + }; + try block.statements.append(&node.base); + + stack.append(State { .Semicolon = &node.base }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = &node.expr } }); + continue; + }, Token.Id.LBrace => { const inner_block = try self.createBlock(arena, (?Token)(null), next); try block.statements.append(&inner_block.base); @@ -2218,6 +2236,10 @@ pub const Parser = struct { const else_node = @fieldParentPtr(ast.NodeElse, "base", n); n = else_node.body; }, + ast.Node.Id.Defer => { + const defer_node = @fieldParentPtr(ast.NodeDefer, "base", n); + n = defer_node.expr; + }, else => return true, } } @@ -2912,6 +2934,11 @@ pub const Parser = struct { } } }, + ast.Node.Id.Defer => { + const defer_node = @fieldParentPtr(ast.NodeDefer, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(defer_node.defer_token)); + try stack.append(RenderState { .Expression = defer_node.expr }); + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 422aa2062..4cce31bae 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -26,6 +26,7 @@ pub const Token = struct { KeywordId{.bytes="defer", .id = Id.Keyword_defer}, KeywordId{.bytes="else", .id = Id.Keyword_else}, KeywordId{.bytes="enum", .id = Id.Keyword_enum}, + KeywordId{.bytes="errdefer", .id = Id.Keyword_errdefer}, KeywordId{.bytes="error", .id = Id.Keyword_error}, KeywordId{.bytes="export", .id = Id.Keyword_export}, KeywordId{.bytes="extern", .id = Id.Keyword_extern}, @@ -151,6 +152,7 @@ pub const Token = struct { Keyword_defer, Keyword_else, Keyword_enum, + Keyword_errdefer, Keyword_error, Keyword_export, Keyword_extern, From 7d32c9521fff3d2a18293d38562636c5a0e3408e Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 13:24:47 +0200 Subject: [PATCH 109/132] std.zig.parser now parses comptime --- std/zig/ast.zig | 27 +++++++++++++++++++++++++++ std/zig/parser.zig | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 29eaa56be..2da819556 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -21,6 +21,7 @@ pub const Node = struct { ParamDecl, Block, Defer, + Comptime, ErrorPayload, ValuePayload, ValueIndexPayload, @@ -67,6 +68,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).iterate(index), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), @@ -114,6 +116,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).firstToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), @@ -161,6 +164,7 @@ pub const Node = struct { Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), + Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).lastToken(), Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), @@ -606,6 +610,29 @@ pub const NodeDefer = struct { } }; +pub const NodeComptime = struct { + base: Node, + comptime_token: Token, + expr: &Node, + + pub fn iterate(self: &NodeComptime, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeComptime) Token { + return self.comptime_token; + } + + pub fn lastToken(self: &NodeComptime) Token { + return self.expr.lastToken(); + } +}; + pub const NodeErrorPayload = struct { base: Node, lpipe: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index b783af3ea..aa04ad68d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -309,6 +309,18 @@ pub const Parser = struct { }); continue; }, + Token.Id.Keyword_comptime => { + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + try root_node.decls.append(&node.base); + stack.append(State.TopLevel) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, else => { self.putBackToken(token); stack.append(State.TopLevel) catch unreachable; @@ -1523,7 +1535,15 @@ pub const Parser = struct { try stack.append(State { .ExpectToken = Token.Id.LParen }); }, Token.Id.Keyword_comptime => { - @panic("TODO: inline comptime"); + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); @@ -2131,7 +2151,10 @@ pub const Parser = struct { continue; } else { self.putBackToken(mut_token); - @panic("TODO: comptime block"); + self.putBackToken(next); + const statememt = try block.statements.addOne(); + stack.append(State { .Semicolon = statememt }) catch unreachable; + try stack.append(State { .Expression = DestPtr{.Field = statememt } }); } }, Token.Id.Keyword_var, Token.Id.Keyword_const => { @@ -2240,6 +2263,10 @@ pub const Parser = struct { const defer_node = @fieldParentPtr(ast.NodeDefer, "base", n); n = defer_node.expr; }, + ast.Node.Id.Comptime => { + const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", n); + n = comptime_node.expr; + }, else => return true, } } @@ -2824,6 +2851,12 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = value}); } }, + ast.Node.Id.Comptime => { + if (requireSemiColon(decl)) { + try stack.append(RenderState { .Text = ";" }); + } + try stack.append(RenderState { .Expression = decl }); + }, else => unreachable, } }, @@ -2939,6 +2972,11 @@ pub const Parser = struct { try stream.print("{} ", self.tokenizer.getTokenSlice(defer_node.defer_token)); try stack.append(RenderState { .Expression = defer_node.expr }); }, + ast.Node.Id.Comptime => { + const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_node.comptime_token)); + try stack.append(RenderState { .Expression = comptime_node.expr }); + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); From aa552633cc0bc66d46f61ea2105f7a4392df37be Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 14:02:03 +0200 Subject: [PATCH 110/132] std.zig.parser now parses fn types --- std/zig/parser.zig | 134 +++++++++++++++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 35 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index aa04ad68d..7bdff56b8 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1413,6 +1413,17 @@ pub const Parser = struct { }) catch unreachable; }, Token.Id.Keyword_extern => { + const next = self.getNextToken(); + if (next.id == Token.Id.Keyword_fn) { + // TODO shouldn't need this cast + const fn_proto = try self.createFnProto(arena, next, + (?Token)(token), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + continue; + } + + self.putBackToken(next); stack.append(State { .ContainerExtern = ContainerExternCtx { .dest_ptr = dest_ptr, @@ -1455,7 +1466,26 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_fn => { - @panic("TODO: fn proto"); + // TODO shouldn't need these casts + const fn_proto = try self.createFnProto(arena, token, + (?Token)(null), (?&ast.Node)(null), (?Token)(null), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + continue; + }, + Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { + // TODO shouldn't need this cast + const fn_proto = try self.createFnProto(arena, undefined, + (?Token)(null), (?&ast.Node)(null), (?Token)(token), (?Token)(null), (?Token)(null)); + dest_ptr.store(&fn_proto.base); + stack.append(State { .FnProto = fn_proto }) catch unreachable; + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + continue; }, Token.Id.Keyword_asm => { @panic("TODO: inline asm"); @@ -2781,41 +2811,14 @@ pub const Parser = struct { ast.Node.Id.FnProto => { const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl); - if (fn_proto.body_node == null) { - try stack.append(RenderState { .Text = ";" }); + if (fn_proto.body_node) |body_node| { + stack.append(RenderState { .Expression = body_node}) catch unreachable; + try stack.append(RenderState { .Text = " "}); + } else { + stack.append(RenderState { .Text = ";" }) catch unreachable; } - try stack.append(RenderState { .FnProtoRParen = fn_proto}); - var i = fn_proto.params.len; - while (i != 0) { - i -= 1; - const param_decl_node = fn_proto.params.items[i]; - try stack.append(RenderState { .ParamDecl = param_decl_node}); - if (i != 0) { - try stack.append(RenderState { .Text = ", " }); - } - } - - try stack.append(RenderState { .Text = "(" }); - if (fn_proto.name_token) |name_token| { - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); - } - - try stack.append(RenderState { .Text = "fn " }); - if (fn_proto.lib_name) |lib_name| { - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Expression = lib_name }); - } - if (fn_proto.extern_token) |extern_token| { - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); - } - - if (fn_proto.visib_token) |visib_token| { - assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); - try stack.append(RenderState { .Text = " " }); - try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); - } + try stack.append(RenderState { .Expression = decl }); }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); @@ -3386,7 +3389,65 @@ pub const Parser = struct { } } }, - ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"), + ast.Node.Id.FnProto => { + const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", base); + + switch (fn_proto.return_type) { + ast.NodeFnProto.ReturnType.Explicit => |node| { + try stack.append(RenderState { .Expression = node}); + }, + ast.NodeFnProto.ReturnType.Infer => { + try stack.append(RenderState { .Text = "var"}); + }, + ast.NodeFnProto.ReturnType.InferErrorSet => |node| { + try stack.append(RenderState { .Expression = node}); + try stack.append(RenderState { .Text = "!"}); + }, + } + + if (fn_proto.align_expr != null) { + @panic("TODO"); + } + + try stack.append(RenderState { .Text = ") " }); + var i = fn_proto.params.len; + while (i != 0) { + i -= 1; + const param_decl_node = fn_proto.params.items[i]; + try stack.append(RenderState { .ParamDecl = param_decl_node}); + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + + try stack.append(RenderState { .Text = "(" }); + if (fn_proto.name_token) |name_token| { + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(name_token) }); + try stack.append(RenderState { .Text = " " }); + } + + try stack.append(RenderState { .Text = "fn" }); + + if (fn_proto.cc_token) |cc_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(cc_token) }); + } + + if (fn_proto.lib_name) |lib_name| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = lib_name }); + } + if (fn_proto.extern_token) |extern_token| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) }); + } + + if (fn_proto.visib_token) |visib_token| { + assert(visib_token.id == Token.Id.Keyword_pub or visib_token.id == Token.Id.Keyword_export); + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(visib_token) }); + } + }, ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"), ast.Node.Id.Switch => { const switch_node = @fieldParentPtr(ast.NodeSwitch, "base", base); @@ -4461,6 +4522,9 @@ test "zig fmt: fn type" { \\ return i + 1; \\} \\ + \\const a: fn(u8) u8 = undefined; + \\const b: extern fn(u8) u8 = undefined; + \\const c: nakedcc fn(u8) u8 = undefined; \\const ap: fn(u8) u8 = a; \\ ); From a09bb408a20fcfc0575a9298b318803b07517b0a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Mon, 9 Apr 2018 15:40:16 +0200 Subject: [PATCH 111/132] std.zig.parser now parses asm expressions * We cannot render asm expressions yet --- std/zig/ast.zig | 41 ++++++++++++++ std/zig/parser.zig | 138 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2da819556..d048f4ed4 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -47,6 +47,7 @@ pub const Node = struct { NullLiteral, UndefinedLiteral, ThisLiteral, + Asm, Unreachable, ErrorType, BuiltinCall, @@ -94,6 +95,7 @@ pub const Node = struct { Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).iterate(index), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), @@ -143,6 +145,7 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -190,6 +193,7 @@ pub const Node = struct { Id.NullLiteral => @fieldParentPtr(NodeNullLiteral, "base", base).lastToken(), Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), + Id.Asm => @fieldParentPtr(NodeAsm, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), @@ -1512,6 +1516,43 @@ pub const NodeThisLiteral = struct { } }; +pub const NodeAsm = struct { + base: Node, + asm_token: Token, + is_volatile: bool, + template: Token, + //tokens: ArrayList(AsmToken), + outputs: ArrayList(AsmOutput), + inputs: ArrayList(AsmInput), + cloppers: ArrayList(&NodeStringLiteral), + rparen: Token, + + const AsmOutput = struct { + symbolic_name: Token, + constraint: Token, + variable_name: ?Token, + return_type: ?&Node, + }; + + const AsmInput = struct { + symbolic_name: Token, + constraint: Token, + expr: &Node, + }; + + pub fn iterate(self: &NodeAsm, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeAsm) Token { + return self.asm_token; + } + + pub fn lastToken(self: &NodeAsm) Token { + return self.rparen; + } +}; + pub const NodeUnreachable = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7bdff56b8..7f5427146 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -164,6 +164,9 @@ pub const Parser = struct { WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, Semicolon: &const &const ast.Node, + AsmOutputItems: &ArrayList(ast.NodeAsm.AsmOutput), + AsmInputItems: &ArrayList(ast.NodeAsm.AsmInput), + AsmClopperItems: &ArrayList(&ast.NodeStringLiteral), ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer), @@ -1488,7 +1491,44 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_asm => { - @panic("TODO: inline asm"); + const is_volatile = blk: { + const volatile_token = self.getNextToken(); + if (volatile_token.id != Token.Id.Keyword_volatile) { + self.putBackToken(volatile_token); + break :blk false; + } + break :blk true; + }; + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + const template = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + // TODO parse template + + const node = try arena.create(ast.NodeAsm); + *node = ast.NodeAsm { + .base = self.initNode(ast.Node.Id.Asm), + .asm_token = token, + .is_volatile = is_volatile, + .template = template, + //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), + .outputs = ArrayList(ast.NodeAsm.AsmOutput).init(arena), + .inputs = ArrayList(ast.NodeAsm.AsmInput).init(arena), + .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), + .rparen = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.RParen, + .ptr = &node.rparen, + } + }) catch unreachable; + try stack.append(State { .AsmClopperItems = &node.cloppers }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { .AsmInputItems = &node.inputs }); + try stack.append(State { .IfToken = Token.Id.Colon }); + try stack.append(State { .AsmOutputItems = &node.outputs }); + try stack.append(State { .IfToken = Token.Id.Colon }); }, Token.Id.Keyword_if => { const node = try arena.create(ast.NodeIf); @@ -1621,6 +1661,84 @@ pub const Parser = struct { } }, + + State.AsmOutputItems => |items| { + const lbracket = self.getNextToken(); + if (lbracket.id != Token.Id.LBracket) { + self.putBackToken(lbracket); + continue; + } + + stack.append(State { .AsmOutputItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + + const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const res = try items.addOne(); + *res = ast.NodeAsm.AsmOutput { + .symbolic_name = symbolic_name, + .constraint = constraint, + .variable_name = null, + .return_type = null, + }; + const symbol_or_arrow = self.getNextToken(); + switch (symbol_or_arrow.id) { + Token.Id.Identifier => res.variable_name = symbol_or_arrow, + Token.Id.Arrow => { + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &res.return_type } }); + }, + else => { + try self.parseError(&stack, symbol_or_arrow, "expected '->' or {}, found {}", + @tagName(Token.Id.Identifier), + @tagName(symbol_or_arrow.id)); + continue; + }, + } + }, + + State.AsmInputItems => |items| { + const lbracket = self.getNextToken(); + if (lbracket.id != Token.Id.LBracket) { + self.putBackToken(lbracket); + continue; + } + + stack.append(State { .AsmInputItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + + const symbolic_name = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; + _ = (try self.eatToken(&stack, Token.Id.RBracket)) ?? continue; + const constraint = (try self.eatToken(&stack, Token.Id.StringLiteral)) ?? continue; + + _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; + try stack.append(State { .ExpectToken = Token.Id.RParen }); + + const res = try items.addOne(); + *res = ast.NodeAsm.AsmInput { + .symbolic_name = symbolic_name, + .constraint = constraint, + .expr = undefined, + }; + try stack.append(State { .Expression = DestPtr { .Field = &res.expr } }); + }, + + State.AsmClopperItems => |items| { + const string = self.getNextToken(); + if (string.id != Token.Id.StringLiteral) { + self.putBackToken(string); + continue; + } + + try items.append(try self.createStringLiteral(arena, string)); + stack.append(State { .AsmClopperItems = items }) catch unreachable; + try stack.append(State { .IfToken = Token.Id.Comma }); + }, + State.ExprListItemOrEnd => |list_state| { var token = self.getNextToken(); @@ -3675,6 +3793,24 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = if_node.condition }); try stack.append(RenderState { .Text = "(" }); }, + ast.Node.Id.Asm => { + const asm_node = @fieldParentPtr(ast.NodeAsm, "base", base); + try stream.print("{} ", self.tokenizer.getTokenSlice(asm_node.asm_token)); + + if (asm_node.is_volatile) { + try stream.write("volatile "); + } + + try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template)); + + try stack.append(RenderState { .Text = ")" }); + @panic("TODO: Render asm"); + //\\ return asm volatile ("syscall" + //\\ : [ret] "={rax}" (-> usize) + //\\ : [number] "{rax}" (number), + //\\ [arg1] "{rdi}" (arg1) + //\\ : "rcx", "r11"); + },, ast.Node.Id.StructField, ast.Node.Id.UnionTag, From 4545be360a723a4c5b141cbefcc07ac7f17f0757 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 9 Apr 2018 21:14:55 -0400 Subject: [PATCH 112/132] fix std.io.readline to work on windows closes #882 --- std/io.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/std/io.zig b/std/io.zig index 93d50e670..7b72af15e 100644 --- a/std/io.zig +++ b/std/io.zig @@ -486,6 +486,11 @@ pub fn readLine(buf: []u8) !usize { while (true) { const byte = stream.readByte() catch return error.EndOfFile; switch (byte) { + '\r' => { + // trash the following \n + _ = stream.readByte() catch return error.EndOfFile; + return index; + }, '\n' => return index, else => { if (index == buf.len) return error.InputTooLong; From 2c7996f4006b94f81fc3b1c1e3f73a4a7a291782 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 09:27:11 +0200 Subject: [PATCH 113/132] std.zig.parser can now render asm expressions --- std/zig/ast.zig | 112 +++++++++++++++++++++++++++----- std/zig/parser.zig | 156 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 228 insertions(+), 40 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index d048f4ed4..86a706f1c 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -48,6 +48,8 @@ pub const Node = struct { UndefinedLiteral, ThisLiteral, Asm, + AsmInput, + AsmOutput, Unreachable, ErrorType, BuiltinCall, @@ -96,6 +98,8 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).iterate(index), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).iterate(index), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).iterate(index), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), @@ -146,6 +150,8 @@ pub const Node = struct { Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).firstToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).firstToken(), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).firstToken(), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).firstToken(), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), @@ -194,6 +200,8 @@ pub const Node = struct { Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(), Id.ThisLiteral => @fieldParentPtr(NodeThisLiteral, "base", base).lastToken(), Id.Asm => @fieldParentPtr(NodeAsm, "base", base).lastToken(), + Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).lastToken(), + Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), @@ -1516,31 +1524,105 @@ pub const NodeThisLiteral = struct { } }; +pub const NodeAsmOutput = struct { + base: Node, + symbolic_name: &NodeIdentifier, + constraint: &NodeStringLiteral, + kind: Kind, + + const Kind = union(enum) { + Variable: &NodeIdentifier, + Return: &Node + }; + + pub fn iterate(self: &NodeAsmOutput, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbolic_name.base; + i -= 1; + + if (i < 1) return &self.constraint.base; + i -= 1; + + switch (self.kind) { + Kind.Variable => |variable_name| { + if (i < 1) return &variable_name.base; + i -= 1; + }, + Kind.Return => |return_type| { + if (i < 1) return return_type; + i -= 1; + } + } + + return null; + } + + pub fn firstToken(self: &NodeAsmOutput) Token { + return self.symbolic_name.firstToken(); + } + + pub fn lastToken(self: &NodeAsmOutput) Token { + return switch (self.kind) { + Kind.Variable => |variable_name| variable_name.lastToken(), + Kind.Return => |return_type| return_type.lastToken(), + }; + } +}; + +pub const NodeAsmInput = struct { + base: Node, + symbolic_name: &NodeIdentifier, + constraint: &NodeStringLiteral, + expr: &Node, + + pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node { + var i = index; + + if (i < 1) return &self.symbolic_name.base; + i -= 1; + + if (i < 1) return &self.constraint.base; + i -= 1; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeAsmInput) Token { + return self.symbolic_name.firstToken(); + } + + pub fn lastToken(self: &NodeAsmInput) Token { + return self.expr.lastToken(); + } +}; + pub const NodeAsm = struct { base: Node, asm_token: Token, is_volatile: bool, template: Token, //tokens: ArrayList(AsmToken), - outputs: ArrayList(AsmOutput), - inputs: ArrayList(AsmInput), + outputs: ArrayList(&NodeAsmOutput), + inputs: ArrayList(&NodeAsmInput), cloppers: ArrayList(&NodeStringLiteral), rparen: Token, - const AsmOutput = struct { - symbolic_name: Token, - constraint: Token, - variable_name: ?Token, - return_type: ?&Node, - }; - - const AsmInput = struct { - symbolic_name: Token, - constraint: Token, - expr: &Node, - }; - pub fn iterate(self: &NodeAsm, index: usize) ?&Node { + var i = index; + + if (i < self.outputs.len) return &self.outputs.at(index).base; + i -= self.outputs.len; + + if (i < self.inputs.len) return &self.inputs.at(index).base; + i -= self.inputs.len; + + if (i < self.cloppers.len) return &self.cloppers.at(index).base; + i -= self.cloppers.len; + return null; } diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 7f5427146..9851f6cc3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -164,8 +164,8 @@ pub const Parser = struct { WhileContinueExpr: &?&ast.Node, Statement: &ast.NodeBlock, Semicolon: &const &const ast.Node, - AsmOutputItems: &ArrayList(ast.NodeAsm.AsmOutput), - AsmInputItems: &ArrayList(ast.NodeAsm.AsmInput), + AsmOutputItems: &ArrayList(&ast.NodeAsmOutput), + AsmInputItems: &ArrayList(&ast.NodeAsmInput), AsmClopperItems: &ArrayList(&ast.NodeStringLiteral), ExprListItemOrEnd: ExprListCtx, ExprListCommaOrEnd: ExprListCtx, @@ -1510,8 +1510,8 @@ pub const Parser = struct { .is_volatile = is_volatile, .template = template, //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena), - .outputs = ArrayList(ast.NodeAsm.AsmOutput).init(arena), - .inputs = ArrayList(ast.NodeAsm.AsmInput).init(arena), + .outputs = ArrayList(&ast.NodeAsmOutput).init(arena), + .inputs = ArrayList(&ast.NodeAsmInput).init(arena), .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena), .rparen = undefined, }; @@ -1679,18 +1679,23 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const res = try items.addOne(); - *res = ast.NodeAsm.AsmOutput { - .symbolic_name = symbolic_name, - .constraint = constraint, - .variable_name = null, - .return_type = null, + const node = try arena.create(ast.NodeAsmOutput); + *node = ast.NodeAsmOutput { + .base = self.initNode(ast.Node.Id.AsmOutput), + .symbolic_name = try self.createIdentifier(arena, symbolic_name), + .constraint = try self.createStringLiteral(arena, constraint), + .kind = undefined, }; + try items.append(node); + const symbol_or_arrow = self.getNextToken(); switch (symbol_or_arrow.id) { - Token.Id.Identifier => res.variable_name = symbol_or_arrow, + Token.Id.Identifier => { + node.kind = ast.NodeAsmOutput.Kind { .Variable = try self.createIdentifier(arena, symbol_or_arrow) }; + }, Token.Id.Arrow => { - try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &res.return_type } }); + node.kind = ast.NodeAsmOutput.Kind { .Return = undefined }; + try stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.kind.Return } }); }, else => { try self.parseError(&stack, symbol_or_arrow, "expected '->' or {}, found {}", @@ -1718,13 +1723,15 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; try stack.append(State { .ExpectToken = Token.Id.RParen }); - const res = try items.addOne(); - *res = ast.NodeAsm.AsmInput { - .symbolic_name = symbolic_name, - .constraint = constraint, + const node = try arena.create(ast.NodeAsmInput); + *node = ast.NodeAsmInput { + .base = self.initNode(ast.Node.Id.AsmInput), + .symbolic_name = try self.createIdentifier(arena, symbolic_name), + .constraint = try self.createStringLiteral(arena, constraint), .expr = undefined, }; - try stack.append(State { .Expression = DestPtr { .Field = &res.expr } }); + try items.append(node); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); }, State.AsmClopperItems => |items| { @@ -3803,14 +3810,113 @@ pub const Parser = struct { try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template)); + try stack.append(RenderState { .Indent = indent }); try stack.append(RenderState { .Text = ")" }); - @panic("TODO: Render asm"); - //\\ return asm volatile ("syscall" - //\\ : [ret] "={rax}" (-> usize) - //\\ : [number] "{rax}" (number), - //\\ [arg1] "{rdi}" (arg1) - //\\ : "rcx", "r11"); - },, + { + const cloppers = asm_node.cloppers.toSliceConst(); + var i = cloppers.len; + while (i != 0) { + i -= 1; + try stack.append(RenderState { .Expression = &cloppers[i].base }); + + if (i != 0) { + try stack.append(RenderState { .Text = ", " }); + } + } + } + try stack.append(RenderState { .Text = ": " }); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta }); + try stack.append(RenderState { .Text = "\n" }); + { + const inputs = asm_node.inputs.toSliceConst(); + var i = inputs.len; + while (i != 0) { + i -= 1; + const node = inputs[i]; + try stack.append(RenderState { .Expression = &node.base}); + + if (i != 0) { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + const prev_node = inputs[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + try stack.append(RenderState { .Text = "," }); + } + } + } + try stack.append(RenderState { .Indent = indent + indent_delta + 2}); + try stack.append(RenderState { .Text = ": "}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "\n" }); + { + const outputs = asm_node.outputs.toSliceConst(); + var i = outputs.len; + while (i != 0) { + i -= 1; + const node = outputs[i]; + try stack.append(RenderState { .Expression = &node.base}); + + if (i != 0) { + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { + .Text = blk: { + const prev_node = outputs[i - 1]; + const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken()); + if (loc.line >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + try stack.append(RenderState { .Text = "," }); + } + } + } + try stack.append(RenderState { .Indent = indent + indent_delta + 2}); + try stack.append(RenderState { .Text = ": "}); + try stack.append(RenderState.PrintIndent); + try stack.append(RenderState { .Indent = indent + indent_delta}); + try stack.append(RenderState { .Text = "\n" }); + }, + ast.Node.Id.AsmInput => { + const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base); + + try stack.append(RenderState { .Text = ")"}); + try stack.append(RenderState { .Expression = asm_input.expr}); + try stack.append(RenderState { .Text = " ("}); + try stack.append(RenderState { .Expression = &asm_input.constraint.base}); + try stack.append(RenderState { .Text = "] "}); + try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base}); + try stack.append(RenderState { .Text = "["}); + }, + ast.Node.Id.AsmOutput => { + const asm_output = @fieldParentPtr(ast.NodeAsmOutput, "base", base); + + try stack.append(RenderState { .Text = ")"}); + switch (asm_output.kind) { + ast.NodeAsmOutput.Kind.Variable => |variable_name| { + try stack.append(RenderState { .Expression = &variable_name.base}); + }, + ast.NodeAsmOutput.Kind.Return => |return_type| { + try stack.append(RenderState { .Expression = return_type}); + try stack.append(RenderState { .Text = "-> "}); + }, + } + try stack.append(RenderState { .Text = " ("}); + try stack.append(RenderState { .Expression = &asm_output.constraint.base}); + try stack.append(RenderState { .Text = "] "}); + try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base}); + try stack.append(RenderState { .Text = "["}); + }, ast.Node.Id.StructField, ast.Node.Id.UnionTag, @@ -4672,7 +4778,7 @@ test "zig fmt: inline asm" { \\ return asm volatile ("syscall" \\ : [ret] "={rax}" (-> usize) \\ : [number] "{rax}" (number), - \\ [arg1] "{rdi}" (arg1) + \\ [arg1] "{rdi}" (arg1) \\ : "rcx", "r11"); \\} \\ From 5cd69ee6a4b7d02dbc48d93e8f8f95bd608d9d7c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 09:37:29 +0200 Subject: [PATCH 114/132] std.zig.parser changed assign expr to only be allowed in some contexts * Only allowed in while continue expr and statement expr --- std/zig/parser.zig | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9851f6cc3..ff3b35c19 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -721,7 +721,7 @@ pub const Parser = struct { }, else => { self.putBackToken(token); - stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable; + stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; continue; } } @@ -750,7 +750,7 @@ pub const Parser = struct { State.AssignmentExpressionBegin => |dest_ptr| { stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .UnwrapExpressionBegin = dest_ptr }); + try stack.append(State { .Expression = dest_ptr }); continue; }, @@ -762,7 +762,7 @@ pub const Parser = struct { dest_ptr.store(&node.base); stack.append(State { .AssignmentExpressionEnd = dest_ptr }) catch unreachable; - try stack.append(State { .UnwrapExpressionBegin = DestPtr { .Field = &node.rhs } }); + try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); continue; } else { self.putBackToken(token); @@ -1895,7 +1895,7 @@ pub const Parser = struct { _ = (try self.eatToken(&stack, Token.Id.LParen)) ?? continue; stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .NullableField = dest } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = dest } }); }, State.ErrorPayload => |dest| { @@ -2333,7 +2333,7 @@ pub const Parser = struct { try block.statements.append(&node.base); stack.append(State { .Semicolon = &node.base }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = &node.expr } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr{.Field = &node.expr } }); continue; }, Token.Id.LBrace => { @@ -2347,7 +2347,7 @@ pub const Parser = struct { self.putBackToken(next); const statememt = try block.statements.addOne(); stack.append(State { .Semicolon = statememt }) catch unreachable; - try stack.append(State { .Expression = DestPtr{.Field = statememt } }); + try stack.append(State { .AssignmentExpressionBegin = DestPtr{.Field = statememt } }); continue; } } @@ -4260,8 +4260,6 @@ test "zig fmt: precedence" { \\ a or b and c; \\ (a or b) and c; \\ (a or b) and c; - \\ a = b or c; - \\ (a = b) or c; \\} \\ ); From f85b9f2bf3ac0c5af7804921d2003c2992558cb7 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 11:25:58 +0200 Subject: [PATCH 115/132] std.zig.parser now parses coroutine code --- std/zig/ast.zig | 157 +++++++++++++++++++------ std/zig/parser.zig | 277 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 345 insertions(+), 89 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 86a706f1c..9043a66ac 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -17,14 +17,15 @@ pub const Node = struct { UnionTag, EnumTag, Identifier, + AsyncAttribute, FnProto, ParamDecl, Block, Defer, Comptime, - ErrorPayload, - ValuePayload, - ValueIndexPayload, + Payload, + PointerPayload, + PointerIndexPayload, Else, Switch, SwitchCase, @@ -37,6 +38,7 @@ pub const Node = struct { SuffixOp, GroupedExpression, ControlFlowExpression, + Suspend, FieldInitializer, IntegerLiteral, FloatLiteral, @@ -67,14 +69,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).iterate(index), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).iterate(index), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).iterate(index), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index), Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).iterate(index), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).iterate(index), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).iterate(index), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).iterate(index), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).iterate(index), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).iterate(index), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).iterate(index), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index), @@ -87,6 +90,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).iterate(index), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).iterate(index), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).iterate(index), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index), @@ -118,14 +122,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).firstToken(), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).firstToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).firstToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).firstToken(), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).firstToken(), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).firstToken(), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).firstToken(), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).firstToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).firstToken(), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).firstToken(), Id.Else => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(), @@ -138,6 +143,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).firstToken(), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).firstToken(), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).firstToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), @@ -169,14 +175,15 @@ pub const Node = struct { Id.UnionTag => @fieldParentPtr(NodeUnionTag, "base", base).lastToken(), Id.EnumTag => @fieldParentPtr(NodeEnumTag, "base", base).lastToken(), Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), + Id.AsyncAttribute => @fieldParentPtr(NodeAsyncAttribute, "base", base).lastToken(), Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), Id.Defer => @fieldParentPtr(NodeDefer, "base", base).lastToken(), Id.Comptime => @fieldParentPtr(NodeComptime, "base", base).lastToken(), - Id.ErrorPayload => @fieldParentPtr(NodeErrorPayload, "base", base).lastToken(), - Id.ValuePayload => @fieldParentPtr(NodeValuePayload, "base", base).lastToken(), - Id.ValueIndexPayload => @fieldParentPtr(NodeValueIndexPayload, "base", base).lastToken(), + Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(), + Id.PointerPayload => @fieldParentPtr(NodePointerPayload, "base", base).lastToken(), + Id.PointerIndexPayload => @fieldParentPtr(NodePointerIndexPayload, "base", base).lastToken(), Id.Else => @fieldParentPtr(NodeElse, "base", base).lastToken(), Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(), Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(), @@ -189,6 +196,7 @@ pub const Node = struct { Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(), Id.GroupedExpression => @fieldParentPtr(NodeGroupedExpression, "base", base).lastToken(), Id.ControlFlowExpression => @fieldParentPtr(NodeControlFlowExpression, "base", base).lastToken(), + Id.Suspend => @fieldParentPtr(NodeSuspend, "base", base).lastToken(), Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(), Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), @@ -456,6 +464,36 @@ pub const NodeIdentifier = struct { } }; +pub const NodeAsyncAttribute = struct { + base: Node, + async_token: Token, + allocator_type: ?&Node, + rangle_bracket: ?Token, + + pub fn iterate(self: &NodeAsyncAttribute, index: usize) ?&Node { + var i = index; + + if (self.allocator_type) |allocator_type| { + if (i < 1) return allocator_type; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeAsyncAttribute) Token { + return self.async_token; + } + + pub fn lastToken(self: &NodeAsyncAttribute) Token { + if (self.rangle_bracket) |rangle_bracket| { + return rangle_bracket; + } + + return self.async_token; + } +}; + pub const NodeFnProto = struct { base: Node, visib_token: ?Token, @@ -467,6 +505,7 @@ pub const NodeFnProto = struct { extern_token: ?Token, inline_token: ?Token, cc_token: ?Token, + async_attr: ?&NodeAsyncAttribute, body_node: ?&Node, lib_name: ?&Node, // populated if this is an extern declaration align_expr: ?&Node, // populated if align(A) is present @@ -511,6 +550,14 @@ pub const NodeFnProto = struct { i -= 1; } + switch (self.call_convetion) { + CallConvetion.Async => |attr| { + if (i < 1) return &attr.base; + i -= 1; + }, + else => {}, + } + return null; } @@ -645,13 +692,13 @@ pub const NodeComptime = struct { } }; -pub const NodeErrorPayload = struct { +pub const NodePayload = struct { base: Node, lpipe: Token, error_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeErrorPayload, index: usize) ?&Node { + pub fn iterate(self: &NodePayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.error_symbol.base; @@ -660,23 +707,23 @@ pub const NodeErrorPayload = struct { return null; } - pub fn firstToken(self: &NodeErrorPayload) Token { + pub fn firstToken(self: &NodePayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeErrorPayload) Token { + pub fn lastToken(self: &NodePayload) Token { return self.rpipe; } }; -pub const NodeValuePayload = struct { +pub const NodePointerPayload = struct { base: Node, lpipe: Token, is_ptr: bool, value_symbol: &NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeValuePayload, index: usize) ?&Node { + pub fn iterate(self: &NodePointerPayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.value_symbol.base; @@ -685,16 +732,16 @@ pub const NodeValuePayload = struct { return null; } - pub fn firstToken(self: &NodeValuePayload) Token { + pub fn firstToken(self: &NodePointerPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeValuePayload) Token { + pub fn lastToken(self: &NodePointerPayload) Token { return self.rpipe; } }; -pub const NodeValueIndexPayload = struct { +pub const NodePointerIndexPayload = struct { base: Node, lpipe: Token, is_ptr: bool, @@ -702,7 +749,7 @@ pub const NodeValueIndexPayload = struct { index_symbol: ?&NodeIdentifier, rpipe: Token, - pub fn iterate(self: &NodeValueIndexPayload, index: usize) ?&Node { + pub fn iterate(self: &NodePointerIndexPayload, index: usize) ?&Node { var i = index; if (i < 1) return &self.value_symbol.base; @@ -716,11 +763,11 @@ pub const NodeValueIndexPayload = struct { return null; } - pub fn firstToken(self: &NodeValueIndexPayload) Token { + pub fn firstToken(self: &NodePointerIndexPayload) Token { return self.lpipe; } - pub fn lastToken(self: &NodeValueIndexPayload) Token { + pub fn lastToken(self: &NodePointerIndexPayload) Token { return self.rpipe; } }; @@ -728,7 +775,7 @@ pub const NodeValueIndexPayload = struct { pub const NodeElse = struct { base: Node, else_token: Token, - payload: ?&NodeErrorPayload, + payload: ?&NodePayload, body: &Node, pub fn iterate(self: &NodeElse, index: usize) ?&Node { @@ -785,7 +832,7 @@ pub const NodeSwitch = struct { pub const NodeSwitchCase = struct { base: Node, items: ArrayList(&Node), - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, expr: &Node, pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node { @@ -837,7 +884,7 @@ pub const NodeWhile = struct { inline_token: ?Token, while_token: Token, condition: &Node, - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, continue_expr: ?&Node, body: &Node, @"else": ?&NodeElse, @@ -896,7 +943,7 @@ pub const NodeFor = struct { inline_token: ?Token, for_token: Token, array_expr: &Node, - payload: ?&NodeValueIndexPayload, + payload: ?&NodePointerIndexPayload, body: &Node, @"else": ?&NodeElse, @@ -947,7 +994,7 @@ pub const NodeIf = struct { base: Node, if_token: Token, condition: &Node, - payload: ?&NodeValuePayload, + payload: ?&NodePointerPayload, body: &Node, @"else": ?&NodeElse, @@ -1020,7 +1067,7 @@ pub const NodeInfixOp = struct { BitXor, BoolAnd, BoolOr, - Catch: ?&NodeErrorPayload, + Catch: ?&NodePayload, Div, EqualEqual, ErrorUnion, @@ -1113,13 +1160,16 @@ pub const NodePrefixOp = struct { const PrefixOp = union(enum) { AddrOf: AddrOfInfo, + ArrayType: &Node, + Await, BitNot, BoolNot, + Cancel, Deref, MaybeType, Negation, NegationWrap, - ArrayType: &Node, + Resume, SliceType: AddrOfInfo, Try, UnwrapMaybe, @@ -1153,13 +1203,16 @@ pub const NodePrefixOp = struct { if (i < 1) return size_expr; i -= 1; }, + PrefixOp.Await, PrefixOp.BitNot, PrefixOp.BoolNot, + PrefixOp.Cancel, PrefixOp.Deref, PrefixOp.Negation, PrefixOp.NegationWrap, PrefixOp.Return, PrefixOp.Try, + PrefixOp.Resume, PrefixOp.UnwrapMaybe => {}, } @@ -1218,8 +1271,7 @@ pub const NodeSuffixOp = struct { const CallInfo = struct { params: ArrayList(&Node), - is_async: bool, - allocator: ?&Node, + async_attr: ?&NodeAsyncAttribute, }; const SliceRange = struct { @@ -1347,6 +1399,45 @@ pub const NodeControlFlowExpression = struct { } }; +pub const NodeSuspend = struct { + base: Node, + suspend_token: Token, + payload: ?&NodePayload, + body: ?&Node, + + pub fn iterate(self: &NodeSuspend, index: usize) ?&Node { + var i = index; + + if (self.payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + + if (self.body) |body| { + if (i < 1) return body; + i -= 1; + } + + return null; + } + + pub fn firstToken(self: &NodeSuspend) Token { + return self.suspend_token; + } + + pub fn lastToken(self: &NodeSuspend) Token { + if (self.body) |body| { + return body.lastToken(); + } + + if (self.payload) |payload| { + return payload.lastToken(); + } + + return self.suspend_token; + } +}; + pub const NodeIntegerLiteral = struct { base: Node, token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index ff3b35c19..9d6d5de6d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -134,6 +134,11 @@ pub const Parser = struct { dest_ptr: DestPtr, }; + const AsyncEndCtx = struct { + dest_ptr: DestPtr, + attribute: &ast.NodeAsyncAttribute, + }; + const State = union(enum) { TopLevel, TopLevelExtern: TopLevelDeclCtx, @@ -173,9 +178,11 @@ pub const Parser = struct { FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer), FieldListCommaOrEnd: &ast.NodeContainerDecl, SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase), - ErrorPayload: &?&ast.NodeErrorPayload, - ValuePayload: &?&ast.NodeValuePayload, - ValueIndexPayload: &?&ast.NodeValueIndexPayload, + SuspendBody: &ast.NodeSuspend, + AsyncEnd: AsyncEndCtx, + Payload: &?&ast.NodePayload, + PointerPayload: &?&ast.NodePointerPayload, + PointerIndexPayload: &?&ast.NodePointerIndexPayload, SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase), SwitchCaseItem: &ArrayList(&ast.Node), SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node), @@ -399,6 +406,45 @@ pub const Parser = struct { }); continue; }, + Token.Id.Keyword_async => { + // TODO shouldn't need this cast + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + ctx.extern_token, ctx.lib_name, (?Token)(null), (?Token)(null), (?Token)(null)); + + const async_node = try arena.create(ast.NodeAsyncAttribute); + *async_node = ast.NodeAsyncAttribute { + .base = self.initNode(ast.Node.Id.AsyncAttribute), + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + }; + + fn_proto.async_attr = async_node; + stack.append(State { .FnDef = fn_proto }) catch unreachable; + try stack.append(State { .FnProto = fn_proto }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Keyword_fn, + .ptr = &fn_proto.fn_token, + } + }); + + const langle_bracket = self.getNextToken(); + if (langle_bracket.id != Token.Id.AngleBracketLeft) { + self.putBackToken(langle_bracket); + continue; + } + + async_node.rangle_bracket = Token(undefined); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.AngleBracketRight, + .ptr = &??async_node.rangle_bracket, + } + }); + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &async_node.allocator_type } }); + continue; + }, else => { try self.parseError(&stack, token, "expected variable declaration or function, found {}", @tagName(token.id)); continue; @@ -711,13 +757,14 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_cancel => { - @panic("TODO: cancel"); + const cancel_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Cancel); + dest_ptr.store(&cancel_node.base); + stack.append(State { .Expression = DestPtr { .Field = &cancel_node.rhs } }) catch unreachable; }, Token.Id.Keyword_resume => { - @panic("TODO: resume"); - }, - Token.Id.Keyword_await => { - @panic("TODO: await"); + const resume_node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp.Resume); + dest_ptr.store(&resume_node.base); + stack.append(State { .Expression = DestPtr { .Field = &resume_node.rhs } }) catch unreachable; }, else => { self.putBackToken(token); @@ -786,7 +833,7 @@ pub const Parser = struct { stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }); - try stack.append(State { .ErrorPayload = &node.op.Catch }); + try stack.append(State { .Payload = &node.op.Catch }); continue; }, Token.Id.QuestionMarkQuestionMark => { @@ -1113,6 +1160,9 @@ pub const Parser = struct { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); } continue; + //Token.Id.Keyword_await => { + // @panic("TODO: await"); + //}, } else { self.putBackToken(token); stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; @@ -1124,7 +1174,38 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_async => { - @panic("TODO: Parse async"); + const async_node = try arena.create(ast.NodeAsyncAttribute); + *async_node = ast.NodeAsyncAttribute { + .base = self.initNode(ast.Node.Id.AsyncAttribute), + .async_token = token, + .allocator_type = null, + .rangle_bracket = null, + }; + + stack.append(State { + .AsyncEnd = AsyncEndCtx { + .dest_ptr = dest_ptr, + .attribute = async_node, + } + }) catch unreachable; + try stack.append(State { .SuffixOpExpressionEnd = dest_ptr }); + try stack.append(State { .PrimaryExpression = dest_ptr }); + + const langle_bracket = self.getNextToken(); + if (langle_bracket.id != Token.Id.AngleBracketLeft) { + self.putBackToken(langle_bracket); + continue; + } + + async_node.rangle_bracket = Token(undefined); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.AngleBracketRight, + .ptr = &??async_node.rangle_bracket, + } + }); + try stack.append(State { .TypeExprBegin = DestPtr { .NullableField = &async_node.allocator_type } }); + continue; }, else => { self.putBackToken(token); @@ -1142,8 +1223,7 @@ pub const Parser = struct { const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp { .Call = ast.NodeSuffixOp.CallInfo { .params = ArrayList(&ast.Node).init(arena), - .is_async = false, // TODO: ASYNC - .allocator = null, + .async_attr = null, } }); node.lhs = dest_ptr.get(); @@ -1257,6 +1337,19 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, + Token.Id.Keyword_suspend => { + const node = try arena.create(ast.NodeSuspend); + *node = ast.NodeSuspend { + .base = self.initNode(ast.Node.Id.Suspend), + .suspend_token = token, + .payload = null, + .body = null, + }; + dest_ptr.store(&node.base); + stack.append(State { .SuspendBody = node }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + continue; + }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1477,17 +1570,12 @@ pub const Parser = struct { continue; }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { + const fn_token = (try self.eatToken(&stack, Token.Id.Keyword_fn)) ?? continue; // TODO shouldn't need this cast - const fn_proto = try self.createFnProto(arena, undefined, + const fn_proto = try self.createFnProto(arena, fn_token, (?Token)(null), (?&ast.Node)(null), (?Token)(token), (?Token)(null), (?Token)(null)); dest_ptr.store(&fn_proto.base); stack.append(State { .FnProto = fn_proto }) catch unreachable; - try stack.append(State { - .ExpectTokenSave = ExpectTokenSave { - .id = Token.Id.Keyword_fn, - .ptr = &fn_proto.fn_token, - } - }); continue; }, Token.Id.Keyword_asm => { @@ -1544,7 +1632,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -1816,7 +1904,7 @@ pub const Parser = struct { try list_state.list.append(node); stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable; try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); const maybe_else = self.getNextToken(); if (maybe_else.id == Token.Id.Keyword_else) { @@ -1883,7 +1971,7 @@ pub const Parser = struct { *dest = node; stack.append(State { .Expression = DestPtr { .Field = &node.body } }) catch unreachable; - try stack.append(State { .ErrorPayload = &node.payload }); + try stack.append(State { .Payload = &node.payload }); }, State.WhileContinueExpr => |dest| { @@ -1898,7 +1986,41 @@ pub const Parser = struct { try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = dest } }); }, - State.ErrorPayload => |dest| { + State.SuspendBody => |suspend_node| { + if (suspend_node.payload != null) { + try stack.append(State { .AssignmentExpressionBegin = DestPtr { .NullableField = &suspend_node.body } }); + } + continue; + }, + + State.AsyncEnd => |ctx| { + const node = ctx.dest_ptr.get(); + + switch (node.id) { + ast.Node.Id.FnProto => { + const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", node); + fn_proto.async_attr = ctx.attribute; + }, + ast.Node.Id.SuffixOp => { + const suffix_op = @fieldParentPtr(ast.NodeSuffixOp, "base", node); + if (suffix_op.op == ast.NodeSuffixOp.SuffixOp.Call) { + suffix_op.op.Call.async_attr = ctx.attribute; + continue; + } + + try self.parseError(&stack, node.firstToken(), "expected call or fn proto, found {}.", + @tagName(suffix_op.op)); + continue; + }, + else => { + try self.parseError(&stack, node.firstToken(), "expected call or fn proto, found {}.", + @tagName(node.id)); + continue; + } + } + }, + + State.Payload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1907,9 +2029,9 @@ pub const Parser = struct { const error_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeErrorPayload); - *node = ast.NodeErrorPayload { - .base = self.initNode(ast.Node.Id.ErrorPayload), + const node = try arena.create(ast.NodePayload); + *node = ast.NodePayload { + .base = self.initNode(ast.Node.Id.Payload), .lpipe = lpipe, .error_symbol = try self.createIdentifier(arena, error_symbol), .rpipe = rpipe @@ -1917,7 +2039,7 @@ pub const Parser = struct { *dest = node; }, - State.ValuePayload => |dest| { + State.PointerPayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1936,9 +2058,9 @@ pub const Parser = struct { const value_symbol = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeValuePayload); - *node = ast.NodeValuePayload { - .base = self.initNode(ast.Node.Id.ValuePayload), + const node = try arena.create(ast.NodePointerPayload); + *node = ast.NodePointerPayload { + .base = self.initNode(ast.Node.Id.PointerPayload), .lpipe = lpipe, .is_ptr = is_ptr, .value_symbol = try self.createIdentifier(arena, value_symbol), @@ -1947,7 +2069,7 @@ pub const Parser = struct { *dest = node; }, - State.ValueIndexPayload => |dest| { + State.PointerIndexPayload => |dest| { const lpipe = self.getNextToken(); if (lpipe.id != Token.Id.Pipe) { self.putBackToken(lpipe); @@ -1977,9 +2099,9 @@ pub const Parser = struct { }; const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue; - const node = try arena.create(ast.NodeValueIndexPayload); - *node = ast.NodeValueIndexPayload { - .base = self.initNode(ast.Node.Id.ValueIndexPayload), + const node = try arena.create(ast.NodePointerIndexPayload); + *node = ast.NodePointerIndexPayload { + .base = self.initNode(ast.Node.Id.PointerIndexPayload), .lpipe = lpipe, .is_ptr = is_ptr, .value_symbol = try self.createIdentifier(arena, value_symbol), @@ -2249,7 +2371,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); try stack.append(State { .WhileContinueExpr = &node.continue_expr }); - try stack.append(State { .ValuePayload = &node.payload }); + try stack.append(State { .PointerPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -2271,7 +2393,7 @@ pub const Parser = struct { stack.append(State { .Else = &node.@"else" }) catch unreachable; try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .ValueIndexPayload = &node.payload }); + try stack.append(State { .PointerIndexPayload = &node.payload }); try stack.append(State { .ExpectToken = Token.Id.RParen }); try stack.append(State { .Expression = DestPtr { .Field = &node.array_expr } }); try stack.append(State { .ExpectToken = Token.Id.LParen }); @@ -2374,9 +2496,9 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.ParamDecl, ast.Node.Id.Block, - ast.Node.Id.ErrorPayload, - ast.Node.Id.ValuePayload, - ast.Node.Id.ValueIndexPayload, + ast.Node.Id.Payload, + ast.Node.Id.PointerPayload, + ast.Node.Id.PointerIndexPayload, ast.Node.Id.Switch, ast.Node.Id.SwitchCase, ast.Node.Id.SwitchElse, @@ -2422,6 +2544,15 @@ pub const Parser = struct { const comptime_node = @fieldParentPtr(ast.NodeComptime, "base", n); n = comptime_node.expr; }, + ast.Node.Id.Suspend => { + const suspend_node = @fieldParentPtr(ast.NodeSuspend, "base", n); + if (suspend_node.body) |body| { + n = body; + continue; + } + + return true; + }, else => return true, } } @@ -2540,6 +2671,7 @@ pub const Parser = struct { }, Token.Id.QuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.MaybeType), Token.Id.QuestionMarkQuestionMark => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.UnwrapMaybe), + Token.Id.Keyword_await => (ast.NodePrefixOp.PrefixOp)(ast.NodePrefixOp.PrefixOp.Await), else => null, }; } @@ -2628,6 +2760,7 @@ pub const Parser = struct { .extern_token = *extern_token, .inline_token = *inline_token, .cc_token = *cc_token, + .async_attr = null, .body_node = null, .lib_name = lib_name, .align_expr = null, @@ -3105,6 +3238,30 @@ pub const Parser = struct { try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_node.comptime_token)); try stack.append(RenderState { .Expression = comptime_node.expr }); }, + ast.Node.Id.AsyncAttribute => { + const async_attr = @fieldParentPtr(ast.NodeAsyncAttribute, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(async_attr.async_token)); + + if (async_attr.allocator_type) |allocator_type| { + try stack.append(RenderState { .Text = ">" }); + try stack.append(RenderState { .Expression = allocator_type }); + try stack.append(RenderState { .Text = "<" }); + } + }, + ast.Node.Id.Suspend => { + const suspend_node = @fieldParentPtr(ast.NodeSuspend, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(suspend_node.suspend_token)); + + if (suspend_node.body) |body| { + try stack.append(RenderState { .Expression = body }); + try stack.append(RenderState { .Text = " " }); + } + + if (suspend_node.payload) |payload| { + try stack.append(RenderState { .Expression = &payload.base }); + try stack.append(RenderState { .Text = " " }); + } + }, ast.Node.Id.InfixOp => { const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base); try stack.append(RenderState { .Expression = prefix_op_node.rhs }); @@ -3211,6 +3368,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Try => try stream.write("try "), ast.NodePrefixOp.PrefixOp.UnwrapMaybe => try stream.write("??"), ast.NodePrefixOp.PrefixOp.MaybeType => try stream.write("?"), + ast.NodePrefixOp.PrefixOp.Await => try stream.write("await "), + ast.NodePrefixOp.PrefixOp.Cancel => try stream.write("cancel "), + ast.NodePrefixOp.PrefixOp.Resume => try stream.write("resume "), } }, ast.Node.Id.SuffixOp => { @@ -3229,11 +3389,18 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "("}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); + + if (call_info.async_attr) |async_attr| { + try stack.append(RenderState { .Text = " "}); + try stack.append(RenderState { .Expression = &async_attr.base }); + } }, ast.NodeSuffixOp.SuffixOp.ArrayAccess => |index_expr| { try stack.append(RenderState { .Text = "]"}); try stack.append(RenderState { .Expression = index_expr}); try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.Slice => |range| { try stack.append(RenderState { .Text = "]"}); @@ -3243,6 +3410,7 @@ pub const Parser = struct { try stack.append(RenderState { .Text = ".."}); try stack.append(RenderState { .Expression = range.start}); try stack.append(RenderState { .Text = "["}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.StructInitializer => |field_inits| { try stack.append(RenderState { .Text = " }"}); @@ -3257,6 +3425,7 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "{"}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.NodeSuffixOp.SuffixOp.ArrayInitializer => |exprs| { try stack.append(RenderState { .Text = " }"}); @@ -3271,10 +3440,9 @@ pub const Parser = struct { } } try stack.append(RenderState { .Text = "{"}); + try stack.append(RenderState { .Expression = suffix_op.lhs }); }, } - - try stack.append(RenderState { .Expression = suffix_op.lhs }); }, ast.Node.Id.ControlFlowExpression => { const flow_expr = @fieldParentPtr(ast.NodeControlFlowExpression, "base", base); @@ -3302,14 +3470,14 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = rhs }); } }, - ast.Node.Id.ErrorPayload => { - const payload = @fieldParentPtr(ast.NodeErrorPayload, "base", base); + ast.Node.Id.Payload => { + const payload = @fieldParentPtr(ast.NodePayload, "base", base); try stack.append(RenderState { .Text = "|"}); try stack.append(RenderState { .Expression = &payload.error_symbol.base }); try stack.append(RenderState { .Text = "|"}); }, - ast.Node.Id.ValuePayload => { - const payload = @fieldParentPtr(ast.NodeValuePayload, "base", base); + ast.Node.Id.PointerPayload => { + const payload = @fieldParentPtr(ast.NodePointerPayload, "base", base); try stack.append(RenderState { .Text = "|"}); try stack.append(RenderState { .Expression = &payload.value_symbol.base }); @@ -3319,8 +3487,8 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "|"}); }, - ast.Node.Id.ValueIndexPayload => { - const payload = @fieldParentPtr(ast.NodeValueIndexPayload, "base", base); + ast.Node.Id.PointerIndexPayload => { + const payload = @fieldParentPtr(ast.NodePointerIndexPayload, "base", base); try stack.append(RenderState { .Text = "|"}); if (payload.index_symbol) |index_symbol| { @@ -3553,6 +3721,11 @@ pub const Parser = struct { try stack.append(RenderState { .Text = "fn" }); + if (fn_proto.async_attr) |async_attr| { + try stack.append(RenderState { .Text = " " }); + try stack.append(RenderState { .Expression = &async_attr.base }); + } + if (fn_proto.cc_token) |cc_token| { try stack.append(RenderState { .Text = " " }); try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(cc_token) }); @@ -4789,8 +4962,7 @@ test "zig fmt: coroutines" { \\ x += 1; \\ suspend; \\ x += 1; - \\ suspend |p| { - \\ } + \\ suspend |p| {} \\ const p = async simpleAsyncFn() catch unreachable; \\ await p; \\} @@ -4803,10 +4975,3 @@ test "zig fmt: coroutines" { \\ ); } - -test "zig fmt: zig fmt" { - try testCanonical(@embedFile("ast.zig")); - try testCanonical(@embedFile("index.zig")); - try testCanonical(@embedFile("parser.zig")); - try testCanonical(@embedFile("tokenizer.zig")); -} From 34af38e09b33957a9f677e42d57e9cd96f859b76 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 11:35:41 +0200 Subject: [PATCH 116/132] std.zig.tokinizer: fixed failing tests --- std/zig/tokenizer.zig | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 4cce31bae..7b1f86712 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -613,17 +613,25 @@ pub const Tokenizer = struct { '\\' => { state = State.CharLiteralBackslash; }, - '\'' => break, // Look for this error later. + '\'' => { + result.id = Token.Id.Invalid; + break; + }, else => { - if (c < 0x20 or c == 0x7f) - break; // Look for this error later. + if (c < 0x20 or c == 0x7f) { + result.id = Token.Id.Invalid; + break; + } state = State.CharLiteralEnd; } }, State.CharLiteralBackslash => switch (c) { - '\n' => break, // Look for this error later. + '\n' => { + result.id = Token.Id.Invalid; + break; + }, else => { state = State.CharLiteralEnd; }, @@ -635,7 +643,10 @@ pub const Tokenizer = struct { self.index += 1; break; }, - else => break, // Look for this error later. + else => { + result.id = Token.Id.Invalid; + break; + }, }, State.MultilineStringLiteralLine => switch (c) { @@ -903,7 +914,6 @@ pub const Tokenizer = struct { State.FloatExponentNumber, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, - State.CharLiteralEnd, State.Builtin => {}, State.Identifier => { @@ -922,6 +932,7 @@ pub const Tokenizer = struct { State.MultilineStringLiteralLineBackslash, State.CharLiteral, State.CharLiteralBackslash, + State.CharLiteralEnd, State.StringLiteralBackslash => { result.id = Token.Id.Invalid; }, @@ -1073,7 +1084,7 @@ test "tokenizer - invalid token characters" { testTokenize("`", []Token.Id{Token.Id.Invalid}); testTokenize("'c", []Token.Id {Token.Id.Invalid}); testTokenize("'", []Token.Id {Token.Id.Invalid}); - testTokenize("''", []Token.Id {Token.Id.Invalid}); + testTokenize("''", []Token.Id {Token.Id.Invalid, Token.Id.Invalid}); } test "tokenizer - invalid literal/comment characters" { @@ -1147,6 +1158,7 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { const token = tokenizer.next(); + std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id)); std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id)); switch (expected_token_id) { Token.Id.StringLiteral => |expected_kind| { From 1b81e406f0ca285738404120178b7b3824cf84bc Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 13:43:20 +0200 Subject: [PATCH 117/132] std.zig: fixed compiler errors --- std/zig/ast.zig | 23 +- std/zig/parser.zig | 1510 ++++++++++++++++++++++---------------------- 2 files changed, 768 insertions(+), 765 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 9043a66ac..2128b9976 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -297,7 +297,7 @@ pub const NodeErrorSetDecl = struct { pub fn iterate(self: &NodeErrorSetDecl, index: usize) ?&Node { var i = index; - if (i < self.decls.len) return self.decls.at(i); + if (i < self.decls.len) return &self.decls.at(i).base; i -= self.decls.len; return null; @@ -550,14 +550,6 @@ pub const NodeFnProto = struct { i -= 1; } - switch (self.call_convetion) { - CallConvetion.Async => |attr| { - if (i < 1) return &attr.base; - i -= 1; - }, - else => {}, - } - return null; } @@ -814,7 +806,7 @@ pub const NodeSwitch = struct { if (i < 1) return self.expr; i -= 1; - if (i < self.cases.len) return self.cases.at(i); + if (i < self.cases.len) return &self.cases.at(i).base; i -= self.cases.len; return null; @@ -842,7 +834,7 @@ pub const NodeSwitchCase = struct { i -= self.items.len; if (self.payload) |payload| { - if (i < 1) return payload; + if (i < 1) return &payload.base; i -= 1; } @@ -1093,6 +1085,13 @@ pub const NodeInfixOp = struct { i -= 1; switch (self.op) { + InfixOp.Catch => |maybe_payload| { + if (maybe_payload) |payload| { + if (i < 1) return &payload.base; + i -= 1; + } + }, + InfixOp.Add, InfixOp.AddWrap, InfixOp.ArrayCat, @@ -1208,9 +1207,9 @@ pub const NodePrefixOp = struct { PrefixOp.BoolNot, PrefixOp.Cancel, PrefixOp.Deref, + PrefixOp.MaybeType, PrefixOp.Negation, PrefixOp.NegationWrap, - PrefixOp.Return, PrefixOp.Try, PrefixOp.Resume, PrefixOp.UnwrapMaybe => {}, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 9d6d5de6d..33408bcc3 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1421,7 +1421,7 @@ pub const Parser = struct { } }); dest_ptr.store(&node.base); - stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .AddrOfModifiers = &node.op.SliceType }); continue; } @@ -1432,7 +1432,7 @@ pub const Parser = struct { .ArrayType = undefined, }); dest_ptr.store(&node.base); - stack.append(State { .Expression = DestPtr { .Field = &node.rhs } }) catch unreachable; + stack.append(State { .TypeExprBegin = DestPtr { .Field = &node.rhs } }) catch unreachable; try stack.append(State { .ExpectToken = Token.Id.RBracket }); try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayType } }); @@ -4222,756 +4222,760 @@ fn testCanonical(source: []const u8) !void { } } -test "zig fmt: get stdout or fail" { - try testCanonical( - \\const std = @import("std"); - \\ - \\pub fn main() !void { - \\ // If this program is run without stdout attached, exit with an error. - \\ // another comment - \\ var stdout_file = try std.io.getStdOut; - \\} - \\ - ); -} +//test "zig fmt: get stdout or fail" { +// try testCanonical( +// \\const std = @import("std"); +// \\ +// \\pub fn main() !void { +// \\ // If this program is run without stdout attached, exit with an error. +// \\ // another comment +// \\ var stdout_file = try std.io.getStdOut; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: preserve spacing" { +// try testCanonical( +// \\const std = @import("std"); +// \\ +// \\pub fn main() !void { +// \\ var stdout_file = try std.io.getStdOut; +// \\ var stdout_file = try std.io.getStdOut; +// \\ +// \\ var stdout_file = try std.io.getStdOut; +// \\ var stdout_file = try std.io.getStdOut; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: return types" { +// try testCanonical( +// \\pub fn main() !void {} +// \\pub fn main() var {} +// \\pub fn main() i32 {} +// \\ +// ); +//} +// +//test "zig fmt: imports" { +// try testCanonical( +// \\const std = @import("std"); +// \\const std = @import(); +// \\ +// ); +//} +// +//test "zig fmt: global declarations" { +// try testCanonical( +// \\const a = b; +// \\pub const a = b; +// \\var a = b; +// \\pub var a = b; +// \\const a: i32 = b; +// \\pub const a: i32 = b; +// \\var a: i32 = b; +// \\pub var a: i32 = b; +// \\extern const a: i32 = b; +// \\pub extern const a: i32 = b; +// \\extern var a: i32 = b; +// \\pub extern var a: i32 = b; +// \\extern "a" const a: i32 = b; +// \\pub extern "a" const a: i32 = b; +// \\extern "a" var a: i32 = b; +// \\pub extern "a" var a: i32 = b; +// \\ +// ); +//} +// +//test "zig fmt: extern declaration" { +// try testCanonical( +// \\extern var foo: c_int; +// \\ +// ); +//} +// +//test "zig fmt: alignment" { +// try testCanonical( +// \\var foo: c_int align(1); +// \\ +// ); +//} +// +//test "zig fmt: C main" { +// try testCanonical( +// \\fn main(argc: c_int, argv: &&u8) c_int { +// \\ const a = b; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: return" { +// try testCanonical( +// \\fn foo(argc: c_int, argv: &&u8) c_int { +// \\ return 0; +// \\} +// \\ +// \\fn bar() void { +// \\ return; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: pointer attributes" { +// try testCanonical( +// \\extern fn f1(s: &align(&u8) u8) c_int; +// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; +// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; +// \\extern fn f4(s: &align(1) const volatile u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: slice attributes" { +// try testCanonical( +// \\extern fn f1(s: &align(&u8) u8) c_int; +// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; +// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; +// \\extern fn f4(s: &align(1) const volatile u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: test declaration" { +// try testCanonical( +// \\test "test name" { +// \\ const a = 1; +// \\ var b = 1; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: infix operators" { +// try testCanonical( +// \\test "infix operators" { +// \\ var i = undefined; +// \\ i = 2; +// \\ i *= 2; +// \\ i |= 2; +// \\ i ^= 2; +// \\ i <<= 2; +// \\ i >>= 2; +// \\ i &= 2; +// \\ i *= 2; +// \\ i *%= 2; +// \\ i -= 2; +// \\ i -%= 2; +// \\ i += 2; +// \\ i +%= 2; +// \\ i /= 2; +// \\ i %= 2; +// \\ _ = i == i; +// \\ _ = i != i; +// \\ _ = i != i; +// \\ _ = i.i; +// \\ _ = i || i; +// \\ _ = i!i; +// \\ _ = i ** i; +// \\ _ = i ++ i; +// \\ _ = i ?? i; +// \\ _ = i % i; +// \\ _ = i / i; +// \\ _ = i *% i; +// \\ _ = i * i; +// \\ _ = i -% i; +// \\ _ = i - i; +// \\ _ = i +% i; +// \\ _ = i + i; +// \\ _ = i << i; +// \\ _ = i >> i; +// \\ _ = i & i; +// \\ _ = i ^ i; +// \\ _ = i | i; +// \\ _ = i >= i; +// \\ _ = i <= i; +// \\ _ = i > i; +// \\ _ = i < i; +// \\ _ = i and i; +// \\ _ = i or i; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: precedence" { +// try testCanonical( +// \\test "precedence" { +// \\ a!b(); +// \\ (a!b)(); +// \\ !a!b; +// \\ !(a!b); +// \\ !a{ }; +// \\ !(a{ }); +// \\ a + b{ }; +// \\ (a + b){ }; +// \\ a << b + c; +// \\ (a << b) + c; +// \\ a & b << c; +// \\ (a & b) << c; +// \\ a ^ b & c; +// \\ (a ^ b) & c; +// \\ a | b ^ c; +// \\ (a | b) ^ c; +// \\ a == b | c; +// \\ (a == b) | c; +// \\ a and b == c; +// \\ (a and b) == c; +// \\ a or b and c; +// \\ (a or b) and c; +// \\ (a or b) and c; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: prefix operators" { +// try testCanonical( +// \\test "prefix operators" { +// \\ try return --%~??!*&0; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: call expression" { +// try testCanonical( +// \\test "test calls" { +// \\ a(); +// \\ a(1); +// \\ a(1, 2); +// \\ a(1, 2) + a(1, 2); +// \\} +// \\ +// ); +//} +// +//test "zig fmt: var args" { +// try testCanonical( +// \\fn print(args: ...) void {} +// \\ +// ); +//} +// +//test "zig fmt: extern function" { +// try testCanonical( +// \\extern fn puts(s: &const u8) c_int; +// \\extern "c" fn puts(s: &const u8) c_int; +// \\ +// ); +//} +// +//test "zig fmt: multiline string" { +// try testCanonical( +// \\const s = +// \\ \\ something +// \\ \\ something else +// \\ ; +// \\ +// ); +//} +// +//test "zig fmt: values" { +// try testCanonical( +// \\test "values" { +// \\ 1; +// \\ 1.0; +// \\ "string"; +// \\ c"cstring"; +// \\ 'c'; +// \\ true; +// \\ false; +// \\ null; +// \\ undefined; +// \\ error; +// \\ this; +// \\ unreachable; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: indexing" { +// try testCanonical( +// \\test "test index" { +// \\ a[0]; +// \\ a[0 + 5]; +// \\ a[0..]; +// \\ a[0..5]; +// \\ a[a[0]]; +// \\ a[a[0..]]; +// \\ a[a[0..5]]; +// \\ a[a[0]..]; +// \\ a[a[0..5]..]; +// \\ a[a[0]..a[0]]; +// \\ a[a[0..5]..a[0]]; +// \\ a[a[0..5]..a[0..5]]; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: struct declaration" { +// try testCanonical( +// \\const S = struct { +// \\ const Self = this; +// \\ f1: u8, +// \\ +// \\ fn method(self: &Self) Self { +// \\ return *self; +// \\ } +// \\ +// \\ f2: u8 +// \\}; +// \\ +// \\const Ps = packed struct { +// \\ a: u8, +// \\ b: u8, +// \\ +// \\ c: u8 +// \\}; +// \\ +// \\const Es = extern struct { +// \\ a: u8, +// \\ b: u8, +// \\ +// \\ c: u8 +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: enum declaration" { +// try testCanonical( +// \\const E = enum { +// \\ Ok, +// \\ SomethingElse = 0 +// \\}; +// \\ +// \\const E2 = enum(u8) { +// \\ Ok, +// \\ SomethingElse = 255, +// \\ SomethingThird +// \\}; +// \\ +// \\const Ee = extern enum { +// \\ Ok, +// \\ SomethingElse, +// \\ SomethingThird +// \\}; +// \\ +// \\const Ep = packed enum { +// \\ Ok, +// \\ SomethingElse, +// \\ SomethingThird +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: union declaration" { +// try testCanonical( +// \\const U = union { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const Ue = union(enum) { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const E = enum { +// \\ Int, +// \\ Float, +// \\ None, +// \\ Bool +// \\}; +// \\ +// \\const Ue2 = union(E) { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// \\const Eu = extern union { +// \\ Int: u8, +// \\ Float: f32, +// \\ None, +// \\ Bool: bool +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: error set declaration" { +// try testCanonical( +// \\const E = error { +// \\ A, +// \\ B, +// \\ +// \\ C +// \\}; +// \\ +// ); +//} +// +//test "zig fmt: arrays" { +// try testCanonical( +// \\test "test array" { +// \\ const a: [2]u8 = [2]u8{ 1, 2 }; +// \\ const a: [2]u8 = []u8{ 1, 2 }; +// \\ const a: [0]u8 = []u8{ }; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: container initializers" { +// try testCanonical( +// \\const a1 = []u8{ }; +// \\const a2 = []u8{ 1, 2, 3, 4 }; +// \\const s1 = S{ }; +// \\const s2 = S{ .a = 1, .b = 2 }; +// \\ +// ); +//} +// +//test "zig fmt: catch" { +// try testCanonical( +// \\test "catch" { +// \\ const a: error!u8 = 0; +// \\ _ = a catch return; +// \\ _ = a catch |err| return; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: blocks" { +// try testCanonical( +// \\test "blocks" { +// \\ { +// \\ const a = 0; +// \\ const b = 0; +// \\ } +// \\ +// \\ blk: { +// \\ const a = 0; +// \\ const b = 0; +// \\ } +// \\ +// \\ const r = blk: { +// \\ const a = 0; +// \\ const b = 0; +// \\ }; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: switch" { +// try testCanonical( +// \\test "switch" { +// \\ switch (0) { +// \\ 0 => {}, +// \\ 1 => unreachable, +// \\ 2, 3 => {}, +// \\ 4 ... 7 => {}, +// \\ 1 + 4 * 3 + 22 => {}, +// \\ else => { +// \\ const a = 1; +// \\ const b = a; +// \\ } +// \\ } +// \\ +// \\ const res = switch (0) { +// \\ 0 => 0, +// \\ 1 => 2, +// \\ else => 4 +// \\ }; +// \\ +// \\ const Union = union(enum) { +// \\ Int: i64, +// \\ Float: f64 +// \\ }; +// \\ +// \\ const u = Union{ .Int = 0 }; +// \\ switch (u) { +// \\ Union.Int => |int| {}, +// \\ Union.Float => |*float| unreachable +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: while" { +// try testCanonical( +// \\test "while" { +// \\ while (10 < 1) { +// \\ unreachable; +// \\ } +// \\ +// \\ while (10 < 1) +// \\ unreachable; +// \\ +// \\ var i: usize = 0; +// \\ while (i < 10) : (i += 1) { +// \\ continue; +// \\ } +// \\ +// \\ i = 0; +// \\ while (i < 10) : (i += 1) +// \\ continue; +// \\ +// \\ i = 0; +// \\ var j: usize = 0; +// \\ while (i < 10) : ({ +// \\ i += 1; +// \\ j += 1; +// \\ }) { +// \\ continue; +// \\ } +// \\ +// \\ var a: ?u8 = 2; +// \\ while (a) |v| : (a = null) { +// \\ continue; +// \\ } +// \\ +// \\ while (a) |v| : (a = null) +// \\ unreachable; +// \\ +// \\ label: while (10 < 0) { +// \\ unreachable; +// \\ } +// \\ +// \\ const res = while (0 < 10) { +// \\ break 7; +// \\ } else { +// \\ unreachable; +// \\ }; +// \\ +// \\ var a: error!u8 = 0; +// \\ while (a) |v| { +// \\ a = error.Err; +// \\ } else |err| { +// \\ i = 1; +// \\ } +// \\ +// \\ comptime var k: usize = 0; +// \\ inline while (i < 10) : (i += 1) +// \\ j += 2; +// \\} +// \\ +// ); +//} +// +//test "zig fmt: for" { +// try testCanonical( +// \\test "for" { +// \\ const a = []u8{ 1, 2, 3 }; +// \\ for (a) |v| { +// \\ continue; +// \\ } +// \\ +// \\ for (a) |v| +// \\ continue; +// \\ +// \\ for (a) |*v| +// \\ continue; +// \\ +// \\ for (a) |v, i| { +// \\ continue; +// \\ } +// \\ +// \\ for (a) |v, i| +// \\ continue; +// \\ +// \\ const res = for (a) |v, i| { +// \\ break v; +// \\ } else { +// \\ unreachable; +// \\ }; +// \\ +// \\ var num: usize = 0; +// \\ inline for (a) |v, i| { +// \\ num += v; +// \\ num += i; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: if" { +// try testCanonical( +// \\test "if" { +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } +// \\ +// \\ if (10 < 0) unreachable; +// \\ +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } else { +// \\ const a = 20; +// \\ } +// \\ +// \\ if (10 < 0) { +// \\ unreachable; +// \\ } else if (5 < 0) { +// \\ unreachable; +// \\ } else { +// \\ const a = 20; +// \\ } +// \\ +// \\ const is_world_broken = if (10 < 0) true else false; +// \\ +// \\ const a: ?u8 = 10; +// \\ const b: ?u8 = null; +// \\ if (a) |v| { +// \\ const some = v; +// \\ } else if (b) |*v| { +// \\ unreachable; +// \\ } else { +// \\ const some = 10; +// \\ } +// \\ +// \\ const non_null_a = if (a) |v| v else 0; +// \\ +// \\ const a_err: error!u8 = 0; +// \\ if (a_err) |v| { +// \\ const p = v; +// \\ } else |err| { +// \\ unreachable; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: defer" { +// try testCanonical( +// \\test "defer" { +// \\ var i: usize = 0; +// \\ defer i = 1; +// \\ defer { +// \\ i += 2; +// \\ i *= i; +// \\ } +// \\ +// \\ errdefer i += 3; +// \\ errdefer { +// \\ i += 2; +// \\ i /= i; +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: comptime" { +// try testCanonical( +// \\fn a() u8 { +// \\ return 5; +// \\} +// \\ +// \\fn b(comptime i: u8) u8 { +// \\ return i; +// \\} +// \\ +// \\const av = comptime a(); +// \\const av2 = comptime blk: { +// \\ var res = a(); +// \\ res *= b(2); +// \\ break :blk res; +// \\}; +// \\ +// \\comptime { +// \\ _ = a(); +// \\} +// \\ +// \\test "comptime" { +// \\ const av3 = comptime a(); +// \\ const av4 = comptime blk: { +// \\ var res = a(); +// \\ res *= a(); +// \\ break :blk res; +// \\ }; +// \\ +// \\ comptime var i = 0; +// \\ comptime { +// \\ i = a(); +// \\ i += b(i); +// \\ } +// \\} +// \\ +// ); +//} +// +//test "zig fmt: fn type" { +// try testCanonical( +// \\fn a(i: u8) u8 { +// \\ return i + 1; +// \\} +// \\ +// \\const a: fn(u8) u8 = undefined; +// \\const b: extern fn(u8) u8 = undefined; +// \\const c: nakedcc fn(u8) u8 = undefined; +// \\const ap: fn(u8) u8 = a; +// \\ +// ); +//} +// +//test "zig fmt: inline asm" { +// try testCanonical( +// \\pub fn syscall1(number: usize, arg1: usize) usize { +// \\ return asm volatile ("syscall" +// \\ : [ret] "={rax}" (-> usize) +// \\ : [number] "{rax}" (number), +// \\ [arg1] "{rdi}" (arg1) +// \\ : "rcx", "r11"); +// \\} +// \\ +// ); +//} +// +//test "zig fmt: coroutines" { +// try testCanonical( +// \\async fn simpleAsyncFn() void { +// \\ x += 1; +// \\ suspend; +// \\ x += 1; +// \\ suspend |p| {} +// \\ const p = async simpleAsyncFn() catch unreachable; +// \\ await p; +// \\} +// \\ +// \\test "coroutine suspend, resume, cancel" { +// \\ const p = try async testAsyncSeq(); +// \\ resume p; +// \\ cancel p; +// \\} +// \\ +// ); +//} -test "zig fmt: preserve spacing" { - try testCanonical( - \\const std = @import("std"); - \\ - \\pub fn main() !void { - \\ var stdout_file = try std.io.getStdOut; - \\ var stdout_file = try std.io.getStdOut; - \\ - \\ var stdout_file = try std.io.getStdOut; - \\ var stdout_file = try std.io.getStdOut; - \\} - \\ - ); -} - -test "zig fmt: return types" { - try testCanonical( - \\pub fn main() !void {} - \\pub fn main() var {} - \\pub fn main() i32 {} - \\ - ); -} - -test "zig fmt: imports" { - try testCanonical( - \\const std = @import("std"); - \\const std = @import(); - \\ - ); -} - -test "zig fmt: global declarations" { - try testCanonical( - \\const a = b; - \\pub const a = b; - \\var a = b; - \\pub var a = b; - \\const a: i32 = b; - \\pub const a: i32 = b; - \\var a: i32 = b; - \\pub var a: i32 = b; - \\extern const a: i32 = b; - \\pub extern const a: i32 = b; - \\extern var a: i32 = b; - \\pub extern var a: i32 = b; - \\extern "a" const a: i32 = b; - \\pub extern "a" const a: i32 = b; - \\extern "a" var a: i32 = b; - \\pub extern "a" var a: i32 = b; - \\ - ); -} - -test "zig fmt: extern declaration" { - try testCanonical( - \\extern var foo: c_int; - \\ - ); -} - -test "zig fmt: alignment" { - try testCanonical( - \\var foo: c_int align(1); - \\ - ); -} - -test "zig fmt: C main" { - try testCanonical( - \\fn main(argc: c_int, argv: &&u8) c_int { - \\ const a = b; - \\} - \\ - ); -} - -test "zig fmt: return" { - try testCanonical( - \\fn foo(argc: c_int, argv: &&u8) c_int { - \\ return 0; - \\} - \\ - \\fn bar() void { - \\ return; - \\} - \\ - ); -} - -test "zig fmt: pointer attributes" { - try testCanonical( - \\extern fn f1(s: &align(&u8) u8) c_int; - \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f4(s: &align(1) const volatile u8) c_int; - \\ - ); -} - -test "zig fmt: slice attributes" { - try testCanonical( - \\extern fn f1(s: &align(&u8) u8) c_int; - \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; - \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; - \\extern fn f4(s: &align(1) const volatile u8) c_int; - \\ - ); -} - -test "zig fmt: test declaration" { - try testCanonical( - \\test "test name" { - \\ const a = 1; - \\ var b = 1; - \\} - \\ - ); -} - -test "zig fmt: infix operators" { - try testCanonical( - \\test "infix operators" { - \\ var i = undefined; - \\ i = 2; - \\ i *= 2; - \\ i |= 2; - \\ i ^= 2; - \\ i <<= 2; - \\ i >>= 2; - \\ i &= 2; - \\ i *= 2; - \\ i *%= 2; - \\ i -= 2; - \\ i -%= 2; - \\ i += 2; - \\ i +%= 2; - \\ i /= 2; - \\ i %= 2; - \\ _ = i == i; - \\ _ = i != i; - \\ _ = i != i; - \\ _ = i.i; - \\ _ = i || i; - \\ _ = i!i; - \\ _ = i ** i; - \\ _ = i ++ i; - \\ _ = i ?? i; - \\ _ = i % i; - \\ _ = i / i; - \\ _ = i *% i; - \\ _ = i * i; - \\ _ = i -% i; - \\ _ = i - i; - \\ _ = i +% i; - \\ _ = i + i; - \\ _ = i << i; - \\ _ = i >> i; - \\ _ = i & i; - \\ _ = i ^ i; - \\ _ = i | i; - \\ _ = i >= i; - \\ _ = i <= i; - \\ _ = i > i; - \\ _ = i < i; - \\ _ = i and i; - \\ _ = i or i; - \\} - \\ - ); -} - -test "zig fmt: precedence" { - try testCanonical( - \\test "precedence" { - \\ a!b(); - \\ (a!b)(); - \\ !a!b; - \\ !(a!b); - \\ !a{ }; - \\ !(a{ }); - \\ a + b{ }; - \\ (a + b){ }; - \\ a << b + c; - \\ (a << b) + c; - \\ a & b << c; - \\ (a & b) << c; - \\ a ^ b & c; - \\ (a ^ b) & c; - \\ a | b ^ c; - \\ (a | b) ^ c; - \\ a == b | c; - \\ (a == b) | c; - \\ a and b == c; - \\ (a and b) == c; - \\ a or b and c; - \\ (a or b) and c; - \\ (a or b) and c; - \\} - \\ - ); -} - -test "zig fmt: prefix operators" { - try testCanonical( - \\test "prefix operators" { - \\ try return --%~??!*&0; - \\} - \\ - ); -} - -test "zig fmt: call expression" { - try testCanonical( - \\test "test calls" { - \\ a(); - \\ a(1); - \\ a(1, 2); - \\ a(1, 2) + a(1, 2); - \\} - \\ - ); -} - -test "zig fmt: var args" { - try testCanonical( - \\fn print(args: ...) void {} - \\ - ); -} - -test "zig fmt: extern function" { - try testCanonical( - \\extern fn puts(s: &const u8) c_int; - \\extern "c" fn puts(s: &const u8) c_int; - \\ - ); -} - -test "zig fmt: multiline string" { - try testCanonical( - \\const s = - \\ \\ something - \\ \\ something else - \\ ; - \\ - ); -} - -test "zig fmt: values" { - try testCanonical( - \\test "values" { - \\ 1; - \\ 1.0; - \\ "string"; - \\ c"cstring"; - \\ 'c'; - \\ true; - \\ false; - \\ null; - \\ undefined; - \\ error; - \\ this; - \\ unreachable; - \\} - \\ - ); -} - -test "zig fmt: indexing" { - try testCanonical( - \\test "test index" { - \\ a[0]; - \\ a[0 + 5]; - \\ a[0..]; - \\ a[0..5]; - \\ a[a[0]]; - \\ a[a[0..]]; - \\ a[a[0..5]]; - \\ a[a[0]..]; - \\ a[a[0..5]..]; - \\ a[a[0]..a[0]]; - \\ a[a[0..5]..a[0]]; - \\ a[a[0..5]..a[0..5]]; - \\} - \\ - ); -} - -test "zig fmt: struct declaration" { - try testCanonical( - \\const S = struct { - \\ const Self = this; - \\ f1: u8, - \\ - \\ fn method(self: &Self) Self { - \\ return *self; - \\ } - \\ - \\ f2: u8 - \\}; - \\ - \\const Ps = packed struct { - \\ a: u8, - \\ b: u8, - \\ - \\ c: u8 - \\}; - \\ - \\const Es = extern struct { - \\ a: u8, - \\ b: u8, - \\ - \\ c: u8 - \\}; - \\ - ); -} - -test "zig fmt: enum declaration" { - try testCanonical( - \\const E = enum { - \\ Ok, - \\ SomethingElse = 0 - \\}; - \\ - \\const E2 = enum(u8) { - \\ Ok, - \\ SomethingElse = 255, - \\ SomethingThird - \\}; - \\ - \\const Ee = extern enum { - \\ Ok, - \\ SomethingElse, - \\ SomethingThird - \\}; - \\ - \\const Ep = packed enum { - \\ Ok, - \\ SomethingElse, - \\ SomethingThird - \\}; - \\ - ); -} - -test "zig fmt: union declaration" { - try testCanonical( - \\const U = union { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const Ue = union(enum) { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const E = enum { - \\ Int, - \\ Float, - \\ None, - \\ Bool - \\}; - \\ - \\const Ue2 = union(E) { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - \\const Eu = extern union { - \\ Int: u8, - \\ Float: f32, - \\ None, - \\ Bool: bool - \\}; - \\ - ); -} - -test "zig fmt: error set declaration" { - try testCanonical( - \\const E = error { - \\ A, - \\ B, - \\ - \\ C - \\}; - \\ - ); -} - -test "zig fmt: arrays" { - try testCanonical( - \\test "test array" { - \\ const a: [2]u8 = [2]u8{ 1, 2 }; - \\ const a: [2]u8 = []u8{ 1, 2 }; - \\ const a: [0]u8 = []u8{ }; - \\} - \\ - ); -} - -test "zig fmt: container initializers" { - try testCanonical( - \\const a1 = []u8{ }; - \\const a2 = []u8{ 1, 2, 3, 4 }; - \\const s1 = S{ }; - \\const s2 = S{ .a = 1, .b = 2 }; - \\ - ); -} - -test "zig fmt: catch" { - try testCanonical( - \\test "catch" { - \\ const a: error!u8 = 0; - \\ _ = a catch return; - \\ _ = a catch |err| return; - \\} - \\ - ); -} - -test "zig fmt: blocks" { - try testCanonical( - \\test "blocks" { - \\ { - \\ const a = 0; - \\ const b = 0; - \\ } - \\ - \\ blk: { - \\ const a = 0; - \\ const b = 0; - \\ } - \\ - \\ const r = blk: { - \\ const a = 0; - \\ const b = 0; - \\ }; - \\} - \\ - ); -} - -test "zig fmt: switch" { - try testCanonical( - \\test "switch" { - \\ switch (0) { - \\ 0 => {}, - \\ 1 => unreachable, - \\ 2, 3 => {}, - \\ 4 ... 7 => {}, - \\ 1 + 4 * 3 + 22 => {}, - \\ else => { - \\ const a = 1; - \\ const b = a; - \\ } - \\ } - \\ - \\ const res = switch (0) { - \\ 0 => 0, - \\ 1 => 2, - \\ else => 4 - \\ }; - \\ - \\ const Union = union(enum) { - \\ Int: i64, - \\ Float: f64 - \\ }; - \\ - \\ const u = Union{ .Int = 0 }; - \\ switch (u) { - \\ Union.Int => |int| {}, - \\ Union.Float => |*float| unreachable - \\ } - \\} - \\ - ); -} - -test "zig fmt: while" { - try testCanonical( - \\test "while" { - \\ while (10 < 1) { - \\ unreachable; - \\ } - \\ - \\ while (10 < 1) - \\ unreachable; - \\ - \\ var i: usize = 0; - \\ while (i < 10) : (i += 1) { - \\ continue; - \\ } - \\ - \\ i = 0; - \\ while (i < 10) : (i += 1) - \\ continue; - \\ - \\ i = 0; - \\ var j: usize = 0; - \\ while (i < 10) : ({ - \\ i += 1; - \\ j += 1; - \\ }) { - \\ continue; - \\ } - \\ - \\ var a: ?u8 = 2; - \\ while (a) |v| : (a = null) { - \\ continue; - \\ } - \\ - \\ while (a) |v| : (a = null) - \\ unreachable; - \\ - \\ label: while (10 < 0) { - \\ unreachable; - \\ } - \\ - \\ const res = while (0 < 10) { - \\ break 7; - \\ } else { - \\ unreachable; - \\ }; - \\ - \\ var a: error!u8 = 0; - \\ while (a) |v| { - \\ a = error.Err; - \\ } else |err| { - \\ i = 1; - \\ } - \\ - \\ comptime var k: usize = 0; - \\ inline while (i < 10) : (i += 1) - \\ j += 2; - \\} - \\ - ); -} - -test "zig fmt: for" { - try testCanonical( - \\test "for" { - \\ const a = []u8{ 1, 2, 3 }; - \\ for (a) |v| { - \\ continue; - \\ } - \\ - \\ for (a) |v| - \\ continue; - \\ - \\ for (a) |*v| - \\ continue; - \\ - \\ for (a) |v, i| { - \\ continue; - \\ } - \\ - \\ for (a) |v, i| - \\ continue; - \\ - \\ const res = for (a) |v, i| { - \\ break v; - \\ } else { - \\ unreachable; - \\ }; - \\ - \\ var num: usize = 0; - \\ inline for (a) |v, i| { - \\ num += v; - \\ num += i; - \\ } - \\} - \\ - ); -} - -test "zig fmt: if" { - try testCanonical( - \\test "if" { - \\ if (10 < 0) { - \\ unreachable; - \\ } - \\ - \\ if (10 < 0) unreachable; - \\ - \\ if (10 < 0) { - \\ unreachable; - \\ } else { - \\ const a = 20; - \\ } - \\ - \\ if (10 < 0) { - \\ unreachable; - \\ } else if (5 < 0) { - \\ unreachable; - \\ } else { - \\ const a = 20; - \\ } - \\ - \\ const is_world_broken = if (10 < 0) true else false; - \\ - \\ const a: ?u8 = 10; - \\ const b: ?u8 = null; - \\ if (a) |v| { - \\ const some = v; - \\ } else if (b) |*v| { - \\ unreachable; - \\ } else { - \\ const some = 10; - \\ } - \\ - \\ const non_null_a = if (a) |v| v else 0; - \\ - \\ const a_err: error!u8 = 0; - \\ if (a_err) |v| { - \\ const p = v; - \\ } else |err| { - \\ unreachable; - \\ } - \\} - \\ - ); -} - -test "zig fmt: defer" { - try testCanonical( - \\test "defer" { - \\ var i: usize = 0; - \\ defer i = 1; - \\ defer { - \\ i += 2; - \\ i *= i; - \\ } - \\ - \\ errdefer i += 3; - \\ errdefer { - \\ i += 2; - \\ i /= i; - \\ } - \\} - \\ - ); -} - -test "zig fmt: comptime" { - try testCanonical( - \\fn a() u8 { - \\ return 5; - \\} - \\ - \\fn b(comptime i: u8) u8 { - \\ return i; - \\} - \\ - \\const av = comptime a(); - \\const av2 = comptime blk: { - \\ var res = a(); - \\ res *= b(2); - \\ break :blk res; - \\}; - \\ - \\comptime { - \\ _ = a(); - \\} - \\ - \\test "comptime" { - \\ const av3 = comptime a(); - \\ const av4 = comptime blk: { - \\ var res = a(); - \\ res *= a(); - \\ break :blk res; - \\ }; - \\ - \\ comptime var i = 0; - \\ comptime { - \\ i = a(); - \\ i += b(i); - \\ } - \\} - \\ - ); -} - -test "zig fmt: fn type" { - try testCanonical( - \\fn a(i: u8) u8 { - \\ return i + 1; - \\} - \\ - \\const a: fn(u8) u8 = undefined; - \\const b: extern fn(u8) u8 = undefined; - \\const c: nakedcc fn(u8) u8 = undefined; - \\const ap: fn(u8) u8 = a; - \\ - ); -} - -test "zig fmt: inline asm" { - try testCanonical( - \\pub fn syscall1(number: usize, arg1: usize) usize { - \\ return asm volatile ("syscall" - \\ : [ret] "={rax}" (-> usize) - \\ : [number] "{rax}" (number), - \\ [arg1] "{rdi}" (arg1) - \\ : "rcx", "r11"); - \\} - \\ - ); -} - -test "zig fmt: coroutines" { - try testCanonical( - \\async fn simpleAsyncFn() void { - \\ x += 1; - \\ suspend; - \\ x += 1; - \\ suspend |p| {} - \\ const p = async simpleAsyncFn() catch unreachable; - \\ await p; - \\} - \\ - \\test "coroutine suspend, resume, cancel" { - \\ const p = try async testAsyncSeq(); - \\ resume p; - \\ cancel p; - \\} - \\ - ); +test "1" { + try testCanonical(@embedFile("../array_list.zig")); } From 706e0d739e5bb4a6e7a0e4f6e168eb71069e4df2 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 13:49:52 +0200 Subject: [PATCH 118/132] std.zig.parser readded all tests * Ops! --- std/zig/parser.zig | 1527 ++++++++++++++++++++++---------------------- 1 file changed, 761 insertions(+), 766 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 33408bcc3..5f7412b5c 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1119,12 +1119,6 @@ pub const Parser = struct { }, State.TypeExprBegin => |dest_ptr| { - const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_var) { - @panic("TODO param with type var"); - } - self.putBackToken(token); - stack.append(State { .TypeExprEnd = dest_ptr }) catch unreachable; try stack.append(State { .PrefixOpExpression = dest_ptr }); continue; @@ -1160,9 +1154,6 @@ pub const Parser = struct { try stack.append(State { .AddrOfModifiers = &node.op.AddrOf }); } continue; - //Token.Id.Keyword_await => { - // @panic("TODO: await"); - //}, } else { self.putBackToken(token); stack.append(State { .SuffixOpExpressionBegin = dest_ptr }) catch unreachable; @@ -1408,7 +1399,6 @@ pub const Parser = struct { continue; }, Token.Id.LBracket => { - // TODO: option("align" "(" Expression option(":" Integer ":" Integer) ")")) option("const") option("volatile") const rbracket_token = self.getNextToken(); if (rbracket_token.id == Token.Id.RBracket) { const node = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ @@ -4222,760 +4212,765 @@ fn testCanonical(source: []const u8) !void { } } -//test "zig fmt: get stdout or fail" { -// try testCanonical( -// \\const std = @import("std"); -// \\ -// \\pub fn main() !void { -// \\ // If this program is run without stdout attached, exit with an error. -// \\ // another comment -// \\ var stdout_file = try std.io.getStdOut; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: preserve spacing" { -// try testCanonical( -// \\const std = @import("std"); -// \\ -// \\pub fn main() !void { -// \\ var stdout_file = try std.io.getStdOut; -// \\ var stdout_file = try std.io.getStdOut; -// \\ -// \\ var stdout_file = try std.io.getStdOut; -// \\ var stdout_file = try std.io.getStdOut; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: return types" { -// try testCanonical( -// \\pub fn main() !void {} -// \\pub fn main() var {} -// \\pub fn main() i32 {} -// \\ -// ); -//} -// -//test "zig fmt: imports" { -// try testCanonical( -// \\const std = @import("std"); -// \\const std = @import(); -// \\ -// ); -//} -// -//test "zig fmt: global declarations" { -// try testCanonical( -// \\const a = b; -// \\pub const a = b; -// \\var a = b; -// \\pub var a = b; -// \\const a: i32 = b; -// \\pub const a: i32 = b; -// \\var a: i32 = b; -// \\pub var a: i32 = b; -// \\extern const a: i32 = b; -// \\pub extern const a: i32 = b; -// \\extern var a: i32 = b; -// \\pub extern var a: i32 = b; -// \\extern "a" const a: i32 = b; -// \\pub extern "a" const a: i32 = b; -// \\extern "a" var a: i32 = b; -// \\pub extern "a" var a: i32 = b; -// \\ -// ); -//} -// -//test "zig fmt: extern declaration" { -// try testCanonical( -// \\extern var foo: c_int; -// \\ -// ); -//} -// -//test "zig fmt: alignment" { -// try testCanonical( -// \\var foo: c_int align(1); -// \\ -// ); -//} -// -//test "zig fmt: C main" { -// try testCanonical( -// \\fn main(argc: c_int, argv: &&u8) c_int { -// \\ const a = b; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: return" { -// try testCanonical( -// \\fn foo(argc: c_int, argv: &&u8) c_int { -// \\ return 0; -// \\} -// \\ -// \\fn bar() void { -// \\ return; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: pointer attributes" { -// try testCanonical( -// \\extern fn f1(s: &align(&u8) u8) c_int; -// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; -// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; -// \\extern fn f4(s: &align(1) const volatile u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: slice attributes" { -// try testCanonical( -// \\extern fn f1(s: &align(&u8) u8) c_int; -// \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; -// \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; -// \\extern fn f4(s: &align(1) const volatile u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: test declaration" { -// try testCanonical( -// \\test "test name" { -// \\ const a = 1; -// \\ var b = 1; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: infix operators" { -// try testCanonical( -// \\test "infix operators" { -// \\ var i = undefined; -// \\ i = 2; -// \\ i *= 2; -// \\ i |= 2; -// \\ i ^= 2; -// \\ i <<= 2; -// \\ i >>= 2; -// \\ i &= 2; -// \\ i *= 2; -// \\ i *%= 2; -// \\ i -= 2; -// \\ i -%= 2; -// \\ i += 2; -// \\ i +%= 2; -// \\ i /= 2; -// \\ i %= 2; -// \\ _ = i == i; -// \\ _ = i != i; -// \\ _ = i != i; -// \\ _ = i.i; -// \\ _ = i || i; -// \\ _ = i!i; -// \\ _ = i ** i; -// \\ _ = i ++ i; -// \\ _ = i ?? i; -// \\ _ = i % i; -// \\ _ = i / i; -// \\ _ = i *% i; -// \\ _ = i * i; -// \\ _ = i -% i; -// \\ _ = i - i; -// \\ _ = i +% i; -// \\ _ = i + i; -// \\ _ = i << i; -// \\ _ = i >> i; -// \\ _ = i & i; -// \\ _ = i ^ i; -// \\ _ = i | i; -// \\ _ = i >= i; -// \\ _ = i <= i; -// \\ _ = i > i; -// \\ _ = i < i; -// \\ _ = i and i; -// \\ _ = i or i; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: precedence" { -// try testCanonical( -// \\test "precedence" { -// \\ a!b(); -// \\ (a!b)(); -// \\ !a!b; -// \\ !(a!b); -// \\ !a{ }; -// \\ !(a{ }); -// \\ a + b{ }; -// \\ (a + b){ }; -// \\ a << b + c; -// \\ (a << b) + c; -// \\ a & b << c; -// \\ (a & b) << c; -// \\ a ^ b & c; -// \\ (a ^ b) & c; -// \\ a | b ^ c; -// \\ (a | b) ^ c; -// \\ a == b | c; -// \\ (a == b) | c; -// \\ a and b == c; -// \\ (a and b) == c; -// \\ a or b and c; -// \\ (a or b) and c; -// \\ (a or b) and c; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: prefix operators" { -// try testCanonical( -// \\test "prefix operators" { -// \\ try return --%~??!*&0; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: call expression" { -// try testCanonical( -// \\test "test calls" { -// \\ a(); -// \\ a(1); -// \\ a(1, 2); -// \\ a(1, 2) + a(1, 2); -// \\} -// \\ -// ); -//} -// -//test "zig fmt: var args" { -// try testCanonical( -// \\fn print(args: ...) void {} -// \\ -// ); -//} -// -//test "zig fmt: extern function" { -// try testCanonical( -// \\extern fn puts(s: &const u8) c_int; -// \\extern "c" fn puts(s: &const u8) c_int; -// \\ -// ); -//} -// -//test "zig fmt: multiline string" { -// try testCanonical( -// \\const s = -// \\ \\ something -// \\ \\ something else -// \\ ; -// \\ -// ); -//} -// -//test "zig fmt: values" { -// try testCanonical( -// \\test "values" { -// \\ 1; -// \\ 1.0; -// \\ "string"; -// \\ c"cstring"; -// \\ 'c'; -// \\ true; -// \\ false; -// \\ null; -// \\ undefined; -// \\ error; -// \\ this; -// \\ unreachable; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: indexing" { -// try testCanonical( -// \\test "test index" { -// \\ a[0]; -// \\ a[0 + 5]; -// \\ a[0..]; -// \\ a[0..5]; -// \\ a[a[0]]; -// \\ a[a[0..]]; -// \\ a[a[0..5]]; -// \\ a[a[0]..]; -// \\ a[a[0..5]..]; -// \\ a[a[0]..a[0]]; -// \\ a[a[0..5]..a[0]]; -// \\ a[a[0..5]..a[0..5]]; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: struct declaration" { -// try testCanonical( -// \\const S = struct { -// \\ const Self = this; -// \\ f1: u8, -// \\ -// \\ fn method(self: &Self) Self { -// \\ return *self; -// \\ } -// \\ -// \\ f2: u8 -// \\}; -// \\ -// \\const Ps = packed struct { -// \\ a: u8, -// \\ b: u8, -// \\ -// \\ c: u8 -// \\}; -// \\ -// \\const Es = extern struct { -// \\ a: u8, -// \\ b: u8, -// \\ -// \\ c: u8 -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: enum declaration" { -// try testCanonical( -// \\const E = enum { -// \\ Ok, -// \\ SomethingElse = 0 -// \\}; -// \\ -// \\const E2 = enum(u8) { -// \\ Ok, -// \\ SomethingElse = 255, -// \\ SomethingThird -// \\}; -// \\ -// \\const Ee = extern enum { -// \\ Ok, -// \\ SomethingElse, -// \\ SomethingThird -// \\}; -// \\ -// \\const Ep = packed enum { -// \\ Ok, -// \\ SomethingElse, -// \\ SomethingThird -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: union declaration" { -// try testCanonical( -// \\const U = union { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const Ue = union(enum) { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const E = enum { -// \\ Int, -// \\ Float, -// \\ None, -// \\ Bool -// \\}; -// \\ -// \\const Ue2 = union(E) { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// \\const Eu = extern union { -// \\ Int: u8, -// \\ Float: f32, -// \\ None, -// \\ Bool: bool -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: error set declaration" { -// try testCanonical( -// \\const E = error { -// \\ A, -// \\ B, -// \\ -// \\ C -// \\}; -// \\ -// ); -//} -// -//test "zig fmt: arrays" { -// try testCanonical( -// \\test "test array" { -// \\ const a: [2]u8 = [2]u8{ 1, 2 }; -// \\ const a: [2]u8 = []u8{ 1, 2 }; -// \\ const a: [0]u8 = []u8{ }; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: container initializers" { -// try testCanonical( -// \\const a1 = []u8{ }; -// \\const a2 = []u8{ 1, 2, 3, 4 }; -// \\const s1 = S{ }; -// \\const s2 = S{ .a = 1, .b = 2 }; -// \\ -// ); -//} -// -//test "zig fmt: catch" { -// try testCanonical( -// \\test "catch" { -// \\ const a: error!u8 = 0; -// \\ _ = a catch return; -// \\ _ = a catch |err| return; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: blocks" { -// try testCanonical( -// \\test "blocks" { -// \\ { -// \\ const a = 0; -// \\ const b = 0; -// \\ } -// \\ -// \\ blk: { -// \\ const a = 0; -// \\ const b = 0; -// \\ } -// \\ -// \\ const r = blk: { -// \\ const a = 0; -// \\ const b = 0; -// \\ }; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: switch" { -// try testCanonical( -// \\test "switch" { -// \\ switch (0) { -// \\ 0 => {}, -// \\ 1 => unreachable, -// \\ 2, 3 => {}, -// \\ 4 ... 7 => {}, -// \\ 1 + 4 * 3 + 22 => {}, -// \\ else => { -// \\ const a = 1; -// \\ const b = a; -// \\ } -// \\ } -// \\ -// \\ const res = switch (0) { -// \\ 0 => 0, -// \\ 1 => 2, -// \\ else => 4 -// \\ }; -// \\ -// \\ const Union = union(enum) { -// \\ Int: i64, -// \\ Float: f64 -// \\ }; -// \\ -// \\ const u = Union{ .Int = 0 }; -// \\ switch (u) { -// \\ Union.Int => |int| {}, -// \\ Union.Float => |*float| unreachable -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: while" { -// try testCanonical( -// \\test "while" { -// \\ while (10 < 1) { -// \\ unreachable; -// \\ } -// \\ -// \\ while (10 < 1) -// \\ unreachable; -// \\ -// \\ var i: usize = 0; -// \\ while (i < 10) : (i += 1) { -// \\ continue; -// \\ } -// \\ -// \\ i = 0; -// \\ while (i < 10) : (i += 1) -// \\ continue; -// \\ -// \\ i = 0; -// \\ var j: usize = 0; -// \\ while (i < 10) : ({ -// \\ i += 1; -// \\ j += 1; -// \\ }) { -// \\ continue; -// \\ } -// \\ -// \\ var a: ?u8 = 2; -// \\ while (a) |v| : (a = null) { -// \\ continue; -// \\ } -// \\ -// \\ while (a) |v| : (a = null) -// \\ unreachable; -// \\ -// \\ label: while (10 < 0) { -// \\ unreachable; -// \\ } -// \\ -// \\ const res = while (0 < 10) { -// \\ break 7; -// \\ } else { -// \\ unreachable; -// \\ }; -// \\ -// \\ var a: error!u8 = 0; -// \\ while (a) |v| { -// \\ a = error.Err; -// \\ } else |err| { -// \\ i = 1; -// \\ } -// \\ -// \\ comptime var k: usize = 0; -// \\ inline while (i < 10) : (i += 1) -// \\ j += 2; -// \\} -// \\ -// ); -//} -// -//test "zig fmt: for" { -// try testCanonical( -// \\test "for" { -// \\ const a = []u8{ 1, 2, 3 }; -// \\ for (a) |v| { -// \\ continue; -// \\ } -// \\ -// \\ for (a) |v| -// \\ continue; -// \\ -// \\ for (a) |*v| -// \\ continue; -// \\ -// \\ for (a) |v, i| { -// \\ continue; -// \\ } -// \\ -// \\ for (a) |v, i| -// \\ continue; -// \\ -// \\ const res = for (a) |v, i| { -// \\ break v; -// \\ } else { -// \\ unreachable; -// \\ }; -// \\ -// \\ var num: usize = 0; -// \\ inline for (a) |v, i| { -// \\ num += v; -// \\ num += i; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: if" { -// try testCanonical( -// \\test "if" { -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } -// \\ -// \\ if (10 < 0) unreachable; -// \\ -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } else { -// \\ const a = 20; -// \\ } -// \\ -// \\ if (10 < 0) { -// \\ unreachable; -// \\ } else if (5 < 0) { -// \\ unreachable; -// \\ } else { -// \\ const a = 20; -// \\ } -// \\ -// \\ const is_world_broken = if (10 < 0) true else false; -// \\ -// \\ const a: ?u8 = 10; -// \\ const b: ?u8 = null; -// \\ if (a) |v| { -// \\ const some = v; -// \\ } else if (b) |*v| { -// \\ unreachable; -// \\ } else { -// \\ const some = 10; -// \\ } -// \\ -// \\ const non_null_a = if (a) |v| v else 0; -// \\ -// \\ const a_err: error!u8 = 0; -// \\ if (a_err) |v| { -// \\ const p = v; -// \\ } else |err| { -// \\ unreachable; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: defer" { -// try testCanonical( -// \\test "defer" { -// \\ var i: usize = 0; -// \\ defer i = 1; -// \\ defer { -// \\ i += 2; -// \\ i *= i; -// \\ } -// \\ -// \\ errdefer i += 3; -// \\ errdefer { -// \\ i += 2; -// \\ i /= i; -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: comptime" { -// try testCanonical( -// \\fn a() u8 { -// \\ return 5; -// \\} -// \\ -// \\fn b(comptime i: u8) u8 { -// \\ return i; -// \\} -// \\ -// \\const av = comptime a(); -// \\const av2 = comptime blk: { -// \\ var res = a(); -// \\ res *= b(2); -// \\ break :blk res; -// \\}; -// \\ -// \\comptime { -// \\ _ = a(); -// \\} -// \\ -// \\test "comptime" { -// \\ const av3 = comptime a(); -// \\ const av4 = comptime blk: { -// \\ var res = a(); -// \\ res *= a(); -// \\ break :blk res; -// \\ }; -// \\ -// \\ comptime var i = 0; -// \\ comptime { -// \\ i = a(); -// \\ i += b(i); -// \\ } -// \\} -// \\ -// ); -//} -// -//test "zig fmt: fn type" { -// try testCanonical( -// \\fn a(i: u8) u8 { -// \\ return i + 1; -// \\} -// \\ -// \\const a: fn(u8) u8 = undefined; -// \\const b: extern fn(u8) u8 = undefined; -// \\const c: nakedcc fn(u8) u8 = undefined; -// \\const ap: fn(u8) u8 = a; -// \\ -// ); -//} -// -//test "zig fmt: inline asm" { -// try testCanonical( -// \\pub fn syscall1(number: usize, arg1: usize) usize { -// \\ return asm volatile ("syscall" -// \\ : [ret] "={rax}" (-> usize) -// \\ : [number] "{rax}" (number), -// \\ [arg1] "{rdi}" (arg1) -// \\ : "rcx", "r11"); -// \\} -// \\ -// ); -//} -// -//test "zig fmt: coroutines" { -// try testCanonical( -// \\async fn simpleAsyncFn() void { -// \\ x += 1; -// \\ suspend; -// \\ x += 1; -// \\ suspend |p| {} -// \\ const p = async simpleAsyncFn() catch unreachable; -// \\ await p; -// \\} -// \\ -// \\test "coroutine suspend, resume, cancel" { -// \\ const p = try async testAsyncSeq(); -// \\ resume p; -// \\ cancel p; -// \\} -// \\ -// ); -//} - -test "1" { - try testCanonical(@embedFile("../array_list.zig")); +test "zig fmt: get stdout or fail" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ // If this program is run without stdout attached, exit with an error. + \\ // another comment + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); } + +test "zig fmt: preserve spacing" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\ + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); +} + +test "zig fmt: return types" { + try testCanonical( + \\pub fn main() !void {} + \\pub fn main() var {} + \\pub fn main() i32 {} + \\ + ); +} + +test "zig fmt: imports" { + try testCanonical( + \\const std = @import("std"); + \\const std = @import(); + \\ + ); +} + +test "zig fmt: global declarations" { + try testCanonical( + \\const a = b; + \\pub const a = b; + \\var a = b; + \\pub var a = b; + \\const a: i32 = b; + \\pub const a: i32 = b; + \\var a: i32 = b; + \\pub var a: i32 = b; + \\extern const a: i32 = b; + \\pub extern const a: i32 = b; + \\extern var a: i32 = b; + \\pub extern var a: i32 = b; + \\extern "a" const a: i32 = b; + \\pub extern "a" const a: i32 = b; + \\extern "a" var a: i32 = b; + \\pub extern "a" var a: i32 = b; + \\ + ); +} + +test "zig fmt: extern declaration" { + try testCanonical( + \\extern var foo: c_int; + \\ + ); +} + +test "zig fmt: alignment" { + try testCanonical( + \\var foo: c_int align(1); + \\ + ); +} + +test "zig fmt: C main" { + try testCanonical( + \\fn main(argc: c_int, argv: &&u8) c_int { + \\ const a = b; + \\} + \\ + ); +} + +test "zig fmt: return" { + try testCanonical( + \\fn foo(argc: c_int, argv: &&u8) c_int { + \\ return 0; + \\} + \\ + \\fn bar() void { + \\ return; + \\} + \\ + ); +} + +test "zig fmt: pointer attributes" { + try testCanonical( + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: slice attributes" { + try testCanonical( + \\extern fn f1(s: &align(&u8) u8) c_int; + \\extern fn f2(s: &&align(1) &const &volatile u8) c_int; + \\extern fn f3(s: &align(1) const &align(1) volatile &const volatile u8) c_int; + \\extern fn f4(s: &align(1) const volatile u8) c_int; + \\ + ); +} + +test "zig fmt: test declaration" { + try testCanonical( + \\test "test name" { + \\ const a = 1; + \\ var b = 1; + \\} + \\ + ); +} + +test "zig fmt: infix operators" { + try testCanonical( + \\test "infix operators" { + \\ var i = undefined; + \\ i = 2; + \\ i *= 2; + \\ i |= 2; + \\ i ^= 2; + \\ i <<= 2; + \\ i >>= 2; + \\ i &= 2; + \\ i *= 2; + \\ i *%= 2; + \\ i -= 2; + \\ i -%= 2; + \\ i += 2; + \\ i +%= 2; + \\ i /= 2; + \\ i %= 2; + \\ _ = i == i; + \\ _ = i != i; + \\ _ = i != i; + \\ _ = i.i; + \\ _ = i || i; + \\ _ = i!i; + \\ _ = i ** i; + \\ _ = i ++ i; + \\ _ = i ?? i; + \\ _ = i % i; + \\ _ = i / i; + \\ _ = i *% i; + \\ _ = i * i; + \\ _ = i -% i; + \\ _ = i - i; + \\ _ = i +% i; + \\ _ = i + i; + \\ _ = i << i; + \\ _ = i >> i; + \\ _ = i & i; + \\ _ = i ^ i; + \\ _ = i | i; + \\ _ = i >= i; + \\ _ = i <= i; + \\ _ = i > i; + \\ _ = i < i; + \\ _ = i and i; + \\ _ = i or i; + \\} + \\ + ); +} + +test "zig fmt: precedence" { + try testCanonical( + \\test "precedence" { + \\ a!b(); + \\ (a!b)(); + \\ !a!b; + \\ !(a!b); + \\ !a{ }; + \\ !(a{ }); + \\ a + b{ }; + \\ (a + b){ }; + \\ a << b + c; + \\ (a << b) + c; + \\ a & b << c; + \\ (a & b) << c; + \\ a ^ b & c; + \\ (a ^ b) & c; + \\ a | b ^ c; + \\ (a | b) ^ c; + \\ a == b | c; + \\ (a == b) | c; + \\ a and b == c; + \\ (a and b) == c; + \\ a or b and c; + \\ (a or b) and c; + \\ (a or b) and c; + \\} + \\ + ); +} + +test "zig fmt: prefix operators" { + try testCanonical( + \\test "prefix operators" { + \\ try return --%~??!*&0; + \\} + \\ + ); +} + +test "zig fmt: call expression" { + try testCanonical( + \\test "test calls" { + \\ a(); + \\ a(1); + \\ a(1, 2); + \\ a(1, 2) + a(1, 2); + \\} + \\ + ); +} + +test "zig fmt: var args" { + try testCanonical( + \\fn print(args: ...) void {} + \\ + ); +} + +test "zig fmt: extern function" { + try testCanonical( + \\extern fn puts(s: &const u8) c_int; + \\extern "c" fn puts(s: &const u8) c_int; + \\ + ); +} + +test "zig fmt: multiline string" { + try testCanonical( + \\const s = + \\ \\ something + \\ \\ something else + \\ ; + \\ + ); +} + +test "zig fmt: values" { + try testCanonical( + \\test "values" { + \\ 1; + \\ 1.0; + \\ "string"; + \\ c"cstring"; + \\ 'c'; + \\ true; + \\ false; + \\ null; + \\ undefined; + \\ error; + \\ this; + \\ unreachable; + \\} + \\ + ); +} + +test "zig fmt: indexing" { + try testCanonical( + \\test "test index" { + \\ a[0]; + \\ a[0 + 5]; + \\ a[0..]; + \\ a[0..5]; + \\ a[a[0]]; + \\ a[a[0..]]; + \\ a[a[0..5]]; + \\ a[a[0]..]; + \\ a[a[0..5]..]; + \\ a[a[0]..a[0]]; + \\ a[a[0..5]..a[0]]; + \\ a[a[0..5]..a[0..5]]; + \\} + \\ + ); +} + +test "zig fmt: struct declaration" { + try testCanonical( + \\const S = struct { + \\ const Self = this; + \\ f1: u8, + \\ + \\ fn method(self: &Self) Self { + \\ return *self; + \\ } + \\ + \\ f2: u8 + \\}; + \\ + \\const Ps = packed struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8 + \\}; + \\ + \\const Es = extern struct { + \\ a: u8, + \\ b: u8, + \\ + \\ c: u8 + \\}; + \\ + ); +} + +test "zig fmt: enum declaration" { + try testCanonical( + \\const E = enum { + \\ Ok, + \\ SomethingElse = 0 + \\}; + \\ + \\const E2 = enum(u8) { + \\ Ok, + \\ SomethingElse = 255, + \\ SomethingThird + \\}; + \\ + \\const Ee = extern enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird + \\}; + \\ + \\const Ep = packed enum { + \\ Ok, + \\ SomethingElse, + \\ SomethingThird + \\}; + \\ + ); +} + +test "zig fmt: union declaration" { + try testCanonical( + \\const U = union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const Ue = union(enum) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const E = enum { + \\ Int, + \\ Float, + \\ None, + \\ Bool + \\}; + \\ + \\const Ue2 = union(E) { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + \\const Eu = extern union { + \\ Int: u8, + \\ Float: f32, + \\ None, + \\ Bool: bool + \\}; + \\ + ); +} + +test "zig fmt: error set declaration" { + try testCanonical( + \\const E = error { + \\ A, + \\ B, + \\ + \\ C + \\}; + \\ + ); +} + +test "zig fmt: arrays" { + try testCanonical( + \\test "test array" { + \\ const a: [2]u8 = [2]u8{ 1, 2 }; + \\ const a: [2]u8 = []u8{ 1, 2 }; + \\ const a: [0]u8 = []u8{ }; + \\} + \\ + ); +} + +test "zig fmt: container initializers" { + try testCanonical( + \\const a1 = []u8{ }; + \\const a2 = []u8{ 1, 2, 3, 4 }; + \\const s1 = S{ }; + \\const s2 = S{ .a = 1, .b = 2 }; + \\ + ); +} + +test "zig fmt: catch" { + try testCanonical( + \\test "catch" { + \\ const a: error!u8 = 0; + \\ _ = a catch return; + \\ _ = a catch |err| return; + \\} + \\ + ); +} + +test "zig fmt: blocks" { + try testCanonical( + \\test "blocks" { + \\ { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ blk: { + \\ const a = 0; + \\ const b = 0; + \\ } + \\ + \\ const r = blk: { + \\ const a = 0; + \\ const b = 0; + \\ }; + \\} + \\ + ); +} + +test "zig fmt: switch" { + try testCanonical( + \\test "switch" { + \\ switch (0) { + \\ 0 => {}, + \\ 1 => unreachable, + \\ 2, 3 => {}, + \\ 4 ... 7 => {}, + \\ 1 + 4 * 3 + 22 => {}, + \\ else => { + \\ const a = 1; + \\ const b = a; + \\ } + \\ } + \\ + \\ const res = switch (0) { + \\ 0 => 0, + \\ 1 => 2, + \\ else => 4 + \\ }; + \\ + \\ const Union = union(enum) { + \\ Int: i64, + \\ Float: f64 + \\ }; + \\ + \\ const u = Union{ .Int = 0 }; + \\ switch (u) { + \\ Union.Int => |int| {}, + \\ Union.Float => |*float| unreachable + \\ } + \\} + \\ + ); +} + +test "zig fmt: while" { + try testCanonical( + \\test "while" { + \\ while (10 < 1) { + \\ unreachable; + \\ } + \\ + \\ while (10 < 1) + \\ unreachable; + \\ + \\ var i: usize = 0; + \\ while (i < 10) : (i += 1) { + \\ continue; + \\ } + \\ + \\ i = 0; + \\ while (i < 10) : (i += 1) + \\ continue; + \\ + \\ i = 0; + \\ var j: usize = 0; + \\ while (i < 10) : ({ + \\ i += 1; + \\ j += 1; + \\ }) { + \\ continue; + \\ } + \\ + \\ var a: ?u8 = 2; + \\ while (a) |v| : (a = null) { + \\ continue; + \\ } + \\ + \\ while (a) |v| : (a = null) + \\ unreachable; + \\ + \\ label: while (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ const res = while (0 < 10) { + \\ break 7; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ var a: error!u8 = 0; + \\ while (a) |v| { + \\ a = error.Err; + \\ } else |err| { + \\ i = 1; + \\ } + \\ + \\ comptime var k: usize = 0; + \\ inline while (i < 10) : (i += 1) + \\ j += 2; + \\} + \\ + ); +} + +test "zig fmt: for" { + try testCanonical( + \\test "for" { + \\ const a = []u8{ 1, 2, 3 }; + \\ for (a) |v| { + \\ continue; + \\ } + \\ + \\ for (a) |v| + \\ continue; + \\ + \\ for (a) |*v| + \\ continue; + \\ + \\ for (a) |v, i| { + \\ continue; + \\ } + \\ + \\ for (a) |v, i| + \\ continue; + \\ + \\ const res = for (a) |v, i| { + \\ break v; + \\ } else { + \\ unreachable; + \\ }; + \\ + \\ var num: usize = 0; + \\ inline for (a) |v, i| { + \\ num += v; + \\ num += i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: if" { + try testCanonical( + \\test "if" { + \\ if (10 < 0) { + \\ unreachable; + \\ } + \\ + \\ if (10 < 0) unreachable; + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ if (10 < 0) { + \\ unreachable; + \\ } else if (5 < 0) { + \\ unreachable; + \\ } else { + \\ const a = 20; + \\ } + \\ + \\ const is_world_broken = if (10 < 0) true else false; + \\ + \\ const a: ?u8 = 10; + \\ const b: ?u8 = null; + \\ if (a) |v| { + \\ const some = v; + \\ } else if (b) |*v| { + \\ unreachable; + \\ } else { + \\ const some = 10; + \\ } + \\ + \\ const non_null_a = if (a) |v| v else 0; + \\ + \\ const a_err: error!u8 = 0; + \\ if (a_err) |v| { + \\ const p = v; + \\ } else |err| { + \\ unreachable; + \\ } + \\} + \\ + ); +} + +test "zig fmt: defer" { + try testCanonical( + \\test "defer" { + \\ var i: usize = 0; + \\ defer i = 1; + \\ defer { + \\ i += 2; + \\ i *= i; + \\ } + \\ + \\ errdefer i += 3; + \\ errdefer { + \\ i += 2; + \\ i /= i; + \\ } + \\} + \\ + ); +} + +test "zig fmt: comptime" { + try testCanonical( + \\fn a() u8 { + \\ return 5; + \\} + \\ + \\fn b(comptime i: u8) u8 { + \\ return i; + \\} + \\ + \\const av = comptime a(); + \\const av2 = comptime blk: { + \\ var res = a(); + \\ res *= b(2); + \\ break :blk res; + \\}; + \\ + \\comptime { + \\ _ = a(); + \\} + \\ + \\test "comptime" { + \\ const av3 = comptime a(); + \\ const av4 = comptime blk: { + \\ var res = a(); + \\ res *= a(); + \\ break :blk res; + \\ }; + \\ + \\ comptime var i = 0; + \\ comptime { + \\ i = a(); + \\ i += b(i); + \\ } + \\} + \\ + ); +} + +test "zig fmt: fn type" { + try testCanonical( + \\fn a(i: u8) u8 { + \\ return i + 1; + \\} + \\ + \\const a: fn(u8) u8 = undefined; + \\const b: extern fn(u8) u8 = undefined; + \\const c: nakedcc fn(u8) u8 = undefined; + \\const ap: fn(u8) u8 = a; + \\ + ); +} + +test "zig fmt: inline asm" { + try testCanonical( + \\pub fn syscall1(number: usize, arg1: usize) usize { + \\ return asm volatile ("syscall" + \\ : [ret] "={rax}" (-> usize) + \\ : [number] "{rax}" (number), + \\ [arg1] "{rdi}" (arg1) + \\ : "rcx", "r11"); + \\} + \\ + ); +} + +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| {} + \\ const p = async simpleAsyncFn() catch unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + +//{ +// var it = self.link_libs.iterator(); +// while (true) { +// const entry = it.next() ?? break; +// zig_args.append("--library") catch unreachable; +// zig_args.append(entry.key) catch unreachable; +// } +//} From db0812d4b7f856425e0bd26cd6f579468f3ac8ab Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 14:22:01 +0200 Subject: [PATCH 119/132] std.zig.parser: changed block exprs from primary expr to expr --- std/zig/parser.zig | 232 +++++++++++++++++++++++------------------- std/zig/tokenizer.zig | 1 - 2 files changed, 129 insertions(+), 104 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5f7412b5c..65e8056ad 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -766,6 +766,101 @@ pub const Parser = struct { dest_ptr.store(&resume_node.base); stack.append(State { .Expression = DestPtr { .Field = &resume_node.rhs } }) catch unreachable; }, + Token.Id.Keyword_suspend => { + const node = try arena.create(ast.NodeSuspend); + *node = ast.NodeSuspend { + .base = self.initNode(ast.Node.Id.Suspend), + .suspend_token = token, + .payload = null, + .body = null, + }; + dest_ptr.store(&node.base); + stack.append(State { .SuspendBody = node }) catch unreachable; + try stack.append(State { .Payload = &node.payload }); + continue; + }, + Token.Id.Keyword_if => { + const node = try arena.create(ast.NodeIf); + *node = ast.NodeIf { + .base = self.initNode(ast.Node.Id.If), + .if_token = token, + .condition = undefined, + .payload = null, + .body = undefined, + .@"else" = null, + }; + dest_ptr.store(&node.base); + + stack.append(State { .Else = &node.@"else" }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); + try stack.append(State { .PointerPayload = &node.payload }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + continue; + }, + Token.Id.Keyword_while => { + stack.append(State { + .While = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_for => { + stack.append(State { + .For = LoopCtx { + .label = null, + .inline_token = null, + .loop_token = token, + .dest_ptr = dest_ptr, + } + }) catch unreachable; + continue; + }, + Token.Id.Keyword_switch => { + const node = try arena.create(ast.NodeSwitch); + *node = ast.NodeSwitch { + .base = self.initNode(ast.Node.Id.Switch), + .switch_token = token, + .expr = undefined, + .cases = ArrayList(&ast.NodeSwitchCase).init(arena), + .rbrace = undefined, + }; + dest_ptr.store(&node.base); + + stack.append(State { + .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { + .list = &node.cases, + .ptr = &node.rbrace, + }, + }) catch unreachable; + try stack.append(State { .ExpectToken = Token.Id.LBrace }); + try stack.append(State { .ExpectToken = Token.Id.RParen }); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + try stack.append(State { .ExpectToken = Token.Id.LParen }); + }, + Token.Id.Keyword_comptime => { + const node = try arena.create(ast.NodeComptime); + *node = ast.NodeComptime { + .base = self.initNode(ast.Node.Id.Comptime), + .comptime_token = token, + .expr = undefined, + }; + dest_ptr.store(&node.base); + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.LBrace => { + const block = try self.createBlock(arena, (?Token)(null), token); + dest_ptr.store(&block.base); + + stack.append(State { .Block = block }) catch unreachable; + continue; + }, else => { self.putBackToken(token); stack.append(State { .UnwrapExpressionBegin = dest_ptr }) catch unreachable; @@ -1328,19 +1423,6 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, - Token.Id.Keyword_suspend => { - const node = try arena.create(ast.NodeSuspend); - *node = ast.NodeSuspend { - .base = self.initNode(ast.Node.Id.Suspend), - .suspend_token = token, - .payload = null, - .body = null, - }; - dest_ptr.store(&node.base); - stack.append(State { .SuspendBody = node }) catch unreachable; - try stack.append(State { .Payload = &node.payload }); - continue; - }, Token.Id.MultilineStringLiteralLine => { const node = try arena.create(ast.NodeMultilineStringLiteral); *node = ast.NodeMultilineStringLiteral { @@ -1544,13 +1626,6 @@ pub const Parser = struct { }) catch unreachable; continue; }, - Token.Id.LBrace => { - const block = try self.createBlock(arena, (?Token)(null), token); - dest_ptr.store(&block.base); - - stack.append(State { .Block = block }) catch unreachable; - continue; - }, Token.Id.Keyword_fn => { // TODO shouldn't need these casts const fn_proto = try self.createFnProto(arena, token, @@ -1608,26 +1683,6 @@ pub const Parser = struct { try stack.append(State { .AsmOutputItems = &node.outputs }); try stack.append(State { .IfToken = Token.Id.Colon }); }, - Token.Id.Keyword_if => { - const node = try arena.create(ast.NodeIf); - *node = ast.NodeIf { - .base = self.initNode(ast.Node.Id.If), - .if_token = token, - .condition = undefined, - .payload = null, - .body = undefined, - .@"else" = null, - }; - dest_ptr.store(&node.base); - - stack.append(State { .Else = &node.@"else" }) catch unreachable; - try stack.append(State { .Expression = DestPtr { .Field = &node.body } }); - try stack.append(State { .PointerPayload = &node.payload }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.condition } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - continue; - }, Token.Id.Keyword_inline => { stack.append(State { .Inline = InlineCtx { @@ -1638,61 +1693,6 @@ pub const Parser = struct { }) catch unreachable; continue; }, - Token.Id.Keyword_while => { - stack.append(State { - .While = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_for => { - stack.append(State { - .For = LoopCtx { - .label = null, - .inline_token = null, - .loop_token = token, - .dest_ptr = dest_ptr, - } - }) catch unreachable; - continue; - }, - Token.Id.Keyword_switch => { - const node = try arena.create(ast.NodeSwitch); - *node = ast.NodeSwitch { - .base = self.initNode(ast.Node.Id.Switch), - .switch_token = token, - .expr = undefined, - .cases = ArrayList(&ast.NodeSwitchCase).init(arena), - .rbrace = undefined, - }; - dest_ptr.store(&node.base); - - stack.append(State { - .SwitchCaseOrEnd = ListSave(&ast.NodeSwitchCase) { - .list = &node.cases, - .ptr = &node.rbrace, - }, - }) catch unreachable; - try stack.append(State { .ExpectToken = Token.Id.LBrace }); - try stack.append(State { .ExpectToken = Token.Id.RParen }); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - try stack.append(State { .ExpectToken = Token.Id.LParen }); - }, - Token.Id.Keyword_comptime => { - const node = try arena.create(ast.NodeComptime); - *node = ast.NodeComptime { - .base = self.initNode(ast.Node.Id.Comptime), - .comptime_token = token, - .expr = undefined, - }; - dest_ptr.store(&node.base); - try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); - continue; - }, else => { try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id)); continue; @@ -4966,11 +4966,37 @@ test "zig fmt: coroutines" { ); } -//{ -// var it = self.link_libs.iterator(); -// while (true) { -// const entry = it.next() ?? break; -// zig_args.append("--library") catch unreachable; -// zig_args.append(entry.key) catch unreachable; -// } -//} +test "zig fmt: coroutines" { + try testCanonical( + \\async fn simpleAsyncFn() void { + \\ x += 1; + \\ suspend; + \\ x += 1; + \\ suspend |p| {} + \\ const p = async simpleAsyncFn() catch unreachable; + \\ await p; + \\} + \\ + \\test "coroutine suspend, resume, cancel" { + \\ const p = try async testAsyncSeq(); + \\ resume p; + \\ cancel p; + \\} + \\ + ); +} + +test "zig fmt: Block after if" { + try testCanonical( + \\test "Block after if" { + \\ if (true) { + \\ const a = 0; + \\ } + \\ + \\ { + \\ const a = 0; + \\ } + \\} + \\ + ); +} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 7b1f86712..91fb20974 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -1158,7 +1158,6 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { const token = tokenizer.next(); - std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id)); std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id)); switch (expected_token_id) { Token.Id.StringLiteral => |expected_kind| { From 3b80e665074cd56d9e24fb8ae0fb9d86e8cd841a Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 14:52:47 +0200 Subject: [PATCH 120/132] std.zig.parser now parses toplevel use --- std/zig/ast.zig | 29 ++++++++++++++++ std/zig/parser.zig | 87 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 93 insertions(+), 23 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 2128b9976..1096086e3 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -11,6 +11,7 @@ pub const Node = struct { pub const Id = enum { Root, VarDecl, + Use, ErrorSetDecl, ContainerDecl, StructField, @@ -63,6 +64,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index), + Id.Use => @fieldParentPtr(NodeUse, "base", base).iterate(index), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).iterate(index), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).iterate(index), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).iterate(index), @@ -116,6 +118,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.Use => @fieldParentPtr(NodeUse, "base", base).firstToken(), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).firstToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).firstToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).firstToken(), @@ -169,6 +172,7 @@ pub const Node = struct { return switch (base.id) { Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.Use => @fieldParentPtr(NodeUse, "base", base).lastToken(), Id.ErrorSetDecl => @fieldParentPtr(NodeErrorSetDecl, "base", base).lastToken(), Id.ContainerDecl => @fieldParentPtr(NodeContainerDecl, "base", base).lastToken(), Id.StructField => @fieldParentPtr(NodeStructField, "base", base).lastToken(), @@ -288,6 +292,31 @@ pub const NodeVarDecl = struct { } }; +pub const NodeUse = struct { + base: Node, + visib_token: ?Token, + expr: &Node, + semicolon_token: Token, + + pub fn iterate(self: &NodeUse, index: usize) ?&Node { + var i = index; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: &NodeUse) Token { + if (self.visib_token) |visib_token| return visib_token; + return self.expr.firstToken(); + } + + pub fn lastToken(self: &NodeUse) Token { + return self.semicolon_token; + } +}; + pub const NodeErrorSetDecl = struct { base: Node, error_token: Token, diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 65e8056ad..e11bcfef0 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -348,31 +348,54 @@ pub const Parser = struct { }, State.TopLevelExtern => |ctx| { const token = self.getNextToken(); - if (token.id == Token.Id.Keyword_extern) { - const lib_name_token = self.getNextToken(); - const lib_name = blk: { - if (lib_name_token.id == Token.Id.StringLiteral) { - const res = try self.createStringLiteral(arena, lib_name_token); - break :blk &res.base; - } else { - self.putBackToken(lib_name_token); - break :blk null; - } - }; - - stack.append(State { - .TopLevelDecl = TopLevelDeclCtx { - .decls = ctx.decls, + switch (token.id) { + Token.Id.Keyword_use => { + const node = try arena.create(ast.NodeUse); + *node = ast.NodeUse { + .base = self.initNode(ast.Node.Id.Use), .visib_token = ctx.visib_token, - .extern_token = token, - .lib_name = lib_name, - }, - }) catch unreachable; - continue; + .expr = undefined, + .semicolon_token = undefined, + }; + try ctx.decls.append(&node.base); + + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Semicolon, + .ptr = &node.semicolon_token, + } + }) catch unreachable; + try stack.append(State { .Expression = DestPtr { .Field = &node.expr } }); + continue; + }, + Token.Id.Keyword_extern => { + const lib_name_token = self.getNextToken(); + const lib_name = blk: { + if (lib_name_token.id == Token.Id.StringLiteral) { + const res = try self.createStringLiteral(arena, lib_name_token); + break :blk &res.base; + } else { + self.putBackToken(lib_name_token); + break :blk null; + } + }; + + stack.append(State { + .TopLevelDecl = TopLevelDeclCtx { + .decls = ctx.decls, + .visib_token = ctx.visib_token, + .extern_token = token, + .lib_name = lib_name, + }, + }) catch unreachable; + continue; + }, + else => { + self.putBackToken(token); + stack.append(State { .TopLevelDecl = ctx }) catch unreachable; + continue; + } } - self.putBackToken(token); - stack.append(State { .TopLevelDecl = ctx }) catch unreachable; - continue; }, State.TopLevelDecl => |ctx| { const token = self.getNextToken(); @@ -3068,6 +3091,15 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = decl }); }, + ast.Node.Id.Use => { + const use_decl = @fieldParentPtr(ast.NodeUse, "base", decl); + if (use_decl.visib_token) |visib_token| { + try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token)); + } + try stream.print("use "); + try stack.append(RenderState { .Text = ";" }); + try stack.append(RenderState { .Expression = use_decl.expr }); + }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); try stack.append(RenderState { .VarDecl = var_decl}); @@ -4086,6 +4118,7 @@ pub const Parser = struct { ast.Node.Id.EnumTag, ast.Node.Id.Root, ast.Node.Id.VarDecl, + ast.Node.Id.Use, ast.Node.Id.TestDecl, ast.Node.Id.ParamDecl => unreachable, }, @@ -5000,3 +5033,11 @@ test "zig fmt: Block after if" { \\ ); } + +test "zig fmt: use" { + try testCanonical( + \\use @import("std"); + \\pub use @import("std"); + \\ + ); +} From aa09e7b63995639084d25329954b1972a72ad12d Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:01:21 +0200 Subject: [PATCH 121/132] std.zig.tokinizer now treats string identifiers as identifiers --- std/zig/parser.zig | 8 ++++++++ std/zig/tokenizer.zig | 5 ++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index e11bcfef0..a01a0d748 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -5041,3 +5041,11 @@ test "zig fmt: use" { \\ ); } + +test "zig fmt: string identifier" { + try testCanonical( + \\const @"a b" = @"c d".@"e f"; + \\fn @"g h"() void {} + \\ + ); +} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index 91fb20974..a2c4def9e 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -78,7 +78,6 @@ pub const Token = struct { StringLiteral: StrLitKind, MultilineStringLiteralLine: StrLitKind, CharLiteral, - StringIdentifier, Eof, Builtin, Bang, @@ -434,7 +433,7 @@ pub const Tokenizer = struct { State.SawAtSign => switch (c) { '"' => { - result.id = Token.Id.StringIdentifier; + result.id = Token.Id.Identifier; state = State.StringLiteral; }, else => { @@ -1136,7 +1135,7 @@ test "tokenizer - string identifier and builtin fns" { , []Token.Id{ Token.Id.Keyword_const, - Token.Id.StringIdentifier, + Token.Id.Identifier, Token.Id.Equal, Token.Id.Builtin, Token.Id.LParen, From db9a9f3a6c248e74953d9f7cbf6bedc78e3c23ff Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:16:31 +0200 Subject: [PATCH 122/132] std.zig.parser now parses the `var` type * I parse it as a type in all contexts. This is not how the C++ compiler does it, but I think typechecking should catch this --- std/zig/ast.zig | 23 +++++++++++++++++++++-- std/zig/parser.zig | 30 +++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 1096086e3..d296b3b52 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -55,6 +55,7 @@ pub const Node = struct { AsmOutput, Unreachable, ErrorType, + VarType, BuiltinCall, LineComment, TestDecl, @@ -108,6 +109,7 @@ pub const Node = struct { Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).iterate(index), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).iterate(index), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).iterate(index), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).iterate(index), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).iterate(index), @@ -162,6 +164,7 @@ pub const Node = struct { Id.AsmInput => @fieldParentPtr(NodeAsmInput, "base", base).firstToken(), Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).firstToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).firstToken(), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).firstToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).firstToken(), @@ -216,6 +219,7 @@ pub const Node = struct { Id.AsmOutput => @fieldParentPtr(NodeAsmOutput, "base", base).lastToken(), Id.Unreachable => @fieldParentPtr(NodeUnreachable, "base", base).lastToken(), Id.ErrorType => @fieldParentPtr(NodeErrorType, "base", base).lastToken(), + Id.VarType => @fieldParentPtr(NodeVarType, "base", base).lastToken(), Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(), Id.TestDecl => @fieldParentPtr(NodeTestDecl, "base", base).lastToken(), @@ -541,7 +545,6 @@ pub const NodeFnProto = struct { pub const ReturnType = union(enum) { Explicit: &Node, - Infer: Token, InferErrorSet: &Node, }; @@ -597,7 +600,6 @@ pub const NodeFnProto = struct { // TODO allow this and next prong to share bodies since the types are the same ReturnType.Explicit => |node| return node.lastToken(), ReturnType.InferErrorSet => |node| return node.lastToken(), - ReturnType.Infer => |token| return token, } } }; @@ -1788,6 +1790,23 @@ pub const NodeErrorType = struct { } }; +pub const NodeVarType = struct { + base: Node, + token: Token, + + pub fn iterate(self: &NodeVarType, index: usize) ?&Node { + return null; + } + + pub fn firstToken(self: &NodeVarType) Token { + return self.token; + } + + pub fn lastToken(self: &NodeVarType) Token { + return self.token; + } +}; + pub const NodeLineComment = struct { base: Node, lines: ArrayList(Token), diff --git a/std/zig/parser.zig b/std/zig/parser.zig index a01a0d748..79c90e6e6 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -1437,6 +1437,14 @@ pub const Parser = struct { dest_ptr.store(&node.base); continue; }, + Token.Id.Keyword_var => { + const node = try arena.create(ast.NodeVarType); + *node = ast.NodeVarType { + .base = self.initNode(ast.Node.Id.VarType), + .token = token, + }; + dest_ptr.store(&node.base); + }, Token.Id.Keyword_unreachable => { const node = try arena.create(ast.NodeUnreachable); *node = ast.NodeUnreachable { @@ -2192,9 +2200,6 @@ pub const Parser = struct { State.FnProtoReturnType => |fn_proto| { const token = self.getNextToken(); switch (token.id) { - Token.Id.Keyword_var => { - fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token }; - }, Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; stack.append(State { @@ -3573,6 +3578,10 @@ pub const Parser = struct { const error_type = @fieldParentPtr(ast.NodeErrorType, "base", base); try stream.print("{}", self.tokenizer.getTokenSlice(error_type.token)); }, + ast.Node.Id.VarType => { + const var_type = @fieldParentPtr(ast.NodeVarType, "base", base); + try stream.print("{}", self.tokenizer.getTokenSlice(var_type.token)); + }, ast.Node.Id.ContainerDecl => { const container_decl = @fieldParentPtr(ast.NodeContainerDecl, "base", base); @@ -3711,9 +3720,6 @@ pub const Parser = struct { ast.NodeFnProto.ReturnType.Explicit => |node| { try stack.append(RenderState { .Expression = node}); }, - ast.NodeFnProto.ReturnType.Infer => { - try stack.append(RenderState { .Text = "var"}); - }, ast.NodeFnProto.ReturnType.InferErrorSet => |node| { try stack.append(RenderState { .Expression = node}); try stack.append(RenderState { .Text = "!"}); @@ -4136,9 +4142,6 @@ pub const Parser = struct { ast.NodeFnProto.ReturnType.Explicit => |node| { try stack.append(RenderState { .Expression = node}); }, - ast.NodeFnProto.ReturnType.Infer => { - try stream.print("var"); - }, ast.NodeFnProto.ReturnType.InferErrorSet => |node| { try stream.print("!"); try stack.append(RenderState { .Expression = node}); @@ -4489,6 +4492,15 @@ test "zig fmt: var args" { ); } +test "zig fmt: var type" { + try testCanonical( + \\fn print(args: var) var {} + \\const Var = var; + \\const i: var = 0; + \\ + ); +} + test "zig fmt: extern function" { try testCanonical( \\extern fn puts(s: &const u8) c_int; From b9cccce26d616c2ff570a1e1ba7073d3e8f79672 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 15:56:37 +0200 Subject: [PATCH 123/132] std.zig.ast: fixed none compiling code --- std/zig/ast.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/std/zig/ast.zig b/std/zig/ast.zig index d296b3b52..045548d62 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -566,7 +566,6 @@ pub const NodeFnProto = struct { if (i < 1) return node; i -= 1; }, - ReturnType.Infer => {}, } if (self.align_expr) |align_expr| { From c6aa637146c6695f382553f66d311599a90d7425 Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 16:33:43 +0200 Subject: [PATCH 124/132] std.zig.parser: removed dublicate "zig fmt: coroutines" test --- std/zig/parser.zig | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 79c90e6e6..76eb2d29d 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -5011,26 +5011,6 @@ test "zig fmt: coroutines" { ); } -test "zig fmt: coroutines" { - try testCanonical( - \\async fn simpleAsyncFn() void { - \\ x += 1; - \\ suspend; - \\ x += 1; - \\ suspend |p| {} - \\ const p = async simpleAsyncFn() catch unreachable; - \\ await p; - \\} - \\ - \\test "coroutine suspend, resume, cancel" { - \\ const p = try async testAsyncSeq(); - \\ resume p; - \\ cancel p; - \\} - \\ - ); -} - test "zig fmt: Block after if" { try testCanonical( \\test "Block after if" { From 477ded9042f045a322b3e92c632e322834a0cdd9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 11:00:26 -0400 Subject: [PATCH 125/132] add missing call in zig fmt to commit results to disk --- src-self-hosted/main.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index a04faaec4..d125b05b2 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -741,6 +741,7 @@ fn fmtMain(allocator: &mem.Allocator, file_paths: []const []const u8) !void { defer baf.destroy(); try parser.renderSource(baf.stream(), tree.root_node); + try baf.finish(); } } From 0ba85ea6ff910c7a49ae036625b945c475c0f58c Mon Sep 17 00:00:00 2001 From: Jimmi Holst Christensen Date: Tue, 10 Apr 2018 17:46:17 +0200 Subject: [PATCH 126/132] std.zig.parser fixed segfault when parsing cc for fn decl --- std/zig/parser.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 76eb2d29d..92b461d20 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -417,7 +417,7 @@ pub const Parser = struct { }, Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => { // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), ctx.extern_token, ctx.lib_name, (?Token)(token), (?Token)(null), (?Token)(null)); stack.append(State { .FnDef = fn_proto }) catch unreachable; try stack.append(State { .FnProto = fn_proto }); From ee3e2790aa85c624fc952bc8326b82d72843bb17 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 20:57:37 -0400 Subject: [PATCH 127/132] cmake defaults stage1 to install in build directory --- CMakeLists.txt | 5 +++++ README.md | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5171d726..021fd43cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,11 @@ if(NOT CMAKE_BUILD_TYPE) "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) endif() +if(NOT CMAKE_INSTALL_PREFIX) + set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}" CACHE STRING + "Directory to install zig to" FORCE) +endif() + project(zig C CXX) set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) diff --git a/README.md b/README.md index d1e7e8f6d..1f23e133f 100644 --- a/README.md +++ b/README.md @@ -141,10 +141,10 @@ libc. Create demo games using Zig. ``` mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) +cmake .. make make install -./zig build --build-file ../build.zig test +bin/zig build --build-file ../build.zig test ``` ##### MacOS @@ -154,9 +154,9 @@ brew install cmake llvm@6 brew outdated llvm@6 || brew upgrade llvm@6 mkdir build cd build -cmake .. -DCMAKE_PREFIX_PATH=/usr/local/opt/llvm@6/ -DCMAKE_INSTALL_PREFIX=$(pwd) +cmake .. -DCMAKE_PREFIX_PATH=/usr/local/opt/llvm@6/ make install -./zig build --build-file ../build.zig test +bin/zig build --build-file ../build.zig test ``` ##### Windows From 27e881c2d7bebbf06b9932559ff2677ed8b6088b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 21:58:04 -0400 Subject: [PATCH 128/132] fix another undefined deref see 0ba85ea6ff91 --- std/zig/parser.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 92b461d20..5d5b1ceab 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -431,7 +431,7 @@ pub const Parser = struct { }, Token.Id.Keyword_async => { // TODO shouldn't need this cast - const fn_proto = try self.createAttachFnProto(arena, ctx.decls, undefined, + const fn_proto = try self.createAttachFnProto(arena, ctx.decls, Token(undefined), ctx.extern_token, ctx.lib_name, (?Token)(null), (?Token)(null), (?Token)(null)); const async_node = try arena.create(ast.NodeAsyncAttribute); From f6c77746d6be1bb238bf632f74c6dfc28de034ed Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 22:24:01 -0400 Subject: [PATCH 129/132] add memmove to builtin.o related: #514 --- std/special/builtin.zig | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/std/special/builtin.zig b/std/special/builtin.zig index 268d0ab54..9de0aa767 100644 --- a/std/special/builtin.zig +++ b/std/special/builtin.zig @@ -14,26 +14,43 @@ pub fn panic(msg: []const u8, error_return_trace: ?&builtin.StackTrace) noreturn } } -// Note that memset does not return `dest`, like the libc API. -// The semantics of memset is dictated by the corresponding -// LLVM intrinsics, not by the libc API. -export fn memset(dest: ?&u8, c: u8, n: usize) void { +export fn memset(dest: ?&u8, c: u8, n: usize) ?&u8 { @setRuntimeSafety(false); var index: usize = 0; while (index != n) : (index += 1) (??dest)[index] = c; + + return dest; } -// Note that memcpy does not return `dest`, like the libc API. -// The semantics of memcpy is dictated by the corresponding -// LLVM intrinsics, not by the libc API. -export fn memcpy(noalias dest: ?&u8, noalias src: ?&const u8, n: usize) void { +export fn memcpy(noalias dest: ?&u8, noalias src: ?&const u8, n: usize) ?&u8 { @setRuntimeSafety(false); var index: usize = 0; while (index != n) : (index += 1) (??dest)[index] = (??src)[index]; + + return dest; +} + +export fn memmove(dest: ?&u8, src: ?&const u8, n: usize) ?&u8 { + @setRuntimeSafety(false); + + if (@ptrToInt(dest) < @ptrToInt(src)) { + var index: usize = 0; + while (index != n) : (index += 1) { + (??dest)[index] = (??src)[index]; + } + } else { + var index = n; + while (index != 0) { + index -= 1; + (??dest)[index] = (??src)[index]; + } + } + + return dest; } comptime { From 405a2390f09c78fb5db435e97d8ff23c0b44753b Mon Sep 17 00:00:00 2001 From: Josh Wolfe Date: Tue, 10 Apr 2018 22:44:55 -0400 Subject: [PATCH 130/132] zig fmt while-else with no blocks --- std/zig/parser.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 5d5b1ceab..11b551fec 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -3839,12 +3839,13 @@ pub const Parser = struct { }, ast.Node.Id.Else => { const else_node = @fieldParentPtr(ast.NodeElse, "base", base); - try stream.print("{} ", self.tokenizer.getTokenSlice(else_node.else_token)); + try stream.print("{}", self.tokenizer.getTokenSlice(else_node.else_token)); switch (else_node.body.id) { ast.Node.Id.Block, ast.Node.Id.If, ast.Node.Id.For, ast.Node.Id.While, ast.Node.Id.Switch => { + try stream.print(" "); try stack.append(RenderState { .Expression = else_node.body }); }, else => { @@ -4805,6 +4806,11 @@ test "zig fmt: while" { \\ unreachable; \\ }; \\ + \\ const res = while (0 < 10) + \\ break 7 + \\ else + \\ unreachable; + \\ \\ var a: error!u8 = 0; \\ while (a) |v| { \\ a = error.Err; From 2ec1cec92d018d25f720c43a7586a12eafd2cbeb Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Apr 2018 23:27:27 -0400 Subject: [PATCH 131/132] add more linux syscalls and constants Based on #904 by tgshultz --- std/os/linux/index.zig | 372 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) diff --git a/std/os/linux/index.zig b/std/os/linux/index.zig index 7f27fc83d..aa2a6d85d 100644 --- a/std/os/linux/index.zig +++ b/std/os/linux/index.zig @@ -478,6 +478,126 @@ pub const CLOCK_BOOTTIME_ALARM = 9; pub const CLOCK_SGI_CYCLE = 10; pub const CLOCK_TAI = 11; +pub const CSIGNAL = 0x000000ff; +pub const CLONE_VM = 0x00000100; +pub const CLONE_FS = 0x00000200; +pub const CLONE_FILES = 0x00000400; +pub const CLONE_SIGHAND = 0x00000800; +pub const CLONE_PTRACE = 0x00002000; +pub const CLONE_VFORK = 0x00004000; +pub const CLONE_PARENT = 0x00008000; +pub const CLONE_THREAD = 0x00010000; +pub const CLONE_NEWNS = 0x00020000; +pub const CLONE_SYSVSEM = 0x00040000; +pub const CLONE_SETTLS = 0x00080000; +pub const CLONE_PARENT_SETTID = 0x00100000; +pub const CLONE_CHILD_CLEARTID = 0x00200000; +pub const CLONE_DETACHED = 0x00400000; +pub const CLONE_UNTRACED = 0x00800000; +pub const CLONE_CHILD_SETTID = 0x01000000; +pub const CLONE_NEWCGROUP = 0x02000000; +pub const CLONE_NEWUTS = 0x04000000; +pub const CLONE_NEWIPC = 0x08000000; +pub const CLONE_NEWUSER = 0x10000000; +pub const CLONE_NEWPID = 0x20000000; +pub const CLONE_NEWNET = 0x40000000; +pub const CLONE_IO = 0x80000000; + +pub const MS_RDONLY = 1; +pub const MS_NOSUID = 2; +pub const MS_NODEV = 4; +pub const MS_NOEXEC = 8; +pub const MS_SYNCHRONOUS = 16; +pub const MS_REMOUNT = 32; +pub const MS_MANDLOCK = 64; +pub const MS_DIRSYNC = 128; +pub const MS_NOATIME = 1024; +pub const MS_NODIRATIME = 2048; +pub const MS_BIND = 4096; +pub const MS_MOVE = 8192; +pub const MS_REC = 16384; +pub const MS_SILENT = 32768; +pub const MS_POSIXACL = (1<<16); +pub const MS_UNBINDABLE = (1<<17); +pub const MS_PRIVATE = (1<<18); +pub const MS_SLAVE = (1<<19); +pub const MS_SHARED = (1<<20); +pub const MS_RELATIME = (1<<21); +pub const MS_KERNMOUNT = (1<<22); +pub const MS_I_VERSION = (1<<23); +pub const MS_STRICTATIME = (1<<24); +pub const MS_LAZYTIME = (1<<25); +pub const MS_NOREMOTELOCK = (1<<27); +pub const MS_NOSEC = (1<<28); +pub const MS_BORN = (1<<29); +pub const MS_ACTIVE = (1<<30); +pub const MS_NOUSER = (1<<31); + +pub const MS_RMT_MASK = (MS_RDONLY|MS_SYNCHRONOUS|MS_MANDLOCK|MS_I_VERSION|MS_LAZYTIME); + +pub const MS_MGC_VAL = 0xc0ed0000; +pub const MS_MGC_MSK = 0xffff0000; + +pub const MNT_FORCE = 1; +pub const MNT_DETACH = 2; +pub const MNT_EXPIRE = 4; +pub const UMOUNT_NOFOLLOW = 8; + + +pub const S_IFMT = 0o170000; + +pub const S_IFDIR = 0o040000; +pub const S_IFCHR = 0o020000; +pub const S_IFBLK = 0o060000; +pub const S_IFREG = 0o100000; +pub const S_IFIFO = 0o010000; +pub const S_IFLNK = 0o120000; +pub const S_IFSOCK = 0o140000; + +pub const S_ISUID = 0o4000; +pub const S_ISGID = 0o2000; +pub const S_ISVTX = 0o1000; +pub const S_IRUSR = 0o400; +pub const S_IWUSR = 0o200; +pub const S_IXUSR = 0o100; +pub const S_IRWXU = 0o700; +pub const S_IRGRP = 0o040; +pub const S_IWGRP = 0o020; +pub const S_IXGRP = 0o010; +pub const S_IRWXG = 0o070; +pub const S_IROTH = 0o004; +pub const S_IWOTH = 0o002; +pub const S_IXOTH = 0o001; +pub const S_IRWXO = 0o007; + +pub fn S_ISREG(m: u32) bool { + return m & S_IFMT == S_IFREG; +} + +pub fn S_ISDIR(m: u32) bool { + return m & S_IFMT == S_IFDIR; +} + +pub fn S_ISCHR(m: u32) bool { + return m & S_IFMT == S_IFCHR; +} + +pub fn S_ISBLK(m: u32) bool { + return m & S_IFMT == S_IFBLK; +} + +pub fn S_ISFIFO(m: u32) bool { + return m & S_IFMT == S_IFIFO; +} + +pub fn S_ISLNK(m: u32) bool { + return m & S_IFMT == S_IFLNK; +} + +pub fn S_ISSOCK(m: u32) bool { + return m & S_IFMT == S_IFSOCK; +} + pub const TFD_NONBLOCK = O_NONBLOCK; pub const TFD_CLOEXEC = O_CLOEXEC; @@ -515,6 +635,10 @@ pub fn chdir(path: &const u8) usize { return syscall1(SYS_chdir, @ptrToInt(path)); } +pub fn chroot(path: &const u8) usize { + return syscall1(SYS_chroot, @ptrToInt(path)); +} + pub fn execve(path: &const u8, argv: &const ?&const u8, envp: &const ?&const u8) usize { return syscall3(SYS_execve, @ptrToInt(path), @ptrToInt(argv), @ptrToInt(envp)); } @@ -544,6 +668,18 @@ pub fn mkdir(path: &const u8, mode: u32) usize { return syscall2(SYS_mkdir, @ptrToInt(path), mode); } +pub fn mount(special: &const u8, dir: &const u8, fstype: &const u8, flags: usize, data: usize) usize { + return syscall5(SYS_mount, @ptrToInt(special), @ptrToInt(dir), @ptrToInt(fstype), flags, data); +} + +pub fn umount(special: &const u8) usize { + return syscall2(SYS_umount2, @ptrToInt(special), 0); +} + +pub fn umount2(special: &const u8, flags: u32) usize { + return syscall2(SYS_umount2, @ptrToInt(special), flags); +} + pub fn mmap(address: ?&u8, length: usize, prot: usize, flags: usize, fd: i32, offset: isize) usize { return syscall6(SYS_mmap, @ptrToInt(address), length, prot, flags, usize(fd), @bitCast(usize, offset)); @@ -650,6 +786,58 @@ pub fn setregid(rgid: u32, egid: u32) usize { return syscall2(SYS_setregid, rgid, egid); } +pub fn getuid() u32 { + return u32(syscall0(SYS_getuid)); +} + +pub fn getgid() u32 { + return u32(syscall0(SYS_getgid)); +} + +pub fn geteuid() u32 { + return u32(syscall0(SYS_geteuid)); +} + +pub fn getegid() u32 { + return u32(syscall0(SYS_getegid)); +} + +pub fn seteuid(euid: u32) usize { + return syscall1(SYS_seteuid, euid); +} + +pub fn setegid(egid: u32) usize { + return syscall1(SYS_setegid, egid); +} + +pub fn getresuid(ruid: &u32, euid: &u32, suid: &u32) usize { + return syscall3(SYS_getresuid, @ptrToInt(ruid), @ptrToInt(euid), @ptrToInt(suid)); +} + +pub fn getresgid(rgid: &u32, egid: &u32, sgid: &u32) usize { + return syscall3(SYS_getresgid, @ptrToInt(rgid), @ptrToInt(egid), @ptrToInt(sgid)); +} + +pub fn setresuid(ruid: u32, euid: u32, suid: u32) usize { + return syscall3(SYS_setresuid, ruid, euid, suid); +} + +pub fn setresgid(rgid: u32, egid: u32, sgid: u32) usize { + return syscall3(SYS_setresgid, rgid, egid, sgid); +} + +pub fn getgroups(size: usize, list: &u32) usize { + return syscall2(SYS_getgroups, size, @ptrToInt(list)); +} + +pub fn setgroups(size: usize, list: &const u32) usize { + return syscall2(SYS_setgroups, size, @ptrToInt(list)); +} + +pub fn getpid() i32 { + return @bitCast(i32, u32(syscall0(SYS_getpid))); +} + pub fn sigprocmask(flags: u32, noalias set: &const sigset_t, noalias oldset: ?&sigset_t) usize { return syscall4(SYS_rt_sigprocmask, flags, @ptrToInt(set), @ptrToInt(oldset), NSIG/8); } @@ -833,6 +1021,71 @@ pub fn fstat(fd: i32, stat_buf: &Stat) usize { return syscall2(SYS_fstat, usize(fd), @ptrToInt(stat_buf)); } +pub fn stat(pathname: &const u8, statbuf: &Stat) usize { + return syscall2(SYS_stat, @ptrToInt(pathname), @ptrToInt(statbuf)); +} + +pub fn lstat(pathname: &const u8, statbuf: &Stat) usize { + return syscall2(SYS_lstat, @ptrToInt(pathname), @ptrToInt(statbuf)); +} + +pub fn listxattr(path: &const u8, list: &u8, size: usize) usize { + return syscall3(SYS_listxattr, @ptrToInt(path), @ptrToInt(list), size); +} + +pub fn llistxattr(path: &const u8, list: &u8, size: usize) usize { + return syscall3(SYS_llistxattr, @ptrToInt(path), @ptrToInt(list), size); +} + +pub fn flistxattr(fd: usize, list: &u8, size: usize) usize { + return syscall3(SYS_flistxattr, fd, @ptrToInt(list), size); +} + +pub fn getxattr(path: &const u8, name: &const u8, value: &void, size: usize) usize { + return syscall4(SYS_getxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), size); +} + +pub fn lgetxattr(path: &const u8, name: &const u8, value: &void, size: usize) usize { + return syscall4(SYS_lgetxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), size); +} + +pub fn fgetxattr(fd: usize, name: &const u8, value: &void, size: usize) usize { + return syscall4(SYS_lgetxattr, fd, @ptrToInt(name), @ptrToInt(value), size); +} + +pub fn setxattr(path: &const u8, name: &const u8, value: &const void, + size: usize, flags: usize) usize { + + return syscall5(SYS_setxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), + size, flags); +} + +pub fn lsetxattr(path: &const u8, name: &const u8, value: &const void, + size: usize, flags: usize) usize { + + return syscall5(SYS_lsetxattr, @ptrToInt(path), @ptrToInt(name), @ptrToInt(value), + size, flags); +} + +pub fn fsetxattr(fd: usize, name: &const u8, value: &const void, + size: usize, flags: usize) usize { + + return syscall5(SYS_fsetxattr, fd, @ptrToInt(name), @ptrToInt(value), + size, flags); +} + +pub fn removexattr(path: &const u8, name: &const u8) usize { + return syscall2(SYS_removexattr, @ptrToInt(path), @ptrToInt(name)); +} + +pub fn lremovexattr(path: &const u8, name: &const u8) usize { + return syscall2(SYS_lremovexattr, @ptrToInt(path), @ptrToInt(name)); +} + +pub fn fremovexattr(fd: usize, name: &const u8) usize { + return syscall2(SYS_fremovexattr, fd, @ptrToInt(name)); +} + pub const epoll_data = packed union { ptr: usize, fd: i32, @@ -878,6 +1131,125 @@ pub fn timerfd_settime(fd: i32, flags: u32, new_value: &const itimerspec, old_va return syscall4(SYS_timerfd_settime, usize(fd), usize(flags), @ptrToInt(new_value), @ptrToInt(old_value)); } +pub const _LINUX_CAPABILITY_VERSION_1 = 0x19980330; +pub const _LINUX_CAPABILITY_U32S_1 = 1; + +pub const _LINUX_CAPABILITY_VERSION_2 = 0x20071026; +pub const _LINUX_CAPABILITY_U32S_2 = 2; + +pub const _LINUX_CAPABILITY_VERSION_3 = 0x20080522; +pub const _LINUX_CAPABILITY_U32S_3 = 2; + +pub const VFS_CAP_REVISION_MASK = 0xFF000000; +pub const VFS_CAP_REVISION_SHIFT = 24; +pub const VFS_CAP_FLAGS_MASK = ~VFS_CAP_REVISION_MASK; +pub const VFS_CAP_FLAGS_EFFECTIVE = 0x000001; + +pub const VFS_CAP_REVISION_1 = 0x01000000; +pub const VFS_CAP_U32_1 = 1; +pub const XATTR_CAPS_SZ_1 = @sizeOf(u32)*(1 + 2*VFS_CAP_U32_1); + +pub const VFS_CAP_REVISION_2 = 0x02000000; +pub const VFS_CAP_U32_2 = 2; +pub const XATTR_CAPS_SZ_2 = @sizeOf(u32)*(1 + 2*VFS_CAP_U32_2); + +pub const XATTR_CAPS_SZ = XATTR_CAPS_SZ_2; +pub const VFS_CAP_U32 = VFS_CAP_U32_2; +pub const VFS_CAP_REVISION = VFS_CAP_REVISION_2; + +pub const vfs_cap_data = extern struct { + //all of these are mandated as little endian + //when on disk. + const Data = struct { + permitted: u32, + inheritable: u32, + }; + + magic_etc: u32, + data: [VFS_CAP_U32]Data, +}; + + +pub const CAP_CHOWN = 0; +pub const CAP_DAC_OVERRIDE = 1; +pub const CAP_DAC_READ_SEARCH = 2; +pub const CAP_FOWNER = 3; +pub const CAP_FSETID = 4; +pub const CAP_KILL = 5; +pub const CAP_SETGID = 6; +pub const CAP_SETUID = 7; +pub const CAP_SETPCAP = 8; +pub const CAP_LINUX_IMMUTABLE = 9; +pub const CAP_NET_BIND_SERVICE = 10; +pub const CAP_NET_BROADCAST = 11; +pub const CAP_NET_ADMIN = 12; +pub const CAP_NET_RAW = 13; +pub const CAP_IPC_LOCK = 14; +pub const CAP_IPC_OWNER = 15; +pub const CAP_SYS_MODULE = 16; +pub const CAP_SYS_RAWIO = 17; +pub const CAP_SYS_CHROOT = 18; +pub const CAP_SYS_PTRACE = 19; +pub const CAP_SYS_PACCT = 20; +pub const CAP_SYS_ADMIN = 21; +pub const CAP_SYS_BOOT = 22; +pub const CAP_SYS_NICE = 23; +pub const CAP_SYS_RESOURCE = 24; +pub const CAP_SYS_TIME = 25; +pub const CAP_SYS_TTY_CONFIG = 26; +pub const CAP_MKNOD = 27; +pub const CAP_LEASE = 28; +pub const CAP_AUDIT_WRITE = 29; +pub const CAP_AUDIT_CONTROL = 30; +pub const CAP_SETFCAP = 31; +pub const CAP_MAC_OVERRIDE = 32; +pub const CAP_MAC_ADMIN = 33; +pub const CAP_SYSLOG = 34; +pub const CAP_WAKE_ALARM = 35; +pub const CAP_BLOCK_SUSPEND = 36; +pub const CAP_AUDIT_READ = 37; +pub const CAP_LAST_CAP = CAP_AUDIT_READ; + +pub fn cap_valid(u8: x) bool { + return x >= 0 and x <= CAP_LAST_CAP; +} + +pub fn CAP_TO_MASK(cap: u8) u32 { + return u32(1) << u5(cap & 31); +} + +pub fn CAP_TO_INDEX(cap: u8) u8 { + return cap >> 5; +} + +pub const cap_t = extern struct { + hdrp: &cap_user_header_t, + datap: &cap_user_data_t, +}; + +pub const cap_user_header_t = extern struct { + version: u32, + pid: usize, +}; + +pub const cap_user_data_t = extern struct { + effective: u32, + permitted: u32, + inheritable: u32, +}; + +pub fn unshare(flags: usize) usize { + return syscall1(SYS_unshare, usize(flags)); +} + +pub fn capget(hdrp: &cap_user_header_t, datap: &cap_user_data_t) usize { + return syscall2(SYS_capget, @ptrToInt(hdrp), @ptrToInt(datap)); +} + +pub fn capset(hdrp: &cap_user_header_t, datap: &const cap_user_data_t) usize { + return syscall2(SYS_capset, @ptrToInt(hdrp), @ptrToInt(datap)); +} + test "import linux test" { // TODO lazy analysis should prevent this test from being compiled on windows, but // it is still compiled on windows From 58c6424d4fe64f88e25714d20d5755d31a7775c1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Apr 2018 00:32:42 -0400 Subject: [PATCH 132/132] simplify and fix BufMap logic --- std/buf_map.zig | 33 ++++++++++++--------------------- std/hash_map.zig | 6 +++++- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/std/buf_map.zig b/std/buf_map.zig index 7e2ea99f1..3e12d9a7d 100644 --- a/std/buf_map.zig +++ b/std/buf_map.zig @@ -1,6 +1,8 @@ -const HashMap = @import("hash_map.zig").HashMap; -const mem = @import("mem.zig"); +const std = @import("index.zig"); +const HashMap = std.HashMap; +const mem = std.mem; const Allocator = mem.Allocator; +const assert = std.debug.assert; /// BufMap copies keys and values before they go into the map, and /// frees them when they get removed. @@ -28,18 +30,12 @@ pub const BufMap = struct { } pub fn set(self: &BufMap, key: []const u8, value: []const u8) !void { - if (self.hash_map.get(key)) |entry| { - const value_copy = try self.copy(value); - errdefer self.free(value_copy); - const old_value = ??(try self.hash_map.put(key, value_copy)); - self.free(old_value); - } else { - const key_copy = try self.copy(key); - errdefer self.free(key_copy); - const value_copy = try self.copy(value); - errdefer self.free(value_copy); - _ = try self.hash_map.put(key_copy, value_copy); - } + self.delete(key); + const key_copy = try self.copy(key); + errdefer self.free(key_copy); + const value_copy = try self.copy(value); + errdefer self.free(value_copy); + _ = try self.hash_map.put(key_copy, value_copy); } pub fn get(self: &BufMap, key: []const u8) ?[]const u8 { @@ -66,17 +62,12 @@ pub const BufMap = struct { } fn copy(self: &BufMap, value: []const u8) ![]const u8 { - const result = try self.hash_map.allocator.alloc(u8, value.len); - mem.copy(u8, result, value); - return result; + return mem.dupe(self.hash_map.allocator, u8, value); } }; -const assert = @import("debug/index.zig").assert; -const heap = @import("heap.zig"); - test "BufMap" { - var direct_allocator = heap.DirectAllocator.init(); + var direct_allocator = std.heap.DirectAllocator.init(); defer direct_allocator.deinit(); var bufmap = BufMap.init(&direct_allocator.allocator); diff --git a/std/hash_map.zig b/std/hash_map.zig index becced64f..29dd23375 100644 --- a/std/hash_map.zig +++ b/std/hash_map.zig @@ -114,6 +114,7 @@ pub fn HashMap(comptime K: type, comptime V: type, } pub fn remove(hm: &Self, key: K) ?&Entry { + if (hm.entries.len == 0) return null; hm.incrementModificationCount(); const start_index = hm.keyToIndex(key); {var roll_over: usize = 0; while (roll_over <= hm.max_distance_from_start_index) : (roll_over += 1) { @@ -236,7 +237,10 @@ pub fn HashMap(comptime K: type, comptime V: type, } test "basic hash map usage" { - var map = HashMap(i32, i32, hash_i32, eql_i32).init(debug.global_allocator); + var direct_allocator = std.heap.DirectAllocator.init(); + defer direct_allocator.deinit(); + + var map = HashMap(i32, i32, hash_i32, eql_i32).init(&direct_allocator.allocator); defer map.deinit(); assert((map.put(1, 11) catch unreachable) == null);