self hosted compiler: move tokenization and parsing to std lib

CMakeLists.txt

@@ -477,6 +477,10 @@ set(ZIG_STD_FILES
     "special/panic.zig"
     "special/test_runner.zig"
     "unicode.zig"
+    "zig/ast.zig"
+    "zig/index.zig"
+    "zig/parser.zig"
+    "zig/tokenizer.zig"
 )
 
 set(ZIG_C_HEADER_FILES

build.zig

@@ -108,10 +108,6 @@ pub fn build(b: &Builder) !void {
         "std/special/compiler_rt/index.zig", "compiler-rt", "Run the compiler_rt tests",
         with_lldb));
 
-    test_step.dependOn(tests.addPkgTests(b, test_filter,
-        "src-self-hosted/main.zig", "fmt", "Run the fmt tests",
-        with_lldb));
-
     test_step.dependOn(tests.addCompareOutputTests(b, test_filter));
     test_step.dependOn(tests.addBuildExampleTests(b, test_filter));
     test_step.dependOn(tests.addCompileErrorTests(b, test_filter));

src-self-hosted/main.zig

@@ -622,8 +622,3 @@ fn findZigLibDir(allocator: &mem.Allocator) ![]u8 {
 
     return error.FileNotFound;
 }
-
-test "import tests" {
-    _ = @import("tokenizer.zig");
-    _ = @import("parser.zig");
-}

src-self-hosted/module.zig

@@ -8,9 +8,9 @@ const c = @import("c.zig");
 const builtin = @import("builtin");
 const Target = @import("target.zig").Target;
 const warn = std.debug.warn;
-const Tokenizer = @import("tokenizer.zig").Tokenizer;
-const Token = @import("tokenizer.zig").Token;
-const Parser = @import("parser.zig").Parser;
+const Tokenizer = std.zig.Tokenizer;
+const Token = std.zig.Token;
+const Parser = std.zig.Parser;
 const ArrayList = std.ArrayList;
 
 pub const Module = struct {

std/index.zig

@@ -28,6 +28,7 @@ pub const os = @import("os/index.zig");
 pub const rand = @import("rand.zig");
 pub const sort = @import("sort.zig");
 pub const unicode = @import("unicode.zig");
+pub const zig = @import("zig/index.zig");
 
 test "std" {
     // run tests from these
@@ -58,4 +59,5 @@ test "std" {
     _ = @import("rand.zig");
     _ = @import("sort.zig");
     _ = @import("unicode.zig");
+    _ = @import("zig/index.zig");
 }

std/zig/ast.zig

@@ -1,7 +1,7 @@
-const std = @import("std");
+const std = @import("../index.zig");
 const assert = std.debug.assert;
 const ArrayList = std.ArrayList;
-const Token = @import("tokenizer.zig").Token;
+const Token = std.zig.Token;
 const mem = std.mem;
 
 pub const Node = struct {

std/zig/index.zig

@@ -0,0 +1,11 @@
+const tokenizer = @import("tokenizer.zig");
+pub const Token = tokenizer.Token;
+pub const Tokenizer = tokenizer.Tokenizer;
+pub const Parser = @import("parser.zig").Parser;
+pub const ast = @import("ast.zig");
+
+test "std.zig tests" {
+    _ = @import("tokenizer.zig");
+    _ = @import("parser.zig");
+    _ = @import("ast.zig");
+}

std/zig/parser.zig

@@ -1,10 +1,10 @@
-const std = @import("std");
+const std = @import("../index.zig");
 const assert = std.debug.assert;
 const ArrayList = std.ArrayList;
 const mem = std.mem;
-const ast = @import("ast.zig");
-const Tokenizer = @import("tokenizer.zig").Tokenizer;
-const Token = @import("tokenizer.zig").Token;
+const ast = std.zig.ast;
+const Tokenizer = std.zig.Tokenizer;
+const Token = std.zig.Token;
 const builtin = @import("builtin");
 const io = std.io;
 

std/zig/tokenizer.zig

@@ -1,4 +1,4 @@
-const std = @import("std");
+const std = @import("../index.zig");
 const mem = std.mem;
 
 pub const Token = struct {