From 257cf09472ce5f4a51bf39808e119717fa0e4280 Mon Sep 17 00:00:00 2001
From: Andrew Kelley <superjoe30@gmail.com>
Date: Tue, 1 Dec 2015 00:50:11 -0700
Subject: [PATCH] colored error messages that tell the source file

---
 CMakeLists.txt        |   1 +
 src/analyze.cpp       |  18 ++++---
 src/buffer.cpp        |  14 +++--
 src/buffer.hpp        |   2 +
 src/codegen.cpp       |  33 +++++++++---
 src/codegen.hpp       |   9 ----
 src/errmsg.cpp        |  38 ++++++++++++++
 src/errmsg.hpp        |  27 ++++++++++
 src/os.cpp            |   4 ++
 src/os.hpp            |   2 +
 src/parser.cpp        | 118 +++++++++++++++++++++++-------------------
 src/parser.hpp        |   5 +-
 src/semantic_info.hpp |   5 +-
 src/tokenizer.cpp     |  43 +++++++++------
 src/tokenizer.hpp     |  12 ++++-
 test/run_tests.cpp    |  22 ++++----
 16 files changed, 240 insertions(+), 113 deletions(-)
 create mode 100644 src/errmsg.cpp
 create mode 100644 src/errmsg.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e8c78213e..0f980bd16 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,6 +31,7 @@ set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/main.cpp"
     "${CMAKE_SOURCE_DIR}/src/os.cpp"
     "${CMAKE_SOURCE_DIR}/src/util.cpp"
+    "${CMAKE_SOURCE_DIR}/src/errmsg.cpp"
     "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp"
 )
 
diff --git a/src/analyze.cpp b/src/analyze.cpp
index e1a960a69..fccbad8bb 100644
--- a/src/analyze.cpp
+++ b/src/analyze.cpp
@@ -12,13 +12,17 @@
 #include "os.hpp"
 
 static void add_node_error(CodeGen *g, AstNode *node, Buf *msg) {
-    g->errors.add_one();
-    ErrorMsg *last_msg = &g->errors.last();
-    last_msg->line_start = node->line;
-    last_msg->column_start = node->column;
-    last_msg->line_end = -1;
-    last_msg->column_end = -1;
-    last_msg->msg = msg;
+    ErrorMsg *err = allocate<ErrorMsg>(1);
+    err->line_start = node->line;
+    err->column_start = node->column;
+    err->line_end = -1;
+    err->column_end = -1;
+    err->msg = msg;
+    err->path = node->owner->path;
+    err->source = node->owner->source_code;
+    err->line_offsets = node->owner->line_offsets;
+
+    g->errors.append(err);
 }
 
 static int parse_version_string(Buf *buf, int *major, int *minor, int *patch) {
diff --git a/src/buffer.cpp b/src/buffer.cpp
index 197837103..a03ab333f 100644
--- a/src/buffer.cpp
+++ b/src/buffer.cpp
@@ -3,9 +3,8 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-Buf *buf_sprintf(const char *format, ...) {
-    va_list ap, ap2;
-    va_start(ap, format);
+Buf *buf_vprintf(const char *format, va_list ap) {
+    va_list ap2;
     va_copy(ap2, ap);
 
     int len1 = vsnprintf(nullptr, 0, format, ap);
@@ -19,11 +18,18 @@ Buf *buf_sprintf(const char *format, ...) {
     assert(len2 == len1);
 
     va_end(ap2);
-    va_end(ap);
 
     return buf;
 }
 
+Buf *buf_sprintf(const char *format, ...) {
+    va_list ap;
+    va_start(ap, format);
+    Buf *result = buf_vprintf(format, ap);
+    va_end(ap);
+    return result;
+}
+
 void buf_appendf(Buf *buf, const char *format, ...) {
     assert(buf->list.length);
     va_list ap, ap2;
diff --git a/src/buffer.hpp b/src/buffer.hpp
index f9b249754..367033203 100644
--- a/src/buffer.hpp
+++ b/src/buffer.hpp
@@ -13,6 +13,7 @@
 #include <assert.h>
 #include <stdint.h>
 #include <ctype.h>
+#include <stdarg.h>
 
 #define BUF_INIT {{0}}
 
@@ -24,6 +25,7 @@ struct Buf {
 
 Buf *buf_sprintf(const char *format, ...)
     __attribute__ ((format (printf, 1, 2)));
+Buf *buf_vprintf(const char *format, va_list ap);
 
 static inline int buf_len(Buf *buf) {
     assert(buf->list.length);
diff --git a/src/codegen.cpp b/src/codegen.cpp
index ee15e2a2e..3717fb63d 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -13,6 +13,7 @@
 #include "error.hpp"
 #include "semantic_info.hpp"
 #include "analyze.hpp"
+#include "errmsg.hpp"
 
 #include <stdio.h>
 #include <errno.h>
@@ -678,24 +679,42 @@ static ImportTableEntry *codegen_add_code(CodeGen *g, Buf *source_path, Buf *sou
         fprintf(stderr, "---------\n");
     }
 
-    ZigList<Token> *tokens = tokenize(source_code);
+    Tokenization tokenization = {0};
+    tokenize(source_code, &tokenization);
+
+    if (tokenization.err) {
+        ErrorMsg *err = allocate<ErrorMsg>(1);
+        err->line_start = tokenization.err_line;
+        err->column_start = tokenization.err_column;
+        err->line_end = -1;
+        err->column_end = -1;
+        err->msg = tokenization.err;
+        err->path = source_path;
+        err->source = source_code;
+        err->line_offsets = tokenization.line_offsets;
+
+        print_err_msg(err);
+        exit(1);
+    }
 
     if (g->verbose) {
-        print_tokens(source_code, tokens);
+        print_tokens(source_code, tokenization.tokens);
 
         fprintf(stderr, "\nAST:\n");
         fprintf(stderr, "------\n");
     }
 
     ImportTableEntry *import_entry = allocate<ImportTableEntry>(1);
+    import_entry->source_code = source_code;
+    import_entry->line_offsets = tokenization.line_offsets;
+    import_entry->path = source_path;
     import_entry->fn_table.init(32);
-    import_entry->root = ast_parse(source_code, tokens);
+    import_entry->root = ast_parse(source_code, tokenization.tokens, import_entry);
     assert(import_entry->root);
     if (g->verbose) {
         ast_print(import_entry->root, 0);
     }
 
-    import_entry->path = source_path;
     import_entry->di_file = LLVMZigCreateFile(g->dbuilder, buf_ptr(&basename), buf_ptr(&dirname));
     g->import_table.put(source_path, import_entry);
 
@@ -736,10 +755,8 @@ void codegen_add_root_code(CodeGen *g, Buf *source_path, Buf *source_code) {
         }
     } else {
         for (int i = 0; i < g->errors.length; i += 1) {
-            ErrorMsg *err = &g->errors.at(i);
-            fprintf(stderr, "Error: Line %d, column %d: %s\n",
-                    err->line_start + 1, err->column_start + 1,
-                    buf_ptr(err->msg));
+            ErrorMsg *err = g->errors.at(i);
+            print_err_msg(err);
         }
         exit(1);
     }
diff --git a/src/codegen.hpp b/src/codegen.hpp
index 447601784..5795a5a94 100644
--- a/src/codegen.hpp
+++ b/src/codegen.hpp
@@ -20,15 +20,6 @@ enum OutType {
 };
 
 
-struct ErrorMsg {
-    int line_start;
-    int column_start;
-    int line_end;
-    int column_end;
-    Buf *msg;
-};
-
-
 CodeGen *codegen_create(Buf *root_source_dir);
 
 enum CodeGenBuildType {
diff --git a/src/errmsg.cpp b/src/errmsg.cpp
new file mode 100644
index 000000000..36d203a48
--- /dev/null
+++ b/src/errmsg.cpp
@@ -0,0 +1,38 @@
+#include "errmsg.hpp"
+#include "os.hpp"
+
+#include <stdio.h>
+
+#define RED "\x1b[31;1m"
+#define WHITE "\x1b[37;1m"
+#define GREEN "\x1b[32;1m"
+#define RESET "\x1b[0m"
+
+void print_err_msg(ErrorMsg *err) {
+    if (os_stderr_tty()) {
+        fprintf(stderr, WHITE "%s:%d:%d: " RED "error:" WHITE " %s" RESET "\n",
+                buf_ptr(err->path),
+                err->line_start + 1, err->column_start + 1,
+                buf_ptr(err->msg));
+
+        assert(err->source);
+        assert(err->line_offsets);
+
+        int line_start_offset = err->line_offsets->at(err->line_start);
+        int line_end_offset = err->line_offsets->at(err->line_start + 1);
+
+        fwrite(buf_ptr(err->source) + line_start_offset, 1, line_end_offset - line_start_offset - 1, stderr);
+        fprintf(stderr, "\n");
+        for (int i = 0; i < err->column_start; i += 1) {
+            fprintf(stderr, " ");
+        }
+        fprintf(stderr, GREEN "^" RESET "\n");
+
+    } else {
+        fprintf(stderr, "%s:%d:%d: error: %s\n",
+                buf_ptr(err->path),
+                err->line_start + 1, err->column_start + 1,
+                buf_ptr(err->msg));
+    }
+}
+
diff --git a/src/errmsg.hpp b/src/errmsg.hpp
new file mode 100644
index 000000000..fa851d44b
--- /dev/null
+++ b/src/errmsg.hpp
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2015 Andrew Kelley
+ *
+ * This file is part of zig, which is MIT licensed.
+ * See http://opensource.org/licenses/MIT
+ */
+
+#ifndef ZIG_ERRMSG_HPP
+#define ZIG_ERRMSG_HPP
+
+#include "buffer.hpp"
+#include "list.hpp"
+
+struct ErrorMsg {
+    int line_start;
+    int column_start;
+    int line_end;
+    int column_end;
+    Buf *msg;
+    Buf *path;
+    Buf *source;
+    ZigList<int> *line_offsets;
+};
+
+void print_err_msg(ErrorMsg *msg);
+
+#endif
diff --git a/src/os.cpp b/src/os.cpp
index 61dd1c01e..bfc8eaa77 100644
--- a/src/os.cpp
+++ b/src/os.cpp
@@ -180,3 +180,7 @@ int os_get_cwd(Buf *out_cwd) {
 
     return 0;
 }
+
+bool os_stderr_tty(void) {
+    return isatty(STDERR_FILENO);
+}
diff --git a/src/os.hpp b/src/os.hpp
index 8bb5d8ea4..7b7143785 100644
--- a/src/os.hpp
+++ b/src/os.hpp
@@ -29,4 +29,6 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents);
 int os_get_cwd(Buf *out_cwd);
 
 
+bool os_stderr_tty(void);
+
 #endif
diff --git a/src/parser.cpp b/src/parser.cpp
index 613c55923..a6809a277 100644
--- a/src/parser.cpp
+++ b/src/parser.cpp
@@ -6,6 +6,8 @@
  */
 
 #include "parser.hpp"
+#include "errmsg.hpp"
+#include "semantic_info.hpp"
 
 #include <stdarg.h>
 #include <stdio.h>
@@ -45,21 +47,6 @@ static const char *prefix_op_str(PrefixOp prefix_op) {
     zig_unreachable();
 }
 
-__attribute__ ((format (printf, 2, 3)))
-__attribute__ ((noreturn))
-static void ast_error(Token *token, const char *format, ...) {
-    int line = token->start_line + 1;
-    int column = token->start_column + 1;
-
-    va_list ap;
-    va_start(ap, format);
-    fprintf(stderr, "Error: Line %d, column %d: ", line, column);
-    vfprintf(stderr, format, ap);
-    fprintf(stderr, "\n");
-    va_end(ap);
-    exit(EXIT_FAILURE);
-}
-
 const char *node_type_str(NodeType node_type) {
     switch (node_type) {
         case NodeTypeRoot:
@@ -254,11 +241,35 @@ struct ParseContext {
     AstNode *root;
     ZigList<Token> *tokens;
     ZigList<AstNode *> *directive_list;
+    ImportTableEntry *owner;
 };
 
-static AstNode *ast_create_node_no_line_info(NodeType type) {
+__attribute__ ((format (printf, 3, 4)))
+__attribute__ ((noreturn))
+static void ast_error(ParseContext *pc, Token *token, const char *format, ...) {
+    ErrorMsg *err = allocate<ErrorMsg>(1);
+    err->line_start = token->start_line;
+    err->column_start = token->start_column;
+    err->line_end = -1;
+    err->column_end = -1;
+
+    va_list ap;
+    va_start(ap, format);
+    err->msg = buf_vprintf(format, ap);
+    va_end(ap);
+
+    err->path = pc->owner->path;
+    err->source = pc->owner->source_code;
+    err->line_offsets = pc->owner->line_offsets;
+
+    print_err_msg(err);
+    exit(EXIT_FAILURE);
+}
+
+static AstNode *ast_create_node_no_line_info(ParseContext *pc, NodeType type) {
     AstNode *node = allocate<AstNode>(1);
     node->type = type;
+    node->owner = pc->owner;
     return node;
 }
 
@@ -267,21 +278,21 @@ static void ast_update_node_line_info(AstNode *node, Token *first_token) {
     node->column = first_token->start_column;
 }
 
-static AstNode *ast_create_node(NodeType type, Token *first_token) {
-    AstNode *node = ast_create_node_no_line_info(type);
+static AstNode *ast_create_node(ParseContext *pc, NodeType type, Token *first_token) {
+    AstNode *node = ast_create_node_no_line_info(pc, type);
     ast_update_node_line_info(node, first_token);
     return node;
 }
 
-static AstNode *ast_create_node_with_node(NodeType type, AstNode *other_node) {
-    AstNode *node = ast_create_node_no_line_info(type);
+static AstNode *ast_create_node_with_node(ParseContext *pc, NodeType type, AstNode *other_node) {
+    AstNode *node = ast_create_node_no_line_info(pc, type);
     node->line = other_node->line;
     node->column = other_node->column;
     return node;
 }
 
 static AstNode *ast_create_void_type_node(ParseContext *pc, Token *token) {
-    AstNode *node = ast_create_node(NodeTypeType, token);
+    AstNode *node = ast_create_node(pc, NodeTypeType, token);
     node->data.type.type = AstNodeTypeTypePrimitive;
     buf_init_from_str(&node->data.type.primitive_name, "void");
     return node;
@@ -331,7 +342,7 @@ __attribute__ ((noreturn))
 static void ast_invalid_token_error(ParseContext *pc, Token *token) {
     Buf token_value = BUF_INIT;
     ast_buf_from_token(pc, token, &token_value);
-    ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));
+    ast_error(pc, token, "invalid token: '%s'", buf_ptr(&token_value));
 }
 
 static AstNode *ast_parse_expression(ParseContext *pc, int *token_index, bool mandatory);
@@ -349,7 +360,7 @@ static AstNode *ast_parse_directive(ParseContext *pc, int token_index, int *new_
     token_index += 1;
     ast_expect_token(pc, number_sign, TokenIdNumberSign);
 
-    AstNode *node = ast_create_node(NodeTypeDirective, number_sign);
+    AstNode *node = ast_create_node(pc, NodeTypeDirective, number_sign);
 
     Token *name_symbol = &pc->tokens->at(token_index);
     token_index += 1;
@@ -399,7 +410,7 @@ static AstNode *ast_parse_type(ParseContext *pc, int token_index, int *new_token
     Token *token = &pc->tokens->at(token_index);
     token_index += 1;
 
-    AstNode *node = ast_create_node(NodeTypeType, token);
+    AstNode *node = ast_create_node(pc, NodeTypeType, token);
 
     if (token->id == TokenIdKeywordUnreachable) {
         node->data.type.type = AstNodeTypeTypePrimitive;
@@ -437,7 +448,7 @@ static AstNode *ast_parse_param_decl(ParseContext *pc, int token_index, int *new
     token_index += 1;
     ast_expect_token(pc, param_name, TokenIdSymbol);
 
-    AstNode *node = ast_create_node(NodeTypeParamDecl, param_name);
+    AstNode *node = ast_create_node(pc, NodeTypeParamDecl, param_name);
 
 
     ast_buf_from_token(pc, param_name, &node->data.param_decl.name);
@@ -544,21 +555,21 @@ static AstNode *ast_parse_primary_expr(ParseContext *pc, int *token_index, bool
     Token *token = &pc->tokens->at(*token_index);
 
     if (token->id == TokenIdNumberLiteral) {
-        AstNode *node = ast_create_node(NodeTypeNumberLiteral, token);
+        AstNode *node = ast_create_node(pc, NodeTypeNumberLiteral, token);
         ast_buf_from_token(pc, token, &node->data.number);
         *token_index += 1;
         return node;
     } else if (token->id == TokenIdStringLiteral) {
-        AstNode *node = ast_create_node(NodeTypeStringLiteral, token);
+        AstNode *node = ast_create_node(pc, NodeTypeStringLiteral, token);
         parse_string_literal(pc, token, &node->data.string);
         *token_index += 1;
         return node;
     } else if (token->id == TokenIdKeywordUnreachable) {
-        AstNode *node = ast_create_node(NodeTypeUnreachable, token);
+        AstNode *node = ast_create_node(pc, NodeTypeUnreachable, token);
         *token_index += 1;
         return node;
     } else if (token->id == TokenIdSymbol) {
-        AstNode *node = ast_create_node(NodeTypeSymbol, token);
+        AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
         ast_buf_from_token(pc, token, &node->data.symbol);
         *token_index += 1;
         return node;
@@ -592,7 +603,7 @@ static AstNode *ast_parse_fn_call_expr(ParseContext *pc, int *token_index, bool
     if (l_paren->id != TokenIdLParen)
         return primary_expr;
 
-    AstNode *node = ast_create_node_with_node(NodeTypeFnCallExpr, primary_expr);
+    AstNode *node = ast_create_node_with_node(pc, NodeTypeFnCallExpr, primary_expr);
     node->data.fn_call_expr.fn_ref_expr = primary_expr;
     ast_parse_fn_call_param_list(pc, *token_index, token_index, &node->data.fn_call_expr.params);
 
@@ -635,7 +646,7 @@ static AstNode *ast_parse_prefix_op_expr(ParseContext *pc, int *token_index, boo
         return ast_parse_fn_call_expr(pc, token_index, mandatory);
 
     AstNode *primary_expr = ast_parse_fn_call_expr(pc, token_index, true);
-    AstNode *node = ast_create_node(NodeTypePrefixOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypePrefixOpExpr, token);
     node->data.prefix_op_expr.primary_expr = primary_expr;
     node->data.prefix_op_expr.prefix_op = prefix_op;
 
@@ -656,7 +667,7 @@ static AstNode *ast_parse_cast_expression(ParseContext *pc, int *token_index, bo
         return prefix_op_expr;
     *token_index += 1;
 
-    AstNode *node = ast_create_node(NodeTypeCastExpr, as_kw);
+    AstNode *node = ast_create_node(pc, NodeTypeCastExpr, as_kw);
     node->data.cast_expr.prefix_op_expr = prefix_op_expr;
 
     node->data.cast_expr.type = ast_parse_type(pc, *token_index, token_index);
@@ -705,7 +716,7 @@ static AstNode *ast_parse_mult_expr(ParseContext *pc, int *token_index, bool man
 
     AstNode *operand_2 = ast_parse_cast_expression(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = mult_op;
     node->data.bin_op_expr.op2 = operand_2;
@@ -753,7 +764,7 @@ static AstNode *ast_parse_add_expr(ParseContext *pc, int *token_index, bool mand
 
     AstNode *operand_2 = ast_parse_mult_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = add_op;
     node->data.bin_op_expr.op2 = operand_2;
@@ -801,7 +812,7 @@ static AstNode *ast_parse_bit_shift_expr(ParseContext *pc, int *token_index, boo
 
     AstNode *operand_2 = ast_parse_add_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = bit_shift_op;
     node->data.bin_op_expr.op2 = operand_2;
@@ -825,7 +836,7 @@ static AstNode *ast_parse_bin_and_expr(ParseContext *pc, int *token_index, bool
 
     AstNode *operand_2 = ast_parse_bit_shift_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = BinOpTypeBinAnd;
     node->data.bin_op_expr.op2 = operand_2;
@@ -848,7 +859,7 @@ static AstNode *ast_parse_bin_xor_expr(ParseContext *pc, int *token_index, bool
 
     AstNode *operand_2 = ast_parse_bin_and_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = BinOpTypeBinXor;
     node->data.bin_op_expr.op2 = operand_2;
@@ -871,7 +882,7 @@ static AstNode *ast_parse_bin_or_expr(ParseContext *pc, int *token_index, bool m
 
     AstNode *operand_2 = ast_parse_bin_xor_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = BinOpTypeBinOr;
     node->data.bin_op_expr.op2 = operand_2;
@@ -920,7 +931,7 @@ static AstNode *ast_parse_comparison_expr(ParseContext *pc, int *token_index, bo
 
     AstNode *operand_2 = ast_parse_bin_or_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = cmp_op;
     node->data.bin_op_expr.op2 = operand_2;
@@ -943,7 +954,7 @@ static AstNode *ast_parse_bool_and_expr(ParseContext *pc, int *token_index, bool
 
     AstNode *operand_2 = ast_parse_comparison_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = BinOpTypeBoolAnd;
     node->data.bin_op_expr.op2 = operand_2;
@@ -958,7 +969,7 @@ static AstNode *ast_parse_return_expr(ParseContext *pc, int *token_index, bool m
     Token *return_tok = &pc->tokens->at(*token_index);
     if (return_tok->id == TokenIdKeywordReturn) {
         *token_index += 1;
-        AstNode *node = ast_create_node(NodeTypeReturnExpr, return_tok);
+        AstNode *node = ast_create_node(pc, NodeTypeReturnExpr, return_tok);
         node->data.return_expr.expr = ast_parse_expression(pc, token_index, false);
         return node;
     } else if (mandatory) {
@@ -983,7 +994,7 @@ static AstNode *ast_parse_bool_or_expr(ParseContext *pc, int *token_index, bool
 
     AstNode *operand_2 = ast_parse_bool_and_expr(pc, token_index, true);
 
-    AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);
+    AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
     node->data.bin_op_expr.op1 = operand_1;
     node->data.bin_op_expr.bin_op = BinOpTypeBoolOr;
     node->data.bin_op_expr.op2 = operand_2;
@@ -1046,7 +1057,7 @@ static AstNode *ast_parse_block(ParseContext *pc, int *token_index, bool mandato
     }
     *token_index += 1;
 
-    AstNode *node = ast_create_node(NodeTypeBlock, l_brace);
+    AstNode *node = ast_create_node(pc, NodeTypeBlock, l_brace);
 
     for (;;) {
         Token *token = &pc->tokens->at(*token_index);
@@ -1092,7 +1103,7 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc, int *token_index, bool mand
         return nullptr;
     }
 
-    AstNode *node = ast_create_node(NodeTypeFnProto, token);
+    AstNode *node = ast_create_node(pc, NodeTypeFnProto, token);
     node->data.fn_proto.visib_mod = visib_mod;
     node->data.fn_proto.directives = pc->directive_list;
     pc->directive_list = nullptr;
@@ -1125,7 +1136,7 @@ static AstNode *ast_parse_fn_def(ParseContext *pc, int *token_index, bool mandat
     AstNode *fn_proto = ast_parse_fn_proto(pc, token_index, mandatory);
     if (!fn_proto)
         return nullptr;
-    AstNode *node = ast_create_node_with_node(NodeTypeFnDef, fn_proto);
+    AstNode *node = ast_create_node_with_node(pc, NodeTypeFnDef, fn_proto);
 
     node->data.fn_def.fn_proto = fn_proto;
     node->data.fn_def.body = ast_parse_block(pc, token_index, true);
@@ -1138,7 +1149,7 @@ FnDecl : FnProto token(Semicolon)
 */
 static AstNode *ast_parse_fn_decl(ParseContext *pc, int token_index, int *new_token_index) {
     AstNode *fn_proto = ast_parse_fn_proto(pc, &token_index, true);
-    AstNode *node = ast_create_node_with_node(NodeTypeFnDecl, fn_proto);
+    AstNode *node = ast_create_node_with_node(pc, NodeTypeFnDecl, fn_proto);
 
     node->data.fn_decl.fn_proto = fn_proto;
 
@@ -1166,7 +1177,7 @@ static AstNode *ast_parse_extern_block(ParseContext *pc, int *token_index, bool
     }
     *token_index += 1;
 
-    AstNode *node = ast_create_node(NodeTypeExternBlock, extern_kw);
+    AstNode *node = ast_create_node(pc, NodeTypeExternBlock, extern_kw);
 
     node->data.extern_block.directives = pc->directive_list;
     pc->directive_list = nullptr;
@@ -1184,7 +1195,7 @@ static AstNode *ast_parse_extern_block(ParseContext *pc, int *token_index, bool
         Token *token = &pc->tokens->at(*token_index);
         if (token->id == TokenIdRBrace) {
             if (pc->directive_list->length > 0) {
-                ast_error(directive_token, "invalid directive");
+                ast_error(pc, directive_token, "invalid directive");
             }
             pc->directive_list = nullptr;
 
@@ -1216,7 +1227,7 @@ static AstNode *ast_parse_root_export_decl(ParseContext *pc, int *token_index, b
 
     *token_index += 2;
 
-    AstNode *node = ast_create_node(NodeTypeRootExportDecl, export_kw);
+    AstNode *node = ast_create_node(pc, NodeTypeRootExportDecl, export_kw);
     node->data.root_export_decl.directives = pc->directive_list;
     pc->directive_list = nullptr;
 
@@ -1254,7 +1265,7 @@ static AstNode *ast_parse_use(ParseContext *pc, int *token_index, bool mandatory
     *token_index += 1;
     ast_expect_token(pc, semicolon, TokenIdSemicolon);
 
-    AstNode *node = ast_create_node(NodeTypeUse, use_kw);
+    AstNode *node = ast_create_node(pc, NodeTypeUse, use_kw);
 
     parse_string_literal(pc, use_name, &node->data.use.path);
 
@@ -1299,7 +1310,7 @@ static void ast_parse_top_level_decls(ParseContext *pc, int *token_index, ZigLis
         }
 
         if (pc->directive_list->length > 0) {
-            ast_error(directive_token, "invalid directive");
+            ast_error(pc, directive_token, "invalid directive");
         }
         pc->directive_list = nullptr;
 
@@ -1312,7 +1323,7 @@ static void ast_parse_top_level_decls(ParseContext *pc, int *token_index, ZigLis
 Root : many(TopLevelDecl) token(EOF)
  */
 static AstNode *ast_parse_root(ParseContext *pc, int *token_index) {
-    AstNode *node = ast_create_node(NodeTypeRoot, &pc->tokens->at(*token_index));
+    AstNode *node = ast_create_node(pc, NodeTypeRoot, &pc->tokens->at(*token_index));
 
     ast_parse_top_level_decls(pc, token_index, &node->data.root.top_level_decls);
 
@@ -1323,8 +1334,9 @@ static AstNode *ast_parse_root(ParseContext *pc, int *token_index) {
     return node;
 }
 
-AstNode *ast_parse(Buf *buf, ZigList<Token> *tokens) {
+AstNode *ast_parse(Buf *buf, ZigList<Token> *tokens, ImportTableEntry *owner) {
     ParseContext pc = {0};
+    pc.owner = owner;
     pc.buf = buf;
     pc.tokens = tokens;
     int token_index = 0;
diff --git a/src/parser.hpp b/src/parser.hpp
index cc7b3dd0b..3346763ac 100644
--- a/src/parser.hpp
+++ b/src/parser.hpp
@@ -14,6 +14,7 @@
 
 struct AstNode;
 struct CodeGenNode;
+struct ImportTableEntry;
 
 enum NodeType {
     NodeTypeRoot,
@@ -166,10 +167,10 @@ struct AstNodeUse {
 
 struct AstNode {
     enum NodeType type;
-    AstNode *parent;
     int line;
     int column;
     CodeGenNode *codegen_node;
+    ImportTableEntry *owner;
     union {
         AstNodeRoot root;
         AstNodeRootExportDecl root_export_decl;
@@ -198,7 +199,7 @@ void ast_token_error(Token *token, const char *format, ...);
 
 
 // This function is provided by generated code, generated by parsergen.cpp
-AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);
+AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens, ImportTableEntry *owner);
 
 const char *node_type_str(NodeType node_type);
 
diff --git a/src/semantic_info.hpp b/src/semantic_info.hpp
index 89ecfabea..5a8ec35cb 100644
--- a/src/semantic_info.hpp
+++ b/src/semantic_info.hpp
@@ -11,6 +11,7 @@
 #include "codegen.hpp"
 #include "hash_map.hpp"
 #include "zig_llvm.hpp"
+#include "errmsg.hpp"
 
 struct FnTableEntry;
 
@@ -30,6 +31,8 @@ struct ImportTableEntry {
     AstNode *root;
     Buf *path; // relative to root_source_dir
     LLVMZigDIFile *di_file;
+    Buf *source_code;
+    ZigList<int> *line_offsets;
 
     // reminder: hash tables must be initialized before use
     HashMap<Buf *, FnTableEntry *, buf_hash, buf_eql_buf> fn_table;
@@ -47,7 +50,7 @@ struct FnTableEntry {
 
 struct CodeGen {
     LLVMModuleRef module;
-    ZigList<ErrorMsg> errors;
+    ZigList<ErrorMsg*> errors;
     LLVMBuilderRef builder;
     LLVMZigDIBuilder *dbuilder;
     LLVMZigDICompileUnit *compile_unit;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 94f3966f0..0a61bb916 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -104,6 +104,7 @@ enum TokenizeState {
     TokenizeStateBang,
     TokenizeStateLessThan,
     TokenizeStateGreaterThan,
+    TokenizeStateError,
 };
 
 
@@ -116,27 +117,25 @@ struct Tokenize {
     int column;
     Token *cur_tok;
     int multi_line_comment_count;
+    Tokenization *out;
 };
 
 __attribute__ ((format (printf, 2, 3)))
 static void tokenize_error(Tokenize *t, const char *format, ...) {
-    int line;
-    int column;
+    t->state = TokenizeStateError;
+
     if (t->cur_tok) {
-        line = t->cur_tok->start_line + 1;
-        column = t->cur_tok->start_column + 1;
+        t->out->err_line = t->cur_tok->start_line;
+        t->out->err_column = t->cur_tok->start_column;
     } else {
-        line = t->line + 1;
-        column = t->column + 1;
+        t->out->err_line = t->line;
+        t->out->err_column = t->column;
     }
 
     va_list ap;
     va_start(ap, format);
-    fprintf(stderr, "Error: Line %d, column %d: ", line, column);
-    vfprintf(stderr, format, ap);
-    fprintf(stderr, "\n");
+    t->out->err = buf_vprintf(format, ap);
     va_end(ap);
-    exit(EXIT_FAILURE);
 }
 
 static void begin_token(Tokenize *t, TokenId id) {
@@ -187,13 +186,20 @@ static void end_token(Tokenize *t) {
     t->cur_tok = nullptr;
 }
 
-ZigList<Token> *tokenize(Buf *buf) {
+void tokenize(Buf *buf, Tokenization *out) {
     Tokenize t = {0};
-    t.tokens = allocate<ZigList<Token>>(1);
+    t.out = out;
+    t.tokens = out->tokens = allocate<ZigList<Token>>(1);
     t.buf = buf;
+
+    out->line_offsets = allocate<ZigList<int>>(1);
+
+    out->line_offsets->append(0);
     for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) {
         uint8_t c = buf_ptr(t.buf)[t.pos];
         switch (t.state) {
+            case TokenizeStateError:
+                break;
             case TokenizeStateStart:
                 switch (c) {
                     case WHITESPACE:
@@ -509,6 +515,7 @@ ZigList<Token> *tokenize(Buf *buf) {
                 break;
         }
         if (c == '\n') {
+            out->line_offsets->append(t.pos + 1);
             t.line += 1;
             t.column = 0;
         } else {
@@ -518,6 +525,7 @@ ZigList<Token> *tokenize(Buf *buf) {
     // EOF
     switch (t.state) {
         case TokenizeStateStart:
+        case TokenizeStateError:
             break;
         case TokenizeStateString:
             tokenize_error(&t, "unterminated string");
@@ -544,11 +552,12 @@ ZigList<Token> *tokenize(Buf *buf) {
             tokenize_error(&t, "unterminated multi-line comment");
             break;
     }
-    t.pos = -1;
-    begin_token(&t, TokenIdEof);
-    end_token(&t);
-    assert(!t.cur_tok);
-    return t.tokens;
+    if (t.state != TokenizeStateError) {
+        t.pos = -1;
+        begin_token(&t, TokenIdEof);
+        end_token(&t);
+        assert(!t.cur_tok);
+    }
 }
 
 static const char * token_name(Token *token) {
diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp
index 94cbb15d4..45bc011e0 100644
--- a/src/tokenizer.hpp
+++ b/src/tokenizer.hpp
@@ -65,7 +65,17 @@ struct Token {
     int start_column;
 };
 
-ZigList<Token> *tokenize(Buf *buf);
+struct Tokenization {
+    ZigList<Token> *tokens;
+    ZigList<int> *line_offsets;
+
+    // if an error occurred
+    Buf *err;
+    int err_line;
+    int err_column;
+};
+
+void tokenize(Buf *buf, Tokenization *out_tokenization);
 
 void print_tokens(Buf *buf, ZigList<Token> *tokens);
 
diff --git a/test/run_tests.cpp b/test/run_tests.cpp
index b54f0bd4c..34a166867 100644
--- a/test/run_tests.cpp
+++ b/test/run_tests.cpp
@@ -139,7 +139,7 @@ static void add_compile_failure_test_cases(void) {
     add_compile_fail_case("multiple function definitions", R"SOURCE(
 fn a() {}
 fn a() {}
-    )SOURCE", 1, "Line 3, column 1: redefinition of 'a'");
+    )SOURCE", 1, ".tmp_source.zig:3:1: error: redefinition of 'a'");
 
     add_compile_fail_case("bad directive", R"SOURCE(
 #bogus1("")
@@ -148,37 +148,37 @@ extern {
 }
 #bogus2("")
 fn a() {}
-    )SOURCE", 2, "Line 2, column 1: invalid directive: 'bogus1'",
-                 "Line 6, column 1: invalid directive: 'bogus2'");
+    )SOURCE", 2, ".tmp_source.zig:2:1: error: invalid directive: 'bogus1'",
+                 ".tmp_source.zig:6:1: error: invalid directive: 'bogus2'");
 
     add_compile_fail_case("unreachable with return", R"SOURCE(
 fn a() -> unreachable {return;}
-    )SOURCE", 1, "Line 2, column 24: return statement in function with unreachable return type");
+    )SOURCE", 1, ".tmp_source.zig:2:24: error: return statement in function with unreachable return type");
 
     add_compile_fail_case("control reaches end of non-void function", R"SOURCE(
 fn a() -> i32 {}
-    )SOURCE", 1, "Line 2, column 1: control reaches end of non-void function");
+    )SOURCE", 1, ".tmp_source.zig:2:1: error: control reaches end of non-void function");
 
     add_compile_fail_case("undefined function call", R"SOURCE(
 fn a() {
     b();
 }
-    )SOURCE", 1, "Line 3, column 5: undefined function: 'b'");
+    )SOURCE", 1, ".tmp_source.zig:3:5: error: undefined function: 'b'");
 
     add_compile_fail_case("wrong number of arguments", R"SOURCE(
 fn a() {
     b(1);
 }
 fn b(a: i32, b: i32, c: i32) { }
-    )SOURCE", 1, "Line 3, column 5: wrong number of arguments. Expected 3, got 1.");
+    )SOURCE", 1, ".tmp_source.zig:3:5: error: wrong number of arguments. Expected 3, got 1.");
 
     add_compile_fail_case("invalid type", R"SOURCE(
 fn a() -> bogus {}
-    )SOURCE", 1, "Line 2, column 11: invalid type name: 'bogus'");
+    )SOURCE", 1, ".tmp_source.zig:2:11: error: invalid type name: 'bogus'");
 
     add_compile_fail_case("pointer to unreachable", R"SOURCE(
 fn a() -> *mut unreachable {}
-    )SOURCE", 1, "Line 2, column 11: pointer to unreachable not allowed");
+    )SOURCE", 1, ".tmp_source.zig:2:11: error: pointer to unreachable not allowed");
 
     add_compile_fail_case("unreachable code", R"SOURCE(
 fn a() {
@@ -187,12 +187,12 @@ fn a() {
 }
 
 fn b() {}
-    )SOURCE", 1, "Line 4, column 5: unreachable code");
+    )SOURCE", 1, ".tmp_source.zig:4:5: error: unreachable code");
 
     add_compile_fail_case("bad version string", R"SOURCE(
 #version("aoeu")
 export executable "test";
-    )SOURCE", 1, "Line 2, column 1: invalid version string");
+    )SOURCE", 1, ".tmp_source.zig:2:1: error: invalid version string");
 }
 
 static void print_compiler_invokation(TestCase *test_case, Buf *zig_stderr) {