Sync go parser with fishman-ctags

New features include:
* struct/interface detection
* struct member parsing
* function prototype parsing

tagmanager/ctags/go.c

@@ -13,6 +13,7 @@
 /*
  *	 MACROS
  */
+#define MAX_SIGNATURE_LENGTH 512
 #define isType(token,t) (boolean) ((token)->type == (t))
 #define isKeyword(token,k) (boolean) ((token)->keyword == (k))
 
@@ -77,6 +78,7 @@ typedef struct sTokenInfo {
 
 static int Lang_go;
 static vString *scope;
+static vString *signature = NULL;
 
 typedef enum {
 	GOTAG_UNDEFINED = -1,
@@ -85,6 +87,9 @@ typedef enum {
 	GOTAG_CONST,
 	GOTAG_TYPE,
 	GOTAG_VAR,
+	GOTAG_STRUCT,
+	GOTAG_INTERFACE,
+	GOTAG_MEMBER
 } goKind;
 
 static kindOption GoKinds[] = {
@@ -92,7 +97,10 @@ static kindOption GoKinds[] = {
 	{TRUE, 'f', "function", "functions"},
 	{TRUE, 'c', "macro", "constants"},
 	{TRUE, 't', "typedef", "types"},
-	{TRUE, 'v', "variable", "variables"}
+	{TRUE, 'v', "variable", "variables"},
+	{TRUE, 's', "struct", "structs"},
+	{TRUE, 'i', "interface", "interfaces"},
+	{TRUE, 'm', "member", "struct members"}
 };
 
 static keywordDesc GoKeywordTable[] = {
@@ -149,6 +157,17 @@ static tokenInfo *newToken (void)
 	return token;
 }
 
+static tokenInfo *copyToken (tokenInfo *other)
+{
+	tokenInfo *const token = xMalloc (1, tokenInfo);
+	token->type = other->type;
+	token->keyword = other->keyword;
+	token->string = vStringNewCopy (other->string);
+	token->lineNumber = other->lineNumber;
+	token->filePosition = other->filePosition;
+	return token;
+}
+
 static void deleteToken (tokenInfo * const token)
 {
 	if (token != NULL)
@@ -201,6 +220,8 @@ static void readToken (tokenInfo *const token)
 {
 	int c;
 	static tokenType lastTokenType = TOKEN_NONE;
+	boolean firstWhitespace = TRUE;
+	boolean whitespace;
 
 	token->type = TOKEN_NONE;
 	token->keyword = KEYWORD_NONE;
@@ -219,11 +240,16 @@ getNextChar:
 						  lastTokenType == TOKEN_CLOSE_CURLY ||
 						  lastTokenType == TOKEN_CLOSE_SQUARE))
 		{
-			token->type = TOKEN_SEMICOLON;
+			c = ';';  // semicolon injection
-			goto done;
+		}
+		whitespace = c == '\t'  ||  c == ' ' ||  c == '\r' || c == '\n';
+		if (signature && whitespace && firstWhitespace && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
+		{
+			firstWhitespace = FALSE;
+			vStringPut(signature, ' ');
 		}
 	}
-	while (c == '\t'  ||  c == ' ' ||  c == '\r' || c == '\n');
+	while (whitespace);
 
 	switch (c)
 	{
@@ -354,71 +380,79 @@ getNextChar:
 			break;
 	}
 
-done:
+	if (signature && vStringLength (signature) < MAX_SIGNATURE_LENGTH)
+	{
+		if (token->type == TOKEN_LEFT_ARROW)
+			vStringCatS(signature, "<-");
+		else if (token->type == TOKEN_STRING)
+		{
+			// only struct member annotations can appear in function prototypes
+			// so only `` type strings are possible
+			vStringPut(signature, '`');
+			vStringCat(signature, token->string);
+			vStringPut(signature, '`');
+		}
+		else if (token->type == TOKEN_IDENTIFIER || token->type == TOKEN_KEYWORD)
+			vStringCat(signature, token->string);
+		else if (c != EOF)
+			vStringPut(signature, c);
+	}
+
 	lastTokenType = token->type;
 }
 
-static void skipToMatched (tokenInfo *const token)
+static boolean skipToMatchedNoRead (tokenInfo *const token)
 {
 	int nest_level = 0;
-	tokenType open_token;
+	tokenType open_token = token->type;
 	tokenType close_token;
 
-	switch (token->type)
+	switch (open_token)
 	{
 		case TOKEN_OPEN_PAREN:
-			open_token = TOKEN_OPEN_PAREN;
 			close_token = TOKEN_CLOSE_PAREN;
 			break;
 		case TOKEN_OPEN_CURLY:
-			open_token = TOKEN_OPEN_CURLY;
 			close_token = TOKEN_CLOSE_CURLY;
 			break;
 		case TOKEN_OPEN_SQUARE:
-			open_token = TOKEN_OPEN_SQUARE;
 			close_token = TOKEN_CLOSE_SQUARE;
 			break;
 		default:
-			return;
+			return FALSE;
 	}
 
 	/*
 	 * This routine will skip to a matching closing token.
-	 * It will also handle nested tokens like the (, ) below.
+	 * It will also handle nested tokens.
-	 *   (  name varchar(30), text binary(10)  )
 	 */
-	if (isType (token, open_token))
+	nest_level++;
+	while (nest_level > 0 && !isType (token, TOKEN_EOF))
 	{
-		nest_level++;
-		while (!(isType (token, close_token) && (nest_level == 0)) &&
-			   !isType (token, TOKEN_EOF))
-		{
-			readToken (token);
-			if (isType (token, open_token))
-			{
-				nest_level++;
-			}
-			if (isType (token, close_token))
-			{
-				if (nest_level > 0)
-				{
-					nest_level--;
-				}
-			}
-		}
 		readToken (token);
+		if (isType (token, open_token))
+			nest_level++;
+		else if (isType (token, close_token))
+			nest_level--;
 	}
+
+	return TRUE;
 }
 
-static void skipType (tokenInfo *const token)
+static void skipToMatched (tokenInfo *const token)
+{
+	if (skipToMatchedNoRead (token))
+		readToken (token);
+}
+
+static boolean skipType (tokenInfo *const token)
 {
-again:
 	// Type      = TypeName | TypeLit | "(" Type ")" .
 	// Skips also function multiple return values "(" Type {"," Type} ")"
 	if (isType (token, TOKEN_OPEN_PAREN))
 	{
 		skipToMatched (token);
-		return;
+		return TRUE;
 	}
 
 	// TypeName  = QualifiedIdent.
@@ -433,7 +467,7 @@ again:
 			if (isType (token, TOKEN_IDENTIFIER))
 				readToken (token);
 		}
-		return;
+		return TRUE;
 	}
 
 	// StructType     = "struct" "{" { FieldDecl ";" } "}"
@@ -443,7 +477,7 @@ again:
 		readToken (token);
 		// skip over "{}"
 		skipToMatched (token);
-		return;
+		return TRUE;
 	}
 
 	// ArrayType   = "[" ArrayLength "]" ElementType .
@@ -452,7 +486,7 @@ again:
 	if (isType (token, TOKEN_OPEN_SQUARE))
 	{
 		skipToMatched (token);
-		goto again;
+		return skipType (token);
 	}
 
 	// PointerType = "*" BaseType .
@@ -461,7 +495,7 @@ again:
 	if (isType (token, TOKEN_STAR) || isKeyword (token, KEYWORD_chan) || isType (token, TOKEN_LEFT_ARROW))
 	{
 		readToken (token);
-		goto again;
+		return skipType (token);
 	}
 
 	// MapType     = "map" "[" KeyType "]" ElementType .
@@ -471,7 +505,7 @@ again:
 		readToken (token);
 		// skip over "[]"
 		skipToMatched (token);
-		goto again;
+		return skipType (token);
 	}
 
 	// FunctionType   = "func" Signature .
@@ -486,11 +520,15 @@ again:
 		// Result is parameters or type or nothing.  skipType treats anything
 		// surrounded by parentheses as a type, and does nothing if what
 		// follows is not a type.
-		goto again;
+		return skipType (token);
 	}
+
+	return FALSE;
 }
 
-static void makeTag (tokenInfo *const token, const goKind kind)
+static void makeTag (tokenInfo *const token, const goKind kind,
+	tokenInfo *const parent_token, const goKind parent_kind,
+	const char *argList)
 {
 	const char *const name = vStringValue (token->string);
 
@@ -504,7 +542,14 @@ static void makeTag (tokenInfo *const token, const goKind kind)
 	e.filePosition = token->filePosition;
 	e.kindName = GoKinds [kind].name;
 	e.kind = GoKinds [kind].letter;
+	if (argList)
+		e.extensionFields.arglist = argList;
 
+	if (parent_kind != GOTAG_UNDEFINED && parent_token != NULL)
+	{
+		e.extensionFields.scope[0] = GoKinds[parent_kind].name;
+		e.extensionFields.scope[1] = vStringValue (parent_token->string);
+	}
 	makeTagEntry (&e);
 
 	if (scope && Option.include.qualifiedTags)
@@ -524,7 +569,7 @@ static void parsePackage (tokenInfo *const token)
 	readToken (token);
 	if (isType (token, TOKEN_IDENTIFIER))
 	{
-		makeTag (token, GOTAG_PACKAGE);
+		makeTag (token, GOTAG_PACKAGE, NULL, GOTAG_UNDEFINED, NULL);
 		if (!scope && Option.include.qualifiedTags)
 		{
 			scope = vStringNew ();
@@ -549,11 +594,25 @@ static void parseFunctionOrMethod (tokenInfo *const token)
 
 	if (isType (token, TOKEN_IDENTIFIER))
 	{
-		makeTag (token, GOTAG_FUNCTION);
+		tokenInfo *functionToken = copyToken (token);
-		
+
+		// Start recording signature
+		signature = vStringNew ();
+
 		// Skip over parameters.
 		readToken (token);
-		skipToMatched (token);
+		skipToMatchedNoRead (token);
+
+		vStringStripLeading (signature);
+		vStringStripTrailing (signature);
+		makeTag (functionToken, GOTAG_FUNCTION, NULL, GOTAG_UNDEFINED, signature->buffer);
+		deleteToken (functionToken);
+		vStringDelete(signature);
+
+		// Stop recording signature
+		signature = NULL;
+
+		readToken (token);
 
 		// Skip over result.
 		skipType (token);
@@ -564,6 +623,75 @@ static void parseFunctionOrMethod (tokenInfo *const token)
 	}
 }
 
+static void parseStructMembers (tokenInfo *const token, tokenInfo *const parent_token)
+{
+	// StructType     = "struct" "{" { FieldDecl ";" } "}" .
+	// FieldDecl      = (IdentifierList Type | AnonymousField) [ Tag ] .
+	// AnonymousField = [ "*" ] TypeName .
+	// Tag            = string_lit .
+
+	readToken (token);
+	if (!isType (token, TOKEN_OPEN_CURLY))
+		return;
+
+	readToken (token);
+	while (!isType (token, TOKEN_EOF) && !isType (token, TOKEN_CLOSE_CURLY))
+	{
+		tokenInfo *memberCandidate = NULL;
+		boolean first = TRUE;
+
+		while (!isType (token, TOKEN_EOF))
+		{
+			if (isType (token, TOKEN_IDENTIFIER))
+			{
+				if (first)
+				{
+					// could be anonymous field like in 'struct {int}' - we don't know yet
+					memberCandidate = copyToken (token);
+					first = FALSE;
+				}
+				else
+				{
+					if (memberCandidate)
+					{
+						// if we are here, there was a comma and memberCandidate isn't an anonymous field
+						makeTag (memberCandidate, GOTAG_MEMBER, parent_token, GOTAG_STRUCT, NULL);
+						deleteToken (memberCandidate);
+						memberCandidate = NULL;
+					}
+					makeTag (token, GOTAG_MEMBER, parent_token, GOTAG_STRUCT, NULL);
+				}
+				readToken (token);
+			}
+			if (!isType (token, TOKEN_COMMA))
+				break;
+			readToken (token);
+		}
+
+		// in the case of  an anonymous field, we already read part of the
+		// type into memberCandidate and skipType() should return FALSE so no tag should
+		// be generated in this case.
+		if (skipType (token) && memberCandidate)
+			makeTag (memberCandidate, GOTAG_MEMBER, parent_token, GOTAG_STRUCT, NULL);
+
+		if (memberCandidate)
+			deleteToken (memberCandidate);
+
+		while (!isType (token, TOKEN_SEMICOLON) && !isType (token, TOKEN_CLOSE_CURLY)
+				&& !isType (token, TOKEN_EOF))
+		{
+			readToken (token);
+			skipToMatched (token);
+		}
+
+		if (!isType (token, TOKEN_CLOSE_CURLY))
+		{
+			// we are at TOKEN_SEMICOLON
+			readToken (token);
+		}
+	}
+}
+
 static void parseConstTypeVar (tokenInfo *const token, goKind kind)
 {
 	// ConstDecl      = "const" ( ConstSpec | "(" { ConstSpec ";" } ")" ) .
@@ -586,11 +714,26 @@ static void parseConstTypeVar (tokenInfo *const token, goKind kind)
 
 	do
 	{
+		tokenInfo *typeToken = NULL;
+
 		while (!isType (token, TOKEN_EOF))
 		{
 			if (isType (token, TOKEN_IDENTIFIER))
 			{
-				makeTag (token, kind);
+				if (kind == GOTAG_TYPE)
+				{
+					typeToken = copyToken (token);
+					readToken (token);
+					if (isKeyword (token, KEYWORD_struct))
+						makeTag (typeToken, GOTAG_STRUCT, NULL, GOTAG_UNDEFINED, NULL);
+					else if (isKeyword (token, KEYWORD_interface))
+						makeTag (typeToken, GOTAG_INTERFACE, NULL, GOTAG_UNDEFINED, NULL);
+					else
+						makeTag (typeToken, kind, NULL, GOTAG_UNDEFINED, NULL);
+					break;
+				}
+				else
+					makeTag (token, kind, NULL, GOTAG_UNDEFINED, NULL);
 				readToken (token);
 			}
 			if (!isType (token, TOKEN_COMMA))
@@ -598,7 +741,17 @@ static void parseConstTypeVar (tokenInfo *const token, goKind kind)
 			readToken (token);
 		}
 
-		skipType (token);
+		if (typeToken)
+		{
+			if (isKeyword (token, KEYWORD_struct))
+				parseStructMembers (token, typeToken);
+			else
+				skipType (token);
+			deleteToken (typeToken);
+		}
+		else
+			skipType (token);
+
 		while (!isType (token, TOKEN_SEMICOLON) && !isType (token, TOKEN_CLOSE_PAREN)
 				&& !isType (token, TOKEN_EOF))
 		{

tests/ctags/test.go

@@ -15,8 +15,11 @@ type (
 )
 
 type T6 struct {
-	a, b, c, d int
+	_a, _b, _c, _d int
-	e float32
+	int
+	T1 `annotation`
+	*T2
+	_e float32
 	//ignored int
 }
 
@@ -44,4 +47,4 @@ func (tt * T7) f4(a func () func ()) (func (), int) {return func (){}, 1};func f
 func main() {
 	go func (){}()
 	fmt.Println("Hello, 世界")
-}
+}

tests/ctags/test.go.tags

@@ -12,23 +12,28 @@ T1
 T2ﾌ4096ﾖ0
 T3ﾌ4096ﾖ0
 T4ﾌ4096ﾖ0
-T5ﾌ4096ﾖ0
+T5ﾌ32ﾖ0
-T6ﾌ4096ﾖ0
+T6ﾌ2048ﾖ0
 T7ﾌ4096ﾖ0
 T8ﾌ4096ﾖ0
 T9ﾌ4096ﾖ0
+_aﾌ64ﾎT6ﾖ0
+_bﾌ64ﾎT6ﾖ0
+_cﾌ64ﾎT6ﾖ0
+_dﾌ64ﾎT6ﾖ0
+_eﾌ64ﾎT6ﾖ0
 aﾌ16384ﾖ0
 bﾌ16384ﾖ0
 cﾌ16384ﾖ0
 dﾌ16384ﾖ0
 eﾌ16384ﾖ0
 fﾌ16384ﾖ0
-f1ﾌ16ﾖ0
+f1ﾌ16ﾍ()ﾖ0
-f2ﾌ16ﾖ0
+f2ﾌ16ﾍ()ﾖ0
-f3ﾌ16ﾖ0
+f3ﾌ16ﾍ()ﾖ0
-f4ﾌ16ﾖ0
+f4ﾌ16ﾍ(a func () func ())ﾖ0
-f5ﾌ16ﾖ0
+f5ﾌ16ﾍ()ﾖ0
 gﾌ16384ﾖ0
 hﾌ16384ﾖ0
-mainﾌ16ﾖ0
+mainﾌ16ﾍ()ﾖ0
 mainﾌ256ﾖ0