translate-c: enough C tokenization/parsing to handle shifting in macros

See #2451

src/c_tokenizer.cpp

@@ -124,6 +124,8 @@ static void begin_token(CTokenize *ctok, CTokId id) {
         case CTokIdAsterisk:
         case CTokIdBang:
         case CTokIdTilde:
+        case CTokIdShl:
+        case CTokIdLt:
             break;
     }
 }
@@ -223,6 +225,10 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
                         begin_token(ctok, CTokIdDot);
                         end_token(ctok);
                         break;
+                    case '<':
+                        begin_token(ctok, CTokIdLt);
+                        ctok->state = CTokStateGotLt;
+                        break;
                     case '(':
                         begin_token(ctok, CTokIdLParen);
                         end_token(ctok);
@@ -251,6 +257,18 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
                         return mark_error(ctok);
                 }
                 break;
+            case CTokStateGotLt:
+                switch (*c) {
+                    case '<':
+                        ctok->cur_tok->id = CTokIdShl;
+                        end_token(ctok);
+                        ctok->state = CTokStateStart;
+                        break;
+                    default:
+                        ctok->state = CTokStateStart;
+                        continue;
+                }
+                break;
             case CTokStateFloat:
                 switch (*c) {
                     case '.':
@@ -791,6 +809,7 @@ found_end_of_macro:
         case CTokStateNumLitIntSuffixL:
         case CTokStateNumLitIntSuffixUL:
         case CTokStateNumLitIntSuffixLL:
+        case CTokStateGotLt:
             end_token(ctok);
             break;
         case CTokStateFloat:

src/c_tokenizer.hpp

@@ -25,6 +25,8 @@ enum CTokId {
     CTokIdAsterisk,
     CTokIdBang,
     CTokIdTilde,
+    CTokIdShl,
+    CTokIdLt,
 };
 
 enum CNumLitSuffix {
@@ -78,6 +80,7 @@ enum CTokState {
     CTokStateNumLitIntSuffixL,
     CTokStateNumLitIntSuffixLL,
     CTokStateNumLitIntSuffixUL,
+    CTokStateGotLt,
 };
 
 struct CTokenize {

src/translate_c.cpp

@@ -4922,6 +4922,8 @@ static AstNode *parse_ctok_primary_expr(Context *c, CTokenize *ctok, size_t *tok
         case CTokIdAsterisk:
         case CTokIdBang:
         case CTokIdTilde:
+        case CTokIdShl:
+        case CTokIdLt:
             // not able to make sense of this
             return nullptr;
     }
@@ -4953,6 +4955,13 @@ static AstNode *parse_ctok_suffix_op_expr(Context *c, CTokenize *ctok, size_t *t
             *tok_i += 1;
 
             node = trans_create_node_ptr_type(c, false, false, node, PtrLenC);
+        } else if (first_tok->id == CTokIdShl) {
+            *tok_i += 1;
+
+            AstNode *rhs_node = parse_ctok_expr(c, ctok, tok_i);
+            if (rhs_node == nullptr)
+                return nullptr;
+            node = trans_create_node_bin_op(c, node, BinOpTypeBitShiftLeft, rhs_node);
         } else {
             return node;
         }

test/translate_c.zig

@@ -2,6 +2,12 @@ const tests = @import("tests.zig");
 const builtin = @import("builtin");
 
 pub fn addCases(cases: *tests.TranslateCContext) void {
+    cases.add("macro with left shift",
+        \\#define REDISMODULE_READ (1<<0)
+    ,
+        \\pub const REDISMODULE_READ = 1 << 0;
+    );
+
     cases.add("casting pointers to ints and ints to pointers",
         \\void foo(void);
         \\void bar(void) {