Rewrite reply parsing to use a read buffer

hiredis.c

@@ -37,6 +37,17 @@
 #include "anet.h"
 #include "sds.h"
 
+typedef struct redisReader {
+    char *buf; /* read buffer */
+    int len; /* buffer length */
+    int avail; /* available bytes for consumption */
+    int pos; /* buffer cursor */
+
+    redisReply **rlist; /* list of items to process */
+    int rlen; /* list length */
+    int rpos; /* list cursor */
+} redisReader;
+
 static redisReply *redisReadReply(int fd);
 static redisReply *createReplyObject(int type, sds reply);
 
@@ -93,109 +104,220 @@ static redisReply *redisIOError(void) {
     return createReplyObject(REDIS_REPLY_ERROR,sdsnew("I/O error"));
 }
 
-/* In a real high performance C client this should be bufferized */
+static char *readBytes(redisReader *r, int bytes) {
-static sds redisReadLine(int fd) {
+    char *p;
-    sds line = sdsempty();
+    if (r->len-r->pos >= bytes) {
+        p = r->buf+r->pos;
+        r->pos += bytes;
+        return p;
+    }
+    return NULL;
+}
 
-    while(1) {
+static char *readLine(redisReader *r, int *_len) {
-        char c;
+    char *p, *s = strstr(r->buf+r->pos,"\r\n");
-        ssize_t ret;
+    int len;
+    if (s != NULL) {
+        p = r->buf+r->pos;
+        len = s-(r->buf+r->pos);
+        r->pos += len+2; /* skip \r\n */
+        if (_len) *_len = len;
+        return p;
+    }
+    return NULL;
+}
 
-        ret = read(fd,&c,1);
+static int processLineItem(redisReader *r) {
-        if (ret == -1) {
+    redisReply *cur = r->rlist[r->rpos];
-            sdsfree(line);
+    char *p;
-            return NULL;
+    int len;
-        } else if ((ret == 0) || (c == '\n')) {
+
-            break;
+    if ((p = readLine(r,&len)) != NULL) {
+        if (cur->type == REDIS_REPLY_INTEGER) {
+            cur->integer = strtoll(p,NULL,10);
         } else {
-            line = sdscatlen(line,&c,1);
+            cur->reply = sdsnewlen(p,len);
+        }
+
+        /* for API compat, set STATUS to STRING */
+        if (cur->type == REDIS_REPLY_STATUS)
+            cur->type = REDIS_REPLY_STRING;
+
+        r->rpos++;
+        return 0;
+    }
+    return -1;
+}
+
+static int processBulkItem(redisReader *r) {
+    redisReply *cur = r->rlist[r->rpos];
+    char *p;
+    int len;
+
+    if (cur->reply == NULL) {
+        if ((p = readLine(r,NULL)) != NULL) {
+            len = atoi(p);
+            if (len == -1) {
+                /* nil means this item is done */
+                cur->type = REDIS_REPLY_NIL;
+                cur->reply = sdsempty();
+                r->rpos++;
+                return 0;
+            } else {
+                cur->reply = sdsnewlen(NULL,len);
+            }
+        } else {
+            return -1;
         }
     }
-    return sdstrim(line,"\r\n");
-}
 
-static redisReply *redisReadSingleLineReply(int fd, int type) {
+    len = sdslen(cur->reply);
-    sds buf = redisReadLine(fd);
+    /* add two bytes for crlf */
-    
+    if ((p = readBytes(r,len+2)) != NULL) {
-    if (buf == NULL) return redisIOError();
+        memcpy(cur->reply,p,len);
-    return createReplyObject(type,buf);
+        r->rpos++;
-}
+        return 0;
-
-static redisReply *redisReadIntegerReply(int fd) {
-    sds buf = redisReadLine(fd);
-    redisReply *r = malloc(sizeof(*r));
-
-    if (r == NULL) redisOOM();
-    if (buf == NULL) {
-        free(r);
-        return redisIOError();
     }
-    r->type = REDIS_REPLY_INTEGER;
+    return -1;
-    r->integer = strtoll(buf,NULL,10);
-    sdsfree(buf);
-    return r;
 }
 
-static redisReply *redisReadBulkReply(int fd) {
+static int processMultiBulkItem(redisReader *r) {
-    sds replylen = redisReadLine(fd);
+    redisReply *cur = r->rlist[r->rpos];
-    sds buf;
+    char *p;
-    char crlf[2];
+    int elements, j;
-    int bulklen;
 
-    if (replylen == NULL) return redisIOError();
+    if ((p = readLine(r,NULL)) != NULL) {
-    bulklen = atoi(replylen);
+        elements = atoi(p);
-    sdsfree(replylen);
+        if (elements == -1) {
-    if (bulklen == -1)
+            /* empty */
-        return createReplyObject(REDIS_REPLY_NIL,sdsempty());
+            cur->type = REDIS_REPLY_NIL;
+            cur->reply = sdsempty();
+            r->rpos++;
+            return 0;
+        }
+    } else {
+        return -1;
+    }
 
-    buf = sdsnewlen(NULL,bulklen);
+    cur->elements = elements;
-    anetRead(fd,buf,bulklen);
+    r->rlen += elements;
-    anetRead(fd,crlf,2);
+    r->rpos++;
-    return createReplyObject(REDIS_REPLY_STRING,buf);
+
+    /* create placeholder items */
+    if ((cur->element = malloc(sizeof(redisReply*)*elements)) == NULL)
+        redisOOM();
+    if ((r->rlist = realloc(r->rlist,sizeof(redisReply*)*r->rlen)) == NULL)
+        redisOOM();
+
+    /* move existing items backwards */
+    memmove(&(r->rlist[r->rpos+elements]),
+            &(r->rlist[r->rpos]),
+            (r->rlen-(r->rpos+elements))*sizeof(redisReply*));
+
+    /* populate item list */
+    for (j = 0; j < elements; j++) {
+        cur->element[j] = createReplyObject(-1,NULL);
+        r->rlist[r->rpos+j] = cur->element[j];
+    }
+    return 0;
 }
 
-static redisReply *redisReadMultiBulkReply(int fd) {
+static int processItem(redisReader *r) {
-    sds replylen = redisReadLine(fd);
+    redisReply *cur = r->rlist[r->rpos];
-    long elements, j;
+    char *p;
-    redisReply *r;
 
-    if (replylen == NULL) return redisIOError();
+    /* check if we need to read type */
-    elements = strtol(replylen,NULL,10);
+    if (cur->type < 0) {
-    sdsfree(replylen);
+        if ((p = readBytes(r,1)) != NULL) {
+            switch (p[0]) {
+            case '-':
+                cur->type = REDIS_REPLY_ERROR;
+                break;
+            case '+':
+                cur->type = REDIS_REPLY_STATUS;
+                break;
+            case ':':
+                cur->type = REDIS_REPLY_INTEGER;
+                break;
+            case '$':
+                cur->type = REDIS_REPLY_STRING;
+                break;
+            case '*':
+                cur->type = REDIS_REPLY_ARRAY;
+                break;
+            default:
+                printf("protocol error, got '%c' as reply type byte\n", p[0]);
+                exit(1);
+            }
+        } else {
+            /* could not consume 1 byte */
+            return -1;
+        }
+    }
 
-    if (elements == -1)
+    /* process typed item */
-        return createReplyObject(REDIS_REPLY_NIL,sdsempty());
+    switch(cur->type) {
-
+    case REDIS_REPLY_ERROR:
-    if ((r = malloc(sizeof(*r))) == NULL) redisOOM();
+    case REDIS_REPLY_STATUS:
-    r->type = REDIS_REPLY_ARRAY;
+    case REDIS_REPLY_INTEGER:
-    r->elements = elements;
+        return processLineItem(r);
-    if ((r->element = malloc(sizeof(*r)*elements)) == NULL) redisOOM();
+    case REDIS_REPLY_STRING:
-    for (j = 0; j < elements; j++)
+        return processBulkItem(r);
-        r->element[j] = redisReadReply(fd);
+    case REDIS_REPLY_ARRAY:
-    return r;
+        return processMultiBulkItem(r);
-}
-
-static redisReply *redisReadReply(int fd) {
-    char type;
-
-    if (anetRead(fd,&type,1) <= 0) return redisIOError();
-    switch(type) {
-    case '-':
-        return redisReadSingleLineReply(fd,REDIS_REPLY_ERROR);
-    case '+':
-        return redisReadSingleLineReply(fd,REDIS_REPLY_STRING);
-    case ':':
-        return redisReadIntegerReply(fd);
-    case '$':
-        return redisReadBulkReply(fd);
-    case '*':
-        return redisReadMultiBulkReply(fd);
     default:
-        printf("protocol error, got '%c' as reply type byte\n", type);
+        printf("unknown item type: %d\n", cur->type);
         exit(1);
     }
 }
 
+#define READ_BUFFER_SIZE 2048
+static redisReply *redisReadReply(int fd) {
+    redisReader r;
+    int bytes;
+
+    /* setup read buffer */
+    r.buf = malloc(READ_BUFFER_SIZE+1);
+    r.len = READ_BUFFER_SIZE;
+    r.avail = 0;
+    r.pos = 0;
+
+    /* setup list of items to process */
+    r.rlist = malloc(sizeof(redisReply*));
+    r.rlist[0] = createReplyObject(-1,NULL);
+    r.rlen = 1;
+    r.rpos = 0;
+
+    while (r.rpos < r.rlen) {
+        /* discard the buffer upto pos */
+        if (r.pos > 0) {
+            memmove(r.buf,r.buf+r.pos,r.len-r.pos);
+            r.avail -= r.pos;
+            r.pos = 0;
+        }
+
+        /* make sure there is room for at least BUFFER_SIZE */
+        if (r.len-r.avail < READ_BUFFER_SIZE) {
+            r.buf = realloc(r.buf,r.avail+READ_BUFFER_SIZE+1);
+            r.len = r.avail+READ_BUFFER_SIZE;
+        }
+
+        /* read from socket into buffer */
+        if ((bytes = read(fd,r.buf+r.avail,READ_BUFFER_SIZE)) <= 0)
+            return redisIOError();
+        r.avail += bytes;
+        r.buf[r.avail] = '\0';
+
+        /* process items in reply */
+        while (r.rpos < r.rlen)
+            if (processItem(&r) < 0)
+                break;
+    }
+    free(r.buf);
+    free(r.rlist);
+    return r.rlist[0];
+}
+
 /* Helper function for redisCommand(). It's used to append the next argument
  * to the argument vector. */
 static void addArgument(sds a, char ***argv, int *argc) {

hiredis.h

@@ -35,6 +35,7 @@
 #define REDIS_REPLY_ARRAY 2
 #define REDIS_REPLY_INTEGER 3
 #define REDIS_REPLY_NIL 4
+#define REDIS_REPLY_STATUS 5
 
 #include "sds.h"
 

test.c

@@ -99,6 +99,23 @@ int main(void) {
               !memcmp(reply->element[1]->reply,"foo",3))
     freeReplyObject(reply);
 
+    /* test 9 (m/e with multi bulk reply *before* other reply).
+     * specifically test ordering of reply items to parse. */
+    printf("#10 can handle nested multi bulk replies: ");
+    freeReplyObject(redisCommand(fd,"MULTI"));
+    freeReplyObject(redisCommand(fd,"LRANGE mylist 0 -1"));
+    freeReplyObject(redisCommand(fd,"PING"));
+    reply = (redisCommand(fd,"EXEC"));
+    test_cond(reply->type == REDIS_REPLY_ARRAY &&
+              reply->elements == 2 &&
+              reply->element[0]->type == REDIS_REPLY_ARRAY &&
+              reply->element[0]->elements == 2 &&
+              !memcmp(reply->element[0]->element[0]->reply,"bar",3) &&
+              !memcmp(reply->element[0]->element[1]->reply,"foo",3) &&
+              reply->element[1]->type == REDIS_REPLY_STRING &&
+              strcasecmp(reply->element[1]->reply,"pong") == 0);
+    freeReplyObject(reply);
+
     if (fails == 0) {
         printf("ALL TESTS PASSED\n");
     } else {