Merge pull request #1777 from darxsys/improvDataGen

[Fuzz] Improve data generation #1723

tests/fuzz/Makefile

@@ -40,8 +40,8 @@ FUZZ_LDFLAGS := -pthread $(LDFLAGS)
 FUZZ_ARFLAGS := $(ARFLAGS)
 FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS)
 
-FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h
+FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h fuzz_data_producer.h
-FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c
+FUZZ_SRC := $(PRGDIR)/util.c zstd_helpers.c fuzz_data_producer.c
 
 ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c
 ZSTDCOMP_SRC   := $(ZSTDDIR)/compress/*.c

tests/fuzz/README.md

@@ -90,7 +90,7 @@ CC=afl-clang CXX=afl-clang++ ./fuzz.py build all --enable-asan --enable-ubsan
 
 ## Regression Testing
 
-The regression rest supports the `all` target to run all the fuzzers in one
+The regression test supports the `all` target to run all the fuzzers in one
 command.
 
 ```

tests/fuzz/fuzz_data_producer.c

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+#include "fuzz_data_producer.h"
+
+struct FUZZ_dataProducer_s{
+  const uint8_t *data;
+  size_t size;
+};
+
+FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size) {
+    FUZZ_dataProducer_t *producer = malloc(sizeof(FUZZ_dataProducer_t));
+
+    FUZZ_ASSERT(producer != NULL);
+
+    producer->data = data;
+    producer->size = size;
+    return producer;
+}
+
+void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer) { free(producer); }
+
+uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t min,
+                                  uint32_t max) {
+    FUZZ_ASSERT(min <= max);
+
+    uint32_t range = max - min;
+    uint32_t rolling = range;
+    uint32_t result = 0;
+
+    while (rolling > 0 && producer->size > 0) {
+      uint8_t next = *(producer->data + producer->size - 1);
+      producer->size -= 1;
+      result = (result << 8) | next;
+      rolling >>= 8;
+    }
+
+    if (range == 0xffffffff) {
+      return result;
+    }
+
+    return min + result % (range + 1);
+}
+
+uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) {
+  return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff);
+}
+
+size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){
+    return producer->size;
+}

tests/fuzz/fuzz_data_producer.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+
+/**
+ * Helper APIs for generating random data from input data stream.
+ The producer reads bytes from the end of the input and appends them together
+ to generate  a random number in the requested range. If it runs out of input
+ data, it will keep returning the same value (min) over and over again.
+
+ */
+
+#ifndef FUZZ_DATA_PRODUCER_H
+#define FUZZ_DATA_PRODUCER_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "fuzz_helpers.h"
+
+/* Struct used for maintaining the state of the data */
+typedef struct FUZZ_dataProducer_s FUZZ_dataProducer_t;
+
+/* Returns a data producer state struct. Use for producer initialization. */
+FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size);
+
+/* Frees the data producer */
+void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer);
+
+/* Returns value between [min, max] */
+uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t min,
+                                  uint32_t max);
+
+/* Returns a uint32 value */
+uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer);
+
+/* Returns the size of the remaining bytes of data in the producer */
+size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer);
+
+#endif // FUZZ_DATA_PRODUCER_H

tests/fuzz/simple_decompress.c

@@ -17,26 +17,32 @@
 #include <stdio.h>
 #include "fuzz_helpers.h"
 #include "zstd.h"
+#include "fuzz_data_producer.h"
 
 static ZSTD_DCtx *dctx = NULL;
 
 int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 {
+  FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size);
 
-    uint32_t seed = FUZZ_seed(&src, &size);
   int i;
   if (!dctx) {
       dctx = ZSTD_createDCtx();
       FUZZ_ASSERT(dctx);
   }
-    /* Run it 10 times over 10 output sizes. Reuse the context. */
+
-    for (i = 0; i < 10; ++i) {
+  size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size);
-        size_t const bufSize = FUZZ_rand32(&seed, 0, 2 * size);
   void *rBuf = malloc(bufSize);
   FUZZ_ASSERT(rBuf);
+
+  /* Restrict to remaining data. If we run out of data while generating params,
+   we should still continue and let decompression happen on empty data. */
+  size = FUZZ_dataProducer_remainingBytes(producer);
+
   ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size);
   free(rBuf);
-    }
+
+  FUZZ_dataProducer_free(producer);
 
 #ifndef STATEFUL_FUZZING
     ZSTD_freeDCtx(dctx); dctx = NULL;