added parameter ZSTD_p_refDictContent

doc/zstd_manual.html

@@ -816,13 +816,11 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
     ZSTD_p_checksumFlag,     </b>/* A 32-bits checksum of content is written at end of frame (default:0) */<b>
     ZSTD_p_dictIDFlag,       </b>/* When applicable, dictID of dictionary is provided in frame header (default:1) */<b>
 
-    </b>/* dictionary parameters */<b>
+    </b>/* dictionary parameters (must be set before loading) */<b>
-    ZSTD_p_refDictContent=300, </b>/* Content of dictionary content will be referenced, instead of copied (default:0).<b>
+    ZSTD_p_dictMode=300,     </b>/* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.<b>
-                              * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its users */
-                             </b>/* Not ready yet ! <=================================== */<b>
-    ZSTD_p_dictMode,         </b>/* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.<b>
                               * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
+    ZSTD_p_refDictContent,   </b>/* Dictionary content will be referenced, instead of copied (default:0==byCopy).<b>
+                              * It requires that dictionary buffer outlives its users */
 
     </b>/* multi-threading parameters */<b>
     ZSTD_p_nbThreads=400,    </b>/* Select how many threads a compression job can spawn (default:1)<b>
@@ -837,9 +835,9 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
 
     </b>/* advanced parameters - may not remain available after API update */<b>
-    ZSTD_p_forceMaxWindow=1100, </b>/* Force back-references to remain < windowSize,<b>
+    ZSTD_p_forceMaxWindow=1100, </b>/* Force back-reference distances to remain < windowSize,<b>
-                              * even when referencing into Dictionary content.
+                              * even when referencing into Dictionary content (default:0) */
-                              * default : 0 when using a CDict, 1 when using a Prefix */
+
 } ZSTD_cParameter;
 </b></pre><BR>
 <pre><b>size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value);
@@ -865,8 +863,8 @@ void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
  @result : 0, or an error code (which can be tested with ZSTD_isError()).
   Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
             meaning "return to no-dictionary mode".
-  Note 1 : Dictionary content will be copied internally,
+  Note 1 : `dict` content will be copied internally,
-           except if ZSTD_p_refDictContent is set.
+           except if ZSTD_p_refDictContent is set before loading.
   Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
            For this reason, compression parameters cannot be changed anymore after loading a dictionary.
            It's also a CPU-heavy operation, with non-negligible impact on latency.

lib/compress/zstd_compress.c

@@ -89,6 +89,7 @@ struct ZSTD_CCtx_s {
     U32   loadedDictEnd;    /* index of end of dictionary */
     U32   forceWindow;      /* force back-references to respect limit of 1<<wLog, even for dictionary */
     ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
+    U32   dictContentByRef;
     ZSTD_compressionStage_e stage;
     U32   rep[ZSTD_REP_NUM];
     U32   repToConfirm[ZSTD_REP_NUM];
@@ -322,11 +323,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
         return 0;
 
-#if 0
-    case ZSTD_p_windowSize :   /* to be done later */
-        return ERROR(compressionParameter_unsupported);
-#endif
-
     case ZSTD_p_contentSizeFlag :
         DEBUGLOG(5, "set content size flag = %u", (value>0));
         /* Content size written in frame header _when known_ (default:1) */
@@ -343,10 +339,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         cctx->requestedParams.fParams.noDictIDFlag = (value==0);
         return 0;
 
-    case ZSTD_p_refDictContent :   /* to be done later */
+    /* Dictionary parameters */
-        return ERROR(compressionParameter_unsupported);
-
     case ZSTD_p_dictMode :
+        if (cctx->cdict) return ERROR(stage_wrong);  /* must be set before loading */
         /* restrict dictionary mode, to "rawContent" or "fullDict" only */
         ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
         if (value > (unsigned)ZSTD_dm_fullDict)
@@ -354,6 +349,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
         cctx->dictMode = (ZSTD_dictMode_e)value;
         return 0;
 
+    case ZSTD_p_refDictContent :
+        if (cctx->cdict) return ERROR(stage_wrong);  /* must be set before loading */
+        /* dictionary content will be referenced, instead of copied */
+        cctx->dictContentByRef = value>0;
+        return 0;
+
     case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
                                    * even when referencing into Dictionary content
                                    * default : 0 when using a CDict, 1 when using a Prefix */
@@ -417,7 +418,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
                 ZSTD_getCParams(cctx->compressionLevel, 0, dictSize);
         cctx->cdictLocal = ZSTD_createCDict_advanced(
                                 dict, dictSize,
-                                0 /* byReference */, cctx->dictMode,
+                                cctx->dictContentByRef, cctx->dictMode,
                                 cParams, cctx->customMem);
         cctx->cdict = cctx->cdictLocal;
         if (cctx->cdictLocal == NULL)

lib/zstd.h

@@ -956,13 +956,11 @@ typedef enum {
     ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 
-    /* dictionary parameters */
+    /* dictionary parameters (must be set before loading) */
-    ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
+    ZSTD_p_dictMode=300,     /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
-                              * This avoids duplicating dictionary content.
-                              * But it also requires that dictionary buffer outlives its users */
-                             /* Not ready yet ! <=================================== */
-    ZSTD_p_dictMode,         /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.
                               * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
+    ZSTD_p_refDictContent,   /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
+                              * It requires that dictionary buffer outlives its users */
 
     /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
@@ -977,9 +975,9 @@ typedef enum {
                               * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
 
     /* advanced parameters - may not remain available after API update */
-    ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
+    ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
-                              * even when referencing into Dictionary content.
+                              * even when referencing into Dictionary content (default:0) */
-                              * default : 0 when using a CDict, 1 when using a Prefix */
+
 } ZSTD_cParameter;
 
 
@@ -1006,8 +1004,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
  *            meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary content will be copied internally,
+ *  Note 1 : `dict` content will be copied internally,
- *           except if ZSTD_p_refDictContent is set.
+ *           except if ZSTD_p_refDictContent is set before loading.
  *  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
  *           For this reason, compression parameters cannot be changed anymore after loading a dictionary.
  *           It's also a CPU-heavy operation, with non-negligible impact on latency.

tests/zstreamtest.c

@@ -1294,7 +1294,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
 
                 /* mess with compression parameters */
-                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* always cancel previous dict, to make user it's possible to pass compression parameters */
+                CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) );   /* cancel previous dict, to allow new compression parameters */
                 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
                 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
@@ -1311,6 +1311,7 @@ static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double
                 if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) );
 
                 /* unconditionally set, to be sync with decoder */
+                if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) );
                 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
 
                 if (dict && dictSize) {