Localize 'dictMode' from cctx to function param

lib/common/zstd_internal.h

@@ -293,9 +293,6 @@ struct ZSTD_CCtx_params_s {
     int compressionLevel;
     U32 forceWindow;           /* force back-references to respect limit of
                                 * 1<<wLog, even for dictionary */
-    /* Dictionary */
-    ZSTD_dictMode_e dictMode;   /* select restricting dictionary to "rawContent"
-                                 * or "fullDict" only */
 
     /* Multithreading: used to pass parameters to mtctx */
     U32 nbThreads;
@@ -376,6 +373,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict);
 /* INTERNAL */
 size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
                                     const void* dict, size_t dictSize,
+                                    ZSTD_dictMode_e dictMode,
                                     ZSTD_CCtx_params params,
                                     unsigned long long pledgedSrcSize);
 

lib/compress/zstd_compress.c

@@ -70,6 +70,12 @@ struct ZSTD_CDict_s {
     ZSTD_CCtx* refContext;
 };  /* typedef'd to ZSTD_CDict within "zstd.h" */
 
+typedef struct ZSTD_prefixDict_s {
+    const void* dict;
+    size_t dictSize;
+    ZSTD_dictMode_e dictMode;
+} ZSTD_prefixDict;
+
 struct ZSTD_CCtx_s {
     const BYTE* nextSrc;    /* next block here to continue on current prefix */
     const BYTE* base;       /* All regular indexes relative to this position */
@@ -116,8 +122,7 @@ struct ZSTD_CCtx_s {
     /* Dictionary */
     ZSTD_CDict* cdictLocal;
     const ZSTD_CDict* cdict;
-    const void* prefix;
+    ZSTD_prefixDict prefixDict;
-    size_t prefixSize;
 
     /* Multi-threading */
     ZSTDMT_CCtx* mtctx;
@@ -328,10 +333,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
     case ZSTD_p_dictIDFlag:
         return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
 
-    case ZSTD_p_dictMode:
-        if (cctx->cdict) return ERROR(stage_wrong);  /* must be set before loading */
-        return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value);
-
     case ZSTD_p_forceMaxWindow :  /* Force back-references to remain < windowSize,
                                    * even when referencing into Dictionary content
                                    * default : 0 when using a CDict, 1 when using a Prefix */
@@ -447,15 +448,6 @@ size_t ZSTD_CCtxParam_setParameter(
         params->fParams.noDictIDFlag = (value == 0);
         return 0;
 
-    case ZSTD_p_dictMode :
-        /* restrict dictionary mode to "rawContent" or "fullDict" only */
-        ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
-        if (value > (unsigned)ZSTD_dm_fullDict) {
-            return ERROR(parameter_outOfBound);
-        }
-        params->dictMode = (ZSTD_dictMode_e)value;
-        return 0;
-
     case ZSTD_p_forceMaxWindow :
         params->forceWindow = value > 0;
         return 0;
@@ -496,9 +488,6 @@ size_t ZSTD_CCtx_applyCCtxParams(ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params
     cctx->requestedParams.fParams = params->fParams;
     cctx->requestedParams.compressionLevel = params->compressionLevel;
 
-    /* Assume dictionary parameters are validated */
-    cctx->requestedParams.dictMode = params->dictMode;
-
     /* Set force window explicitly since it sets cctx->loadedDictEnd */
     CHECK_F( ZSTD_CCtx_setParameter(
                    cctx, ZSTD_p_forceMaxWindow, params->forceWindow) );
@@ -524,9 +513,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
     return 0;
 }
 
-size_t ZSTD_CCtx_loadDictionary_internal(
+size_t ZSTD_CCtx_loadDictionary_advanced(
-      ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
+        ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
-      ZSTD_dictLoadMethod_e dictLoadMethod)
+        ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode)
 {
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     if (cctx->staticSize) return ERROR(memory_allocation);  /* no malloc for static CCtx */
@@ -541,7 +530,7 @@ size_t ZSTD_CCtx_loadDictionary_internal(
         cctx->cdictLocal = ZSTD_createCDict_advanced(
                                 dict, dictSize,
                                 dictLoadMethod,
-                                cctx->requestedParams.dictMode,
+                                dictMode,
                                 cParams, cctx->customMem);
         cctx->cdict = cctx->cdictLocal;
         if (cctx->cdictLocal == NULL)
@@ -553,12 +542,14 @@ size_t ZSTD_CCtx_loadDictionary_internal(
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(
       ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
 {
-    return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, ZSTD_dlm_byRef);
+    return ZSTD_CCtx_loadDictionary_advanced(
+            cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto);
 }
 
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize)
 {
-    return ZSTD_CCtx_loadDictionary_internal(cctx, dict, dictSize, ZSTD_dlm_byCopy);
+    return ZSTD_CCtx_loadDictionary_advanced(
+            cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto);
 }
 
 
@@ -566,20 +557,27 @@ size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     cctx->cdict = cdict;
-    cctx->prefix = NULL;   /* exclusive */
+    memset(&cctx->prefixDict, 0, sizeof(ZSTD_prefixDict));  /* exclusive */
-    cctx->prefixSize = 0;
     return 0;
 }
 
 size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
+{
+    return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dm_rawContent);
+}
+
+size_t ZSTD_CCtx_refPrefix_advanced(
+        ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode)
 {
     if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
     cctx->cdict = NULL;   /* prefix discards any prior cdict */
-    cctx->prefix = prefix;
+    cctx->prefixDict.dict = prefix;
-    cctx->prefixSize = prefixSize;
+    cctx->prefixDict.dictSize = prefixSize;
+    cctx->prefixDict.dictMode = dictMode;
     return 0;
 }
 
+
 static void ZSTD_startNewCompression(ZSTD_CCtx* cctx)
 {
     cctx->streamStage = zcss_init;
@@ -3387,13 +3385,14 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
  * @return : 0, or an error code */
 static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
+                             ZSTD_dictMode_e dictMode,
                              const ZSTD_CDict* cdict,
                                    ZSTD_CCtx_params params, U64 pledgedSrcSize,
                                    ZSTD_buffered_policy_e zbuff)
 {
     DEBUGLOG(4, "ZSTD_compressBegin_internal");
     DEBUGLOG(4, "dict ? %s", dict ? "dict" : (cdict ? "cdict" : "none"));
-    DEBUGLOG(4, "dictMode : %u", (U32)(params.dictMode));
+    DEBUGLOG(4, "dictMode : %u", (U32)dictMode);
     /* params are supposed to be fully validated at this point */
     assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
@@ -3406,18 +3405,19 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
 
     CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
                                      ZSTDcrp_continue, zbuff) );
-    return ZSTD_compress_insertDictionary(cctx, dict, dictSize, params.dictMode);
+    return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode);
 }
 
 size_t ZSTD_compressBegin_advanced_internal(
                                     ZSTD_CCtx* cctx,
                                     const void* dict, size_t dictSize,
+                                    ZSTD_dictMode_e dictMode,
                                     ZSTD_CCtx_params params,
                                     unsigned long long pledgedSrcSize)
 {
     /* compression parameters verification and optimization */
     CHECK_F( ZSTD_checkCParams(params.cParams) );
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL,
                                        params, pledgedSrcSize,
                                        ZSTDb_not_buffered);
 }
@@ -3430,8 +3430,8 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 {
     ZSTD_CCtx_params cctxParams =
             ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
-    cctxParams.dictMode = ZSTD_dm_auto;
+    return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dm_auto,
-    return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, cctxParams,
+                                                cctxParams,
                                                 pledgedSrcSize);
 }
 
@@ -3440,8 +3440,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
     ZSTD_CCtx_params cctxParams =
             ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
-    cctxParams.dictMode = ZSTD_dm_auto;
+    return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
-    return ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
                                        cctxParams, 0, ZSTDb_not_buffered);
 }
 
@@ -3523,7 +3522,6 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
 {
     ZSTD_CCtx_params cctxParams =
             ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params);
-    cctxParams.dictMode = ZSTD_dm_auto;
     return ZSTD_compress_advanced_internal(cctx,
                                           dst, dstCapacity,
                                           src, srcSize,
@@ -3549,7 +3547,7 @@ size_t ZSTD_compress_advanced_internal(
         const void* dict,size_t dictSize,
         ZSTD_CCtx_params params)
 {
-    CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, NULL,
+    CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL,
                                          params, srcSize, ZSTDb_not_buffered) );
     return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
 }
@@ -3633,9 +3631,8 @@ static size_t ZSTD_initCDict_internal(
         ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams;
         cctxParams.fParams = fParams;
         cctxParams.cParams = cParams;
-        cctxParams.dictMode = dictMode;
         CHECK_F( ZSTD_compressBegin_internal(cdict->refContext,
-                                        cdict->dictContent, dictSize,
+                                        cdict->dictContent, dictSize, dictMode,
                                         NULL,
                                         cctxParams, ZSTD_CONTENTSIZE_UNKNOWN,
                                         ZSTDb_not_buffered) );
@@ -3759,10 +3756,9 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
     {   ZSTD_CCtx_params params = cctx->requestedParams;
         params.cParams = ZSTD_getCParamsFromCDict(cdict);
         params.fParams = fParams;
-        params.dictMode = ZSTD_dm_auto;
         DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced");
         return ZSTD_compressBegin_internal(cctx,
-                                           NULL, 0,
+                                           NULL, 0, ZSTD_dm_auto,
                                            cdict,
                                            params, pledgedSrcSize,
                                            ZSTDb_not_buffered);
@@ -3841,7 +3837,7 @@ size_t ZSTD_CStreamOutSize(void)
 
 static size_t ZSTD_resetCStream_internal(
         ZSTD_CStream* zcs,
-        const void* dict, size_t dictSize,
+        const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
         const ZSTD_CDict* cdict,
         const ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
 {
@@ -3851,7 +3847,7 @@ static size_t ZSTD_resetCStream_internal(
     assert(!((dict) && (cdict)));  /* either dict or cdict, not both */
 
     CHECK_F( ZSTD_compressBegin_internal(zcs,
-                                        dict, dictSize,
+                                        dict, dictSize, dictMode,
                                         cdict,
                                         params, pledgedSrcSize,
                                         ZSTDb_buffered) );
@@ -3871,7 +3867,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
     params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
     params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0);
     DEBUGLOG(5, "ZSTD_resetCStream");
-    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
 }
 
 /*! ZSTD_initCStream_internal() :
@@ -3895,7 +3891,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
         ZSTD_freeCDict(zcs->cdictLocal);
         zcs->cdictLocal = ZSTD_createCDict_advanced(
                                             dict, dictSize,
-                                            ZSTD_dlm_byCopy, params.dictMode,
+                                            ZSTD_dlm_byCopy, ZSTD_dm_auto,
                                             params.cParams, zcs->customMem);
         zcs->cdict = zcs->cdictLocal;
         if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@@ -3911,7 +3907,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
     params.compressionLevel = ZSTD_CLEVEL_CUSTOM;
     zcs->requestedParams = params;
 
-    return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize);
+    return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
 }
 
 /* ZSTD_initCStream_usingCDict_advanced() :
@@ -4157,23 +4153,27 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
 
     /* transparent initialization stage */
     if (cctx->streamStage == zcss_init) {
-        const void* const prefix = cctx->prefix;
+        ZSTD_prefixDict const prefixDict = cctx->prefixDict;
-        size_t const prefixSize = cctx->prefixSize;
         ZSTD_CCtx_params params = cctx->requestedParams;
         params.cParams = ZSTD_getCParamsFromCCtxParams(
                 cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
-        cctx->prefix = NULL; cctx->prefixSize = 0;   /* single usage */
+        memset(&cctx->prefixDict, 0, sizeof(ZSTD_prefixDict));  /* single usage */
-        assert(prefix==NULL || cctx->cdict==NULL);   /* only one can be set */
+        assert(prefixDict.dict==NULL || cctx->cdict==NULL);   /* only one can be set */
 
 #ifdef ZSTD_MULTITHREAD
         if (params.nbThreads > 1) {
             DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads);
-            CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
+            CHECK_F( ZSTDMT_initCStream_internal(
+                             cctx->mtctx, prefixDict.dict, prefixDict.dictSize,
+                             cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
             cctx->streamStage = zcss_load;
         } else
 #endif
         {
-            CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
+            CHECK_F( ZSTD_resetCStream_internal(
+                             cctx, prefixDict.dict, prefixDict.dictSize,
+                             prefixDict.dictMode, cctx->cdict, params,
+                             cctx->pledgedSrcSizePlusOne-1) );
     }   }
 
     /* compression stage */

lib/compress/zstdmt_compress.c

@@ -343,13 +343,13 @@ void ZSTDMT_compressChunk(void* jobDescription)
         if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0;  /* ensure no srcSize control */
         { ZSTD_CCtx_params jobParams = job->params;
           /* Force loading dictionary in "content-only" mode (no header analysis) */
-          size_t const dictModeError =
-              ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_dictMode, (U32)ZSTD_dm_rawContent);
           size_t const forceWindowError =
               ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk);
-          size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, jobParams, job->fullFrameSize);
+          size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize);
-            if (ZSTD_isError(initError) || ZSTD_isError(dictModeError) ||
+            if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) {
-                ZSTD_isError(forceWindowError)) { job->cSize = initError; goto _endJob; }
+                job->cSize = initError;
+                goto _endJob;
+            }
     }   }
     if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
         size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0);

lib/zstd.h

@@ -963,10 +963,6 @@ typedef enum {
     ZSTD_p_checksumFlag,     /* A 32-bits checksum of content is written at end of frame (default:0) */
     ZSTD_p_dictIDFlag,       /* When applicable, dictID of dictionary is provided in frame header (default:1) */
 
-    /* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
-    ZSTD_p_dictMode=300,     /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
-                              * default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
-
     /* multi-threading parameters */
     ZSTD_p_nbThreads=400,    /* Select how many threads a compression job can spawn (default:1)
                               * More threads improve speed, but also increase memory usage.
@@ -1011,19 +1007,19 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
  *            meaning "return to no-dictionary mode".
  *  Note 1 : `dict` content will be copied internally. Use
  *            ZSTD_CCtx_loadDictionary_byReference() to reference dictionary
- *            content instead.
+ *            content instead. The dictionary buffer must then outlive its
+ *            users.
  *  Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
  *           For this reason, compression parameters cannot be changed anymore after loading a dictionary.
  *           It's also a CPU-heavy operation, with non-negligible impact on latency.
  *  Note 3 : Dictionary will be used for all future compression jobs.
- *           To return to "no-dictionary" situation, load a NULL dictionary */
+ *           To return to "no-dictionary" situation, load a NULL dictionary
-ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+ *  Note 5 : Use ZSTD_CCtx_loadDictionary_advanced() to select how dictionary
-
+ *           content will be interpreted.
-/*! ZSTD_CCtx_loadDictionary_byReference() :
- *  Same as ZSTD_CCtx_loadDictionary() except dictionary content will be
- *  referenced, instead of copied. The dictionary buffer must outlive its users.
  */
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
+ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode);
 
 
 /*! ZSTD_CCtx_refCDict() :
@@ -1050,8 +1046,11 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
  *  Note 1 : Prefix buffer is referenced. It must outlive compression job.
  *  Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
  *           It's a CPU-heavy operation, with non-negligible impact on latency.
- *  Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */
+ *  Note 3 : By default, the prefix is treated as raw content
+ *           (ZSTD_dm_rawContent). Use ZSTD_CCtx_refPrefix_advanced() to alter
+ *           dictMode. */
 ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);
+ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode);