Merge pull request #2219 from xxie24/output-dir-mirror
Add output-dir-mirror option
This commit is contained in:
commit
a3296dae38
@ -624,7 +624,6 @@ FIO_openDstFile(FIO_prefs_t* const prefs,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/*! FIO_createDictBuffer() :
|
/*! FIO_createDictBuffer() :
|
||||||
* creates a buffer, pointed by `*bufferPtr`,
|
* creates a buffer, pointed by `*bufferPtr`,
|
||||||
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
|
* loads `filename` content into it, up to DICTSIZE_MAX bytes.
|
||||||
@ -669,15 +668,9 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p
|
|||||||
* Checks for and warns if there are any files that would have the same output path
|
* Checks for and warns if there are any files that would have the same output path
|
||||||
*/
|
*/
|
||||||
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
|
int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
|
||||||
const char **filenameTableSorted, *c, *prevElem, *filename;
|
const char **filenameTableSorted, *prevElem, *filename;
|
||||||
unsigned u;
|
unsigned u;
|
||||||
|
|
||||||
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
|
|
||||||
c = "\\";
|
|
||||||
#else
|
|
||||||
c = "/";
|
|
||||||
#endif
|
|
||||||
|
|
||||||
filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
|
filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles);
|
||||||
if (!filenameTableSorted) {
|
if (!filenameTableSorted) {
|
||||||
DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
|
DISPLAY("Unable to malloc new str array, not checking for name collisions\n");
|
||||||
@ -685,7 +678,7 @@ int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (u = 0; u < nbFiles; ++u) {
|
for (u = 0; u < nbFiles; ++u) {
|
||||||
filename = strrchr(filenameTable[u], c[0]);
|
filename = strrchr(filenameTable[u], PATH_SEP);
|
||||||
if (filename == NULL) {
|
if (filename == NULL) {
|
||||||
filenameTableSorted[u] = filenameTable[u];
|
filenameTableSorted[u] = filenameTable[u];
|
||||||
} else {
|
} else {
|
||||||
@ -1599,6 +1592,7 @@ int FIO_compressFilename(FIO_prefs_t* const prefs, const char* dstFileName,
|
|||||||
cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
|
cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
|
||||||
int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||||
|
|
||||||
|
#define DISPLAY_LEVEL_DEFAULT 2
|
||||||
|
|
||||||
FIO_freeCResources(ress);
|
FIO_freeCResources(ress);
|
||||||
return result;
|
return result;
|
||||||
@ -1663,6 +1657,7 @@ static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsig
|
|||||||
*/
|
*/
|
||||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||||
const char** inFileNamesTable, unsigned nbFiles,
|
const char** inFileNamesTable, unsigned nbFiles,
|
||||||
|
const char* outMirroredRootDirName,
|
||||||
const char* outDirName,
|
const char* outDirName,
|
||||||
const char* outFileName, const char* suffix,
|
const char* outFileName, const char* suffix,
|
||||||
const char* dictFileName, int compressionLevel,
|
const char* dictFileName, int compressionLevel,
|
||||||
@ -1689,12 +1684,30 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
|||||||
ress.dstFile = NULL;
|
ress.dstFile = NULL;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
unsigned u;
|
unsigned int u=0;
|
||||||
|
if (outMirroredRootDirName)
|
||||||
|
UTIL_mirrorSourceFilesDirectories(inFileNamesTable, nbFiles, outMirroredRootDirName);
|
||||||
|
|
||||||
for (u=0; u<nbFiles; u++) {
|
for (u=0; u<nbFiles; u++) {
|
||||||
const char* const srcFileName = inFileNamesTable[u];
|
const char* const srcFileName = inFileNamesTable[u];
|
||||||
const char* const dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
|
const char* dstFileName = NULL;
|
||||||
|
if (outMirroredRootDirName) {
|
||||||
|
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
|
||||||
|
if (validMirroredDirName) {
|
||||||
|
dstFileName = FIO_determineCompressedName(srcFileName, validMirroredDirName, suffix);
|
||||||
|
free(validMirroredDirName);
|
||||||
|
} else {
|
||||||
|
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n", srcFileName, outMirroredRootDirName);
|
||||||
|
error=1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
|
||||||
|
}
|
||||||
|
|
||||||
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (outDirName)
|
if (outDirName)
|
||||||
FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles);
|
FIO_checkFilenameCollisions(inFileNamesTable ,nbFiles);
|
||||||
}
|
}
|
||||||
@ -2536,10 +2549,10 @@ FIO_determineDstName(const char* srcFileName, const char* outDirName)
|
|||||||
/* note : dstFileNameBuffer memory is not going to be free */
|
/* note : dstFileNameBuffer memory is not going to be free */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
int
|
int
|
||||||
FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||||
const char** srcNamesTable, unsigned nbFiles,
|
const char** srcNamesTable, unsigned nbFiles,
|
||||||
|
const char* outMirroredRootDirName,
|
||||||
const char* outDirName, const char* outFileName,
|
const char* outDirName, const char* outFileName,
|
||||||
const char* dictFileName)
|
const char* dictFileName)
|
||||||
{
|
{
|
||||||
@ -2558,12 +2571,25 @@ FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
|||||||
EXM_THROW(72, "Write error : %s : cannot properly close output file",
|
EXM_THROW(72, "Write error : %s : cannot properly close output file",
|
||||||
strerror(errno));
|
strerror(errno));
|
||||||
} else {
|
} else {
|
||||||
unsigned u;
|
unsigned int u = 0;
|
||||||
|
if (outMirroredRootDirName)
|
||||||
|
UTIL_mirrorSourceFilesDirectories(srcNamesTable, nbFiles, outMirroredRootDirName);
|
||||||
|
|
||||||
for (u=0; u<nbFiles; u++) { /* create dstFileName */
|
for (u=0; u<nbFiles; u++) { /* create dstFileName */
|
||||||
const char* const srcFileName = srcNamesTable[u];
|
const char* const srcFileName = srcNamesTable[u];
|
||||||
const char* const dstFileName = FIO_determineDstName(srcFileName, outDirName);
|
const char* dstFileName = NULL;
|
||||||
|
if (outMirroredRootDirName) {
|
||||||
|
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
|
||||||
|
if (validMirroredDirName) {
|
||||||
|
dstFileName = FIO_determineDstName(srcFileName, validMirroredDirName);
|
||||||
|
free(validMirroredDirName);
|
||||||
|
} else {
|
||||||
|
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n", srcFileName, outMirroredRootDirName);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
dstFileName = FIO_determineDstName(srcFileName, outDirName);
|
||||||
|
}
|
||||||
if (dstFileName == NULL) { error=1; continue; }
|
if (dstFileName == NULL) { error=1; continue; }
|
||||||
|
|
||||||
error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
|
error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
|
||||||
}
|
}
|
||||||
if (outDirName)
|
if (outDirName)
|
||||||
|
@ -122,6 +122,7 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
|
|||||||
* @return : nb of missing files */
|
* @return : nb of missing files */
|
||||||
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||||
const char** inFileNamesTable, unsigned nbFiles,
|
const char** inFileNamesTable, unsigned nbFiles,
|
||||||
|
const char* outMirroredDirName,
|
||||||
const char* outDirName,
|
const char* outDirName,
|
||||||
const char* outFileName, const char* suffix,
|
const char* outFileName, const char* suffix,
|
||||||
const char* dictFileName, int compressionLevel,
|
const char* dictFileName, int compressionLevel,
|
||||||
@ -131,6 +132,7 @@ int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
|
|||||||
* @return : nb of missing or skipped files */
|
* @return : nb of missing or skipped files */
|
||||||
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
|
||||||
const char** srcNamesTable, unsigned nbFiles,
|
const char** srcNamesTable, unsigned nbFiles,
|
||||||
|
const char* outMirroredDirName,
|
||||||
const char* outDirName,
|
const char* outDirName,
|
||||||
const char* outFileName,
|
const char* outFileName,
|
||||||
const char* dictFileName);
|
const char* dictFileName);
|
||||||
|
303
programs/util.c
303
programs/util.c
@ -45,7 +45,6 @@ extern "C" {
|
|||||||
# include <string.h> /* strerror, memcpy */
|
# include <string.h> /* strerror, memcpy */
|
||||||
#endif /* #ifdef _WIN32 */
|
#endif /* #ifdef _WIN32 */
|
||||||
|
|
||||||
|
|
||||||
/*-****************************************
|
/*-****************************************
|
||||||
* Internal Macros
|
* Internal Macros
|
||||||
******************************************/
|
******************************************/
|
||||||
@ -130,6 +129,18 @@ int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int UTIL_getDirectoryStat(const char* infilename, stat_t *statbuf)
|
||||||
|
{
|
||||||
|
#if defined(_MSC_VER)
|
||||||
|
int const r = _stat64(infilename, statbuf);
|
||||||
|
if (!r && (statbuf->st_mode & _S_IFDIR)) return 1;
|
||||||
|
#else
|
||||||
|
int const r = stat(infilename, statbuf);
|
||||||
|
if (!r && S_ISDIR(statbuf->st_mode)) return 1;
|
||||||
|
#endif
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* like chmod, but avoid changing permission of /dev/null */
|
/* like chmod, but avoid changing permission of /dev/null */
|
||||||
int UTIL_chmod(char const* filename, mode_t permissions)
|
int UTIL_chmod(char const* filename, mode_t permissions)
|
||||||
{
|
{
|
||||||
@ -178,14 +189,7 @@ int UTIL_setFileStat(const char *filename, stat_t *statbuf)
|
|||||||
int UTIL_isDirectory(const char* infilename)
|
int UTIL_isDirectory(const char* infilename)
|
||||||
{
|
{
|
||||||
stat_t statbuf;
|
stat_t statbuf;
|
||||||
#if defined(_MSC_VER)
|
return UTIL_getDirectoryStat(infilename, &statbuf);
|
||||||
int const r = _stat64(infilename, &statbuf);
|
|
||||||
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
|
|
||||||
#else
|
|
||||||
int const r = stat(infilename, &statbuf);
|
|
||||||
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
|
|
||||||
#endif
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int UTIL_compareStr(const void *p1, const void *p2) {
|
int UTIL_compareStr(const void *p1, const void *p2) {
|
||||||
@ -633,6 +637,287 @@ const char* UTIL_getFileExtension(const char* infilename)
|
|||||||
return extension;
|
return extension;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int pathnameHas2Dots(const char *pathname)
|
||||||
|
{
|
||||||
|
return NULL != strstr(pathname, "..");
|
||||||
|
}
|
||||||
|
|
||||||
|
static int isFileNameValidForMirroredOutput(const char *filename)
|
||||||
|
{
|
||||||
|
return !pathnameHas2Dots(filename);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
#define DIR_DEFAULT_MODE 0755
|
||||||
|
static mode_t getDirMode(const char *dirName)
|
||||||
|
{
|
||||||
|
stat_t st;
|
||||||
|
int ret = UTIL_getDirectoryStat(dirName, &st);
|
||||||
|
if (!ret) {
|
||||||
|
UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno));
|
||||||
|
return DIR_DEFAULT_MODE;
|
||||||
|
}
|
||||||
|
return st.st_mode;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int makeDir(const char *dir, mode_t mode)
|
||||||
|
{
|
||||||
|
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__)
|
||||||
|
int ret = _mkdir(dir);
|
||||||
|
(void) mode;
|
||||||
|
#else
|
||||||
|
int ret = mkdir(dir, mode);
|
||||||
|
#endif
|
||||||
|
if (ret != 0) {
|
||||||
|
if (errno == EEXIST)
|
||||||
|
return 0;
|
||||||
|
UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno));
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* this function requires a mutable input string */
|
||||||
|
static void convertPathnameToDirName(char *pathname)
|
||||||
|
{
|
||||||
|
size_t len = 0;
|
||||||
|
char* pos = NULL;
|
||||||
|
/* get dir name from pathname similar to 'dirname()' */
|
||||||
|
assert(pathname != NULL);
|
||||||
|
|
||||||
|
/* remove trailing '/' chars */
|
||||||
|
len = strlen(pathname);
|
||||||
|
assert(len > 0);
|
||||||
|
while (pathname[len] == PATH_SEP) {
|
||||||
|
pathname[len] = '\0';
|
||||||
|
len--;
|
||||||
|
}
|
||||||
|
if (len == 0) return;
|
||||||
|
|
||||||
|
/* if input is a single file, return '.' instead. i.e.
|
||||||
|
* "xyz/abc/file.txt" => "xyz/abc"
|
||||||
|
"./file.txt" => "."
|
||||||
|
"file.txt" => "."
|
||||||
|
*/
|
||||||
|
pos = strrchr(pathname, PATH_SEP);
|
||||||
|
if (pos == NULL) {
|
||||||
|
pathname[0] = '.';
|
||||||
|
pathname[1] = '\0';
|
||||||
|
} else {
|
||||||
|
*pos = '\0';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* pathname must be valid */
|
||||||
|
static const char* trimLeadingRootChar(const char *pathname)
|
||||||
|
{
|
||||||
|
assert(pathname != NULL);
|
||||||
|
if (pathname[0] == PATH_SEP)
|
||||||
|
return pathname + 1;
|
||||||
|
return pathname;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* pathname must be valid */
|
||||||
|
static const char* trimLeadingCurrentDirConst(const char *pathname)
|
||||||
|
{
|
||||||
|
assert(pathname != NULL);
|
||||||
|
if ((pathname[0] == '.') && (pathname[1] == PATH_SEP))
|
||||||
|
return pathname + 2;
|
||||||
|
return pathname;
|
||||||
|
}
|
||||||
|
|
||||||
|
static char*
|
||||||
|
trimLeadingCurrentDir(char *pathname)
|
||||||
|
{
|
||||||
|
/* 'union charunion' can do const-cast without compiler warning */
|
||||||
|
union charunion {
|
||||||
|
char *chr;
|
||||||
|
const char* cchr;
|
||||||
|
} ptr;
|
||||||
|
ptr.cchr = trimLeadingCurrentDirConst(pathname);
|
||||||
|
return ptr.chr;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* remove leading './' or '/' chars here */
|
||||||
|
static const char * trimPath(const char *pathname)
|
||||||
|
{
|
||||||
|
return trimLeadingRootChar(
|
||||||
|
trimLeadingCurrentDirConst(pathname));
|
||||||
|
}
|
||||||
|
|
||||||
|
static char* mallocAndJoin2Dir(const char *dir1, const char *dir2)
|
||||||
|
{
|
||||||
|
const size_t dir1Size = strlen(dir1);
|
||||||
|
const size_t dir2Size = strlen(dir2);
|
||||||
|
char *outDirBuffer, *buffer, trailingChar;
|
||||||
|
|
||||||
|
assert(dir1 != NULL && dir2 != NULL);
|
||||||
|
outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2);
|
||||||
|
CONTROL(outDirBuffer != NULL);
|
||||||
|
|
||||||
|
strncpy(outDirBuffer, dir1, dir1Size);
|
||||||
|
outDirBuffer[dir1Size] = '\0';
|
||||||
|
|
||||||
|
if (dir2[0] == '.')
|
||||||
|
return outDirBuffer;
|
||||||
|
|
||||||
|
buffer = outDirBuffer + dir1Size;
|
||||||
|
trailingChar = *(buffer - 1);
|
||||||
|
if (trailingChar != PATH_SEP) {
|
||||||
|
*buffer = PATH_SEP;
|
||||||
|
buffer++;
|
||||||
|
}
|
||||||
|
strncpy(buffer, dir2, dir2Size);
|
||||||
|
buffer[dir2Size] = '\0';
|
||||||
|
|
||||||
|
return outDirBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* this function will return NULL if input srcFileName is not valid name for mirrored output path */
|
||||||
|
char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName)
|
||||||
|
{
|
||||||
|
char* pathname = NULL;
|
||||||
|
if (!isFileNameValidForMirroredOutput(srcFileName))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName));
|
||||||
|
|
||||||
|
convertPathnameToDirName(pathname);
|
||||||
|
return pathname;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
mirrorSrcDir(char* srcDirName, const char* outDirName)
|
||||||
|
{
|
||||||
|
mode_t srcMode;
|
||||||
|
int status = 0;
|
||||||
|
char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName));
|
||||||
|
if (!newDir)
|
||||||
|
return -ENOMEM;
|
||||||
|
|
||||||
|
srcMode = getDirMode(srcDirName);
|
||||||
|
status = makeDir(newDir, srcMode);
|
||||||
|
free(newDir);
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
mirrorSrcDirRecursive(char* srcDirName, const char* outDirName)
|
||||||
|
{
|
||||||
|
int status = 0;
|
||||||
|
char* pp = trimLeadingCurrentDir(srcDirName);
|
||||||
|
char* sp = NULL;
|
||||||
|
|
||||||
|
while ((sp = strchr(pp, PATH_SEP)) != NULL) {
|
||||||
|
if (sp != pp) {
|
||||||
|
*sp = '\0';
|
||||||
|
status = mirrorSrcDir(srcDirName, outDirName);
|
||||||
|
if (status != 0)
|
||||||
|
return status;
|
||||||
|
*sp = PATH_SEP;
|
||||||
|
}
|
||||||
|
pp = sp + 1;
|
||||||
|
}
|
||||||
|
status = mirrorSrcDir(srcDirName, outDirName);
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName)
|
||||||
|
{
|
||||||
|
unsigned int i = 0;
|
||||||
|
for (i = 0; i < nbFile; i++)
|
||||||
|
mirrorSrcDirRecursive(srcDirNames[i], outDirName);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir)
|
||||||
|
{
|
||||||
|
size_t firstDirLen = strlen(firstDir),
|
||||||
|
secondDirLen = strlen(secondDir);
|
||||||
|
return firstDirLen <= secondDirLen &&
|
||||||
|
(secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') &&
|
||||||
|
0 == strncmp(firstDir, secondDir, firstDirLen);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int compareDir(const void* pathname1, const void* pathname2) {
|
||||||
|
/* sort it after remove the leading '/' or './'*/
|
||||||
|
const char* s1 = trimPath(*(char * const *) pathname1);
|
||||||
|
const char* s2 = trimPath(*(char * const *) pathname2);
|
||||||
|
return strcmp(s1, s2);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName)
|
||||||
|
{
|
||||||
|
unsigned int i = 0, uniqueDirNr = 0;
|
||||||
|
char** uniqueDirNames = NULL;
|
||||||
|
|
||||||
|
if (nbFile == 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *));
|
||||||
|
CONTROL(uniqueDirNames != NULL);
|
||||||
|
|
||||||
|
/* if dirs is "a/b/c" and "a/b/c/d", we only need call:
|
||||||
|
* we just need "a/b/c/d" */
|
||||||
|
qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir);
|
||||||
|
|
||||||
|
uniqueDirNr = 1;
|
||||||
|
uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0];
|
||||||
|
for (i = 1; i < nbFile; i++) {
|
||||||
|
char* prevDirName = srcDirNames[i - 1];
|
||||||
|
char* currDirName = srcDirNames[i];
|
||||||
|
|
||||||
|
/* note: we alwasy compare trimmed path, i.e.:
|
||||||
|
* src dir of "./foo" and "/foo" will be both saved into:
|
||||||
|
* "outDirName/foo/" */
|
||||||
|
if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName),
|
||||||
|
trimPath(currDirName)))
|
||||||
|
uniqueDirNr++;
|
||||||
|
|
||||||
|
/* we need maintain original src dir name instead of trimmed
|
||||||
|
* dir, so we can retrive the original src dir's mode_t */
|
||||||
|
uniqueDirNames[uniqueDirNr - 1] = currDirName;
|
||||||
|
}
|
||||||
|
|
||||||
|
makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName);
|
||||||
|
|
||||||
|
free(uniqueDirNames);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName)
|
||||||
|
{
|
||||||
|
unsigned int i = 0;
|
||||||
|
for (i = 0; i < nbFile; ++i)
|
||||||
|
convertPathnameToDirName(srcFileNames[i]);
|
||||||
|
makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName)
|
||||||
|
{
|
||||||
|
unsigned int i = 0, validFilenamesNr = 0;
|
||||||
|
char** srcFileNames = (char **) malloc(nbFile * sizeof (char *));
|
||||||
|
CONTROL(srcFileNames != NULL);
|
||||||
|
|
||||||
|
/* check input filenames is valid */
|
||||||
|
for (i = 0; i < nbFile; ++i) {
|
||||||
|
if (isFileNameValidForMirroredOutput(inFileNames[i])) {
|
||||||
|
char* fname = STRDUP(inFileNames[i]);
|
||||||
|
CONTROL(fname != NULL);
|
||||||
|
srcFileNames[validFilenamesNr++] = fname;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (validFilenamesNr > 0) {
|
||||||
|
makeDir(outDirName, DIR_DEFAULT_MODE);
|
||||||
|
makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (i = 0; i < validFilenamesNr; i++)
|
||||||
|
free(srcFileNames[i]);
|
||||||
|
free(srcFileNames);
|
||||||
|
}
|
||||||
|
|
||||||
FileNamesTable*
|
FileNamesTable*
|
||||||
UTIL_createExpandedFNT(const char** inputNames, size_t nbIfns, int followLinks)
|
UTIL_createExpandedFNT(const char** inputNames, size_t nbIfns, int followLinks)
|
||||||
|
@ -104,6 +104,14 @@ extern int g_utilDisplayLevel;
|
|||||||
typedef struct stat stat_t;
|
typedef struct stat stat_t;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */
|
||||||
|
#define PATH_SEP '\\'
|
||||||
|
#define STRDUP(s) _strdup(s)
|
||||||
|
#else
|
||||||
|
#define PATH_SEP '/'
|
||||||
|
#include <libgen.h>
|
||||||
|
#define STRDUP(s) strdup(s)
|
||||||
|
#endif
|
||||||
|
|
||||||
int UTIL_fileExist(const char* filename);
|
int UTIL_fileExist(const char* filename);
|
||||||
int UTIL_isRegularFile(const char* infilename);
|
int UTIL_isRegularFile(const char* infilename);
|
||||||
@ -118,9 +126,13 @@ U64 UTIL_getFileSize(const char* infilename);
|
|||||||
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
|
U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles);
|
||||||
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
|
int UTIL_getFileStat(const char* infilename, stat_t* statbuf);
|
||||||
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
|
int UTIL_setFileStat(const char* filename, stat_t* statbuf);
|
||||||
|
int UTIL_getDirectoryStat(const char* infilename, stat_t* statbuf);
|
||||||
int UTIL_chmod(char const* filename, mode_t permissions); /*< like chmod, but avoid changing permission of /dev/null */
|
int UTIL_chmod(char const* filename, mode_t permissions); /*< like chmod, but avoid changing permission of /dev/null */
|
||||||
int UTIL_compareStr(const void *p1, const void *p2);
|
int UTIL_compareStr(const void *p1, const void *p2);
|
||||||
const char* UTIL_getFileExtension(const char* infilename);
|
const char* UTIL_getFileExtension(const char* infilename);
|
||||||
|
void UTIL_mirrorSourceFilesDirectories(const char** fileNamesTable, unsigned int nbFiles, const char *outDirName);
|
||||||
|
char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*-****************************************
|
/*-****************************************
|
||||||
@ -207,6 +219,7 @@ void UTIL_refFilename(FileNamesTable* fnt, const char* filename);
|
|||||||
# define UTIL_HAS_CREATEFILELIST
|
# define UTIL_HAS_CREATEFILELIST
|
||||||
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
|
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
|
||||||
# define UTIL_HAS_CREATEFILELIST
|
# define UTIL_HAS_CREATEFILELIST
|
||||||
|
# define UTIL_HAS_MIRRORFILELIST
|
||||||
#else
|
#else
|
||||||
/* do not define UTIL_HAS_CREATEFILELIST */
|
/* do not define UTIL_HAS_CREATEFILELIST */
|
||||||
#endif
|
#endif
|
||||||
|
@ -227,6 +227,15 @@ the last one takes effect.
|
|||||||
if multiple files, from different directories, end up having the same name.
|
if multiple files, from different directories, end up having the same name.
|
||||||
Collision resolution ensures first file with a given name will be present in `dir`,
|
Collision resolution ensures first file with a given name will be present in `dir`,
|
||||||
while in combination with `-f`, the last file will be present instead.
|
while in combination with `-f`, the last file will be present instead.
|
||||||
|
* `--output-dir-mirror[=dir]`:
|
||||||
|
similar to `--output-dir-flat`, the output files are stored underneath target
|
||||||
|
`dir` directory, but this option will preserve input directories structure in output `dir`.
|
||||||
|
|
||||||
|
If the input directory has "..", the files in this directory will be ignored. If
|
||||||
|
the input directory is absolute directory (i.e. "/var/tmp/abc"), it will be
|
||||||
|
stored into the "output-dir/var/tmp/abc".
|
||||||
|
If there is multiple input files or directories, the collision resolution will be same as
|
||||||
|
`--output-dir-flat`.
|
||||||
* `--format=FORMAT`:
|
* `--format=FORMAT`:
|
||||||
compress and decompress in other formats. If compiled with
|
compress and decompress in other formats. If compiled with
|
||||||
support, zstd can compress to or decompress from other compression algorithm
|
support, zstd can compress to or decompress from other compression algorithm
|
||||||
|
@ -150,6 +150,10 @@ static void usage_advanced(const char* programName)
|
|||||||
DISPLAYOUT( "--output-dir-flat=DIR : all resulting files are stored into DIR \n");
|
DISPLAYOUT( "--output-dir-flat=DIR : all resulting files are stored into DIR \n");
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef UTIL_HAS_MIRRORFILELIST
|
||||||
|
DISPLAYOUT( "--output-dir-mirror==DIR : all resulting files are stored into DIR in the original directory structure \n");
|
||||||
|
#endif
|
||||||
|
|
||||||
DISPLAYOUT( "-- : All arguments after \"--\" are treated as files \n");
|
DISPLAYOUT( "-- : All arguments after \"--\" are treated as files \n");
|
||||||
|
|
||||||
#ifndef ZSTD_NOCOMPRESS
|
#ifndef ZSTD_NOCOMPRESS
|
||||||
@ -645,6 +649,7 @@ int main(int const argCount, const char* argv[])
|
|||||||
rsyncable = 0,
|
rsyncable = 0,
|
||||||
nextArgumentIsOutFileName = 0,
|
nextArgumentIsOutFileName = 0,
|
||||||
nextArgumentIsOutDirName = 0,
|
nextArgumentIsOutDirName = 0,
|
||||||
|
nextArgumentIsMirroredOutDirName = 0,
|
||||||
nextArgumentIsMaxDict = 0,
|
nextArgumentIsMaxDict = 0,
|
||||||
nextArgumentIsDictID = 0,
|
nextArgumentIsDictID = 0,
|
||||||
nextArgumentsAreFiles = 0,
|
nextArgumentsAreFiles = 0,
|
||||||
@ -672,6 +677,7 @@ int main(int const argCount, const char* argv[])
|
|||||||
const char* programName = argv[0];
|
const char* programName = argv[0];
|
||||||
const char* outFileName = NULL;
|
const char* outFileName = NULL;
|
||||||
const char* outDirName = NULL;
|
const char* outDirName = NULL;
|
||||||
|
const char* outMirroredDirName = NULL;
|
||||||
const char* dictFileName = NULL;
|
const char* dictFileName = NULL;
|
||||||
const char* patchFromDictFileName = NULL;
|
const char* patchFromDictFileName = NULL;
|
||||||
const char* suffix = ZSTD_EXTENSION;
|
const char* suffix = ZSTD_EXTENSION;
|
||||||
@ -768,6 +774,9 @@ int main(int const argCount, const char* argv[])
|
|||||||
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
|
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
|
||||||
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
|
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
|
||||||
if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; }
|
if (!strcmp(argument, "--output-dir-flat")) {nextArgumentIsOutDirName=1; lastCommand=1; continue; }
|
||||||
|
#ifdef UTIL_HAS_MIRRORFILELIST
|
||||||
|
if (!strcmp(argument, "--output-dir-mirror")) {nextArgumentIsMirroredOutDirName=1; lastCommand=1; continue; }
|
||||||
|
#endif
|
||||||
if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; }
|
if (!strcmp(argument, "--show-default-cparams")) { showDefaultCParams = 1; continue; }
|
||||||
if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
|
if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
|
||||||
if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
|
if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
|
||||||
@ -838,6 +847,7 @@ int main(int const argCount, const char* argv[])
|
|||||||
if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; }
|
if (longCommandWArg(&argument, "--target-compressed-block-size=")) { targetCBlockSize = readSizeTFromChar(&argument); continue; }
|
||||||
if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; }
|
if (longCommandWArg(&argument, "--size-hint=")) { srcSizeHint = readSizeTFromChar(&argument); continue; }
|
||||||
if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; }
|
if (longCommandWArg(&argument, "--output-dir-flat=")) { outDirName = argument; continue; }
|
||||||
|
if (longCommandWArg(&argument, "--output-dir-mirror=")) { outMirroredDirName = argument; continue; }
|
||||||
if (longCommandWArg(&argument, "--patch-from=")) { patchFromDictFileName = argument; continue; }
|
if (longCommandWArg(&argument, "--patch-from=")) { patchFromDictFileName = argument; continue; }
|
||||||
if (longCommandWArg(&argument, "--long")) {
|
if (longCommandWArg(&argument, "--long")) {
|
||||||
unsigned ldmWindowLog = 0;
|
unsigned ldmWindowLog = 0;
|
||||||
@ -1066,6 +1076,13 @@ int main(int const argCount, const char* argv[])
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (nextArgumentIsMirroredOutDirName) {
|
||||||
|
nextArgumentIsMirroredOutDirName = 0;
|
||||||
|
lastCommand = 0;
|
||||||
|
outMirroredDirName = argument;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
/* none of the above : add filename to list */
|
/* none of the above : add filename to list */
|
||||||
UTIL_refFilename(filenames, argument);
|
UTIL_refFilename(filenames, argument);
|
||||||
}
|
}
|
||||||
@ -1330,7 +1347,7 @@ int main(int const argCount, const char* argv[])
|
|||||||
if ((filenames->tableSize==1) && outFileName)
|
if ((filenames->tableSize==1) && outFileName)
|
||||||
operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
|
operationResult = FIO_compressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName, cLevel, compressionParams);
|
||||||
else
|
else
|
||||||
operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
operationResult = FIO_compressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
|
||||||
#else
|
#else
|
||||||
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */
|
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */
|
||||||
DISPLAY("Compression not supported \n");
|
DISPLAY("Compression not supported \n");
|
||||||
@ -1340,7 +1357,7 @@ int main(int const argCount, const char* argv[])
|
|||||||
if (filenames->tableSize == 1 && outFileName) {
|
if (filenames->tableSize == 1 && outFileName) {
|
||||||
operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName);
|
operationResult = FIO_decompressFilename(prefs, outFileName, filenames->fileNames[0], dictFileName);
|
||||||
} else {
|
} else {
|
||||||
operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outDirName, outFileName, dictFileName);
|
operationResult = FIO_decompressMultipleFilenames(prefs, filenames->fileNames, (unsigned)filenames->tableSize, outMirroredDirName, outDirName, outFileName, dictFileName);
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
DISPLAY("Decompression not supported \n");
|
DISPLAY("Decompression not supported \n");
|
||||||
|
@ -445,6 +445,31 @@ test -f tmpOutDirDecomp/tmp2
|
|||||||
test -f tmpOutDirDecomp/tmp1
|
test -f tmpOutDirDecomp/tmp1
|
||||||
rm -rf tmp*
|
rm -rf tmp*
|
||||||
|
|
||||||
|
if [ "$isWindows" = false ] ; then
|
||||||
|
println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror"
|
||||||
|
println "test --output-dir-mirror" > tmp1
|
||||||
|
mkdir -p tmpInputTestDir/we/must/go/deeper
|
||||||
|
println cool > tmpInputTestDir/we/must/go/deeper/tmp2
|
||||||
|
zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir
|
||||||
|
test -f tmpOutDir/tmp1.zst
|
||||||
|
test -f tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2.zst
|
||||||
|
|
||||||
|
println "test: compress input dir will be ignored if it has '..'"
|
||||||
|
zstd -r tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'"
|
||||||
|
test ! -d non-exist
|
||||||
|
|
||||||
|
println "test : decompress multiple files into an output directory, --output-dir-mirror"
|
||||||
|
zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp
|
||||||
|
test -f tmpOutDirDecomp/tmpOutDir/tmp1
|
||||||
|
test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/must/go/deeper/tmp2
|
||||||
|
|
||||||
|
println "test: decompress input dir will be ignored if it has '..'"
|
||||||
|
zstd -r tmpOutDir/tmpInputTestDir/we/must/../must --output-dir-mirror non-exist && die "input cannot contain '..'"
|
||||||
|
test ! -d non-exist
|
||||||
|
|
||||||
|
rm -rf tmp*
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
println "test : compress multiple files reading them from a file, --filelist=FILE"
|
println "test : compress multiple files reading them from a file, --filelist=FILE"
|
||||||
println "Hello world!, file1" > tmp1
|
println "Hello world!, file1" > tmp1
|
||||||
|
Loading…
x
Reference in New Issue
Block a user