From 24aa7b423890b5cd4e7cc07d99bed229a6166c31 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 16 Jun 2016 14:15:32 +0200 Subject: [PATCH 1/7] test-zstd-versions.py: create dictionaries for v0.5.1 and newer --- tests/test-zstd-versions.py | 63 +++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 437cd4c0..3c8eac7d 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -9,6 +9,7 @@ import os import shutil import subprocess import sys +from subprocess import Popen, PIPE repo_url = 'https://github.com/Cyan4973/zstd.git' tmp_dir_name = 'tests/versionsTest' @@ -17,6 +18,27 @@ git_cmd = 'git' test_dat_src = 'README.md' test_dat = 'test_dat' head = 'vdevel' +dict_source = 'dict_source' +dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c ' +dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h' + + +def execute(command, print_output=False, print_error=True): + popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) + itout = iter(popen.stdout.readline, b"") + iterr = iter(popen.stderr.readline, b"") + stdout_lines = b''.join(list(itout)).decode("utf-8") + if print_output: + print(stdout_lines) + stderr_lines = b''.join(list(iterr)).decode("utf-8") + if print_output: + print(stderr_lines) + popen.communicate() + if popen.returncode is not None and popen.returncode != 0: + if not print_output and print_error: + print(stderr_lines) + raise RuntimeError(stderr_lines) + return stdout_lines + stderr_lines def proc(cmd_args, pipe=True, dummy=False): @@ -85,7 +107,7 @@ def remove_duplicates(): print('duplicated : {} == {}'.format(ref_zst, compared_zst)) -def decompress_zst(tag): +def decompress_zst(tag, zstd_up_to_v05=False): dec_error = 0 list_zst = sorted(glob.glob('*.zst')) try: @@ -96,7 +118,11 @@ def decompress_zst(tag): print(file_zst, end=' ') print(tag, end=' ') file_dec = file_zst + '_d64_' + tag + '.dec' - if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0: + if zstd_up_to_v05: + params = ['./zstd.' + tag, '-df', file_zst, file_dec] + else: + params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] + if subprocess.call(params, stderr=DEVNULL) == 0: if not filecmp.cmp(file_dec, test_dat): print('ERR !! ') dec_error = 1 @@ -107,12 +133,24 @@ def decompress_zst(tag): return dec_error +def create_dict(tag, dict_source_path): + dict_name = 'dict.' + tag + if not os.path.isfile(dict_name): + cFiles = glob.glob(dict_source_path + "/*.c") + hFiles = glob.glob(dict_source_path + "/*.h") + execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name) + print(dict_name + ' created') + else: + print(dict_name + ' already exists') + + if __name__ == '__main__': error_code = 0 - base_dir = os.getcwd() + '/..' # /path/to/zstd - tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest - clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd - programs_dir = base_dir + '/programs' # /path/to/zstd/programs + base_dir = os.getcwd() + '/..' # /path/to/zstd + tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest + clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd + dict_source_path = tmp_dir + '/' + dict_source # /path/to/zstd/tests/versionsTest/dict_source + programs_dir = base_dir + '/programs' # /path/to/zstd/programs os.makedirs(tmp_dir, exist_ok=True) # since Travis clones limited depth, we should clone full repository @@ -151,15 +189,28 @@ if __name__ == '__main__': for dec in glob.glob("*.dec"): os.remove(dec) + # copy *.c and *.h to a temporary directory ("dict_source") + if not os.path.isdir(dict_source_path): + os.mkdir(dict_source_path) + print('cp ' + dict_files + ' ' + dict_source_path) + subprocess.call(['cp ' + dict_files + ' ' + dict_source_path], shell=True) + + dictFiles = glob.glob("dict*") + print('dictFiles=' + str(dictFiles)) + print('Compress test.dat by all released zstd') error_code = 0 for tag in tags: print(tag) + if tag >= 'v0.5.1': + create_dict(tag, dict_source_path) compress_sample(tag, test_dat) remove_duplicates() if tag >= 'v0.5.1': error_code += decompress_zst(tag) + else: + error_code += decompress_zst(tag, zstd_up_to_v05=True) print('') print('Enumerate different compressed files') From 150152fb8a0b2a5b3938f41a28cbe4eb7129fd16 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 16 Jun 2016 19:29:09 +0200 Subject: [PATCH 2/7] test-zstd-versions.py: test dictiony compression for v0.5.1 and newer --- tests/test-zstd-versions.py | 79 +++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 3c8eac7d..dffa79dd 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -67,21 +67,44 @@ def get_git_tags(): return tags +def create_dict(tag, dict_source_path): + dict_name = 'dict.' + tag + if not os.path.isfile(dict_name): + cFiles = glob.glob(dict_source_path + "/*.c") + hFiles = glob.glob(dict_source_path + "/*.h") + execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name) + print(dict_name + ' created') + else: + print(dict_name + ' already exists') + + +def dict_compress_sample(tag, sample): + dict_name = 'dict.' + tag + try: + from subprocess import DEVNULL # py3k + except ImportError: + DEVNULL = open(os.devnull, 'wb') + subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample, '-o', sample + '_01_64' + tag + '_dict.zst'], stderr=DEVNULL) + # zstdFiles = glob.glob("*.zst*") + # print(zstdFiles) + print(tag + " : dict compression completed") + + def compress_sample(tag, sample): try: from subprocess import DEVNULL # py3k except ImportError: DEVNULL = open(os.devnull, 'wb') if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst') + os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodic.zst') if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst') + os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodic.zst') if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst') + os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodic.zst') if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst') + os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodic.zst') if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst') + os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodic.zst') # zstdFiles = glob.glob("*.zst*") # print(zstdFiles) print(tag + " : compression completed") @@ -94,7 +117,7 @@ def sha1_of_file(filepath): def remove_duplicates(): - list_of_zst = sorted(glob.glob('*.zst')) + list_of_zst = sorted(glob.glob('*_nodic.zst')) for i, ref_zst in enumerate(list_of_zst): if not os.path.isfile(ref_zst): continue @@ -109,7 +132,7 @@ def remove_duplicates(): def decompress_zst(tag, zstd_up_to_v05=False): dec_error = 0 - list_zst = sorted(glob.glob('*.zst')) + list_zst = sorted(glob.glob('*_nodic.zst')) try: from subprocess import DEVNULL # py3k except ImportError: @@ -133,15 +156,37 @@ def decompress_zst(tag, zstd_up_to_v05=False): return dec_error -def create_dict(tag, dict_source_path): - dict_name = 'dict.' + tag - if not os.path.isfile(dict_name): - cFiles = glob.glob(dict_source_path + "/*.c") - hFiles = glob.glob(dict_source_path + "/*.h") - execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name) - print(dict_name + ' created') - else: - print(dict_name + ' already exists') +def decompress_dict(tag, zstd_up_to_v05=False): + dec_error = 0 + list_zst = sorted(glob.glob('*_dict.zst')) + try: + from subprocess import DEVNULL # py3k + except ImportError: + DEVNULL = open(os.devnull, 'wb') + for file_zst in list_zst: + dict_tag = file_zst[0:len(file_zst)-9] # remove "_dict.zst" + dict_tag = dict_tag[dict_tag.rfind('v'):] + if dict_tag == 'vel': + dict_tag = head + dict_name = 'dict.' + dict_tag + #print("dict_tag=" + dict_tag + " dict_name=" + dict_name) + print(file_zst, end=' ') + print(tag, end=' ') + print(dict_tag, end=' ') + file_dec = file_zst + '_d64_' + tag + '.dec' + if zstd_up_to_v05: + params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] + else: + params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] + if subprocess.call(params, stderr=DEVNULL) == 0: + if not filecmp.cmp(file_dec, test_dat): + print('ERR !! ') + dec_error = 1 + else: + print('OK ') + else: + print('command does not work') + return dec_error if __name__ == '__main__': @@ -205,6 +250,8 @@ if __name__ == '__main__': print(tag) if tag >= 'v0.5.1': create_dict(tag, dict_source_path) + dict_compress_sample(tag, test_dat) + decompress_dict(tag) compress_sample(tag, test_dat) remove_duplicates() if tag >= 'v0.5.1': From d1af4e66b6e849ab075768376397f2cc619ffb01 Mon Sep 17 00:00:00 2001 From: inikep Date: Thu, 16 Jun 2016 20:23:11 +0200 Subject: [PATCH 3/7] test-zstd-versions.py: dictionary compression with levels 1, 5, 9, 15, 18 --- tests/test-zstd-versions.py | 38 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index dffa79dd..dd5b8eb8 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -84,7 +84,11 @@ def dict_compress_sample(tag, sample): from subprocess import DEVNULL # py3k except ImportError: DEVNULL = open(os.devnull, 'wb') - subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample, '-o', sample + '_01_64' + tag + '_dict.zst'], stderr=DEVNULL) + subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample, '-o', sample + '_01_64_' + tag + '_dictio.zst'], stderr=DEVNULL) + subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample, '-o', sample + '_05_64_' + tag + '_dictio.zst'], stderr=DEVNULL) + subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample, '-o', sample + '_09_64_' + tag + '_dictio.zst'], stderr=DEVNULL) + subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample, '-o', sample + '_15_64_' + tag + '_dictio.zst'], stderr=DEVNULL) + subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample, '-o', sample + '_18_64_' + tag + '_dictio.zst'], stderr=DEVNULL) # zstdFiles = glob.glob("*.zst*") # print(zstdFiles) print(tag + " : dict compression completed") @@ -96,15 +100,15 @@ def compress_sample(tag, sample): except ImportError: DEVNULL = open(os.devnull, 'wb') if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodic.zst') + os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodic.zst') + os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst') if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodic.zst') + os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst') if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodic.zst') + os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst') if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0: - os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodic.zst') + os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst') # zstdFiles = glob.glob("*.zst*") # print(zstdFiles) print(tag + " : compression completed") @@ -117,7 +121,7 @@ def sha1_of_file(filepath): def remove_duplicates(): - list_of_zst = sorted(glob.glob('*_nodic.zst')) + list_of_zst = sorted(glob.glob('*.zst')) for i, ref_zst in enumerate(list_of_zst): if not os.path.isfile(ref_zst): continue @@ -132,7 +136,7 @@ def remove_duplicates(): def decompress_zst(tag, zstd_up_to_v05=False): dec_error = 0 - list_zst = sorted(glob.glob('*_nodic.zst')) + list_zst = sorted(glob.glob('*_nodict.zst')) try: from subprocess import DEVNULL # py3k except ImportError: @@ -158,21 +162,19 @@ def decompress_zst(tag, zstd_up_to_v05=False): def decompress_dict(tag, zstd_up_to_v05=False): dec_error = 0 - list_zst = sorted(glob.glob('*_dict.zst')) + list_zst = sorted(glob.glob('*_dictio.zst')) try: from subprocess import DEVNULL # py3k except ImportError: DEVNULL = open(os.devnull, 'wb') for file_zst in list_zst: - dict_tag = file_zst[0:len(file_zst)-9] # remove "_dict.zst" - dict_tag = dict_tag[dict_tag.rfind('v'):] - if dict_tag == 'vel': + dict_tag = file_zst[0:len(file_zst)-11] # remove "_dictio.zst" + if head in dict_tag: # find vdevel dict_tag = head + else: + dict_tag = dict_tag[dict_tag.rfind('v'):] dict_name = 'dict.' + dict_tag - #print("dict_tag=" + dict_tag + " dict_name=" + dict_name) - print(file_zst, end=' ') - print(tag, end=' ') - print(dict_tag, end=' ') + print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ') file_dec = file_zst + '_d64_' + tag + '.dec' if zstd_up_to_v05: params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] @@ -186,6 +188,7 @@ def decompress_dict(tag, zstd_up_to_v05=False): print('OK ') else: print('command does not work') + dec_error = 1 return dec_error @@ -251,7 +254,8 @@ if __name__ == '__main__': if tag >= 'v0.5.1': create_dict(tag, dict_source_path) dict_compress_sample(tag, test_dat) - decompress_dict(tag) + remove_duplicates() + error_code += decompress_dict(tag) compress_sample(tag, test_dat) remove_duplicates() if tag >= 'v0.5.1': From 4545671b4445b30f8b67d6e374c2d563db6b0675 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 17 Jun 2016 13:39:43 +0200 Subject: [PATCH 4/7] test-zstd-versions.py: fixed DEVNULL --- tests/test-zstd-versions.py | 44 ++++++++++--------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index dd5b8eb8..499f8618 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -7,9 +7,9 @@ import glob import hashlib import os import shutil -import subprocess import sys -from subprocess import Popen, PIPE +import subprocess +from subprocess import Popen, PIPE, DEVNULL repo_url = 'https://github.com/Cyan4973/zstd.git' tmp_dir_name = 'tests/versionsTest' @@ -25,31 +25,25 @@ dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h def execute(command, print_output=False, print_error=True): popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) - itout = iter(popen.stdout.readline, b"") - iterr = iter(popen.stderr.readline, b"") - stdout_lines = b''.join(list(itout)).decode("utf-8") + stdout_lines, stderr_lines = popen.communicate() + stderr_lines = stderr_lines.decode("utf-8") + stdout_lines = stdout_lines.decode("utf-8") if print_output: print(stdout_lines) - stderr_lines = b''.join(list(iterr)).decode("utf-8") - if print_output: print(stderr_lines) - popen.communicate() if popen.returncode is not None and popen.returncode != 0: if not print_output and print_error: print(stderr_lines) - raise RuntimeError(stderr_lines) - return stdout_lines + stderr_lines + return popen.returncode def proc(cmd_args, pipe=True, dummy=False): if dummy: return if pipe: - subproc = subprocess.Popen(cmd_args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE) else: - subproc = subprocess.Popen(cmd_args) + subproc = Popen(cmd_args) return subproc.communicate() @@ -72,18 +66,16 @@ def create_dict(tag, dict_source_path): if not os.path.isfile(dict_name): cFiles = glob.glob(dict_source_path + "/*.c") hFiles = glob.glob(dict_source_path + "/*.h") - execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name) - print(dict_name + ' created') + if execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False) == 0: + print(dict_name + ' created') + else: + print('ERROR: creating of ' + dict_name + ' failed') else: print(dict_name + ' already exists') def dict_compress_sample(tag, sample): dict_name = 'dict.' + tag - try: - from subprocess import DEVNULL # py3k - except ImportError: - DEVNULL = open(os.devnull, 'wb') subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample, '-o', sample + '_01_64_' + tag + '_dictio.zst'], stderr=DEVNULL) subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample, '-o', sample + '_05_64_' + tag + '_dictio.zst'], stderr=DEVNULL) subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample, '-o', sample + '_09_64_' + tag + '_dictio.zst'], stderr=DEVNULL) @@ -95,10 +87,6 @@ def dict_compress_sample(tag, sample): def compress_sample(tag, sample): - try: - from subprocess import DEVNULL # py3k - except ImportError: - DEVNULL = open(os.devnull, 'wb') if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: @@ -137,10 +125,6 @@ def remove_duplicates(): def decompress_zst(tag, zstd_up_to_v05=False): dec_error = 0 list_zst = sorted(glob.glob('*_nodict.zst')) - try: - from subprocess import DEVNULL # py3k - except ImportError: - DEVNULL = open(os.devnull, 'wb') for file_zst in list_zst: print(file_zst, end=' ') print(tag, end=' ') @@ -163,10 +147,6 @@ def decompress_zst(tag, zstd_up_to_v05=False): def decompress_dict(tag, zstd_up_to_v05=False): dec_error = 0 list_zst = sorted(glob.glob('*_dictio.zst')) - try: - from subprocess import DEVNULL # py3k - except ImportError: - DEVNULL = open(os.devnull, 'wb') for file_zst in list_zst: dict_tag = file_zst[0:len(file_zst)-11] # remove "_dictio.zst" if head in dict_tag: # find vdevel From 2ef16501cacc0c40b0c1d0d0da28707358b3a00b Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 17 Jun 2016 14:07:42 +0200 Subject: [PATCH 5/7] test-zstd-versions.py: improved error handling --- tests/test-zstd-versions.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index 499f8618..e65c753b 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -23,8 +23,8 @@ dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h' -def execute(command, print_output=False, print_error=True): - popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True) +def execute(command, print_output=False, print_error=True, param_shell=False): + popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell) stdout_lines, stderr_lines = popen.communicate() stderr_lines = stderr_lines.decode("utf-8") stdout_lines = stdout_lines.decode("utf-8") @@ -66,7 +66,7 @@ def create_dict(tag, dict_source_path): if not os.path.isfile(dict_name): cFiles = glob.glob(dict_source_path + "/*.c") hFiles = glob.glob(dict_source_path + "/*.h") - if execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False) == 0: + if execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) == 0: print(dict_name + ' created') else: print('ERROR: creating of ' + dict_name + ' failed') @@ -133,7 +133,7 @@ def decompress_zst(tag, zstd_up_to_v05=False): params = ['./zstd.' + tag, '-df', file_zst, file_dec] else: params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] - if subprocess.call(params, stderr=DEVNULL) == 0: + if execute(params) == 0: if not filecmp.cmp(file_dec, test_dat): print('ERR !! ') dec_error = 1 @@ -160,7 +160,7 @@ def decompress_dict(tag, zstd_up_to_v05=False): params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] else: params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] - if subprocess.call(params, stderr=DEVNULL) == 0: + if execute(params) == 0: if not filecmp.cmp(file_dec, test_dat): print('ERR !! ') dec_error = 1 @@ -221,10 +221,7 @@ if __name__ == '__main__': if not os.path.isdir(dict_source_path): os.mkdir(dict_source_path) print('cp ' + dict_files + ' ' + dict_source_path) - subprocess.call(['cp ' + dict_files + ' ' + dict_source_path], shell=True) - - dictFiles = glob.glob("dict*") - print('dictFiles=' + str(dictFiles)) + execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True) print('Compress test.dat by all released zstd') From 7e3597bf38858c7b4b9fbc77b09c19feca3d128e Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 17 Jun 2016 14:43:24 +0200 Subject: [PATCH 6/7] test-zstd-versions.py: create and test dictionaries for v0.5.0 --- tests/test-zstd-versions.py | 46 ++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index e65c753b..b9a81521 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -9,7 +9,7 @@ import os import shutil import sys import subprocess -from subprocess import Popen, PIPE, DEVNULL +from subprocess import Popen, PIPE repo_url = 'https://github.com/Cyan4973/zstd.git' tmp_dir_name = 'tests/versionsTest' @@ -66,7 +66,11 @@ def create_dict(tag, dict_source_path): if not os.path.isfile(dict_name): cFiles = glob.glob(dict_source_path + "/*.c") hFiles = glob.glob(dict_source_path + "/*.h") - if execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) == 0: + if tag == 'v0.5.0': + result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) + else: + result = execute('./zstd.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) + if result == 0: print(dict_name + ' created') else: print('ERROR: creating of ' + dict_name + ' failed') @@ -76,17 +80,24 @@ def create_dict(tag, dict_source_path): def dict_compress_sample(tag, sample): dict_name = 'dict.' + tag - subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample, '-o', sample + '_01_64_' + tag + '_dictio.zst'], stderr=DEVNULL) - subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample, '-o', sample + '_05_64_' + tag + '_dictio.zst'], stderr=DEVNULL) - subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample, '-o', sample + '_09_64_' + tag + '_dictio.zst'], stderr=DEVNULL) - subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample, '-o', sample + '_15_64_' + tag + '_dictio.zst'], stderr=DEVNULL) - subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample, '-o', sample + '_18_64_' + tag + '_dictio.zst'], stderr=DEVNULL) + DEVNULL = open(os.devnull, 'wb') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst') # zstdFiles = glob.glob("*.zst*") # print(zstdFiles) print(tag + " : dict compression completed") def compress_sample(tag, sample): + DEVNULL = open(os.devnull, 'wb') if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: @@ -122,14 +133,14 @@ def remove_duplicates(): print('duplicated : {} == {}'.format(ref_zst, compared_zst)) -def decompress_zst(tag, zstd_up_to_v05=False): +def decompress_zst(tag): dec_error = 0 list_zst = sorted(glob.glob('*_nodict.zst')) for file_zst in list_zst: print(file_zst, end=' ') print(tag, end=' ') file_dec = file_zst + '_d64_' + tag + '.dec' - if zstd_up_to_v05: + if tag <= 'v0.5.0': params = ['./zstd.' + tag, '-df', file_zst, file_dec] else: params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] @@ -144,7 +155,7 @@ def decompress_zst(tag, zstd_up_to_v05=False): return dec_error -def decompress_dict(tag, zstd_up_to_v05=False): +def decompress_dict(tag): dec_error = 0 list_zst = sorted(glob.glob('*_dictio.zst')) for file_zst in list_zst: @@ -153,10 +164,12 @@ def decompress_dict(tag, zstd_up_to_v05=False): dict_tag = head else: dict_tag = dict_tag[dict_tag.rfind('v'):] + if tag == 'v0.6.0' and dict_tag < 'v0.6.0': + continue dict_name = 'dict.' + dict_tag print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ') file_dec = file_zst + '_d64_' + tag + '.dec' - if zstd_up_to_v05: + if tag <= 'v0.5.0': params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] else: params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] @@ -203,6 +216,10 @@ if __name__ == '__main__': os.makedirs(r_dir, exist_ok=True) os.chdir(clone_dir) git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) + if tag == 'v0.5.0': + os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder + make(['clean', 'dictBuilder'], False) + shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag)) os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest//programs make(['clean', 'zstd'], False) else: @@ -228,17 +245,14 @@ if __name__ == '__main__': error_code = 0 for tag in tags: print(tag) - if tag >= 'v0.5.1': + if tag >= 'v0.5.0': create_dict(tag, dict_source_path) dict_compress_sample(tag, test_dat) remove_duplicates() error_code += decompress_dict(tag) compress_sample(tag, test_dat) remove_duplicates() - if tag >= 'v0.5.1': - error_code += decompress_zst(tag) - else: - error_code += decompress_zst(tag, zstd_up_to_v05=True) + error_code += decompress_zst(tag) print('') print('Enumerate different compressed files') From e16f65675b0096ecc83828e3d6c72bddae4c69f8 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 17 Jun 2016 15:17:35 +0200 Subject: [PATCH 7/7] test-zstd-versions.py: fixed creation of dictionaries for v0.5.1+ --- tests/test-zstd-versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-zstd-versions.py b/tests/test-zstd-versions.py index b9a81521..e5e5ec90 100755 --- a/tests/test-zstd-versions.py +++ b/tests/test-zstd-versions.py @@ -69,7 +69,7 @@ def create_dict(tag, dict_source_path): if tag == 'v0.5.0': result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) else: - result = execute('./zstd.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) + result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) if result == 0: print(dict_name + ' created') else: