facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test-zstd-versions.py: create dictionaries for v0.5.1 and newer

dev
inikep 2016-06-16 14:15:32 +02:00
parent e98b66f8ae
commit 24aa7b4238
1 changed files with 57 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
tests/test-zstd-versions.py
View File

@ -9,6 +9,7 @@ import os
import shutil
import subprocess
import sys
from subprocess import Popen, PIPE
repo_url = 'https://github.com/Cyan4973/zstd.git'
tmp_dir_name = 'tests/versionsTest'
@ -17,6 +18,27 @@ git_cmd = 'git'
test_dat_src = 'README.md'
test_dat = 'test_dat'
head = 'vdevel'
dict_source = 'dict_source'
dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
def execute(command, print_output=False, print_error=True):
popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=True)
itout = iter(popen.stdout.readline, b"")
iterr = iter(popen.stderr.readline, b"")
stdout_lines = b''.join(list(itout)).decode("utf-8")
if print_output:
print(stdout_lines)
stderr_lines = b''.join(list(iterr)).decode("utf-8")
if print_output:
print(stderr_lines)
popen.communicate()
if popen.returncode is not None and popen.returncode != 0:
if not print_output and print_error:
print(stderr_lines)
raise RuntimeError(stderr_lines)
return stdout_lines + stderr_lines
def proc(cmd_args, pipe=True, dummy=False):
@ -85,7 +107,7 @@ def remove_duplicates():
print('duplicated : {} == {}'.format(ref_zst, compared_zst))
def decompress_zst(tag):
def decompress_zst(tag, zstd_up_to_v05=False):
dec_error = 0
list_zst = sorted(glob.glob('*.zst'))
try:
@ -96,7 +118,11 @@ def decompress_zst(tag):
print(file_zst, end=' ')
print(tag, end=' ')
file_dec = file_zst + '_d64_' + tag + '.dec'
if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0:
if zstd_up_to_v05:
params = ['./zstd.' + tag, '-df', file_zst, file_dec]
else:
params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
if subprocess.call(params, stderr=DEVNULL) == 0:
if not filecmp.cmp(file_dec, test_dat):
print('ERR !! ')
dec_error = 1
@ -107,12 +133,24 @@ def decompress_zst(tag):
return dec_error
def create_dict(tag, dict_source_path):
dict_name = 'dict.' + tag
if not os.path.isfile(dict_name):
cFiles = glob.glob(dict_source_path + "/*.c")
hFiles = glob.glob(dict_source_path + "/*.h")
execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name)
print(dict_name + ' created')
else:
print(dict_name + ' already exists')
if __name__ == '__main__':
error_code = 0
base_dir = os.getcwd() + '/..' # /path/to/zstd
tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest
clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd
programs_dir = base_dir + '/programs' # /path/to/zstd/programs
base_dir = os.getcwd() + '/..' # /path/to/zstd
tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest
clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd
dict_source_path = tmp_dir + '/' + dict_source # /path/to/zstd/tests/versionsTest/dict_source
programs_dir = base_dir + '/programs' # /path/to/zstd/programs
os.makedirs(tmp_dir, exist_ok=True)
# since Travis clones limited depth, we should clone full repository
@ -151,15 +189,28 @@ if __name__ == '__main__':
for dec in glob.glob("*.dec"):
os.remove(dec)
# copy *.c and *.h to a temporary directory ("dict_source")
if not os.path.isdir(dict_source_path):
os.mkdir(dict_source_path)
print('cp ' + dict_files + ' ' + dict_source_path)
subprocess.call(['cp ' + dict_files + ' ' + dict_source_path], shell=True)
dictFiles = glob.glob("dict*")
print('dictFiles=' + str(dictFiles))
print('Compress test.dat by all released zstd')
error_code = 0
for tag in tags:
print(tag)
if tag >= 'v0.5.1':
create_dict(tag, dict_source_path)
compress_sample(tag, test_dat)
remove_duplicates()
if tag >= 'v0.5.1':
error_code += decompress_zst(tag)
else:
error_code += decompress_zst(tag, zstd_up_to_v05=True)
print('')
print('Enumerate different compressed files')