Merge pull request #2893 from terrelln/issue-2789
[asm] Share portability macros and restrict ASM furtherdev
commit
486472c453
|
@ -44,7 +44,7 @@
|
|||
#endif
|
||||
#define ZSTD_TRACE 0
|
||||
/* TODO: Can't amalgamate ASM function */
|
||||
#define HUF_DISABLE_ASM 1
|
||||
#define ZSTD_DISABLE_ASM 1
|
||||
|
||||
/* Include zstd_deps.h first with all the options we need enabled. */
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
|
|
|
@ -40,7 +40,7 @@
|
|||
#define ZSTD_STRIP_ERROR_STRINGS
|
||||
#define ZSTD_TRACE 0
|
||||
/* TODO: Can't amalgamate ASM function */
|
||||
#define HUF_DISABLE_ASM 1
|
||||
#define ZSTD_DISABLE_ASM 1
|
||||
|
||||
/* Include zstd_deps.h first with all the options we need enabled. */
|
||||
#define ZSTD_DEPS_NEED_MALLOC
|
||||
|
|
|
@ -16,16 +16,17 @@
|
|||
* decompression.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Disable the ASM Huffman implementation because we need to
|
||||
* include all the sources.
|
||||
*/
|
||||
#define ZSTD_DISABLE_ASM 1
|
||||
|
||||
#include "common/debug.c"
|
||||
#include "common/entropy_common.c"
|
||||
#include "common/error_private.c"
|
||||
#include "common/fse_decompress.c"
|
||||
#include "common/zstd_common.c"
|
||||
/*
|
||||
* Disable the ASM Huffman implementation because we need to
|
||||
* include all the sources.
|
||||
*/
|
||||
#define HUF_DISABLE_ASM 1
|
||||
#include "decompress/huf_decompress.c"
|
||||
#include "decompress/zstd_ddict.c"
|
||||
#include "decompress/zstd_decompress.c"
|
||||
|
|
|
@ -11,6 +11,8 @@
|
|||
#ifndef ZSTD_COMPILER_H
|
||||
#define ZSTD_COMPILER_H
|
||||
|
||||
#include "portability_macros.h"
|
||||
|
||||
/*-*******************************************************
|
||||
* Compiler specifics
|
||||
*********************************************************/
|
||||
|
@ -92,9 +94,6 @@
|
|||
|
||||
|
||||
/* target attribute */
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */
|
||||
#endif
|
||||
#if defined(__GNUC__) || defined(__ICCARM__)
|
||||
# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
|
||||
#else
|
||||
|
@ -107,22 +106,6 @@
|
|||
*/
|
||||
#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2")
|
||||
|
||||
|
||||
/* Enable runtime BMI2 dispatch based on the CPU.
|
||||
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
||||
*/
|
||||
#ifndef DYNAMIC_BMI2
|
||||
#if ((defined(__clang__) && __has_attribute(__target__)) \
|
||||
|| (defined(__GNUC__) \
|
||||
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
|
||||
&& (defined(__x86_64__) || defined(_M_X64)) \
|
||||
&& !defined(__BMI2__)
|
||||
# define DYNAMIC_BMI2 1
|
||||
#else
|
||||
# define DYNAMIC_BMI2 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* prefetch
|
||||
* can be disabled, by declaring NO_PREFETCH build macro */
|
||||
#if defined(NO_PREFETCH)
|
||||
|
@ -221,16 +204,6 @@
|
|||
# endif
|
||||
#endif
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_builtin
|
||||
# define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_feature
|
||||
# define __has_feature(x) 0
|
||||
#endif
|
||||
|
||||
/* C-language Attributes are added in C23. */
|
||||
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute)
|
||||
# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
|
||||
|
@ -267,24 +240,6 @@
|
|||
# endif
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under msan */
|
||||
#ifndef ZSTD_MEMORY_SANITIZER
|
||||
# if __has_feature(memory_sanitizer)
|
||||
# define ZSTD_MEMORY_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_MEMORY_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled undef dfsan */
|
||||
#ifndef ZSTD_DATAFLOW_SANITIZER
|
||||
# if __has_feature(dataflow_sanitizer)
|
||||
# define ZSTD_DATAFLOW_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_DATAFLOW_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/*-**************************************************************
|
||||
* Alignment check
|
||||
*****************************************************************/
|
||||
|
@ -339,17 +294,6 @@ void __msan_poison(const volatile void *a, size_t size);
|
|||
intptr_t __msan_test_shadow(const volatile void *x, size_t size);
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under asan */
|
||||
#ifndef ZSTD_ADDRESS_SANITIZER
|
||||
# if __has_feature(address_sanitizer)
|
||||
# define ZSTD_ADDRESS_SANITIZER 1
|
||||
# elif defined(__SANITIZE_ADDRESS__)
|
||||
# define ZSTD_ADDRESS_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_ADDRESS_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if ZSTD_ADDRESS_SANITIZER
|
||||
/* Not all platforms that support asan provide sanitizers/asan_interface.h.
|
||||
* We therefore declare the functions we need ourselves, rather than trying to
|
||||
|
|
|
@ -0,0 +1,129 @@
|
|||
/*
|
||||
* Copyright (c) Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef ZSTD_PORTABILITY_MACROS_H
|
||||
#define ZSTD_PORTABILITY_MACROS_H
|
||||
|
||||
/**
|
||||
* This header file contains macro defintions to support portability.
|
||||
* This header is shared between C and ASM code, so it MUST only
|
||||
* contain macro definitions. It MUST not contain any C code.
|
||||
*
|
||||
* This header ONLY defines macros to detect platforms/feature support.
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_attribute
|
||||
#define __has_attribute(x) 0
|
||||
#endif
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_builtin
|
||||
# define __has_builtin(x) 0
|
||||
#endif
|
||||
|
||||
/* compat. with non-clang compilers */
|
||||
#ifndef __has_feature
|
||||
# define __has_feature(x) 0
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under msan */
|
||||
#ifndef ZSTD_MEMORY_SANITIZER
|
||||
# if __has_feature(memory_sanitizer)
|
||||
# define ZSTD_MEMORY_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_MEMORY_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under asan */
|
||||
#ifndef ZSTD_ADDRESS_SANITIZER
|
||||
# if __has_feature(address_sanitizer)
|
||||
# define ZSTD_ADDRESS_SANITIZER 1
|
||||
# elif defined(__SANITIZE_ADDRESS__)
|
||||
# define ZSTD_ADDRESS_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_ADDRESS_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/* detects whether we are being compiled under dfsan */
|
||||
#ifndef ZSTD_DATAFLOW_SANITIZER
|
||||
# if __has_feature(dataflow_sanitizer)
|
||||
# define ZSTD_DATAFLOW_SANITIZER 1
|
||||
# else
|
||||
# define ZSTD_DATAFLOW_SANITIZER 0
|
||||
# endif
|
||||
#endif
|
||||
|
||||
|
||||
/* Enable runtime BMI2 dispatch based on the CPU.
|
||||
* Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
|
||||
*/
|
||||
#ifndef DYNAMIC_BMI2
|
||||
#if ((defined(__clang__) && __has_attribute(__target__)) \
|
||||
|| (defined(__GNUC__) \
|
||||
&& (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \
|
||||
&& (defined(__x86_64__) || defined(_M_X64)) \
|
||||
&& !defined(__BMI2__)
|
||||
# define DYNAMIC_BMI2 1
|
||||
#else
|
||||
# define DYNAMIC_BMI2 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Only enable assembly for GNUC comptabile compilers,
|
||||
* because other platforms may not support GAS assembly syntax.
|
||||
*
|
||||
* Only enable assembly for Linux / MacOS, other platforms may
|
||||
* work, but they haven't been tested. This could likely be
|
||||
* extended to BSD systems.
|
||||
*
|
||||
* Disable assembly when MSAN is enabled, because MSAN requires
|
||||
* 100% of code to be instrumented to work.
|
||||
*/
|
||||
#if defined(__GNUC__)
|
||||
# if defined(__linux__) || defined(__linux) || defined(__APPLE__)
|
||||
# if ZSTD_MEMORY_SANITIZER
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
# else
|
||||
# define ZSTD_ASM_SUPPORTED 1
|
||||
# endif
|
||||
# else
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
# endif
|
||||
#else
|
||||
# define ZSTD_ASM_SUPPORTED 0
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Determines whether we should enable assembly for x86-64
|
||||
* with BMI2.
|
||||
*
|
||||
* Enable if all of the following conditions hold:
|
||||
* - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM
|
||||
* - Assembly is supported
|
||||
* - We are compiling for x86-64 and either:
|
||||
* - DYNAMIC_BMI2 is enabled
|
||||
* - BMI2 is supported at compile time
|
||||
*/
|
||||
#if !defined(ZSTD_DISABLE_ASM) && \
|
||||
ZSTD_ASM_SUPPORTED && \
|
||||
defined(__x86_64__) && \
|
||||
(DYNAMIC_BMI2 || defined(__BMI2__))
|
||||
# define ZSTD_ENABLE_ASM_X86_64_BMI2 1
|
||||
#else
|
||||
# define ZSTD_ENABLE_ASM_X86_64_BMI2 0
|
||||
#endif
|
||||
|
||||
#endif /* ZSTD_PORTABILITY_MACROS_H */
|
|
@ -43,31 +43,7 @@
|
|||
#error "Cannot force the use of the X1 and X2 decoders at the same time!"
|
||||
#endif
|
||||
|
||||
/* Only use assembly on Linux / MacOS.
|
||||
* Disable when MSAN is enabled.
|
||||
*/
|
||||
#if defined(__linux__) || defined(__linux) || defined(__APPLE__)
|
||||
# if ZSTD_MEMORY_SANITIZER
|
||||
# define HUF_ASM_SUPPORTED 0
|
||||
# elif ZSTD_DATAFLOW_SANITIZER
|
||||
# define HUF_ASM_SUPPORTED 0
|
||||
# else
|
||||
# define HUF_ASM_SUPPORTED 1
|
||||
# endif
|
||||
#else
|
||||
# define HUF_ASM_SUPPORTED 0
|
||||
#endif
|
||||
|
||||
/* HUF_DISABLE_ASM: Disables all ASM implementations. */
|
||||
#if !defined(HUF_DISABLE_ASM) && \
|
||||
HUF_ASM_SUPPORTED && \
|
||||
defined(__x86_64__) && (DYNAMIC_BMI2 || defined(__BMI2__))
|
||||
# define HUF_ENABLE_ASM_X86_64_BMI2 1
|
||||
#else
|
||||
# define HUF_ENABLE_ASM_X86_64_BMI2 0
|
||||
#endif
|
||||
|
||||
#if HUF_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2
|
||||
# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
|
||||
#else
|
||||
# define HUF_ASM_X86_64_BMI2_ATTRS
|
||||
|
@ -80,13 +56,13 @@
|
|||
#endif
|
||||
#define HUF_ASM_DECL HUF_EXTERN_C
|
||||
|
||||
#if DYNAMIC_BMI2 || (HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
|
||||
#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
|
||||
# define HUF_NEED_BMI2_FUNCTION 1
|
||||
#else
|
||||
# define HUF_NEED_BMI2_FUNCTION 0
|
||||
#endif
|
||||
|
||||
#if !(HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
|
||||
#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
|
||||
# define HUF_NEED_DEFAULT_FUNCTION 1
|
||||
#else
|
||||
# define HUF_NEED_DEFAULT_FUNCTION 0
|
||||
|
@ -162,7 +138,7 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
|
|||
return dtd;
|
||||
}
|
||||
|
||||
#if HUF_ENABLE_ASM_X86_64_BMI2
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
|
||||
static size_t HUF_initDStream(BYTE const* ip) {
|
||||
BYTE const lastByte = ip[7];
|
||||
|
@ -685,7 +661,7 @@ size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize,
|
|||
}
|
||||
#endif
|
||||
|
||||
#if HUF_ENABLE_ASM_X86_64_BMI2
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
|
||||
HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args);
|
||||
|
||||
|
@ -741,7 +717,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
|
|||
/* decoded size */
|
||||
return dstSize;
|
||||
}
|
||||
#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */
|
||||
#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
|
||||
|
||||
typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
|
||||
const void *cSrc,
|
||||
|
@ -755,7 +731,7 @@ static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize,
|
|||
{
|
||||
#if DYNAMIC_BMI2
|
||||
if (bmi2) {
|
||||
# if HUF_ENABLE_ASM_X86_64_BMI2
|
||||
# if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
# else
|
||||
return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
|
@ -765,7 +741,7 @@ static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize,
|
|||
(void)bmi2;
|
||||
#endif
|
||||
|
||||
#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
|
||||
return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
#else
|
||||
return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
|
@ -1401,7 +1377,7 @@ size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize,
|
|||
}
|
||||
#endif
|
||||
|
||||
#if HUF_ENABLE_ASM_X86_64_BMI2
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
|
||||
HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args);
|
||||
|
||||
|
@ -1453,14 +1429,14 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm(
|
|||
/* decoded size */
|
||||
return dstSize;
|
||||
}
|
||||
#endif /* HUF_ENABLE_ASM_X86_64_BMI2 */
|
||||
#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
|
||||
|
||||
static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
|
||||
size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
|
||||
{
|
||||
#if DYNAMIC_BMI2
|
||||
if (bmi2) {
|
||||
# if HUF_ENABLE_ASM_X86_64_BMI2
|
||||
# if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
# else
|
||||
return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
|
@ -1470,7 +1446,7 @@ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize,
|
|||
(void)bmi2;
|
||||
#endif
|
||||
|
||||
#if HUF_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
|
||||
return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
#else
|
||||
return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
#if !defined(HUF_DISABLE_ASM) && defined(__x86_64__)
|
||||
#include "../common/portability_macros.h"
|
||||
|
||||
#if ZSTD_ENABLE_ASM_X86_64_BMI2
|
||||
|
||||
/* Stack marking
|
||||
* ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart
|
||||
|
|
|
@ -114,7 +114,7 @@ ZSTD_LEGACY_FILES :=
|
|||
ZSTD_DECOMPRESS_AMD64_ASM_FILES := $(sort $(wildcard $(LIBZSTD)/decompress/*_amd64.S))
|
||||
|
||||
ifneq ($(ZSTD_NO_ASM), 0)
|
||||
CPPFLAGS += -DHUF_DISABLE_ASM
|
||||
CPPFLAGS += -DZSTD_DISABLE_ASM
|
||||
else
|
||||
# Unconditionally add the ASM files they are disabled by
|
||||
# macros in the .S file.
|
||||
|
|
Loading…
Reference in New Issue