diff --git a/lib/zstd/common/compiler.h b/lib/zstd/common/compiler.h --- a/lib/zstd/common/compiler.h +++ b/lib/zstd/common/compiler.h @@ -14,6 +14,7 @@ #include +#include "zstd_deps.h" #include "portability_macros.h" /*-******************************************************* @@ -96,6 +97,17 @@ */ #define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") +#if !DYNAMIC_BMI2 +# define ZSTD_USE_BMI2(bmi2) 0 +# define ZSTD_SET_BMI2(state, value) do { } while (0) +#elif defined(ZSTD_USE_KERNEL_CPU_FEATURES) +# define ZSTD_USE_BMI2(bmi2) cpu_feature_enabled(X86_FEATURE_BMI2) +# define ZSTD_SET_BMI2(state, value) do { } while (0) +#else +# define ZSTD_USE_BMI2(bmi2) (bmi2) +# define ZSTD_SET_BMI2(state, value) do { (state) = (value); } while (0) +#endif + /* prefetch * can be disabled, by declaring NO_PREFETCH build macro */ #if ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c --- a/lib/zstd/common/entropy_common.c +++ b/lib/zstd/common/entropy_common.c @@ -16,6 +16,10 @@ /* ************************************* * Dependencies ***************************************/ +#include "zstd_deps.h" +#if defined(ZSTD_USE_KERNEL_CPU_FEATURES) +#include +#endif #include "mem.h" #include "error_private.h" /* ERR_*, ERROR */ #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ @@ -210,11 +214,9 @@ size_t FSE_readNCount_bmi2( short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize, int bmi2) { -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); } -#endif (void)bmi2; return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); } @@ -335,11 +337,9 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, void* workSpace, size_t wkspSize, int flags) { -#if DYNAMIC_BMI2 - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } -#endif (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } diff --git a/lib/zstd/common/fse_decompress.c b/lib/zstd/common/fse_decompress.c --- a/lib/zstd/common/fse_decompress.c +++ b/lib/zstd/common/fse_decompress.c @@ -24,6 +24,9 @@ #include "fse.h" #include "error_private.h" #include "zstd_deps.h" /* ZSTD_memcpy */ +#if defined(ZSTD_USE_KERNEL_CPU_FEATURES) +#include +#endif #include "bits.h" /* ZSTD_highbit32 */ @@ -306,11 +309,9 @@ BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, siz size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) { -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } -#endif (void)bmi2; return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } diff --git a/lib/zstd/common/zstd_deps.h b/lib/zstd/common/zstd_deps.h --- a/lib/zstd/common/zstd_deps.h +++ b/lib/zstd/common/zstd_deps.h @@ -26,6 +26,11 @@ #ifndef ZSTD_DEPS_COMMON #define ZSTD_DEPS_COMMON +#if defined(__KERNEL__) && defined(CONFIG_X86) && \ + !defined(__DISABLE_EXPORTS) +#define ZSTD_USE_KERNEL_CPU_FEATURES +#endif + #include #include diff --git a/lib/zstd/compress/huf_compress.c b/lib/zstd/compress/huf_compress.c --- a/lib/zstd/compress/huf_compress.c +++ b/lib/zstd/compress/huf_compress.c @@ -22,6 +22,9 @@ * Includes ****************************************************************/ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#if defined(ZSTD_USE_KERNEL_CPU_FEATURES) +#include +#endif #include "../common/compiler.h" #include "../common/bitstream.h" #include "hist.h" @@ -1138,9 +1141,10 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, const int flags) { - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); } + (void)flags; return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); } diff --git a/lib/zstd/compress/zstd_compress.c b/lib/zstd/compress/zstd_compress.c --- a/lib/zstd/compress/zstd_compress.c +++ b/lib/zstd/compress/zstd_compress.c @@ -102,7 +102,7 @@ static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) assert(cctx != NULL); ZSTD_memset(cctx, 0, sizeof(*cctx)); cctx->customMem = memManager; - cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + ZSTD_SET_BMI2(cctx->bmi2, ZSTD_cpuSupportsBmi2()); { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); assert(!ZSTD_isError(err)); (void)err; @@ -142,7 +142,7 @@ ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); cctx->tmpWorkspace = ZSTD_cwksp_reserve_object(&cctx->workspace, TMP_WORKSPACE_SIZE); cctx->tmpWkspSize = TMP_WORKSPACE_SIZE; - cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + ZSTD_SET_BMI2(cctx->bmi2, ZSTD_cpuid_bmi2(ZSTD_cpuid())); return cctx; } @@ -4042,7 +4042,7 @@ ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, srcSize, zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, - zc->bmi2); + ZSTD_CCtx_get_bmi2(zc)); FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); if (!zc->isFirstBlock && @@ -4332,7 +4332,7 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc, dst, dstCapacity, srcSize, zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */, - zc->bmi2); + ZSTD_CCtx_get_bmi2(zc)); if (frame && /* We don't want to emit our first block as a RLE even if it qualifies because @@ -6796,7 +6796,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, blockSize, cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, - cctx->bmi2); + ZSTD_CCtx_get_bmi2(cctx)); FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); @@ -7321,7 +7321,7 @@ ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx, &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, &cctx->appliedParams, cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, - cctx->bmi2); + ZSTD_CCtx_get_bmi2(cctx)); FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); /* note: the spec forbids for any compressed block to be larger than maximum block size */ if (compressedSeqsSize > cctx->blockSizeMax) compressedSeqsSize = 0; diff --git a/lib/zstd/compress/zstd_compress_internal.h b/lib/zstd/compress/zstd_compress_internal.h --- a/lib/zstd/compress/zstd_compress_internal.h +++ b/lib/zstd/compress/zstd_compress_internal.h @@ -537,6 +537,15 @@ struct ZSTD_CCtx_s { size_t extSeqBufCapacity; }; +MEM_STATIC int ZSTD_CCtx_get_bmi2(const struct ZSTD_CCtx_s *cctx) { +#if DYNAMIC_BMI2 && !defined(ZSTD_USE_KERNEL_CPU_FEATURES) + return cctx->bmi2; +#else + (void)cctx; + return 0; +#endif +} + typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e; typedef enum { ZSTD_tfp_forCCtx, ZSTD_tfp_forCDict } ZSTD_tableFillPurpose_e; diff --git a/lib/zstd/compress/zstd_compress_sequences.c b/lib/zstd/compress/zstd_compress_sequences.c --- a/lib/zstd/compress/zstd_compress_sequences.c +++ b/lib/zstd/compress/zstd_compress_sequences.c @@ -12,6 +12,10 @@ /*-************************************* * Dependencies ***************************************/ +#include "../common/zstd_deps.h" +#if defined(ZSTD_USE_KERNEL_CPU_FEATURES) +#include +#endif #include "zstd_compress_sequences.h" /* @@ -429,15 +433,13 @@ size_t ZSTD_encodeSequences( SeqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) { DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { return ZSTD_encodeSequences_bmi2(dst, dstCapacity, CTable_MatchLength, mlCodeTable, CTable_OffsetBits, ofCodeTable, CTable_LitLength, llCodeTable, sequences, nbSeq, longOffsets); } -#endif (void)bmi2; return ZSTD_encodeSequences_default(dst, dstCapacity, CTable_MatchLength, mlCodeTable, diff --git a/lib/zstd/compress/zstd_compress_superblock.c b/lib/zstd/compress/zstd_compress_superblock.c --- a/lib/zstd/compress/zstd_compress_superblock.c +++ b/lib/zstd/compress/zstd_compress_superblock.c @@ -684,6 +684,6 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, &zc->appliedParams, dst, dstCapacity, src, srcSize, - zc->bmi2, lastBlock, + ZSTD_CCtx_get_bmi2(zc), lastBlock, zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */); } diff --git a/lib/zstd/decompress/huf_decompress.c b/lib/zstd/decompress/huf_decompress.c --- a/lib/zstd/decompress/huf_decompress.c +++ b/lib/zstd/decompress/huf_decompress.c @@ -17,6 +17,9 @@ * Dependencies ****************************************************************/ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#if defined(ZSTD_USE_KERNEL_CPU_FEATURES) +#include +#endif #include "../common/compiler.h" #include "../common/bitstream.h" /* BIT_* */ #include "../common/fse.h" /* to compress headers */ @@ -113,9 +116,10 @@ typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - if (flags & HUF_flags_bmi2) { \ + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { \ return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ } \ + (void)flags; \ return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ } @@ -899,18 +903,16 @@ static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; -#if DYNAMIC_BMI2 - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; # if ZSTD_ENABLE_ASM_X86_64_BMI2 if (!(flags & HUF_flags_disableAsm)) { loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; } # endif - } else { + } else if (DYNAMIC_BMI2) { return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) if (!(flags & HUF_flags_disableAsm)) { @@ -1723,18 +1725,16 @@ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; -#if DYNAMIC_BMI2 - if (flags & HUF_flags_bmi2) { + if (ZSTD_USE_BMI2(flags & HUF_flags_bmi2)) { fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; # if ZSTD_ENABLE_ASM_X86_64_BMI2 if (!(flags & HUF_flags_disableAsm)) { loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; } # endif - } else { + } else if (DYNAMIC_BMI2) { return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) if (!(flags & HUF_flags_disableAsm)) { diff --git a/lib/zstd/decompress/zstd_decompress.c b/lib/zstd/decompress/zstd_decompress.c --- a/lib/zstd/decompress/zstd_decompress.c +++ b/lib/zstd/decompress/zstd_decompress.c @@ -259,9 +259,7 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) dctx->noForwardProgress = 0; dctx->oversizedDuration = 0; dctx->isFrameDecompression = 1; -#if DYNAMIC_BMI2 - dctx->bmi2 = ZSTD_cpuSupportsBmi2(); -#endif + ZSTD_SET_BMI2(dctx->bmi2, ZSTD_cpuSupportsBmi2()); dctx->ddictSet = NULL; ZSTD_DCtx_resetParameters(dctx); #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION diff --git a/lib/zstd/decompress/zstd_decompress_block.c b/lib/zstd/decompress/zstd_decompress_block.c --- a/lib/zstd/decompress/zstd_decompress_block.c +++ b/lib/zstd/decompress/zstd_decompress_block.c @@ -16,6 +16,9 @@ * Dependencies *********************************************************/ #include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#if defined(ZSTD_USE_KERNEL_CPU_FEATURES) +#include +#endif #include "../common/compiler.h" /* prefetch */ #include "../common/cpu.h" /* bmi2 */ #include "../common/mem.h" /* low level memory routines */ @@ -631,13 +634,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2) { -#if DYNAMIC_BMI2 - if (bmi2) { + if (ZSTD_USE_BMI2(bmi2)) { ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); return; } -#endif (void)bmi2; ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize); @@ -1955,11 +1956,9 @@ ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequences"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { + if (ZSTD_USE_BMI2(ZSTD_DCtx_get_bmi2(dctx))) { return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } -#endif return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } static size_t @@ -1968,11 +1967,9 @@ ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDst const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { + if (ZSTD_USE_BMI2(ZSTD_DCtx_get_bmi2(dctx))) { return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } -#endif return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ @@ -1991,11 +1988,9 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, const ZSTD_longOffset_e isLongOffset) { DEBUGLOG(5, "ZSTD_decompressSequencesLong"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { + if (ZSTD_USE_BMI2(ZSTD_DCtx_get_bmi2(dctx))) { return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } -#endif return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ diff --git a/lib/zstd/decompress/zstd_decompress_internal.h b/lib/zstd/decompress/zstd_decompress_internal.h --- a/lib/zstd/decompress/zstd_decompress_internal.h +++ b/lib/zstd/decompress/zstd_decompress_internal.h @@ -204,7 +204,7 @@ struct ZSTD_DCtx_s }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { -#if DYNAMIC_BMI2 +#if DYNAMIC_BMI2 && !defined(ZSTD_USE_KERNEL_CPU_FEATURES) return dctx->bmi2; #else (void)dctx;