diff --git a/mindspore/lite/nnacl/CMakeLists.txt b/mindspore/lite/nnacl/CMakeLists.txt index cfcf2fa94d..875644d1a4 100644 --- a/mindspore/lite/nnacl/CMakeLists.txt +++ b/mindspore/lite/nnacl/CMakeLists.txt @@ -9,6 +9,10 @@ if (PLATFORM_ARM32 OR PLATFORM_ARM64) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fomit-frame-pointer -fstrict-aliasing -ffunction-sections -fdata-sections -ffast-math") endif() endif () +if ("${X86_64_SIMD}" STREQUAL "avx") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.1 -mavx -mavx2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.1 -mavx -mavx2") +endif () ########################### files ########################### file(GLOB KERNEL_SRC @@ -39,6 +43,7 @@ endif() if ("${X86_64_SIMD}" STREQUAL "avx") file(GLOB ASSEMBLY_SRC ${NNACL_DIR}/x86_64_sse/*.c + ${NNACL_DIR}/x86_64_avx/*.c ${NNACL_DIR}/assembly/avx/*.S) set_property(SOURCE ${ASSEMBLY_SRC} PROPERTY LANGUAGE C) endif() diff --git a/mindspore/lite/nnacl/int8/add_int8.c b/mindspore/lite/nnacl/int8/add_int8.c index aacbf42c25..662b47ab7c 100644 --- a/mindspore/lite/nnacl/int8/add_int8.c +++ b/mindspore/lite/nnacl/int8/add_int8.c @@ -18,16 +18,19 @@ #ifdef ENABLE_NEON #include #endif +#ifdef ENABLE_AVX +#include +#include "nnacl/x86_64_avx/common_utils.h" +#endif #include "nnacl/int8/fixed_point.h" void AddInt8(const int8_t *input0, const int8_t *input1, int8_t *output, int size, AddQuantParameter *params) { int in0_left_shift = (1 << params->left_shift_) * (1 << params->in0_args_.left_shift_); int in1_left_shift = (1 << params->left_shift_) * (1 << params->in1_args_.left_shift_); int index = 0; - #ifdef ENABLE_ARM const int8x16_t min_vec = vdupq_n_s8(params->min_); - const int8x16_t max_vac = vdupq_n_s8(params->max_); + const int8x16_t max_vec = vdupq_n_s8(params->max_); const int16x8_t in0_zp_vec = vdupq_n_s16(params->in0_args_.zp_); const int16x8_t in1_zp_vec = vdupq_n_s16(params->in1_args_.zp_); @@ -142,12 +145,11 @@ void AddInt8(const int8_t *input0, const int8_t *input1, int8_t *output, int siz const int16x8_t out_s16_2 = vaddq_s16(vcombine_s16(out3_s16, out4_s16), out_zp_vec); const int8x16_t out = vcombine_s8(vqmovn_s16(out_s16_1), vqmovn_s16(out_s16_2)); - const int8x16_t int8_out = vmaxq_s8(min_vec, vminq_s8(max_vac, out)); + const int8x16_t int8_out = vmaxq_s8(min_vec, vminq_s8(max_vec, out)); vst1q_s8(output + index, int8_out); } #endif - for (; index < size; index++) { const int32_t in0_left = (input0[index] + params->in0_args_.zp_) * in0_left_shift; const int32_t in1_left = (input1[index] + params->in1_args_.zp_) * in1_left_shift; @@ -173,7 +175,7 @@ void AddOptInt8(const int8_t *ptr_in, const int8_t element_in, int8_t *output, i #ifdef ENABLE_ARM /* const value init */ const int8x16_t min_vec = vdupq_n_s8(params->min_); - const int8x16_t max_vac = vdupq_n_s8(params->max_); + const int8x16_t max_vec = vdupq_n_s8(params->max_); const int16x8_t ptr_zp_vec = vdupq_n_s16(ptr_args->zp_); const int16x8_t ele_zp_vec = vdupq_n_s16(ele_args->zp_); @@ -293,7 +295,7 @@ void AddOptInt8(const int8_t *ptr_in, const int8_t element_in, int8_t *output, i const int16x8_t out_s16_2 = vaddq_s16(vcombine_s16(out3_s16, out4_s16), out_zp_vec); const int8x16_t out = vcombine_s8(vqmovn_s16(out_s16_1), vqmovn_s16(out_s16_2)); - const int8x16_t int8_out = vmaxq_s8(min_vec, vminq_s8(max_vac, out)); + const int8x16_t int8_out = vmaxq_s8(min_vec, vminq_s8(max_vec, out)); vst1q_s8(output + index, int8_out); } @@ -325,3 +327,357 @@ int BroadcastAddInt8(const int8_t *in0, const int8_t *in1, int8_t *tile_in0, int TileDimensionsInt8(in0, in1, tile_in0, tile_in1, param); return ElementAddInt8(tile_in0, tile_in1, out, size); } + +#ifdef ENABLE_AVX +void AddInt8_AVX2(const int8_t *input0, const int8_t *input1, int8_t *output, int size, AddQuantParameter *params) { + const int in0_left_shift = (1 << params->left_shift_) * (1 << params->in0_args_.left_shift_); + const int in1_left_shift = (1 << params->left_shift_) * (1 << params->in1_args_.left_shift_); + const __m128i min_vec = _mm_set1_epi8(params->min_); + const __m128i max_vec = _mm_set1_epi8(params->max_); + const __m128i in0_zp_vec = _mm_set1_epi16(params->in0_args_.zp_); + const __m128i in1_zp_vec = _mm_set1_epi16(params->in1_args_.zp_); + const __m128i out_zp_vec = _mm_set1_epi16(params->out_zp_); + const __m128i in0_left_vec = _mm_set1_epi32(in0_left_shift); + const __m128i in1_left_vec = _mm_set1_epi32(in1_left_shift); + const __m128i in0_multiplier = _mm_set1_epi32(params->in0_args_.multiplier_); + const __m128i in1_multiplier = _mm_set1_epi32(params->in1_args_.multiplier_); + const __m128i out_multiplier = _mm_set1_epi32(params->out_multiplier_); + int index = 0; + for (; index <= size - 16; index += 16) { + const __m128i in0_src = _mm_loadu_si128((__m128i_u *)(input0 + index)); + const __m128i in1_src = _mm_loadu_si128((__m128i_u *)(input1 + index)); + + const __m256i in0_s16 = _mm256_cvtepi8_epi16(in0_src); + const __m128i in0_s16_low = _mm256_extractf128_si256(in0_s16, 0); + const __m128i in0_s16_high = _mm256_extractf128_si256(in0_s16, 1); + const __m256i in1_s16 = _mm256_cvtepi8_epi16(in1_src); + const __m128i in1_s16_low = _mm256_extractf128_si256(in1_s16, 0); + const __m128i in1_s16_high = _mm256_extractf128_si256(in1_s16, 1); + + const __m128i in0_zp_low = _mm_add_epi16(in0_s16_low, in0_zp_vec); + const __m128i in0_zp_high = _mm_add_epi16(in0_s16_high, in0_zp_vec); + const __m128i in1_zp_low = _mm_add_epi16(in1_s16_low, in1_zp_vec); + const __m128i in1_zp_high = _mm_add_epi16(in1_s16_high, in1_zp_vec); + + __m256i tmp_in0 = _mm256_cvtepi16_epi32(in0_zp_low); + __m128i in0_1 = _mm256_extractf128_si256(tmp_in0, 0); + __m128i in0_2 = _mm256_extractf128_si256(tmp_in0, 1); + tmp_in0 = _mm256_cvtepi16_epi32(in0_zp_high); + __m128i in0_3 = _mm256_extractf128_si256(tmp_in0, 0); + __m128i in0_4 = _mm256_extractf128_si256(tmp_in0, 1); + __m256i tmp_in1 = _mm256_cvtepi16_epi32(in1_zp_low); + __m128i in1_1 = _mm256_extractf128_si256(tmp_in1, 0); + __m128i in1_2 = _mm256_extractf128_si256(tmp_in1, 1); + tmp_in1 = _mm256_cvtepi16_epi32(in1_zp_high); + __m128i in1_3 = _mm256_extractf128_si256(tmp_in1, 0); + __m128i in1_4 = _mm256_extractf128_si256(tmp_in1, 1); + + // Apply left shift + in0_1 = _mm_mullo_epi32(in0_1, in0_left_vec); + in0_2 = _mm_mullo_epi32(in0_2, in0_left_vec); + in0_3 = _mm_mullo_epi32(in0_3, in0_left_vec); + in0_4 = _mm_mullo_epi32(in0_4, in0_left_vec); + in1_1 = _mm_mullo_epi32(in1_1, in1_left_vec); + in1_2 = _mm_mullo_epi32(in1_2, in1_left_vec); + in1_3 = _mm_mullo_epi32(in1_3, in1_left_vec); + in1_4 = _mm_mullo_epi32(in1_4, in1_left_vec); + + // Apply the fixed-point part of the multiplier. + in0_1 = _mm_qrdmulh_epi32(in0_1, in0_multiplier); + in0_2 = _mm_qrdmulh_epi32(in0_2, in0_multiplier); + in0_3 = _mm_qrdmulh_epi32(in0_3, in0_multiplier); + in0_4 = _mm_qrdmulh_epi32(in0_4, in0_multiplier); + in1_1 = _mm_qrdmulh_epi32(in1_1, in1_multiplier); + in1_2 = _mm_qrdmulh_epi32(in1_2, in1_multiplier); + in1_3 = _mm_qrdmulh_epi32(in1_3, in1_multiplier); + in1_4 = _mm_qrdmulh_epi32(in1_4, in1_multiplier); + + // Apply right shift + int32_t in0_remainder_mask = (1ll << (params->in0_args_.right_shift_)) - 1; + int32_t in0_remainder_threshold = in0_remainder_mask >> 1; + const __m128i vin0_remainder_mask = _mm_set1_epi32(in0_remainder_mask); + const __m128i vin0_remainder_threshold = _mm_set1_epi32(in0_remainder_threshold); + const __m128i in0_1_remainder = + _mm_add_epi32(_mm_and_si128(in0_1, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_1)); + in0_1 = _mm_sub_epi32(_mm_rshr_epi32(in0_1, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_1_remainder, vin0_remainder_threshold)); + const __m128i in0_2_remainder = + _mm_add_epi32(_mm_and_si128(in0_2, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_2)); + in0_2 = _mm_sub_epi32(_mm_rshr_epi32(in0_2, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_2_remainder, vin0_remainder_threshold)); + const __m128i in0_3_remainder = + _mm_add_epi32(_mm_and_si128(in0_3, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_3)); + in0_3 = _mm_sub_epi32(_mm_rshr_epi32(in0_3, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_3_remainder, vin0_remainder_threshold)); + const __m128i in0_4_remainder = + _mm_add_epi32(_mm_and_si128(in0_4, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_4)); + in0_4 = _mm_sub_epi32(_mm_rshr_epi32(in0_4, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_4_remainder, vin0_remainder_threshold)); + + int32_t in1_remainder_mask = (1ll << (params->in1_args_.right_shift_)) - 1; + int32_t in1_remainder_threshold = in1_remainder_mask >> 1; + const __m128i vin1_remainder_mask = _mm_set1_epi32(in1_remainder_mask); + const __m128i vin1_remainder_threshold = _mm_set1_epi32(in1_remainder_threshold); + const __m128i in1_1_remainder = + _mm_add_epi32(_mm_and_si128(in1_1, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_1)); + in1_1 = _mm_sub_epi32(_mm_rshr_epi32(in1_1, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_1_remainder, vin1_remainder_threshold)); + const __m128i in1_2_remainder = + _mm_add_epi32(_mm_and_si128(in1_2, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_2)); + in1_2 = _mm_sub_epi32(_mm_rshr_epi32(in1_2, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_2_remainder, vin1_remainder_threshold)); + const __m128i in1_3_remainder = + _mm_add_epi32(_mm_and_si128(in1_3, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_3)); + in1_3 = _mm_sub_epi32(_mm_rshr_epi32(in1_3, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_3_remainder, vin1_remainder_threshold)); + const __m128i in1_4_remainder = + _mm_add_epi32(_mm_and_si128(in1_4, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_4)); + in1_4 = _mm_sub_epi32(_mm_rshr_epi32(in1_4, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_4_remainder, vin1_remainder_threshold)); + + /* calculate output */ + __m128i out1 = _mm_add_epi32(in0_1, in1_1); + __m128i out2 = _mm_add_epi32(in0_2, in1_2); + __m128i out3 = _mm_add_epi32(in0_3, in1_3); + __m128i out4 = _mm_add_epi32(in0_4, in1_4); + + // Apply left shift + out1 = _mm_slli_epi32(out1, params->out_left_shift_); + out2 = _mm_slli_epi32(out2, params->out_left_shift_); + out3 = _mm_slli_epi32(out3, params->out_left_shift_); + out4 = _mm_slli_epi32(out4, params->out_left_shift_); + + // Apply the fixed-point part of the multiplier. + out1 = _mm_qrdmulh_epi32(out1, out_multiplier); + out2 = _mm_qrdmulh_epi32(out2, out_multiplier); + out3 = _mm_qrdmulh_epi32(out3, out_multiplier); + out4 = _mm_qrdmulh_epi32(out4, out_multiplier); + + // Apply right shift + int32_t out_remainder_mask = (1ll << (params->out_right_shift_)) - 1; + int32_t out_remainder_threshold = out_remainder_mask >> 1; + const __m128i vout_remainder_mask = _mm_set1_epi32(out_remainder_mask); + const __m128i vout_remainder_threshold = _mm_set1_epi32(out_remainder_threshold); + const __m128i out1_remainder = + _mm_add_epi32(_mm_and_si128(out1, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out1)); + out1 = _mm_sub_epi32(_mm_rshr_epi32(out1, params->out_right_shift_), + _mm_cmpgt_epi32(out1_remainder, vout_remainder_threshold)); + const __m128i out2_remainder = + _mm_add_epi32(_mm_and_si128(out2, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out2)); + out2 = _mm_sub_epi32(_mm_rshr_epi32(out2, params->out_right_shift_), + _mm_cmpgt_epi32(out2_remainder, vout_remainder_threshold)); + const __m128i out3_remainder = + _mm_add_epi32(_mm_and_si128(out3, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out3)); + out3 = _mm_sub_epi32(_mm_rshr_epi32(out3, params->out_right_shift_), + _mm_cmpgt_epi32(out3_remainder, vout_remainder_threshold)); + const __m128i out4_remainder = + _mm_add_epi32(_mm_and_si128(out4, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out4)); + out4 = _mm_sub_epi32(_mm_rshr_epi32(out4, params->out_right_shift_), + _mm_cmpgt_epi32(out4_remainder, vout_remainder_threshold)); + + __m128i out1_s16 = _mm_packs_epi32(out1, out2); + __m128i out2_s16 = _mm_packs_epi32(out3, out4); + + __m128i out_s16_1 = _mm_add_epi16(out1_s16, out_zp_vec); + __m128i out_s16_2 = _mm_add_epi16(out2_s16, out_zp_vec); + __m128i out = _mm_packs_epi16(out_s16_1, out_s16_2); + __m128i int8_out = _mm_max_epi8(min_vec, _mm_min_epi8(max_vec, out)); + + _mm_storeu_si128((__m128i_u *)(output + index), int8_out); + } + for (; index < size; index++) { + const int32_t in0_left = (input0[index] + params->in0_args_.zp_) * in0_left_shift; + const int32_t in1_left = (input1[index] + params->in1_args_.zp_) * in1_left_shift; + const int32_t in0 = + MultiplyByMultiplierAndRightShift(in0_left, params->in0_args_.multiplier_, params->in0_args_.right_shift_); + const int32_t in1 = + MultiplyByMultiplierAndRightShift(in1_left, params->in1_args_.multiplier_, params->in1_args_.right_shift_); + + int32_t out = MultiplyByQuantizedMultiplier(in0 + in1, params->out_multiplier_, params->out_left_shift_, + -params->out_right_shift_); + out += params->out_zp_; + output[index] = (int8_t)MSMAX(params->min_, MSMIN(out, params->max_)); + } + return; +} + +void AddOptInt8_AVX2(const int8_t *ptr_in, const int8_t element_in, int8_t *output, int size, AddQuantParameter *params, + AddQuantQrgs *ptr_args, AddQuantQrgs *ele_args) { + // input0: ptr_in + // input1: element_in + // load quant parameters of input0 and input1 + const int in0_left_shift = (1 << params->left_shift_) * (1 << ptr_args->left_shift_); + const int in1_left_shift = (1 << params->left_shift_) * (1 << ele_args->left_shift_); + const __m128i min_vec = _mm_set1_epi8(params->min_); + const __m128i max_vec = _mm_set1_epi8(params->max_); + const __m128i in0_zp_vec = _mm_set1_epi16(ptr_args->zp_); + const __m128i in1_zp_vec = _mm_set1_epi16(ele_args->zp_); + const __m128i out_zp_vec = _mm_set1_epi16(params->out_zp_); + const __m128i in0_left_vec = _mm_set1_epi32(in0_left_shift); + const __m128i in1_left_vec = _mm_set1_epi32(in1_left_shift); + const __m128i in0_multiplier = _mm_set1_epi32(params->in0_args_.multiplier_); + const __m128i in1_multiplier = _mm_set1_epi32(params->in1_args_.multiplier_); + const __m128i out_multiplier = _mm_set1_epi32(params->out_multiplier_); + + // input1 can be processed once because it is const + const __m128i in1_src = _mm_set1_epi8(element_in); + const __m256i in1_s16 = _mm256_cvtepi8_epi16(in1_src); + const __m128i in1_s16_low = _mm256_extractf128_si256(in1_s16, 0); + const __m128i in1_s16_high = _mm256_extractf128_si256(in1_s16, 1); + const __m128i in1_zp_low = _mm_add_epi16(in1_s16_low, in1_zp_vec); + const __m128i in1_zp_high = _mm_add_epi16(in1_s16_high, in1_zp_vec); + __m256i tmp_in1 = _mm256_cvtepi16_epi32(in1_zp_low); + __m128i in1_1 = _mm256_extractf128_si256(tmp_in1, 0); + __m128i in1_2 = _mm256_extractf128_si256(tmp_in1, 1); + tmp_in1 = _mm256_cvtepi16_epi32(in1_zp_high); + __m128i in1_3 = _mm256_extractf128_si256(tmp_in1, 0); + __m128i in1_4 = _mm256_extractf128_si256(tmp_in1, 1); + + // Apply left shift + in1_1 = _mm_mullo_epi32(in1_1, in1_left_vec); + in1_2 = _mm_mullo_epi32(in1_2, in1_left_vec); + in1_3 = _mm_mullo_epi32(in1_3, in1_left_vec); + in1_4 = _mm_mullo_epi32(in1_4, in1_left_vec); + + // Apply the fixed-point part of the multiplier. + in1_1 = _mm_qrdmulh_epi32(in1_1, in1_multiplier); + in1_2 = _mm_qrdmulh_epi32(in1_2, in1_multiplier); + in1_3 = _mm_qrdmulh_epi32(in1_3, in1_multiplier); + in1_4 = _mm_qrdmulh_epi32(in1_4, in1_multiplier); + + // Apply right shift + int32_t in1_remainder_mask = (1ll << (params->in1_args_.right_shift_)) - 1; + int32_t in1_remainder_threshold = in1_remainder_mask >> 1; + const __m128i vin1_remainder_mask = _mm_set1_epi32(in1_remainder_mask); + const __m128i vin1_remainder_threshold = _mm_set1_epi32(in1_remainder_threshold); + const __m128i in1_1_remainder = + _mm_add_epi32(_mm_and_si128(in1_1, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_1)); + in1_1 = _mm_sub_epi32(_mm_rshr_epi32(in1_1, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_1_remainder, vin1_remainder_threshold)); + const __m128i in1_2_remainder = + _mm_add_epi32(_mm_and_si128(in1_2, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_2)); + in1_2 = _mm_sub_epi32(_mm_rshr_epi32(in1_2, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_2_remainder, vin1_remainder_threshold)); + const __m128i in1_3_remainder = + _mm_add_epi32(_mm_and_si128(in1_3, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_3)); + in1_3 = _mm_sub_epi32(_mm_rshr_epi32(in1_3, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_3_remainder, vin1_remainder_threshold)); + const __m128i in1_4_remainder = + _mm_add_epi32(_mm_and_si128(in1_4, vin1_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in1_4)); + in1_4 = _mm_sub_epi32(_mm_rshr_epi32(in1_4, params->in1_args_.right_shift_), + _mm_cmpgt_epi32(in1_4_remainder, vin1_remainder_threshold)); + + int index = 0; + for (; index <= size - 16; index += 16) { + const __m128i in0_src = _mm_loadu_si128((__m128i_u *)(ptr_in + index)); + const __m256i in0_s16 = _mm256_cvtepi8_epi16(in0_src); + const __m128i in0_s16_low = _mm256_extractf128_si256(in0_s16, 0); + const __m128i in0_s16_high = _mm256_extractf128_si256(in0_s16, 1); + const __m128i in0_zp_low = _mm_add_epi16(in0_s16_low, in0_zp_vec); + const __m128i in0_zp_high = _mm_add_epi16(in0_s16_high, in0_zp_vec); + + __m256i tmp_in0 = _mm256_cvtepi16_epi32(in0_zp_low); + __m128i in0_1 = _mm256_extractf128_si256(tmp_in0, 0); + __m128i in0_2 = _mm256_extractf128_si256(tmp_in0, 1); + tmp_in0 = _mm256_cvtepi16_epi32(in0_zp_high); + __m128i in0_3 = _mm256_extractf128_si256(tmp_in0, 0); + __m128i in0_4 = _mm256_extractf128_si256(tmp_in0, 1); + + // Apply left shift + in0_1 = _mm_mullo_epi32(in0_1, in0_left_vec); + in0_2 = _mm_mullo_epi32(in0_2, in0_left_vec); + in0_3 = _mm_mullo_epi32(in0_3, in0_left_vec); + in0_4 = _mm_mullo_epi32(in0_4, in0_left_vec); + + // Apply the fixed-point part of the multiplier. + in0_1 = _mm_qrdmulh_epi32(in0_1, in0_multiplier); + in0_2 = _mm_qrdmulh_epi32(in0_2, in0_multiplier); + in0_3 = _mm_qrdmulh_epi32(in0_3, in0_multiplier); + in0_4 = _mm_qrdmulh_epi32(in0_4, in0_multiplier); + + // Apply right shift + int32_t in0_remainder_mask = (1ll << (params->in0_args_.right_shift_)) - 1; + int32_t in0_remainder_threshold = in0_remainder_mask >> 1; + const __m128i vin0_remainder_mask = _mm_set1_epi32(in0_remainder_mask); + const __m128i vin0_remainder_threshold = _mm_set1_epi32(in0_remainder_threshold); + const __m128i in0_1_remainder = + _mm_add_epi32(_mm_and_si128(in0_1, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_1)); + in0_1 = _mm_sub_epi32(_mm_rshr_epi32(in0_1, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_1_remainder, vin0_remainder_threshold)); + const __m128i in0_2_remainder = + _mm_add_epi32(_mm_and_si128(in0_2, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_2)); + in0_2 = _mm_sub_epi32(_mm_rshr_epi32(in0_2, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_2_remainder, vin0_remainder_threshold)); + const __m128i in0_3_remainder = + _mm_add_epi32(_mm_and_si128(in0_3, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_3)); + in0_3 = _mm_sub_epi32(_mm_rshr_epi32(in0_3, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_3_remainder, vin0_remainder_threshold)); + const __m128i in0_4_remainder = + _mm_add_epi32(_mm_and_si128(in0_4, vin0_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), in0_4)); + in0_4 = _mm_sub_epi32(_mm_rshr_epi32(in0_4, params->in0_args_.right_shift_), + _mm_cmpgt_epi32(in0_4_remainder, vin0_remainder_threshold)); + + /* calculate output */ + __m128i out1 = _mm_add_epi32(in0_1, in1_1); + __m128i out2 = _mm_add_epi32(in0_2, in1_2); + __m128i out3 = _mm_add_epi32(in0_3, in1_3); + __m128i out4 = _mm_add_epi32(in0_4, in1_4); + + // Apply left shift + out1 = _mm_slli_epi32(out1, params->out_left_shift_); + out2 = _mm_slli_epi32(out2, params->out_left_shift_); + out3 = _mm_slli_epi32(out3, params->out_left_shift_); + out4 = _mm_slli_epi32(out4, params->out_left_shift_); + + // Apply the fixed-point part of the multiplier. + out1 = _mm_qrdmulh_epi32(out1, out_multiplier); + out2 = _mm_qrdmulh_epi32(out2, out_multiplier); + out3 = _mm_qrdmulh_epi32(out3, out_multiplier); + out4 = _mm_qrdmulh_epi32(out4, out_multiplier); + + // Apply right shift + int32_t out_remainder_mask = (1ll << (params->out_right_shift_)) - 1; + int32_t out_remainder_threshold = out_remainder_mask >> 1; + const __m128i vout_remainder_mask = _mm_set1_epi32(out_remainder_mask); + const __m128i vout_remainder_threshold = _mm_set1_epi32(out_remainder_threshold); + const __m128i out1_remainder = + _mm_add_epi32(_mm_and_si128(out1, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out1)); + out1 = _mm_sub_epi32(_mm_rshr_epi32(out1, params->out_right_shift_), + _mm_cmpgt_epi32(out1_remainder, vout_remainder_threshold)); + const __m128i out2_remainder = + _mm_add_epi32(_mm_and_si128(out2, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out2)); + out2 = _mm_sub_epi32(_mm_rshr_epi32(out2, params->out_right_shift_), + _mm_cmpgt_epi32(out2_remainder, vout_remainder_threshold)); + const __m128i out3_remainder = + _mm_add_epi32(_mm_and_si128(out3, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out3)); + out3 = _mm_sub_epi32(_mm_rshr_epi32(out3, params->out_right_shift_), + _mm_cmpgt_epi32(out3_remainder, vout_remainder_threshold)); + const __m128i out4_remainder = + _mm_add_epi32(_mm_and_si128(out4, vout_remainder_mask), _mm_cmpgt_epi32(_mm_setzero_si128(), out4)); + out4 = _mm_sub_epi32(_mm_rshr_epi32(out4, params->out_right_shift_), + _mm_cmpgt_epi32(out4_remainder, vout_remainder_threshold)); + + __m128i out1_s16 = _mm_packs_epi32(out1, out2); + __m128i out2_s16 = _mm_packs_epi32(out3, out4); + + __m128i out_s16_1 = _mm_add_epi16(out1_s16, out_zp_vec); + __m128i out_s16_2 = _mm_add_epi16(out2_s16, out_zp_vec); + __m128i out = _mm_packs_epi16(out_s16_1, out_s16_2); + __m128i int8_out = _mm_max_epi8(min_vec, _mm_min_epi8(max_vec, out)); + + _mm_storeu_si128((__m128i_u *)(output + index), int8_out); + } + for (; index < size; index++) { + const int32_t in0_left = (ptr_in[index] + ptr_args->zp_) * in0_left_shift; + const int32_t in1_left = (element_in + ele_args->zp_) * in1_left_shift; + const int32_t in0 = MultiplyByMultiplierAndRightShift(in0_left, ptr_args->multiplier_, ptr_args->right_shift_); + const int32_t in1 = MultiplyByMultiplierAndRightShift(in1_left, ele_args->multiplier_, ele_args->right_shift_); + + int32_t out = MultiplyByQuantizedMultiplier(in0 + in1, params->out_multiplier_, params->out_left_shift_, + -params->out_right_shift_); + out += params->out_zp_; + output[index] = (int8_t)MSMAX(params->min_, MSMIN(out, params->max_)); + } + return; +} +#endif diff --git a/mindspore/lite/nnacl/int8/add_int8.h b/mindspore/lite/nnacl/int8/add_int8.h index f5469b08b9..8663d79a67 100644 --- a/mindspore/lite/nnacl/int8/add_int8.h +++ b/mindspore/lite/nnacl/int8/add_int8.h @@ -17,6 +17,9 @@ #ifndef MINDSPORE_LITE_NNACL_ADD_INT8_H_ #define MINDSPORE_LITE_NNACL_ADD_INT8_H_ +#ifdef ENABLE_AVX +#include +#endif #include "nnacl/op_base.h" #include "nnacl/errorcode.h" #include "nnacl/arithmetic.h" @@ -48,13 +51,21 @@ extern "C" { #endif void AddInt8(const int8_t *input0, const int8_t *input1, int8_t *output, int size, AddQuantParameter *params); + void AddOptInt8(const int8_t *ptr_in, const int8_t element_in, int8_t *output, int size, AddQuantParameter *params, AddQuantQrgs *ptr_args, AddQuantQrgs *ele_args); int ElementAddInt8(const int8_t *in0, const int8_t *in1, int8_t *out, int size); + int BroadcastAddInt8(const int8_t *in0, const int8_t *in1, int8_t *tile_in0, int8_t *tile_in1, int8_t *out, int size, ArithmeticParameter *param); +#ifdef ENABLE_AVX +void AddInt8_AVX2(const int8_t *input0, const int8_t *input1, int8_t *output, int size, AddQuantParameter *params); + +void AddOptInt8_AVX2(const int8_t *ptr_in, const int8_t element_in, int8_t *output, int size, AddQuantParameter *params, + AddQuantQrgs *ptr_args, AddQuantQrgs *ele_args); +#endif #ifdef __cplusplus } #endif diff --git a/mindspore/lite/nnacl/x86_64_avx/common_utils.c b/mindspore/lite/nnacl/x86_64_avx/common_utils.c new file mode 100644 index 0000000000..4a10a3573d --- /dev/null +++ b/mindspore/lite/nnacl/x86_64_avx/common_utils.c @@ -0,0 +1,70 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "nnacl/x86_64_avx/common_utils.h" +#ifdef WIN32 +#ifdef ENABLE_AVX +#include +#endif +#endif + +__m128i _mm_adds_epi32(__m128i a, __m128i b) { + __m128i int_min = _mm_set1_epi32(0x80000000); + __m128i int_max = _mm_set1_epi32(0x7FFFFFFF); + + const __m128i res = _mm_add_epi32(a, b); + const __m128i sign_and = _mm_and_si128(a, b); + const __m128i sign_or = _mm_or_si128(a, b); + + const __m128i min_sat_mask = _mm_andnot_si128(res, sign_and); + const __m128i max_sat_mask = _mm_andnot_si128(sign_or, res); + const __m128 res_temp = + _mm_blendv_ps(_mm_castsi128_ps(res), _mm_castsi128_ps(int_min), _mm_castsi128_ps(min_sat_mask)); + return _mm_castps_si128(_mm_blendv_ps(res_temp, _mm_castsi128_ps(int_max), _mm_castsi128_ps(max_sat_mask))); +} + +__m128i _mm_rshr_epi32(__m128i a, int shift) { + const __m128i vmask = _mm_cmpgt_epi32(_mm_setzero_si128(), a); + const __m128i vabs_a = _mm_sub_epi32(_mm_xor_si128(a, vmask), vmask); + const __m128i tmp_res = _mm_srli_epi32(vabs_a, shift); + return _mm_xor_si128(tmp_res, vmask); +} + +__m128i _mm_qrdmulh_epi32(__m128i a, __m128i b) { + const __m128i tmp_a_lo = _mm_unpacklo_epi32(a, _mm_setzero_si128()); + const __m128i tmp_a_hi = _mm_unpackhi_epi32(a, _mm_setzero_si128()); + const __m256i tmp_a_256 = _mm256_set_m128i(tmp_a_hi, tmp_a_lo); + const __m128i tmp_b_lo = _mm_unpacklo_epi32(b, _mm_setzero_si128()); + const __m128i tmp_b_hi = _mm_unpackhi_epi32(b, _mm_setzero_si128()); + const __m256i tmp_b_256 = _mm256_set_m128i(tmp_b_hi, tmp_b_lo); + __m256i tmp_out = _mm256_mul_epi32(tmp_a_256, tmp_b_256); + tmp_out = _mm256_add_epi64(tmp_out, _mm256_set1_epi64x(1ll << 30)); + const __m256i vmask = _mm256_cmpgt_epi64(_mm256_setzero_si256(), tmp_out); + const __m256i vabs_tmp_out = _mm256_sub_epi64(_mm256_xor_si256(tmp_out, vmask), vmask); + tmp_out = _mm256_srli_epi64(vabs_tmp_out, 31); + const __m256i vtmp_out = _mm256_sub_epi64(_mm256_xor_si256(tmp_out, vmask), vmask); + const int32_t max_32bit = (1ll << 31) - 1; + const int32_t min_32bit = -(1ll << 31); + int64_t *tmp_out_ptr = (int64_t *)(&vtmp_out); + int32_t r1 = tmp_out_ptr[0] > max_32bit ? max_32bit : tmp_out_ptr[0]; + r1 = r1 < min_32bit ? min_32bit : r1; + int32_t r2 = tmp_out_ptr[1] > max_32bit ? max_32bit : tmp_out_ptr[1]; + r2 = r2 < min_32bit ? min_32bit : r2; + int32_t r3 = tmp_out_ptr[2] > max_32bit ? max_32bit : tmp_out_ptr[2]; + r3 = r3 < min_32bit ? min_32bit : r3; + int32_t r4 = tmp_out_ptr[3] > max_32bit ? max_32bit : tmp_out_ptr[3]; + r4 = r4 < min_32bit ? min_32bit : r4; + return _mm_set_epi32(r4, r3, r2, r1); +} diff --git a/mindspore/lite/nnacl/x86_64_avx/common_utils.h b/mindspore/lite/nnacl/x86_64_avx/common_utils.h new file mode 100644 index 0000000000..14589405dc --- /dev/null +++ b/mindspore/lite/nnacl/x86_64_avx/common_utils.h @@ -0,0 +1,44 @@ +/** + * Copyright 2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_NNACL_X86_64_AVX_COMMON_UTILS_H_ +#define MINDSPORE_LITE_NNACL_X86_64_AVX_COMMON_UTILS_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef __GNUC__ +#if __GNUC__ < 8 +#define _mm256_set_m128i(xmm1, xmm2) \ + _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), _mm256_castsi128_si256(xmm2), 2) +#define _mm256_set_m128f(xmm1, xmm2) \ + _mm256_permute2f128_ps(_mm256_castps128_ps256(xmm1), _mm256_castps128_ps256(xmm2), 2) +#endif +#endif + +// Signed saturating Add +__m128i _mm_adds_epi32(__m128i a, __m128i b); + +// Signed rounding shift right +__m128i _mm_rshr_epi32(__m128i a, int shift); + +// Signed saturating Rounding Doubling Multiply return High half +__m128i _mm_qrdmulh_epi32(__m128i a, __m128i b); +#ifdef __cplusplus +} +#endif +#endif // MINDSPORE_LITE_NNACL_X86_64_AVX_COMMON_UTILS_H_ diff --git a/mindspore/lite/src/runtime/kernel/arm/int8/add_int8.cc b/mindspore/lite/src/runtime/kernel/arm/int8/add_int8.cc index c44a4f0b70..1edb796d43 100644 --- a/mindspore/lite/src/runtime/kernel/arm/int8/add_int8.cc +++ b/mindspore/lite/src/runtime/kernel/arm/int8/add_int8.cc @@ -153,7 +153,11 @@ void QuantizedAddCPUKernel::BroadcastRun(int task_id) { cur_in1 = input1_data_ + task_id * stride * in_size_ + i * in_size_; cur_out = output_data_ + task_id * stride * in_size_ + i * in_size_; } +#ifdef ENABLE_AVX + AddInt8_AVX2(cur_in0, cur_in1, cur_out, in_size_, ¶_); +#else AddInt8(cur_in0, cur_in1, cur_out, in_size_, ¶_); +#endif } return; } @@ -180,9 +184,17 @@ int QuantizedAddCPUKernel::DoExecute(int task_id) { int8_t element_in = arith_para_->in_elements_num0_ == 1 ? input0_data_[0] : input1_data_[0]; AddQuantQrgs *ptr_args = arith_para_->in_elements_num0_ == 1 ? ¶_.in1_args_ : ¶_.in0_args_; AddQuantQrgs *ele_args = arith_para_->in_elements_num0_ == 1 ? ¶_.in0_args_ : ¶_.in1_args_; +#ifdef ENABLE_AVX + AddOptInt8_AVX2(ptr_in, element_in, cur_out, rest_count, ¶_, ptr_args, ele_args); +#else AddOptInt8(ptr_in, element_in, cur_out, rest_count, ¶_, ptr_args, ele_args); +#endif } else { +#ifdef ENABLE_AVX + AddInt8_AVX2(cur_in0, cur_in1, cur_out, rest_count, ¶_); +#else AddInt8(cur_in0, cur_in1, cur_out, rest_count, ¶_); +#endif } return RET_OK; diff --git a/mindspore/lite/test/CMakeLists.txt b/mindspore/lite/test/CMakeLists.txt index 037a37de7b..4ea54868d3 100644 --- a/mindspore/lite/test/CMakeLists.txt +++ b/mindspore/lite/test/CMakeLists.txt @@ -75,7 +75,10 @@ if ("${X86_64_SIMD}" STREQUAL "sse") endif() if ("${X86_64_SIMD}" STREQUAL "avx") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.1 -mavx -mavx2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.1 -mavx -mavx2") file(GLOB TEST_ASSEMBLY_SRC ${LITE_DIR}/nnacl/x86_64_sse/*.c + ${LITE_DIR}/nnacl/x86_64_avx/*.c ${LITE_DIR}/nnacl/assembly/avx/*.S) set_property(SOURCE ${TEST_ASSEMBLY_SRC} PROPERTY LANGUAGE C) set(KERNEL_OP_SRC