!5265 [MS][LITE] arm cpu fp16 op: optimize conv depthwise
Merge pull request !5265 from yangruoqi713/lite_fp16pull/5265/MERGE
mindspore-ci-bot
5 years ago
committed by
Gitee
commit
f5429d5da0
@ -0,0 +1,117 @@
|
||||
#ifdef __aarch64__
|
||||
|
||||
.text
|
||||
.align 5
|
||||
.global ConvDwFp16Row
|
||||
#ifndef __APPLE__
|
||||
.type ConvDwFp16Row, %function
|
||||
#endif
|
||||
|
||||
// void ConvDwFp16Row(float16_t* output_ptr, const float16_t* input_ptr,const float16_t* filter_ptr,
|
||||
// size_t num_pixels, size_t input_channel, size_t input_step)
|
||||
// x0: output_ptr, x1: input_ptr, x2: filter_ptr, x3: num_pixels,
|
||||
// x4: input_channel, x5: input_step
|
||||
//
|
||||
ConvDwFp16Row:
|
||||
// registers v8 ~ v15 must be preserved by a callee across subroutine calls, according to
|
||||
// https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst#simd-and-floating-point-registers
|
||||
// x19 ~ x29 should be also preserved
|
||||
// whereas our coding style do not permit such amount of parameters
|
||||
cmp x3, #0
|
||||
beq End
|
||||
|
||||
mov x9, x0
|
||||
mov x12, #2 // sizeof(float16_t)
|
||||
mul x5, x5, x12
|
||||
|
||||
LoopOutPixel:
|
||||
mov x6, x1
|
||||
mov x7, x2
|
||||
mov x8, x4
|
||||
|
||||
LoopInputDepth32In:
|
||||
cmp x8, #32
|
||||
blt Loop8
|
||||
sub x8, x8, #32
|
||||
|
||||
ld1 {v0.8h, v1.8h}, [x6], #32
|
||||
ld1 {v2.8h, v3.8h}, [x7], #32
|
||||
ld1 {v16.8h, v17.8h}, [x0], #32
|
||||
|
||||
cmp x8, #32
|
||||
blt LoopInputDepth32Out
|
||||
LoopInputDepth32:
|
||||
fmla v16.8h, v0.8h, v2.8h
|
||||
fmla v17.8h, v1.8h, v3.8h
|
||||
|
||||
st1 {v16.8h, v17.8h}, [x9], #32
|
||||
|
||||
ld1 {v4.8h, v5.8h}, [x6], #32
|
||||
ld1 {v6.8h, v7.8h}, [x7], #32
|
||||
ld1 {v18.8h, v19.8h}, [x0], #32
|
||||
|
||||
fmla v18.8h, v4.8h, v6.8h
|
||||
fmla v19.8h, v5.8h, v7.8h
|
||||
|
||||
st1 {v18.8h, v19.8h}, [x9], #32
|
||||
|
||||
ld1 {v0.8h, v1.8h}, [x6], #32
|
||||
ld1 {v2.8h, v3.8h}, [x7], #32
|
||||
ld1 {v16.8h, v17.8h}, [x0], #32
|
||||
|
||||
sub x8, x8, #32
|
||||
cmp x8, #32
|
||||
bge LoopInputDepth32
|
||||
|
||||
LoopInputDepth32Out:
|
||||
fmla v16.8h, v0.8h, v2.8h
|
||||
fmla v17.8h, v1.8h, v3.8h
|
||||
st1 {v16.8h, v17.8h}, [x9], #32
|
||||
|
||||
ld1 {v4.8h, v5.8h}, [x6], #32
|
||||
ld1 {v6.8h, v7.8h}, [x7], #32
|
||||
ld1 {v18.8h, v19.8h}, [x0], #32
|
||||
|
||||
fmla v18.8h, v4.8h, v6.8h
|
||||
fmla v19.8h, v5.8h, v7.8h
|
||||
|
||||
st1 {v18.8h, v19.8h}, [x9], #32
|
||||
|
||||
Loop8:
|
||||
cmp x8, #8
|
||||
blt L0
|
||||
|
||||
LoopInputDepth8:
|
||||
ld1 {v0.8h}, [x6], #16
|
||||
ld1 {v2.8h}, [x7], #16
|
||||
ld1 {v16.8h}, [x0], #16
|
||||
fmla v16.8h, v0.8h, v2.8h
|
||||
st1 {v16.8h}, [x9], #16
|
||||
sub x8, x8, #8
|
||||
cmp x8, #8
|
||||
bge LoopInputDepth8
|
||||
|
||||
L0:
|
||||
cmp x8, #0
|
||||
beq Loop8LineEnd
|
||||
|
||||
LoopInputDepth0:
|
||||
ldr h0, [x6], #2
|
||||
ldr h1, [x7], #2
|
||||
ldr h2, [x0], #2
|
||||
fmul h0, h0, h1
|
||||
fadd h2, h2, h0
|
||||
str h2, [x9], #2
|
||||
subs x8, x8, #1
|
||||
bne LoopInputDepth0
|
||||
|
||||
Loop8LineEnd:
|
||||
|
||||
subs x3, x3, #1
|
||||
add x1, x1, x5
|
||||
bne LoopOutPixel
|
||||
|
||||
End:
|
||||
ret
|
||||
|
||||
#endif
|
||||
@ -1,49 +0,0 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_LITE_NNACL_FP16_COMMON_FUNC_H_
|
||||
#define MINDSPORE_LITE_NNACL_FP16_COMMON_FUNC_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "nnacl/op_base.h"
|
||||
#include "nnacl/conv_parameter.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#ifdef ENABLE_ARM64
|
||||
void ConvDwFp16Border(float16_t *dst, const float16_t *src, const float16_t *weight, const float16_t *bias,
|
||||
size_t height, size_t width, size_t in_kh_step, size_t in_kw_step, size_t kernel_w, size_t relu,
|
||||
size_t relu6);
|
||||
void ConvDwFp16Center(float16_t *dst, const float16_t *src, const float16_t *weight, const float16_t *bias,
|
||||
size_t height, size_t width, size_t kernel_h, size_t kernel_w, size_t out_h_step,
|
||||
size_t block_channel, size_t in_sh_step, size_t in_sw_step, size_t in_kh_step, size_t in_kw_step,
|
||||
size_t relu, size_t relu6);
|
||||
void DeconvDwFp16Border(float16_t *dst, const float16_t *src, const float16_t *weight, size_t height, size_t width,
|
||||
size_t in_kh_step, size_t in_kw_step, size_t kernel_w);
|
||||
void DeconvDwFp16Center(float16_t *dst, const float16_t *src, const float16_t *weight, size_t height, size_t width,
|
||||
size_t kernel_h, size_t kernel_w, size_t out_h_step, size_t block_channel, size_t in_sh_step,
|
||||
size_t in_sw_step, size_t in_kh_step, size_t in_kw_step);
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MINDSPORE_LITE_NNACL_FP32_COMMON_FUNC_H_ */
|
||||
@ -0,0 +1,203 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "src/runtime/kernel/arm/fp16/convolution_depthwise_slidewindow_fp16.h"
|
||||
#include "nnacl/fp16/pack_fp16.h"
|
||||
#include "nnacl/fp16/cast_fp16.h"
|
||||
#include "schema/model_generated.h"
|
||||
#include "src/kernel_registry.h"
|
||||
#include "include/errorcode.h"
|
||||
#include "src/runtime/runtime_api.h"
|
||||
|
||||
using mindspore::kernel::KERNEL_ARCH::kCPU;
|
||||
using mindspore::lite::KernelRegistrar;
|
||||
using mindspore::lite::RET_ERROR;
|
||||
using mindspore::lite::RET_OK;
|
||||
using mindspore::schema::PrimitiveType_DepthwiseConv2D;
|
||||
|
||||
namespace mindspore::kernel {
|
||||
ConvolutionDepthwiseSWFp16CPUKernel::~ConvolutionDepthwiseSWFp16CPUKernel() {
|
||||
if (sliding_ != nullptr) {
|
||||
delete sliding_;
|
||||
sliding_ = nullptr;
|
||||
}
|
||||
if (packed_weight_ != nullptr) {
|
||||
delete packed_weight_;
|
||||
packed_weight_ = nullptr;
|
||||
}
|
||||
FreeTmpBuffer();
|
||||
}
|
||||
|
||||
void ConvolutionDepthwiseSWFp16CPUKernel::FreeTmpBuffer() {
|
||||
if (need_align_) {
|
||||
if (packed_input_ != nullptr) {
|
||||
delete packed_input_;
|
||||
packed_input_ = nullptr;
|
||||
}
|
||||
if (packed_output_ != nullptr) {
|
||||
delete packed_output_;
|
||||
packed_output_ = nullptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int ConvolutionDepthwiseSWFp16CPUKernel::InitBuffer() {
|
||||
if (conv_param_->input_channel_ % C4NUM != 0) {
|
||||
need_align_ = true;
|
||||
int C8 = UP_DIV(conv_param_->input_channel_, C8NUM);
|
||||
int pack_input_size = conv_param_->input_batch_ * conv_param_->input_h_ * conv_param_->input_w_ * C8NUM * C8;
|
||||
packed_input_ = reinterpret_cast<float16_t *>(malloc(pack_input_size * sizeof(float16_t)));
|
||||
if (packed_input_ == nullptr) {
|
||||
MS_LOG(ERROR) << "Malloc buffer failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
|
||||
int pack_output_size = conv_param_->output_batch_ * conv_param_->output_h_ * conv_param_->output_w_ * C8NUM * C8;
|
||||
packed_output_ = reinterpret_cast<float16_t *>(malloc(pack_output_size * sizeof(float16_t)));
|
||||
if (packed_output_ == nullptr) {
|
||||
MS_LOG(ERROR) << "Malloc buffer failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
}
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
int ConvolutionDepthwiseSWFp16CPUKernel::InitWeightBias() {
|
||||
// init weight: o, h, w, i; o == group, i == 1
|
||||
auto weight_tensor = in_tensors_[kWeightIndex];
|
||||
int OC8 = UP_DIV(weight_tensor->Batch(), C8NUM);
|
||||
auto origin_weight = reinterpret_cast<float *>(weight_tensor->Data());
|
||||
int pack_weight_size = C8NUM * OC8 * weight_tensor->Height() * weight_tensor->Width();
|
||||
|
||||
packed_weight_ = reinterpret_cast<float16_t *>(malloc(pack_weight_size * sizeof(float16_t)));
|
||||
if (packed_weight_ == nullptr) {
|
||||
MS_LOG(ERROR) << "Malloc buffer failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
PackNCHWFp32ToNC8HW8Fp16(origin_weight, packed_weight_, 1, weight_tensor->Height() * weight_tensor->Width(),
|
||||
weight_tensor->Batch());
|
||||
|
||||
bias_data_ = reinterpret_cast<float16_t *>(malloc(C8NUM * OC8 * sizeof(float16_t)));
|
||||
if (bias_data_ == nullptr) {
|
||||
MS_LOG(ERROR) << "Malloc buffer failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
memset(bias_data_, 0, C8NUM * OC8 * sizeof(float16_t));
|
||||
auto bias_fp16 = reinterpret_cast<float16_t *>(bias_data_);
|
||||
if (in_tensors_.size() == kInputSize2) {
|
||||
auto bias_tensor = in_tensors_.at(kBiasIndex);
|
||||
auto ori_bias = reinterpret_cast<float *>(bias_tensor->Data());
|
||||
for (int i = 0; i < bias_tensor->ElementsNum(); i++) {
|
||||
bias_fp16[i] = (float16_t)ori_bias[i];
|
||||
}
|
||||
}
|
||||
|
||||
conv_param_->thread_num_ = MSMIN(thread_count_, OC8);
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
int ConvolutionDepthwiseSWFp16CPUKernel::Init() {
|
||||
sliding_ = new (std::nothrow) SlidingWindowParam;
|
||||
if (sliding_ == nullptr) {
|
||||
MS_LOG(ERROR) << "new sliding window param failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
|
||||
auto ret = InitWeightBias();
|
||||
if (ret != 0) {
|
||||
MS_LOG(ERROR) << "Convolution depthwise fp16 InitWeightBias failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
|
||||
if (!InferShapeDone()) {
|
||||
return RET_OK;
|
||||
}
|
||||
return ReSize();
|
||||
}
|
||||
|
||||
int ConvolutionDepthwiseSWFp16CPUKernel::ReSize() {
|
||||
FreeTmpBuffer();
|
||||
auto ret = ConvolutionBaseCPUKernel::Init();
|
||||
if (ret != RET_OK) {
|
||||
return ret;
|
||||
}
|
||||
InitSlidingParamConvDw(sliding_, conv_param_, C8NUM);
|
||||
|
||||
ret = InitBuffer();
|
||||
if (ret != 0) {
|
||||
MS_LOG(ERROR) << "Convolution depthwise fp16 InitBuffer failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
int ConvolutionDepthwiseSWFp16CPUKernel::Execute(int task_id) {
|
||||
ConvDwC8Fp16(packed_output_, packed_input_, packed_weight_, reinterpret_cast<float16_t *>(bias_data_), conv_param_,
|
||||
sliding_, task_id);
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
static int ConvDwSWFp16Run(int task_id, LiteParallelGroupEnv *penv, void *cdata) {
|
||||
auto conv_dw_fp16 = reinterpret_cast<ConvolutionDepthwiseSWFp16CPUKernel *>(cdata);
|
||||
auto ret = conv_dw_fp16->Execute(task_id);
|
||||
if (ret != RET_OK) {
|
||||
MS_LOG(ERROR) << "ConvolutionDepthwiseSWFp16Run error task_id[" << task_id << "] error_code[" << ret << "]";
|
||||
return RET_ERROR;
|
||||
}
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
int ConvolutionDepthwiseSWFp16CPUKernel::Run() {
|
||||
auto ret = Prepare();
|
||||
if (ret != RET_OK) {
|
||||
MS_LOG(ERROR) << "Prepare failed.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
if (conv_param_->input_channel_ != conv_param_->output_channel_) {
|
||||
MS_LOG(ERROR) << "Only support input channel equals output channel.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
|
||||
ret = ConvolutionBaseFP16CPUKernel::GetExecuteTensor();
|
||||
if (ret != RET_OK) {
|
||||
MS_LOG(ERROR) << "Get Execute tensor failed.";
|
||||
return ret;
|
||||
}
|
||||
if (need_align_) {
|
||||
PackNHWCToNHWC8Fp16(execute_input_, packed_input_, conv_param_->input_batch_,
|
||||
conv_param_->input_h_ * conv_param_->input_w_, conv_param_->input_channel_);
|
||||
} else {
|
||||
packed_input_ = execute_input_;
|
||||
}
|
||||
if (!need_align_) {
|
||||
packed_output_ = execute_output_;
|
||||
}
|
||||
|
||||
ret = LiteBackendParallelLaunch(ConvDwSWFp16Run, this, conv_param_->thread_num_);
|
||||
if (ret != RET_OK) {
|
||||
MS_LOG(ERROR) << "ConvDwSWFp16Run error: error_code[" << ret << "]";
|
||||
return RET_ERROR;
|
||||
}
|
||||
if (need_align_) {
|
||||
PackNHWC8ToNHWCFp16(packed_output_, execute_output_, conv_param_->output_batch_,
|
||||
conv_param_->output_h_ * conv_param_->output_w_, conv_param_->output_channel_);
|
||||
}
|
||||
ConvolutionBaseFP16CPUKernel::IfCastOutput();
|
||||
ConvolutionBaseFP16CPUKernel::FreeTmpBuffer();
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
} // namespace mindspore::kernel
|
||||
@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SW_FP16_H_
|
||||
#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SW_FP16_H_
|
||||
|
||||
#include <vector>
|
||||
#include "src/lite_kernel.h"
|
||||
#include "src/runtime/kernel/arm/fp16/convolution_base_fp16.h"
|
||||
#include "nnacl/fp16/conv_depthwise_fp16.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
void ConvDwC8Fp16(float16_t *output_data, const float16_t *input_data, const float16_t *weight_data,
|
||||
const float16_t *bias_data, const ConvParameter *conv_param, const SlidingWindowParam *sliding,
|
||||
int task_id);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace mindspore::kernel {
|
||||
class ConvolutionDepthwiseSWFp16CPUKernel : public ConvolutionBaseFP16CPUKernel {
|
||||
public:
|
||||
ConvolutionDepthwiseSWFp16CPUKernel(OpParameter *parameter, const std::vector<lite::tensor::Tensor *> &inputs,
|
||||
const std::vector<lite::tensor::Tensor *> &outputs, const Context *ctx,
|
||||
const mindspore::lite::PrimitiveC *primitive)
|
||||
: ConvolutionBaseFP16CPUKernel(parameter, inputs, outputs, ctx, primitive) {}
|
||||
~ConvolutionDepthwiseSWFp16CPUKernel() override;
|
||||
|
||||
int Init() override;
|
||||
int ReSize() override;
|
||||
int Run() override;
|
||||
|
||||
int InitBuffer();
|
||||
int InitWeightBias();
|
||||
int Execute(int task_id);
|
||||
|
||||
private:
|
||||
void FreeTmpBuffer();
|
||||
SlidingWindowParam *sliding_ = nullptr;
|
||||
float16_t *packed_weight_ = nullptr;
|
||||
float16_t *packed_input_ = nullptr;
|
||||
float16_t *packed_output_ = nullptr;
|
||||
bool need_align_ = false;
|
||||
};
|
||||
} // namespace mindspore::kernel
|
||||
|
||||
#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_ARM_FP16_CONVOLUTION_DEPTHWISE_SW_FP16_H_
|
||||
Loading…
Reference in new issue