|
|
|
|
@ -31,13 +31,12 @@ void ConvFp32(float *input_data, float *packed_input, float *packed_weight, cons
|
|
|
|
|
int out_w = conv_param->output_w_;
|
|
|
|
|
int out_channel = conv_param->output_channel_;
|
|
|
|
|
int thread_count = conv_param->thread_num_;
|
|
|
|
|
int tile_n = 8;
|
|
|
|
|
int output_count = out_h * out_w;
|
|
|
|
|
int output_tile_count = UP_DIV(output_count, tile_n);
|
|
|
|
|
int output_tile_count = UP_DIV(output_count, TILE_NUM);
|
|
|
|
|
int ic4 = UP_DIV(in_channel, C4NUM);
|
|
|
|
|
int kernel_plane = kernel_h * kernel_w;
|
|
|
|
|
int unit_size = kernel_plane * ic4 * C4NUM;
|
|
|
|
|
int packed_input_size = output_tile_count * tile_n * unit_size;
|
|
|
|
|
int packed_input_size = output_tile_count * TILE_NUM * unit_size;
|
|
|
|
|
|
|
|
|
|
// we accumulate 4 channels per time for input blocks
|
|
|
|
|
int conv_depth = kernel_h * kernel_w;
|
|
|
|
|
@ -50,13 +49,13 @@ void ConvFp32(float *input_data, float *packed_input, float *packed_weight, cons
|
|
|
|
|
int out_batch_offset = b * out_channel * out_h * out_w;
|
|
|
|
|
int gemm_in_batch_offset = b * packed_input_size;
|
|
|
|
|
for (int thread_id = task_id; thread_id < output_tile_count; thread_id += thread_count) {
|
|
|
|
|
int start_index = thread_id * tile_n;
|
|
|
|
|
int real_cal_num = (output_count - start_index) < tile_n ? (output_count - start_index) : tile_n;
|
|
|
|
|
float *gemm_input = packed_input + thread_id * unit_size * tile_n + gemm_in_batch_offset;
|
|
|
|
|
int start_index = thread_id * TILE_NUM;
|
|
|
|
|
int real_cal_num = (output_count - start_index) < TILE_NUM ? (output_count - start_index) : TILE_NUM;
|
|
|
|
|
float *gemm_input = packed_input + thread_id * unit_size * TILE_NUM + gemm_in_batch_offset;
|
|
|
|
|
Im2ColPackUnitFp32(input_data + in_batch_offset, conv_param, gemm_input, real_cal_num, start_index);
|
|
|
|
|
|
|
|
|
|
int out_offset = thread_id * tile_n * out_channel + out_batch_offset;
|
|
|
|
|
if (real_cal_num == tile_n) {
|
|
|
|
|
int out_offset = thread_id * TILE_NUM * out_channel + out_batch_offset;
|
|
|
|
|
if (real_cal_num == TILE_NUM) {
|
|
|
|
|
float *gemm_output = output_data + out_offset;
|
|
|
|
|
IndirectGemmFp32_8x8(gemm_output, gemm_input, packed_weight, bias_data, conv_depth, ic4, out_channel,
|
|
|
|
|
output_offset, 0, 0, conv_param->is_relu_, conv_param->is_relu6_);
|
|
|
|
|
@ -121,22 +120,8 @@ void ConvWinogardFp32(float *input_data, float *trans_weight, const float *bias_
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// get real output
|
|
|
|
|
for (int batch = 0; batch < out_batch; batch++) {
|
|
|
|
|
int batch_size = batch * out_channel * conv_param->output_h_ * conv_param->output_w_;
|
|
|
|
|
for (int h = 0; h < conv_param->output_h_; h++) {
|
|
|
|
|
for (int w = 0; w < conv_param->output_w_; w++) {
|
|
|
|
|
for (int c = 0; c < out_channel; c++) {
|
|
|
|
|
int oc4_block = c / C4NUM;
|
|
|
|
|
int oc4_res = c % C4NUM;
|
|
|
|
|
int src_offset = oc4_block * C4NUM * out_w_block * out_h_block * out_unit * out_unit +
|
|
|
|
|
C4NUM * (h * out_w_block * out_unit + w) + oc4_res;
|
|
|
|
|
int dst_offset = (h * conv_param->output_w_ + w) * out_channel + c;
|
|
|
|
|
(output_data + dst_offset)[0] = (tmp_out_data + src_offset)[0];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
UnPackWinogradOutput(tmp_out_data, output_data, out_batch, conv_param->output_h_, conv_param->output_w_, out_channel,
|
|
|
|
|
out_unit);
|
|
|
|
|
int output_num = out_channel * conv_param->output_h_ * conv_param->output_w_ * conv_param->output_batch_;
|
|
|
|
|
if (is_relu) {
|
|
|
|
|
ReluFp32(output_data, output_num);
|
|
|
|
|
@ -147,6 +132,45 @@ void ConvWinogardFp32(float *input_data, float *trans_weight, const float *bias_
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void UnPackWinogradOutput(const float *src, float *dst, int batch, int height, int width, int channel,
|
|
|
|
|
int output_unit) {
|
|
|
|
|
int out_h_block_num = UP_DIV(height, output_unit);
|
|
|
|
|
int out_w_block_num = UP_DIV(width, output_unit);
|
|
|
|
|
int c4 = UP_DIV(channel, C4NUM);
|
|
|
|
|
for (int b = 0; b < batch; b++) {
|
|
|
|
|
int src_batch_offset = b * c4 * C4NUM * out_h_block_num * output_unit * out_w_block_num * output_unit;
|
|
|
|
|
int dst_batch_offset = b * height * width * channel;
|
|
|
|
|
for (int h = 0; h < height; h++) {
|
|
|
|
|
int src_h_offset = src_batch_offset + C4NUM * (h * out_w_block_num * output_unit);
|
|
|
|
|
int dst_h_offset = dst_batch_offset + h * width * channel;
|
|
|
|
|
for (int w = 0; w < width; w++) {
|
|
|
|
|
int src_w_offset = src_h_offset + w * C4NUM;
|
|
|
|
|
int dst_w_offset = dst_h_offset + w * channel;
|
|
|
|
|
for (int c = 0; c < c4 - 1; c++) {
|
|
|
|
|
int src_c4_offset = src_w_offset + c * C4NUM * out_w_block_num * out_h_block_num * output_unit * output_unit;
|
|
|
|
|
int dst_c4_offset = dst_w_offset + c * C4NUM;
|
|
|
|
|
#ifdef ENABLE_NEON
|
|
|
|
|
vst1q_f32(dst + dst_c4_offset, vld1q_f32(src + src_c4_offset));
|
|
|
|
|
#else
|
|
|
|
|
dst[dst_c4_offset] = src[src_c4_offset];
|
|
|
|
|
dst[dst_c4_offset + 1] = src[src_c4_offset + 1];
|
|
|
|
|
dst[dst_c4_offset + 2] = src[src_c4_offset + 2];
|
|
|
|
|
dst[dst_c4_offset + 3] = src[src_c4_offset + 3];
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
int c_res = channel - (c4 - 1) * C4NUM;
|
|
|
|
|
int src_c_res_offset = (c4 - 1) * C4NUM * out_w_block_num * out_h_block_num * output_unit * output_unit;
|
|
|
|
|
int dst_c_res_offset = (c4 - 1) * C4NUM;
|
|
|
|
|
for (int c = 0; c < c_res; c++) {
|
|
|
|
|
int src_c4_res_offset = src_w_offset + src_c_res_offset + c;
|
|
|
|
|
int dst_c4_res_offset = dst_w_offset + dst_c_res_offset + c;
|
|
|
|
|
dst[dst_c4_res_offset] = src[src_c4_res_offset];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// fp32 conv3x3
|
|
|
|
|
void Conv3x3Fp32(float *input_data, float *transed_weight, const float *bias_data, float *output_data,
|
|
|
|
|
TmpBufferAddress *buffer_list, int task_id, ConvParameter *conv_param) {
|
|
|
|
|
@ -182,7 +206,7 @@ void Conv3x3Fp32(float *input_data, float *transed_weight, const float *bias_dat
|
|
|
|
|
}
|
|
|
|
|
PackNC4HW4ToNHWCFp32(nc4hw4_out, output_data, 1, conv_param->output_h_ * conv_param->output_w_, output_channel);
|
|
|
|
|
}
|
|
|
|
|
int output_num = oc4 * C4NUM * conv_param->output_h_ * conv_param->output_w_ * conv_param->output_batch_;
|
|
|
|
|
int output_num = output_channel * conv_param->output_h_ * conv_param->output_w_ * conv_param->output_batch_;
|
|
|
|
|
if (is_relu) {
|
|
|
|
|
ReluFp32(output_data, output_num);
|
|
|
|
|
} else if (is_relu6) {
|
|
|
|
|
@ -191,4 +215,3 @@ void Conv3x3Fp32(float *input_data, float *transed_weight, const float *bias_dat
|
|
|
|
|
// do nothing
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
mindspore-ci-bot
5 years ago
committed by
Gitee