m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[MSLITE] deconv winograd fp16 neon

Browse Source
pull/7648/head
ling 4 years ago
parent b17cc37918
commit d8b928b7f8
8 changed files with 615 additions and 100 deletions
Show Stats Download Patch File Download Diff File

259
mindspore/lite/nnacl/assembly/fp16/TiledC4MatmulFp16.S
View File

File diff suppressed because it is too large Load Diff

136
mindspore/lite/nnacl/assembly/fp16/WinogradTransLeftFp16.S
Unescape View File

@ -0,0 +1,136 @@
.text
.align 5
.global WinogradTransLeftFp16
#ifndef __APPLE__
.type WinogradTransLeftFp16, %function
#endif
WinogradTransLeftFp16:
sub sp, sp, #32
stp x19, x20, [sp], #32
mov x8, #8 // 4 * sizeof(float16)
mul x8, x6, x8
mul x9, x3, x8
sub x9, x9, x8
add x7, x9, x8 // step for S
mov x10, #2
mul x10, x4, x10 // step for B
LoopH:
mov x13, x0
mov x15, x3
LoopW:
mov x14, x13
mov x17, x1
dup v30.4h, wzr
mov x11, x6
InitZero:
st1 {v30.4h}, [x2], #8
subs x11, x11, #1
bne InitZero
sub x2, x2, x8
mov x12, x5
LoopKStart4:
cmp x12, #4
blt LoopKStart3
mov x16, x15
mov x19, x4
LoopK4:
ld1 {v0.h}[0], [x17], x10
ld1 {v0.h}[1], [x17], x10
ld1 {v0.h}[2], [x17], x10
ld1 {v0.h}[3], [x17], x10
mov x11, x6
mov x18, x17
add x18, x14, x7
add x16, x18, x7
add x19, x16, x7
LoopLength4:
ld1 {v16.4h}, [x2]
ld1 {v20.4h}, [x14], #8
fmla v16.4h, v20.4h, v0.h[0]
ld1 {v21.4h}, [x18], #8
fmul v17.4h, v21.4h, v0.h[1]
ld1 {v20.4h}, [x16], #8
fmla v16.4h, v20.4h, v0.h[2]
ld1 {v21.4h}, [x19], #8
fmla v17.4h, v21.4h, v0.h[3]
fadd v17.4h, v16.4h, v17.4h
st1 {v17.4h}, [x2], #8
subs x11, x11, #1
bne LoopLength4
sub x2, x2, x8
sub x12, x12, #4
add x14, x19, x9
cmp x12, #4
bge LoopK4
LoopKStart3:
cmp x12, #3
blt LoopKStart
mov x16, x15
LoopK3:
ld1 {v0.h}[0], [x17], x10
ld1 {v0.h}[1], [x17], x10
ld1 {v0.h}[2], [x17], x10
mov x11, x6
mov x18, x17
add x18, x14, x7
add x16, x18, x7
LoopLength3:
ld1 {v16.4h}, [x2]
ld1 {v20.4h}, [x14], #8
fmla v16.4h, v20.4h, v0.h[0]
ld1 {v21.4h}, [x18], #8
fmul v17.4h, v21.4h, v0.h[1]
ld1 {v20.4h}, [x16], #8
fmla v16.4h, v20.4h, v0.h[2]
fadd v17.4h, v16.4h, v17.4h
st1 {v17.4h}, [x2], #8
subs x11, x11, #1
bne LoopLength3
sub x2, x2, x8
sub x12, x12, #3
add x14, x16, x9
cmp x12, #3
bge LoopK3
LoopKStart:
cmp x12, #0
beq LKEnd
LoopK:
ld1r {v31.4h}, [x17], x10
mov x11, x6
LoopLength:
ld1 {v0.4h}, [x2]
ld1 {v1.4h}, [x14], #8
fmla v0.4h, v1.4h, v31.4h
st1 {v0.4h}, [x2], #8
subs x11, x11, #1
bne LoopLength
subs x12, x12, #1
sub x2, x2, x8
add x14, x14, x9
bne LoopK
LKEnd:
subs x15, x15, #1
add x13, x13, x8
add x2, x2, x8
bne LoopW
add x1, x1, #2 //sizeof(float)
subs x4, x4, #1
bne LoopH
sub sp, sp, #32
ldp x19, x20, [sp], #32
ret

134
mindspore/lite/nnacl/assembly/fp16/WinogradTransRightFp16.S
Unescape View File

@ -0,0 +1,134 @@
.text
.align 5
.global WinogradTransRightFp16
#ifndef __APPLE__
.type WinogradTransRightFp16, %function
#endif
WinogradTransRightFp16:
mov x8, #8 // 4 * sizeof(float16)
mul x8, x6, x8
mul x9, x5, x8 // step for S
mov x10, #2
mul x10, x4, x10 // step for B
LoopH:
mov x7, x1
mov x15, x3
LoopW:
mov x17, x0
mov x13, x7
dup v30.4h, wzr
mov x11, x6
InitZero:
st1 {v30.4h}, [x2], #8
subs x11, x11, #1
bne InitZero
sub x2, x2, x8
mov x12, x5
LoopKStart4:
cmp x12, #4
blt LoopKStart3
mov x16, x15
mov x18, x4
LoopK4:
ld1 {v0.h}[0], [x13], x10
ld1 {v0.h}[1], [x13], x10
ld1 {v0.h}[2], [x13], x10
ld1 {v0.h}[3], [x13], x10
mov x11, x6
mov x14, x13
add x14, x17, x8
add x16, x14, x8
add x18, x16, x8
LoopLength4:
ld1 {v16.4h}, [x2]
ld1 {v20.4h}, [x17], #8
fmla v16.4h, v20.4h, v0.h[0]
ld1 {v21.4h}, [x14], #8
fmul v17.4h, v21.4h, v0.h[1]
ld1 {v20.4h}, [x16], #8
fmla v16.4h, v20.4h, v0.h[2]
ld1 {v21.4h}, [x18], #8
fmla v17.4h, v21.4h, v0.h[3]
fadd v17.4h, v16.4h, v17.4h
st1 {v17.4h}, [x2], #8
subs x11, x11, #1
bne LoopLength4
sub x2, x2, x8
sub x12, x12, #4
mov x17, x18
cmp x12, #4
bge LoopK4
LoopKStart3:
cmp x12, #3
blt LoopKStart
mov x16, x15
LoopK3:
ld1 {v0.h}[0], [x13], x10
ld1 {v0.h}[1], [x13], x10
ld1 {v0.h}[2], [x13], x10
mov x11, x6
mov x14, x13
add x14, x17, x8
add x16, x14, x8
LoopLength3:
ld1 {v16.4h}, [x2]
ld1 {v20.4h}, [x17], #8
fmla v16.4h, v20.4h, v0.h[0]
ld1 {v21.4h}, [x14], #8
fmul v17.4h, v21.4h, v0.h[1]
ld1 {v20.4h}, [x16], #8
fmla v16.4h, v20.4h, v0.h[2]
fadd v17.4h, v16.4h, v17.4h
st1 {v17.4h}, [x2], #8
subs x11, x11, #1
bne LoopLength3
sub x2, x2, x8
sub x12, x12, #3
mov x17, x18
cmp x12, #3
bge LoopK3
LoopKStart:
cmp x12, #0
beq LoopKEnd
LoopK:
ld1r {v31.4h}, [x13], x10
mov x11, x6
LoopLength:
ld1 {v0.4h}, [x2]
ld1 {v1.4h}, [x17], #8
fmla v0.4h, v1.4h, v31.4h
st1 {v0.4h}, [x2], #8
subs x11, x11, #1
bne LoopLength
subs x12, x12, #1
sub x2, x2, x8
bne LoopK
LoopKEnd:
subs x15, x15, #1
add x2, x2, x8
add x7, x7, #2
bne LoopW
add x0, x0, x9
subs x4, x4, #1
bne LoopH
ret

115
mindspore/lite/nnacl/fp16/deconv_winograd_fp16.c
Unescape View File

@ -41,41 +41,73 @@ void DeConvWgInputPackFp16(float16_t *src_ptr, float16_t *dst_ptr, int channel,
return; return;
} }
void C4GemmFp16(float16_t *dst, const float16_t *src, const float16_t *weight, size_t src_depth_quad, size_t dst_step,
size_t dst_depth_quad, size_t width, size_t weight_depth_offset) {
int dx, sz, dz;
int src_depth_step = 4 * width;
for (dz = 0; dz < dst_depth_quad; ++dz) {
float16_t *dst_z = dst + dz * dst_step;
const float16_t *weight_dz = weight + dz * (src_depth_quad * 16 + weight_depth_offset);
for (dx = 0; dx < width; ++dx) {
float16_t *dst_x = dst_z + dx * 4;
dst_x[0] = 0.0f;
dst_x[1] = 0.0f;
dst_x[2] = 0.0f;
dst_x[3] = 0.0f;
const float16_t *src_dx = src + 4 * dx;
for (sz = 0; sz < src_depth_quad; ++sz) {
const float16_t *src_z = src_dx + sz * src_depth_step;
const float16_t *weight_z = weight_dz + sz * 16;
for (int i = 0; i < 4; ++i) {
for (int j = 0; j < 4; ++j) {
dst_x[j] += src_z[i] * weight_z[4 * i + j];
}
}
}
}
}
}
void DeConvWgMergeFp16(const float16_t *src, float16_t *dst, size_t src_stride, size_t dst_stride, size_t count) { void DeConvWgMergeFp16(const float16_t *src, float16_t *dst, size_t src_stride, size_t dst_stride, size_t count) {
for (int i = 0; i < count; ++i) { const float16_t *src_ptr = src;
const float16_t *s = src + i * src_stride; float16_t *dst_ptr = dst;
float16_t *d = dst + i * dst_stride; size_t cuont8 = count / C8NUM * C8NUM;
for (int j = 0; j < 4; ++j) { int i = 0;
d[j] += s[j]; for (; i < cuont8; i += C8NUM) {
} size_t src_step = src_stride * sizeof(float16_t);
size_t dst_step = dst_stride * sizeof(float16_t);
asm volatile(
"mov x7, %[src_ptr]\n"
"mov x8, %[dst_ptr]\n"
"mov x10, x8\n"
"ld1 {v0.4h}, [x7], %[src_step]\n"
"ld1 {v1.4h}, [x8], %[dst_step]\n"
"ld1 {v2.4h}, [x7], %[src_step]\n"
"ld1 {v3.4h}, [x8], %[dst_step]\n"
"fadd v0.4h, v0.4h, v1.4h\n"
"ld1 {v4.4h}, [x7], %[src_step]\n"
"fadd v2.4h, v2.4h, v3.4h\n"
"st1 {v0.4h}, [x10], %[dst_step]\n"
"st1 {v2.4h}, [x10], %[dst_step]\n"
"ld1 {v5.4h}, [x8], %[dst_step]\n"
"ld1 {v6.4h}, [x7], %[src_step]\n"
"fadd v4.4h, v4.4h, v5.4h\n"
"ld1 {v7.4h}, [x8], %[dst_step]\n"
"fadd v6.4h, v6.4h, v7.4h\n"
"ld1 {v0.4h}, [x7], %[src_step]\n"
"st1 {v4.4h}, [x10], %[dst_step]\n"
"st1 {v6.4h}, [x10], %[dst_step]\n"
"ld1 {v1.4h}, [x8], %[dst_step]\n"
"ld1 {v2.4h}, [x7], %[src_step]\n"
"ld1 {v3.4h}, [x8], %[dst_step]\n"
"fadd v0.4h, v0.4h, v1.4h\n"
"fadd v2.4h, v2.4h, v3.4h\n"
"st1 {v0.4h}, [x10], %[dst_step]\n"
"st1 {v2.4h}, [x10], %[dst_step]\n"
"ld1 {v4.4h}, [x7], %[src_step]\n"
"ld1 {v5.4h}, [x8], %[dst_step]\n"
"ld1 {v6.4h}, [x7], %[src_step]\n"
"ld1 {v7.4h}, [x8], %[dst_step]\n"
"fadd v4.4h, v4.4h, v5.4h\n"
"fadd v6.4h, v6.4h, v7.4h\n"
"st1 {v4.4h}, [x10], %[dst_step]\n"
"st1 {v6.4h}, [x10], %[dst_step]\n"
:
: [ src_ptr ] "r"(src_ptr), [ dst_ptr ] "r"(dst_ptr), [ src_step ] "r"(src_step), [ dst_step ] "r"(dst_step)
: "x7", "x8", "x10", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7");
src_ptr += C8NUM * src_stride;
dst_ptr += C8NUM * dst_stride;
}
for (; i < count; i++) {
float16x4_t src_data = vld1_f16(src_ptr);
float16x4_t dst_data = vld1_f16(dst_ptr);
dst_data = vadd_f16(src_data, dst_data);
vst1_f16(dst_ptr, dst_data);
src_ptr += src_stride;
dst_ptr += dst_stride;
} }
return;
} }
void _deConvWinogradFp16(float16_t *tile_in, float16_t *tile_out, float16_t *weight_buf, float16_t *tmp_buf, void _deConvWinogradFp16(float16_t *tile_in, float16_t *tile_out, float16_t *weight_buf, float16_t *tmp_buf,
@ -84,24 +116,24 @@ void _deConvWinogradFp16(float16_t *tile_in, float16_t *tile_out, float16_t *wei
ConvParameter *conv_param, DeConvParam *deconv_param) { ConvParameter *conv_param, DeConvParam *deconv_param) {
int winograd_plane = unit_size * unit_size; int winograd_plane = unit_size * unit_size;
if (!transfered[unit_size]) { if (!transfered[unit_size]) {
WinogradMatrixProductLeftFp16(tile_in, at_buf, a_mid_buf, DECONV_WINOGRAD_DEFAULT_UNIT, unit_size, WinogradTransLeftFp16(tile_in, at_buf, a_mid_buf, DECONV_WINOGRAD_DEFAULT_UNIT, unit_size,
DECONV_WINOGRAD_DEFAULT_UNIT, deconv_param->ic_div4_ * DECONV_WINOGRAD_DEFAULT_TILE); DECONV_WINOGRAD_DEFAULT_UNIT, deconv_param->ic_div4_ * DECONV_WINOGRAD_DEFAULT_TILE);
WinogradMatrixProductRightFp16(a_mid_buf, at_buf, trans_a_buf, unit_size, unit_size, DECONV_WINOGRAD_DEFAULT_UNIT, WinogradTransRightFp16(a_mid_buf, at_buf, trans_a_buf, unit_size, unit_size, DECONV_WINOGRAD_DEFAULT_UNIT,
deconv_param->ic_div4_ * DECONV_WINOGRAD_DEFAULT_TILE); deconv_param->ic_div4_ * DECONV_WINOGRAD_DEFAULT_TILE);
transfered[unit_size] = false; transfered[unit_size] = true;
} }
for (int index = 0; index < winograd_plane; index++) { for (int index = 0; index < winograd_plane; index++) {
float16_t *src = trans_a_buf + index * DECONV_WINOGRAD_DEFAULT_TILE * deconv_param->ic_up4_; float16_t *src = trans_a_buf + index * DECONV_WINOGRAD_DEFAULT_TILE * deconv_param->ic_up4_;
float16_t *dst = tmp_buf + index * deconv_param->oc_up4_ * DECONV_WINOGRAD_DEFAULT_TILE; float16_t *dst = tmp_buf + index * deconv_param->oc_up4_ * DECONV_WINOGRAD_DEFAULT_TILE;
float16_t *weight = weight_buf + index * deconv_param->ic_up4_ * deconv_param->oc_up4_; float16_t *weight = weight_buf + index * deconv_param->ic_up4_ * deconv_param->oc_up4_;
C4GemmFp16(dst, src, weight, deconv_param->ic_div4_, DECONV_WINOGRAD_DEFAULT_TILE * C4NUM, deconv_param->oc_div4_, TiledC4MatmulFp16(dst, src, weight, DECONV_WINOGRAD_DEFAULT_TILE * C4NUM, deconv_param->ic_div4_,
DECONV_WINOGRAD_DEFAULT_TILE, 0); deconv_param->oc_div4_);
} }
WinogradMatrixProductLeftFp16(tmp_buf, bt_buf, b_tmp_buf, unit_size, unit_size, unit_size, WinogradTransLeftFp16(tmp_buf, bt_buf, b_tmp_buf, unit_size, unit_size, unit_size,
deconv_param->oc_div4_ * DECONV_WINOGRAD_DEFAULT_TILE); deconv_param->oc_div4_ * DECONV_WINOGRAD_DEFAULT_TILE);
WinogradMatrixProductRightFp16(b_tmp_buf, bt_buf, tmp_buf, unit_size, unit_size, unit_size, WinogradTransRightFp16(b_tmp_buf, bt_buf, tmp_buf, unit_size, unit_size, unit_size,
deconv_param->oc_div4_ * DECONV_WINOGRAD_DEFAULT_TILE); deconv_param->oc_div4_ * DECONV_WINOGRAD_DEFAULT_TILE);
// Add to dest // Add to dest
@ -128,8 +160,7 @@ void _deConvCommonFp16(float16_t *tile_in, float16_t *tile_out, float16_t *weigh
for (int hi = 0; hi < DECONV_WINOGRAD_DEFAULT_UNIT; hi++) { for (int hi = 0; hi < DECONV_WINOGRAD_DEFAULT_UNIT; hi++) {
for (int wi = 0; wi < DECONV_WINOGRAD_DEFAULT_UNIT; wi++) { for (int wi = 0; wi < DECONV_WINOGRAD_DEFAULT_UNIT; wi++) {
float16_t *src_in = tile_in + (wi + hi * DECONV_WINOGRAD_DEFAULT_UNIT) * in_stride; float16_t *src_in = tile_in + (wi + hi * DECONV_WINOGRAD_DEFAULT_UNIT) * in_stride;
C4GemmFp16(tmp_buf, src_in, weight, deconv_param->ic_div4_, DECONV_WINOGRAD_DEFAULT_TILE * 4, count, TiledC4MatmulFp16(tmp_buf, src_in, weight, DECONV_WINOGRAD_DEFAULT_TILE * 4, deconv_param->ic_div4_, count);
DECONV_WINOGRAD_DEFAULT_TILE, 0);
for (int uhi = 0; uhi < h_size; uhi++) { for (int uhi = 0; uhi < h_size; uhi++) {
for (int uwi = 0; uwi < w_size; uwi++) { for (int uwi = 0; uwi < w_size; uwi++) {

9
mindspore/lite/nnacl/fp16/deconv_winograd_fp16.h
Unescape View File

@ -32,6 +32,15 @@ void DeconvWgFp16(float16_t *nhwc_input_, float16_t *tile_in, float16_t *tile_ou
void DeconvWgPostFp16(float16_t *tile_out, float16_t *nc4hw4_output, ConvParameter *conv_param, void DeconvWgPostFp16(float16_t *tile_out, float16_t *nc4hw4_output, ConvParameter *conv_param,
DeConvParam *deconv_param, int calculate_count, int tile_index); DeConvParam *deconv_param, int calculate_count, int tile_index);
void TiledC4MatmulFp16(float16_t *dst, const float16_t *src, const float16_t *weight, size_t ic4, size_t cal_num,
size_t oc4);
void WinogradTransLeftFp16(const float16_t *S, const float16_t *B, float16_t *M, size_t w, size_t h, size_t k,
size_t length);
void WinogradTransRightFp16(const float16_t *S, const float16_t *B, float16_t *M, size_t w, size_t h, size_t k,
size_t length);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

48
mindspore/lite/nnacl/fp16/matrix_fp16.c
Unescape View File

@ -81,51 +81,3 @@ void MatrixMultiplyVecFp16(const float16x8_t *matrix_a, const float16x8_t *matri
} }
} }
} }
void WinogradMatrixProductLeftFp16(const float16_t *S, const float16_t *B, float16_t *M, size_t w, size_t h, size_t k,
size_t length) {
int unitStep = 4 * length;
for (int y = 0; y < h; ++y) {
float16_t *dstY = M + y * w * unitStep;
for (int x = 0; x < w; ++x) {
float16_t *dstX = dstY + x * unitStep;
const float16_t *srcX = S + x * unitStep;
memset(dstX, 0, unitStep * sizeof(float16_t));
for (int i = 0; i < k; ++i) {
float16_t b = B[i * h + y];
const float16_t *srcY = srcX + i * w * unitStep;
if (0.0f == b) {
continue;
}
for (int j = 0; j < unitStep; ++j) {
dstX[j] += srcY[j] * b;
}
}
}
}
}
// M = S * B , M = w*h * l, S = k*h * l, B = w*k
void WinogradMatrixProductRightFp16(const float16_t *S, const float16_t *B, float16_t *M, size_t w, size_t h, size_t k,
size_t length) {
int unitStep = 4 * length;
for (int y = 0; y < h; ++y) {
float16_t *dstY = M + y * w * unitStep;
const float16_t *srcY = S + y * k * unitStep;
for (int x = 0; x < w; ++x) {
float16_t *dstX = dstY + x * unitStep;
memset(dstX, 0, unitStep * sizeof(float16_t));
for (int i = 0; i < k; ++i) {
const float16_t *srcX = srcY + i * unitStep;
float16_t b = B[i * h + x];
if (0.0f == b) {
continue;
}
for (int j = 0; j < unitStep; ++j) {
dstX[j] += srcX[j] * b;
}
}
}
}
}

7
mindspore/lite/nnacl/fp16/matrix_fp16.h
Unescape View File

@ -29,13 +29,6 @@ void MatrixMultiplyVecFp16(const float16x8_t *matrix_a, const float16x8_t *matri
const float16_t *bias, int m, int k, int n); const float16_t *bias, int m, int k, int n);
void MatrixMultiplyWinogradFp16(const float16_t *matix_a, const float16_t *matrix_b, float16_t *matrix_c, int m, int k, void MatrixMultiplyWinogradFp16(const float16_t *matix_a, const float16_t *matrix_b, float16_t *matrix_c, int m, int k,
int n, int in_channel); int n, int in_channel);
void WinogradMatrixProductLeftFp16(const float16_t *S, const float16_t *B, float16_t *M, size_t w, size_t h, size_t k,
size_t length);
void WinogradMatrixProductRightFp16(const float16_t *S, const float16_t *B, float16_t *M, size_t w, size_t h, size_t k,
size_t length);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
mindspore/lite/src/runtime/kernel/arm/fp16/deconvolution_fp16.cc
Unescape View File

@ -265,4 +265,5 @@ kernel::LiteKernel *CpuDeConvFp16KernelCreator(const std::vector<lite::Tensor *>
return kernel; return kernel;
} }
REG_KERNEL(kCPU, kNumberTypeFloat16, PrimitiveType_DeConv2D, CpuDeConvFp16KernelCreator)
} // namespace mindspore::kernel } // namespace mindspore::kernel

Write Preview
Loading…
Cancel
Save