From 104edd7f784253ff916343c338176a712d7b2b82 Mon Sep 17 00:00:00 2001 From: nihui Date: Thu, 28 May 2026 23:02:28 +0800 Subject: [PATCH] mips/loongarch: remove dead code and optimize transpose pack (#6749) * limit test nT, add gemm int8 nT test --- .../loongarch/convolution_im2col_gemm_bf16s.h | 421 ++++++++------- .../loongarch/convolution_im2col_gemm_int8.h | 489 ++++++++++------- src/layer/loongarch/gemm_bf16s.h | 472 +++++------------ src/layer/loongarch/gemm_int8.h | 18 +- src/layer/loongarch/gemm_loongarch.cpp | 490 +----------------- .../mips/convolution_im2col_gemm_bf16s.h | 212 ++++---- src/layer/mips/convolution_im2col_gemm_int8.h | 364 +++++++++---- src/layer/mips/gemm_bf16s.h | 274 ++-------- src/layer/mips/gemm_int8.h | 18 +- src/layer/mips/gemm_mips.cpp | 336 +----------- tests/test_gemm_nt_int8.cpp | 386 ++++++++++++++ tests/testutil.cpp | 12 +- 12 files changed, 1546 insertions(+), 1946 deletions(-) create mode 100644 tests/test_gemm_nt_int8.cpp diff --git a/src/layer/loongarch/convolution_im2col_gemm_bf16s.h b/src/layer/loongarch/convolution_im2col_gemm_bf16s.h index 89798e68b..55841e35d 100644 --- a/src/layer/loongarch/convolution_im2col_gemm_bf16s.h +++ b/src/layer/loongarch/convolution_im2col_gemm_bf16s.h @@ -3442,43 +3442,42 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl { __builtin_prefetch(p0 + bottom_blob.cstep * 4); - __m128i _zero_bf16 = __lsx_vreplgr2vr_w(0); __m128i _r01 = __lsx_vld(p0, 0); - __m128 _r0 = (__m128)__lsx_vilvl_h(_r01, _zero_bf16); - __m128 _r1 = (__m128)__lsx_vilvh_h(_r01, _zero_bf16); - __m128i _r23_bf16 = __lsx_vld(p0 + 8, 0); - __m128 _r2 = (__m128)__lsx_vilvl_h(_r23_bf16, _zero_bf16); - __m128 _r3 = (__m128)__lsx_vilvh_h(_r23_bf16, _zero_bf16); - __m128i _r45_bf16 = __lsx_vld(p0 + 16, 0); - __m128 _r4 = (__m128)__lsx_vilvl_h(_r45_bf16, _zero_bf16); - __m128 _r5 = (__m128)__lsx_vilvh_h(_r45_bf16, _zero_bf16); - __m128i _r67_bf16 = __lsx_vld(p0 + 24, 0); - __m128 _r6 = (__m128)__lsx_vilvl_h(_r67_bf16, _zero_bf16); - __m128 _r7 = (__m128)__lsx_vilvh_h(_r67_bf16, _zero_bf16); - __m128i _r89_bf16 = __lsx_vld(p0 + 32, 0); - __m128 _r8 = (__m128)__lsx_vilvl_h(_r89_bf16, _zero_bf16); - __m128 _r9 = (__m128)__lsx_vilvh_h(_r89_bf16, _zero_bf16); - __m128i _rab_bf16 = __lsx_vld(p0 + 40, 0); - __m128 _ra = (__m128)__lsx_vilvl_h(_rab_bf16, _zero_bf16); - __m128 _rb = (__m128)__lsx_vilvh_h(_rab_bf16, _zero_bf16); - __m128i _rcd_bf16 = __lsx_vld(p0 + 48, 0); - __m128 _rc = (__m128)__lsx_vilvl_h(_rcd_bf16, _zero_bf16); - __m128 _rd = (__m128)__lsx_vilvh_h(_rcd_bf16, _zero_bf16); - __m128i _ref_bf16 = __lsx_vld(p0 + 56, 0); - __m128 _re = (__m128)__lsx_vilvl_h(_ref_bf16, _zero_bf16); - __m128 _rf = (__m128)__lsx_vilvh_h(_ref_bf16, _zero_bf16); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - transpose4x4_ps(_r8, _r9, _ra, _rb); - transpose4x4_ps(_rc, _rd, _re, _rf); - __lsx_vst(float2bfloat_lsx(_r0, _r4), pp, 0); - __lsx_vst(float2bfloat_lsx(_r8, _rc), pp + 8, 0); - __lsx_vst(float2bfloat_lsx(_r1, _r5), pp + 16, 0); - __lsx_vst(float2bfloat_lsx(_r9, _rd), pp + 24, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r6), pp + 32, 0); - __lsx_vst(float2bfloat_lsx(_ra, _re), pp + 40, 0); - __lsx_vst(float2bfloat_lsx(_r3, _r7), pp + 48, 0); - __lsx_vst(float2bfloat_lsx(_rb, _rf), pp + 56, 0); + __m128i _r23 = __lsx_vld(p0 + 8, 0); + __m128i _r45 = __lsx_vld(p0 + 16, 0); + __m128i _r67 = __lsx_vld(p0 + 24, 0); + __m128i _r0 = __lsx_vilvl_d(_r45, _r01); + __m128i _r1 = __lsx_vilvh_d(_r45, _r01); + __m128i _r2 = __lsx_vilvl_d(_r67, _r23); + __m128i _r3 = __lsx_vilvh_d(_r67, _r23); + transpose8x4_epi16(_r0, _r1, _r2, _r3); + __m128i _c0 = __lsx_vilvl_d(_r2, _r0); + __m128i _c1 = __lsx_vilvh_d(_r2, _r0); + __m128i _c2 = __lsx_vilvl_d(_r3, _r1); + __m128i _c3 = __lsx_vilvh_d(_r3, _r1); + + __m128i _r89 = __lsx_vld(p0 + 32, 0); + __m128i _rab = __lsx_vld(p0 + 40, 0); + __m128i _rcd = __lsx_vld(p0 + 48, 0); + __m128i _ref = __lsx_vld(p0 + 56, 0); + __m128i _r8 = __lsx_vilvl_d(_rcd, _r89); + __m128i _r9 = __lsx_vilvh_d(_rcd, _r89); + __m128i _ra = __lsx_vilvl_d(_ref, _rab); + __m128i _rb = __lsx_vilvh_d(_ref, _rab); + transpose8x4_epi16(_r8, _r9, _ra, _rb); + __m128i _c4 = __lsx_vilvl_d(_ra, _r8); + __m128i _c5 = __lsx_vilvh_d(_ra, _r8); + __m128i _c6 = __lsx_vilvl_d(_rb, _r9); + __m128i _c7 = __lsx_vilvh_d(_rb, _r9); + + __lsx_vst(_c0, pp, 0); + __lsx_vst(_c4, pp + 8, 0); + __lsx_vst(_c1, pp + 16, 0); + __lsx_vst(_c5, pp + 24, 0); + __lsx_vst(_c2, pp + 32, 0); + __lsx_vst(_c6, pp + 40, 0); + __lsx_vst(_c3, pp + 48, 0); + __lsx_vst(_c7, pp + 56, 0); pp += 64; p0 += bottom_blob.cstep * 4; } @@ -3540,25 +3539,19 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl { __builtin_prefetch(p0 + bottom_blob.cstep * 4); - __m128i _zero_bf16 = __lsx_vreplgr2vr_w(0); __m128i _r01 = __lsx_vld(p0, 0); - __m128 _r0 = (__m128)__lsx_vilvl_h(_r01, _zero_bf16); - __m128 _r1 = (__m128)__lsx_vilvh_h(_r01, _zero_bf16); __m128i _r23 = __lsx_vld(p0 + 8, 0); __m128i _r45 = __lsx_vld(p0 + 16, 0); __m128i _r67 = __lsx_vld(p0 + 24, 0); - __m128 _r2 = (__m128)__lsx_vilvl_h(_r23, _zero_bf16); - __m128 _r3 = (__m128)__lsx_vilvh_h(_r23, _zero_bf16); - __m128 _r4 = (__m128)__lsx_vilvl_h(_r45, _zero_bf16); - __m128 _r5 = (__m128)__lsx_vilvh_h(_r45, _zero_bf16); - __m128 _r6 = (__m128)__lsx_vilvl_h(_r67, _zero_bf16); - __m128 _r7 = (__m128)__lsx_vilvh_h(_r67, _zero_bf16); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - __lsx_vst(float2bfloat_lsx(_r0, _r4), pp, 0); - __lsx_vst(float2bfloat_lsx(_r1, _r5), pp + 8, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r6), pp + 16, 0); - __lsx_vst(float2bfloat_lsx(_r3, _r7), pp + 24, 0); + __m128i _r0 = __lsx_vilvl_d(_r45, _r01); + __m128i _r1 = __lsx_vilvh_d(_r45, _r01); + __m128i _r2 = __lsx_vilvl_d(_r67, _r23); + __m128i _r3 = __lsx_vilvh_d(_r67, _r23); + transpose8x4_epi16(_r0, _r1, _r2, _r3); + __lsx_vst(__lsx_vilvl_d(_r2, _r0), pp, 0); + __lsx_vst(__lsx_vilvh_d(_r2, _r0), pp + 8, 0); + __lsx_vst(__lsx_vilvl_d(_r3, _r1), pp + 16, 0); + __lsx_vst(__lsx_vilvh_d(_r3, _r1), pp + 24, 0); pp += 32; p0 += bottom_blob.cstep * 4; } @@ -3611,16 +3604,14 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl { __builtin_prefetch(p0 + bottom_blob.cstep * 4); - __m128i _zero_bf16 = __lsx_vreplgr2vr_w(0); __m128i _r01 = __lsx_vld(p0, 0); - __m128 _r0 = (__m128)__lsx_vilvl_h(_r01, _zero_bf16); - __m128 _r1 = (__m128)__lsx_vilvh_h(_r01, _zero_bf16); __m128i _r23 = __lsx_vld(p0 + 8, 0); - __m128 _r2 = (__m128)__lsx_vilvl_h(_r23, _zero_bf16); - __m128 _r3 = (__m128)__lsx_vilvh_h(_r23, _zero_bf16); - transpose4x4_ps(_r0, _r1, _r2, _r3); - __lsx_vst(float2bfloat_lsx(_r0, _r1), pp, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r3), pp + 8, 0); + __m128i _r02 = __lsx_vilvl_d(_r23, _r01); + __m128i _r13 = __lsx_vilvh_d(_r23, _r01); + __m128i _tmp0 = __lsx_vilvl_h(_r13, _r02); + __m128i _tmp1 = __lsx_vilvh_h(_r13, _r02); + __lsx_vst(__lsx_vilvl_w(_tmp1, _tmp0), pp, 0); + __lsx_vst(__lsx_vilvh_w(_tmp1, _tmp0), pp + 8, 0); pp += 16; p0 += bottom_blob.cstep * 4; } @@ -3671,14 +3662,10 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl int kk = 0; for (; kk + 3 < max_kk; kk += 4) { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[1]; - pp[3] = p0[5]; - pp[4] = p0[2]; - pp[5] = p0[6]; - pp[6] = p0[3]; - pp[7] = p0[7]; + __m128i _r01 = __lsx_vld(p0, 0); + __m128i _r0 = __lsx_vilvl_d(_r01, _r01); + __m128i _r1 = __lsx_vilvh_d(_r01, _r01); + __lsx_vst(__lsx_vilvl_h(_r1, _r0), pp, 0); pp += 8; p0 += bottom_blob.cstep * 4; } @@ -3852,34 +3839,76 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr); - __m128 _r1 = bfloat2float_lsx(sptr + stride_w * 4); - __m128 _r2 = bfloat2float_lsx(sptr + stride_w * 8); - __m128 _r3 = bfloat2float_lsx(sptr + stride_w * 12); - __m128 _r4 = bfloat2float_lsx(sptr + stride_w * 16); - __m128 _r5 = bfloat2float_lsx(sptr + stride_w * 20); - __m128 _r6 = bfloat2float_lsx(sptr + stride_w * 24); - __m128 _r7 = bfloat2float_lsx(sptr + stride_w * 28); - __m128 _r8 = bfloat2float_lsx(sptr + stride_w * 32); - __m128 _r9 = bfloat2float_lsx(sptr + stride_w * 36); - __m128 _ra = bfloat2float_lsx(sptr + stride_w * 40); - __m128 _rb = bfloat2float_lsx(sptr + stride_w * 44); - __m128 _rc = bfloat2float_lsx(sptr + stride_w * 48); - __m128 _rd = bfloat2float_lsx(sptr + stride_w * 52); - __m128 _re = bfloat2float_lsx(sptr + stride_w * 56); - __m128 _rf = bfloat2float_lsx(sptr + stride_w * 60); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - transpose4x4_ps(_r8, _r9, _ra, _rb); - transpose4x4_ps(_rc, _rd, _re, _rf); - __lsx_vst(float2bfloat_lsx(_r0, _r4), pp, 0); - __lsx_vst(float2bfloat_lsx(_r8, _rc), pp + 8, 0); - __lsx_vst(float2bfloat_lsx(_r1, _r5), pp + 16, 0); - __lsx_vst(float2bfloat_lsx(_r9, _rd), pp + 24, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r6), pp + 32, 0); - __lsx_vst(float2bfloat_lsx(_ra, _re), pp + 40, 0); - __lsx_vst(float2bfloat_lsx(_r3, _r7), pp + 48, 0); - __lsx_vst(float2bfloat_lsx(_rb, _rf), pp + 56, 0); + if (stride_w == 1) + { + __m128i _r01 = __lsx_vld(sptr, 0); + __m128i _r23 = __lsx_vld(sptr + 8, 0); + __m128i _r45 = __lsx_vld(sptr + 16, 0); + __m128i _r67 = __lsx_vld(sptr + 24, 0); + __m128i _r0 = __lsx_vilvl_d(_r45, _r01); + __m128i _r1 = __lsx_vilvh_d(_r45, _r01); + __m128i _r2 = __lsx_vilvl_d(_r67, _r23); + __m128i _r3 = __lsx_vilvh_d(_r67, _r23); + transpose8x4_epi16(_r0, _r1, _r2, _r3); + __m128i _c0 = __lsx_vilvl_d(_r2, _r0); + __m128i _c1 = __lsx_vilvh_d(_r2, _r0); + __m128i _c2 = __lsx_vilvl_d(_r3, _r1); + __m128i _c3 = __lsx_vilvh_d(_r3, _r1); + + __m128i _r89 = __lsx_vld(sptr + 32, 0); + __m128i _rab = __lsx_vld(sptr + 40, 0); + __m128i _rcd = __lsx_vld(sptr + 48, 0); + __m128i _ref = __lsx_vld(sptr + 56, 0); + __m128i _r8 = __lsx_vilvl_d(_rcd, _r89); + __m128i _r9 = __lsx_vilvh_d(_rcd, _r89); + __m128i _ra = __lsx_vilvl_d(_ref, _rab); + __m128i _rb = __lsx_vilvh_d(_ref, _rab); + transpose8x4_epi16(_r8, _r9, _ra, _rb); + __m128i _c4 = __lsx_vilvl_d(_ra, _r8); + __m128i _c5 = __lsx_vilvh_d(_ra, _r8); + __m128i _c6 = __lsx_vilvl_d(_rb, _r9); + __m128i _c7 = __lsx_vilvh_d(_rb, _r9); + + __lsx_vst(_c0, pp, 0); + __lsx_vst(_c4, pp + 8, 0); + __lsx_vst(_c1, pp + 16, 0); + __lsx_vst(_c5, pp + 24, 0); + __lsx_vst(_c2, pp + 32, 0); + __lsx_vst(_c6, pp + 40, 0); + __lsx_vst(_c3, pp + 48, 0); + __lsx_vst(_c7, pp + 56, 0); + } + else + { + __m128i _r0 = __lsx_vldrepl_d(sptr, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr + stride_w * 4, 0); + __m128i _r2 = __lsx_vldrepl_d(sptr + stride_w * 8, 0); + __m128i _r3 = __lsx_vldrepl_d(sptr + stride_w * 12, 0); + __m128i _r4 = __lsx_vldrepl_d(sptr + stride_w * 16, 0); + __m128i _r5 = __lsx_vldrepl_d(sptr + stride_w * 20, 0); + __m128i _r6 = __lsx_vldrepl_d(sptr + stride_w * 24, 0); + __m128i _r7 = __lsx_vldrepl_d(sptr + stride_w * 28, 0); + __m128i _r8 = __lsx_vldrepl_d(sptr + stride_w * 32, 0); + __m128i _r9 = __lsx_vldrepl_d(sptr + stride_w * 36, 0); + __m128i _ra = __lsx_vldrepl_d(sptr + stride_w * 40, 0); + __m128i _rb = __lsx_vldrepl_d(sptr + stride_w * 44, 0); + __m128i _rc = __lsx_vldrepl_d(sptr + stride_w * 48, 0); + __m128i _rd = __lsx_vldrepl_d(sptr + stride_w * 52, 0); + __m128i _re = __lsx_vldrepl_d(sptr + stride_w * 56, 0); + __m128i _rf = __lsx_vldrepl_d(sptr + stride_w * 60, 0); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + transpose4x4_epi16(_r4, _r5, _r6, _r7); + transpose4x4_epi16(_r8, _r9, _ra, _rb); + transpose4x4_epi16(_rc, _rd, _re, _rf); + __lsx_vst(__lsx_vilvl_d(_r4, _r0), pp, 0); + __lsx_vst(__lsx_vilvl_d(_rc, _r8), pp + 8, 0); + __lsx_vst(__lsx_vilvh_d(_r4, _r0), pp + 16, 0); + __lsx_vst(__lsx_vilvh_d(_rc, _r8), pp + 24, 0); + __lsx_vst(__lsx_vilvl_d(_r5, _r1), pp + 32, 0); + __lsx_vst(__lsx_vilvl_d(_rd, _r9), pp + 40, 0); + __lsx_vst(__lsx_vilvh_d(_r5, _r1), pp + 48, 0); + __lsx_vst(__lsx_vilvh_d(_rd, _r9), pp + 56, 0); + } pp += 64; } if (elempack == 1) @@ -4006,34 +4035,34 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr0); - __m128 _r1 = bfloat2float_lsx(sptr1); - __m128 _r2 = bfloat2float_lsx(sptr2); - __m128 _r3 = bfloat2float_lsx(sptr3); - __m128 _r4 = bfloat2float_lsx(sptr4); - __m128 _r5 = bfloat2float_lsx(sptr5); - __m128 _r6 = bfloat2float_lsx(sptr6); - __m128 _r7 = bfloat2float_lsx(sptr7); - __m128 _r8 = bfloat2float_lsx(sptr8); - __m128 _r9 = bfloat2float_lsx(sptr9); - __m128 _ra = bfloat2float_lsx(sptra); - __m128 _rb = bfloat2float_lsx(sptrb); - __m128 _rc = bfloat2float_lsx(sptrc); - __m128 _rd = bfloat2float_lsx(sptrd); - __m128 _re = bfloat2float_lsx(sptre); - __m128 _rf = bfloat2float_lsx(sptrf); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - transpose4x4_ps(_r8, _r9, _ra, _rb); - transpose4x4_ps(_rc, _rd, _re, _rf); - __lsx_vst(float2bfloat_lsx(_r0, _r4), pp, 0); - __lsx_vst(float2bfloat_lsx(_r8, _rc), pp + 8, 0); - __lsx_vst(float2bfloat_lsx(_r1, _r5), pp + 16, 0); - __lsx_vst(float2bfloat_lsx(_r9, _rd), pp + 24, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r6), pp + 32, 0); - __lsx_vst(float2bfloat_lsx(_ra, _re), pp + 40, 0); - __lsx_vst(float2bfloat_lsx(_r3, _r7), pp + 48, 0); - __lsx_vst(float2bfloat_lsx(_rb, _rf), pp + 56, 0); + __m128i _r0 = __lsx_vldrepl_d(sptr0, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr1, 0); + __m128i _r2 = __lsx_vldrepl_d(sptr2, 0); + __m128i _r3 = __lsx_vldrepl_d(sptr3, 0); + __m128i _r4 = __lsx_vldrepl_d(sptr4, 0); + __m128i _r5 = __lsx_vldrepl_d(sptr5, 0); + __m128i _r6 = __lsx_vldrepl_d(sptr6, 0); + __m128i _r7 = __lsx_vldrepl_d(sptr7, 0); + __m128i _r8 = __lsx_vldrepl_d(sptr8, 0); + __m128i _r9 = __lsx_vldrepl_d(sptr9, 0); + __m128i _ra = __lsx_vldrepl_d(sptra, 0); + __m128i _rb = __lsx_vldrepl_d(sptrb, 0); + __m128i _rc = __lsx_vldrepl_d(sptrc, 0); + __m128i _rd = __lsx_vldrepl_d(sptrd, 0); + __m128i _re = __lsx_vldrepl_d(sptre, 0); + __m128i _rf = __lsx_vldrepl_d(sptrf, 0); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + transpose4x4_epi16(_r4, _r5, _r6, _r7); + transpose4x4_epi16(_r8, _r9, _ra, _rb); + transpose4x4_epi16(_rc, _rd, _re, _rf); + __lsx_vst(__lsx_vilvl_d(_r4, _r0), pp, 0); + __lsx_vst(__lsx_vilvl_d(_rc, _r8), pp + 8, 0); + __lsx_vst(__lsx_vilvh_d(_r4, _r0), pp + 16, 0); + __lsx_vst(__lsx_vilvh_d(_rc, _r8), pp + 24, 0); + __lsx_vst(__lsx_vilvl_d(_r5, _r1), pp + 32, 0); + __lsx_vst(__lsx_vilvl_d(_rd, _r9), pp + 40, 0); + __lsx_vst(__lsx_vilvh_d(_r5, _r1), pp + 48, 0); + __lsx_vst(__lsx_vilvh_d(_rd, _r9), pp + 56, 0); pp += 64; } if (elempack == 1) @@ -4119,20 +4148,39 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr + stride_w * 0); - __m128 _r1 = bfloat2float_lsx(sptr + stride_w * 4); - __m128 _r2 = bfloat2float_lsx(sptr + stride_w * 8); - __m128 _r3 = bfloat2float_lsx(sptr + stride_w * 12); - __m128 _r4 = bfloat2float_lsx(sptr + stride_w * 16); - __m128 _r5 = bfloat2float_lsx(sptr + stride_w * 20); - __m128 _r6 = bfloat2float_lsx(sptr + stride_w * 24); - __m128 _r7 = bfloat2float_lsx(sptr + stride_w * 28); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - __lsx_vst(float2bfloat_lsx(_r0, _r4), pp + 4 * 0, 0); - __lsx_vst(float2bfloat_lsx(_r1, _r5), pp + 4 * 2, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r6), pp + 4 * 4, 0); - __lsx_vst(float2bfloat_lsx(_r3, _r7), pp + 4 * 6, 0); + if (stride_w == 1) + { + __m128i _r01 = __lsx_vld(sptr, 0); + __m128i _r23 = __lsx_vld(sptr + 8, 0); + __m128i _r45 = __lsx_vld(sptr + 16, 0); + __m128i _r67 = __lsx_vld(sptr + 24, 0); + __m128i _r0 = __lsx_vilvl_d(_r45, _r01); + __m128i _r1 = __lsx_vilvh_d(_r45, _r01); + __m128i _r2 = __lsx_vilvl_d(_r67, _r23); + __m128i _r3 = __lsx_vilvh_d(_r67, _r23); + transpose8x4_epi16(_r0, _r1, _r2, _r3); + __lsx_vst(__lsx_vilvl_d(_r2, _r0), pp, 0); + __lsx_vst(__lsx_vilvh_d(_r2, _r0), pp + 8, 0); + __lsx_vst(__lsx_vilvl_d(_r3, _r1), pp + 16, 0); + __lsx_vst(__lsx_vilvh_d(_r3, _r1), pp + 24, 0); + } + else + { + __m128i _r0 = __lsx_vldrepl_d(sptr, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr + stride_w * 4, 0); + __m128i _r2 = __lsx_vldrepl_d(sptr + stride_w * 8, 0); + __m128i _r3 = __lsx_vldrepl_d(sptr + stride_w * 12, 0); + __m128i _r4 = __lsx_vldrepl_d(sptr + stride_w * 16, 0); + __m128i _r5 = __lsx_vldrepl_d(sptr + stride_w * 20, 0); + __m128i _r6 = __lsx_vldrepl_d(sptr + stride_w * 24, 0); + __m128i _r7 = __lsx_vldrepl_d(sptr + stride_w * 28, 0); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + transpose4x4_epi16(_r4, _r5, _r6, _r7); + __lsx_vst(__lsx_vilvl_d(_r4, _r0), pp, 0); + __lsx_vst(__lsx_vilvh_d(_r4, _r0), pp + 8, 0); + __lsx_vst(__lsx_vilvl_d(_r5, _r1), pp + 16, 0); + __lsx_vst(__lsx_vilvh_d(_r5, _r1), pp + 24, 0); + } pp += 32; } if (elempack == 1) @@ -4210,20 +4258,20 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr0); - __m128 _r1 = bfloat2float_lsx(sptr1); - __m128 _r2 = bfloat2float_lsx(sptr2); - __m128 _r3 = bfloat2float_lsx(sptr3); - __m128 _r4 = bfloat2float_lsx(sptr4); - __m128 _r5 = bfloat2float_lsx(sptr5); - __m128 _r6 = bfloat2float_lsx(sptr6); - __m128 _r7 = bfloat2float_lsx(sptr7); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - __lsx_vst(float2bfloat_lsx(_r0, _r4), pp + 4 * 0, 0); - __lsx_vst(float2bfloat_lsx(_r1, _r5), pp + 4 * 2, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r6), pp + 4 * 4, 0); - __lsx_vst(float2bfloat_lsx(_r3, _r7), pp + 4 * 6, 0); + __m128i _r0 = __lsx_vldrepl_d(sptr0, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr1, 0); + __m128i _r2 = __lsx_vldrepl_d(sptr2, 0); + __m128i _r3 = __lsx_vldrepl_d(sptr3, 0); + __m128i _r4 = __lsx_vldrepl_d(sptr4, 0); + __m128i _r5 = __lsx_vldrepl_d(sptr5, 0); + __m128i _r6 = __lsx_vldrepl_d(sptr6, 0); + __m128i _r7 = __lsx_vldrepl_d(sptr7, 0); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + transpose4x4_epi16(_r4, _r5, _r6, _r7); + __lsx_vst(__lsx_vilvl_d(_r4, _r0), pp, 0); + __lsx_vst(__lsx_vilvh_d(_r4, _r0), pp + 8, 0); + __lsx_vst(__lsx_vilvl_d(_r5, _r1), pp + 16, 0); + __lsx_vst(__lsx_vilvh_d(_r5, _r1), pp + 24, 0); pp += 32; } if (elempack == 1) @@ -4284,13 +4332,27 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr + stride_w * 0); - __m128 _r1 = bfloat2float_lsx(sptr + stride_w * 4); - __m128 _r2 = bfloat2float_lsx(sptr + stride_w * 8); - __m128 _r3 = bfloat2float_lsx(sptr + stride_w * 12); - transpose4x4_ps(_r0, _r1, _r2, _r3); - __lsx_vst(float2bfloat_lsx(_r0, _r1), pp + 4 * 0, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r3), pp + 4 * 2, 0); + if (stride_w == 1) + { + __m128i _r01 = __lsx_vld(sptr, 0); + __m128i _r23 = __lsx_vld(sptr + 8, 0); + __m128i _r02 = __lsx_vilvl_d(_r23, _r01); + __m128i _r13 = __lsx_vilvh_d(_r23, _r01); + __m128i _tmp0 = __lsx_vilvl_h(_r13, _r02); + __m128i _tmp1 = __lsx_vilvh_h(_r13, _r02); + __lsx_vst(__lsx_vilvl_w(_tmp1, _tmp0), pp, 0); + __lsx_vst(__lsx_vilvh_w(_tmp1, _tmp0), pp + 8, 0); + } + else + { + __m128i _r0 = __lsx_vldrepl_d(sptr, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr + stride_w * 4, 0); + __m128i _r2 = __lsx_vldrepl_d(sptr + stride_w * 8, 0); + __m128i _r3 = __lsx_vldrepl_d(sptr + stride_w * 12, 0); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + __lsx_vst(_r0, pp, 0); + __lsx_vst(_r1, pp + 8, 0); + } pp += 16; } if (elempack == 1) @@ -4344,13 +4406,13 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr0); - __m128 _r1 = bfloat2float_lsx(sptr1); - __m128 _r2 = bfloat2float_lsx(sptr2); - __m128 _r3 = bfloat2float_lsx(sptr3); - transpose4x4_ps(_r0, _r1, _r2, _r3); - __lsx_vst(float2bfloat_lsx(_r0, _r1), pp + 4 * 0, 0); - __lsx_vst(float2bfloat_lsx(_r2, _r3), pp + 4 * 2, 0); + __m128i _r0 = __lsx_vldrepl_d(sptr0, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr1, 0); + __m128i _r2 = __lsx_vldrepl_d(sptr2, 0); + __m128i _r3 = __lsx_vldrepl_d(sptr3, 0); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + __lsx_vst(_r0, pp, 0); + __lsx_vst(_r1, pp + 8, 0); pp += 16; } if (elempack == 1) @@ -4402,11 +4464,21 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr + stride_w * 0); - __m128 _r1 = bfloat2float_lsx(sptr + stride_w * 4); - __m128 _tmp0 = (__m128)__lsx_vilvl_w((__m128i)_r1, (__m128i)_r0); - __m128 _tmp1 = (__m128)__lsx_vilvh_w((__m128i)_r1, (__m128i)_r0); - __lsx_vst(float2bfloat_lsx(_tmp0, _tmp1), pp, 0); + __m128i _r0; + __m128i _r1; + if (stride_w == 1) + { + __m128i _r01 = __lsx_vld(sptr, 0); + _r0 = __lsx_vilvl_d(_r01, _r01); + _r1 = __lsx_vilvh_d(_r01, _r01); + } + else + { + _r0 = __lsx_vldrepl_d(sptr, 0); + _r1 = __lsx_vldrepl_d(sptr + stride_w * 4, 0); + } + __m128i _tmp0 = __lsx_vilvl_h(_r1, _r0); + __lsx_vst(_tmp0, pp, 0); pp += 8; } #endif // __loongarch_sx @@ -4451,11 +4523,10 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __m128 _r0 = bfloat2float_lsx(sptr0); - __m128 _r1 = bfloat2float_lsx(sptr1); - __m128 _tmp0 = (__m128)__lsx_vilvl_w((__m128i)_r1, (__m128i)_r0); - __m128 _tmp1 = (__m128)__lsx_vilvh_w((__m128i)_r1, (__m128i)_r0); - __lsx_vst(float2bfloat_lsx(_tmp0, _tmp1), pp, 0); + __m128i _r0 = __lsx_vldrepl_d(sptr0, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr1, 0); + __m128i _tmp0 = __lsx_vilvl_h(_r1, _r0); + __lsx_vst(_tmp0, pp, 0); pp += 8; } #endif // __loongarch_sx diff --git a/src/layer/loongarch/convolution_im2col_gemm_int8.h b/src/layer/loongarch/convolution_im2col_gemm_int8.h index 84b4b8968..e35a43ac2 100644 --- a/src/layer/loongarch/convolution_im2col_gemm_int8.h +++ b/src/layer/loongarch/convolution_im2col_gemm_int8.h @@ -1134,70 +1134,15 @@ static void convolution_im2col_input_tile_conv1x1s1d1_int8(const Mat& bottom_blo const signed char* p2 = p0 + bottom_blob.cstep * 2; const signed char* p3 = p0 + bottom_blob.cstep * 3; - pp[0] = p0[0]; - pp[1] = p1[0]; - pp[2] = p2[0]; - pp[3] = p3[0]; - pp[4] = p0[1]; - pp[5] = p1[1]; - pp[6] = p2[1]; - pp[7] = p3[1]; - pp[8] = p0[2]; - pp[9] = p1[2]; - pp[10] = p2[2]; - pp[11] = p3[2]; - pp[12] = p0[3]; - pp[13] = p1[3]; - pp[14] = p2[3]; - pp[15] = p3[3]; - pp[16] = p0[4]; - pp[17] = p1[4]; - pp[18] = p2[4]; - pp[19] = p3[4]; - pp[20] = p0[5]; - pp[21] = p1[5]; - pp[22] = p2[5]; - pp[23] = p3[5]; - pp[24] = p0[6]; - pp[25] = p1[6]; - pp[26] = p2[6]; - pp[27] = p3[6]; - pp[28] = p0[7]; - pp[29] = p1[7]; - pp[30] = p2[7]; - pp[31] = p3[7]; - pp[32] = p0[8]; - pp[33] = p1[8]; - pp[34] = p2[8]; - pp[35] = p3[8]; - pp[36] = p0[9]; - pp[37] = p1[9]; - pp[38] = p2[9]; - pp[39] = p3[9]; - pp[40] = p0[10]; - pp[41] = p1[10]; - pp[42] = p2[10]; - pp[43] = p3[10]; - pp[44] = p0[11]; - pp[45] = p1[11]; - pp[46] = p2[11]; - pp[47] = p3[11]; - pp[48] = p0[12]; - pp[49] = p1[12]; - pp[50] = p2[12]; - pp[51] = p3[12]; - pp[52] = p0[13]; - pp[53] = p1[13]; - pp[54] = p2[13]; - pp[55] = p3[13]; - pp[56] = p0[14]; - pp[57] = p1[14]; - pp[58] = p2[14]; - pp[59] = p3[14]; - pp[60] = p0[15]; - pp[61] = p1[15]; - pp[62] = p2[15]; - pp[63] = p3[15]; + __m128i _r0 = __lsx_vld(p0, 0); + __m128i _r1 = __lsx_vld(p1, 0); + __m128i _r2 = __lsx_vld(p2, 0); + __m128i _r3 = __lsx_vld(p3, 0); + transpose16x4_epi8(_r3, _r2, _r1, _r0); + __lsx_vst(_r3, pp, 0); + __lsx_vst(_r2, pp + 16, 0); + __lsx_vst(_r1, pp + 32, 0); + __lsx_vst(_r0, pp + 48, 0); pp += 64; p0 += bottom_blob.cstep * 4; } @@ -1284,38 +1229,13 @@ static void convolution_im2col_input_tile_conv1x1s1d1_int8(const Mat& bottom_blo const signed char* p2 = p0 + bottom_blob.cstep * 2; const signed char* p3 = p0 + bottom_blob.cstep * 3; - pp[0] = p0[0]; - pp[1] = p1[0]; - pp[2] = p2[0]; - pp[3] = p3[0]; - pp[4] = p0[1]; - pp[5] = p1[1]; - pp[6] = p2[1]; - pp[7] = p3[1]; - pp[8] = p0[2]; - pp[9] = p1[2]; - pp[10] = p2[2]; - pp[11] = p3[2]; - pp[12] = p0[3]; - pp[13] = p1[3]; - pp[14] = p2[3]; - pp[15] = p3[3]; - pp[16] = p0[4]; - pp[17] = p1[4]; - pp[18] = p2[4]; - pp[19] = p3[4]; - pp[20] = p0[5]; - pp[21] = p1[5]; - pp[22] = p2[5]; - pp[23] = p3[5]; - pp[24] = p0[6]; - pp[25] = p1[6]; - pp[26] = p2[6]; - pp[27] = p3[6]; - pp[28] = p0[7]; - pp[29] = p1[7]; - pp[30] = p2[7]; - pp[31] = p3[7]; + __m128i _r0 = __lsx_vldrepl_d(p0, 0); + __m128i _r1 = __lsx_vldrepl_d(p1, 0); + __m128i _r2 = __lsx_vldrepl_d(p2, 0); + __m128i _r3 = __lsx_vldrepl_d(p3, 0); + transpose16x4_epi8(_r3, _r2, _r1, _r0); + __lsx_vst(_r3, pp, 0); + __lsx_vst(_r2, pp + 16, 0); pp += 32; p0 += bottom_blob.cstep * 4; } @@ -1377,22 +1297,12 @@ static void convolution_im2col_input_tile_conv1x1s1d1_int8(const Mat& bottom_blo const signed char* p2 = p0 + bottom_blob.cstep * 2; const signed char* p3 = p0 + bottom_blob.cstep * 3; - pp[0] = p0[0]; - pp[1] = p1[0]; - pp[2] = p2[0]; - pp[3] = p3[0]; - pp[4] = p0[1]; - pp[5] = p1[1]; - pp[6] = p2[1]; - pp[7] = p3[1]; - pp[8] = p0[2]; - pp[9] = p1[2]; - pp[10] = p2[2]; - pp[11] = p3[2]; - pp[12] = p0[3]; - pp[13] = p1[3]; - pp[14] = p2[3]; - pp[15] = p3[3]; + __m128i _r0 = __lsx_vldrepl_w(p0, 0); + __m128i _r1 = __lsx_vldrepl_w(p1, 0); + __m128i _r2 = __lsx_vldrepl_w(p2, 0); + __m128i _r3 = __lsx_vldrepl_w(p3, 0); + transpose16x4_epi8(_r3, _r2, _r1, _r0); + __lsx_vst(_r3, pp, 0); pp += 16; p0 += bottom_blob.cstep * 4; } @@ -1580,6 +1490,58 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo } if (elempack == 1) { + if (dy[0] == dy[15] && stride_w == 1) + { + for (; kk + 3 < max_kk; kk += 4) + { + const int p0 = (k + kk) / maxk; + const int p1 = (k + kk + 1) / maxk; + const int p2 = (k + kk + 2) / maxk; + const int p3 = (k + kk + 3) / maxk; + const int uv0 = (k + kk) % maxk; + const int uv1 = (k + kk + 1) % maxk; + const int uv2 = (k + kk + 2) % maxk; + const int uv3 = (k + kk + 3) % maxk; + const int u0 = uv0 / kernel_w; + const int u1 = uv1 / kernel_w; + const int u2 = uv2 / kernel_w; + const int u3 = uv3 / kernel_w; + const int v0 = uv0 % kernel_w; + const int v1 = uv1 % kernel_w; + const int v2 = uv2 % kernel_w; + const int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + const int x0 = dx[0] + dilation_w * v0; + const int x1 = dx[0] + dilation_w * v1; + const int x2 = dx[0] + dilation_w * v2; + const int x3 = dx[0] + dilation_w * v3; + const int y0 = stride_h * dy[0] + dilation_h * u0; + const int y1 = stride_h * dy[0] + dilation_h * u1; + const int y2 = stride_h * dy[0] + dilation_h * u2; + const int y3 = stride_h * dy[0] + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + __m128i _r0 = __lsx_vld(sptr0, 0); + __m128i _r1 = __lsx_vld(sptr1, 0); + __m128i _r2 = __lsx_vld(sptr2, 0); + __m128i _r3 = __lsx_vld(sptr3, 0); + transpose16x4_epi8(_r3, _r2, _r1, _r0); + __lsx_vst(_r3, pp, 0); + __lsx_vst(_r2, pp + 16, 0); + __lsx_vst(_r1, pp + 32, 0); + __lsx_vst(_r0, pp + 48, 0); + pp += 64; + } + } for (; kk + 3 < max_kk; kk += 4) { for (int q = 0; q < 4; q++) @@ -1702,12 +1664,134 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo pp += 8; } } - for (; kk + 3 < max_kk / elempack; kk += 4) + if (elempack == 1) { - for (int q = 0; q < 4; q++) + if (stride_w == 1) { - int p = (k / elempack + kk + q) / maxk; - int uv = (k / elempack + kk + q) % maxk; + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = dx0 + dilation_w * v0; + int x1 = dx0 + dilation_w * v1; + int x2 = dx0 + dilation_w * v2; + int x3 = dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + __m128i _r0 = __lsx_vldrepl_d(sptr0, 0); + __m128i _r1 = __lsx_vldrepl_d(sptr1, 0); + __m128i _r2 = __lsx_vldrepl_d(sptr2, 0); + __m128i _r3 = __lsx_vldrepl_d(sptr3, 0); + transpose16x4_epi8(_r3, _r2, _r1, _r0); + __lsx_vst(_r3, pp, 0); + __lsx_vst(_r2, pp + 16, 0); + pp += 32; + } + } + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = stride_w * dx0 + dilation_w * v0; + int x1 = stride_w * dx0 + dilation_w * v1; + int x2 = stride_w * dx0 + dilation_w * v2; + int x3 = stride_w * dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + pp[0] = sptr0[0]; + pp[1] = sptr1[0]; + pp[2] = sptr2[0]; + pp[3] = sptr3[0]; + pp[4] = sptr0[stride_w]; + pp[5] = sptr1[stride_w]; + pp[6] = sptr2[stride_w]; + pp[7] = sptr3[stride_w]; + pp[8] = sptr0[stride_w * 2]; + pp[9] = sptr1[stride_w * 2]; + pp[10] = sptr2[stride_w * 2]; + pp[11] = sptr3[stride_w * 2]; + pp[12] = sptr0[stride_w * 3]; + pp[13] = sptr1[stride_w * 3]; + pp[14] = sptr2[stride_w * 3]; + pp[15] = sptr3[stride_w * 3]; + pp[16] = sptr0[stride_w * 4]; + pp[17] = sptr1[stride_w * 4]; + pp[18] = sptr2[stride_w * 4]; + pp[19] = sptr3[stride_w * 4]; + pp[20] = sptr0[stride_w * 5]; + pp[21] = sptr1[stride_w * 5]; + pp[22] = sptr2[stride_w * 5]; + pp[23] = sptr3[stride_w * 5]; + pp[24] = sptr0[stride_w * 6]; + pp[25] = sptr1[stride_w * 6]; + pp[26] = sptr2[stride_w * 6]; + pp[27] = sptr3[stride_w * 6]; + pp[28] = sptr0[stride_w * 7]; + pp[29] = sptr1[stride_w * 7]; + pp[30] = sptr2[stride_w * 7]; + pp[31] = sptr3[stride_w * 7]; + pp += 32; + } + for (; kk < max_kk; kk++) + { + int p = (k + kk) / maxk; + int uv = (k + kk) % maxk; int u = uv / kernel_w; int v = uv % kernel_w; @@ -1716,38 +1800,8 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo int x0 = stride_w * dx0 + dilation_w * v; int y0 = stride_h * dy0 + dilation_h * u; - const signed char* sptr = img.row(y0) + x0 * elempack; + const signed char* sptr = img.row(y0) + x0; - if (elempack == 1) - { - pp[q] = sptr[0]; - pp[4 + q] = sptr[stride_w]; - pp[8 + q] = sptr[stride_w * 2]; - pp[12 + q] = sptr[stride_w * 3]; - pp[16 + q] = sptr[stride_w * 4]; - pp[20 + q] = sptr[stride_w * 5]; - pp[24 + q] = sptr[stride_w * 6]; - pp[28 + q] = sptr[stride_w * 7]; - } - } - pp += 32; - } - for (; kk < max_kk / elempack; kk++) - { - int p = (k / elempack + kk) / maxk; - int uv = (k / elempack + kk) % maxk; - int u = uv / kernel_w; - int v = uv % kernel_w; - - const Mat img = bottom_blob.channel(p); - - int x0 = stride_w * dx0 + dilation_w * v; - int y0 = stride_h * dy0 + dilation_h * u; - - const signed char* sptr = img.row(y0) + x0 * elempack; - - if (elempack == 1) - { pp[0] = sptr[0]; pp[1] = sptr[stride_w]; pp[2] = sptr[stride_w * 2]; @@ -2032,12 +2086,117 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo pp += 4; } } - for (; kk + 3 < max_kk / elempack; kk += 4) + if (elempack == 1) { - for (int q = 0; q < 4; q++) + if (stride_w == 1) { - int p = (k / elempack + kk + q) / maxk; - int uv = (k / elempack + kk + q) % maxk; + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = dx0 + dilation_w * v0; + int x1 = dx0 + dilation_w * v1; + int x2 = dx0 + dilation_w * v2; + int x3 = dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + __m128i _r0 = __lsx_vldrepl_w(sptr0, 0); + __m128i _r1 = __lsx_vldrepl_w(sptr1, 0); + __m128i _r2 = __lsx_vldrepl_w(sptr2, 0); + __m128i _r3 = __lsx_vldrepl_w(sptr3, 0); + transpose16x4_epi8(_r3, _r2, _r1, _r0); + __lsx_vst(_r3, pp, 0); + pp += 16; + } + } + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = stride_w * dx0 + dilation_w * v0; + int x1 = stride_w * dx0 + dilation_w * v1; + int x2 = stride_w * dx0 + dilation_w * v2; + int x3 = stride_w * dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + pp[0] = sptr0[0]; + pp[1] = sptr1[0]; + pp[2] = sptr2[0]; + pp[3] = sptr3[0]; + pp[4] = sptr0[stride_w]; + pp[5] = sptr1[stride_w]; + pp[6] = sptr2[stride_w]; + pp[7] = sptr3[stride_w]; + pp[8] = sptr0[stride_w * 2]; + pp[9] = sptr1[stride_w * 2]; + pp[10] = sptr2[stride_w * 2]; + pp[11] = sptr3[stride_w * 2]; + pp[12] = sptr0[stride_w * 3]; + pp[13] = sptr1[stride_w * 3]; + pp[14] = sptr2[stride_w * 3]; + pp[15] = sptr3[stride_w * 3]; + pp += 16; + } + for (; kk < max_kk; kk++) + { + int p = (k + kk) / maxk; + int uv = (k + kk) % maxk; int u = uv / kernel_w; int v = uv % kernel_w; @@ -2046,34 +2205,8 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo int x0 = stride_w * dx0 + dilation_w * v; int y0 = stride_h * dy0 + dilation_h * u; - const signed char* sptr = img.row(y0) + x0 * elempack; + const signed char* sptr = img.row(y0) + x0; - if (elempack == 1) - { - pp[q] = sptr[0]; - pp[4 + q] = sptr[stride_w]; - pp[8 + q] = sptr[stride_w * 2]; - pp[12 + q] = sptr[stride_w * 3]; - } - } - pp += 16; - } - for (; kk < max_kk / elempack; kk++) - { - int p = (k / elempack + kk) / maxk; - int uv = (k / elempack + kk) % maxk; - int u = uv / kernel_w; - int v = uv % kernel_w; - - const Mat img = bottom_blob.channel(p); - - int x0 = stride_w * dx0 + dilation_w * v; - int y0 = stride_h * dy0 + dilation_h * u; - - const signed char* sptr = img.row(y0) + x0 * elempack; - - if (elempack == 1) - { pp[0] = sptr[0]; pp[1] = sptr[stride_w]; pp[2] = sptr[stride_w * 2]; diff --git a/src/layer/loongarch/gemm_bf16s.h b/src/layer/loongarch/gemm_bf16s.h index d06e1868a..3258c81c8 100644 --- a/src/layer/loongarch/gemm_bf16s.h +++ b/src/layer/loongarch/gemm_bf16s.h @@ -93,22 +93,6 @@ static void pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, int k, in } for (; ii + 3 < max_ii; ii += 4) { - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const unsigned short* p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - for (int kk = 0; kk < max_kk; kk++) - { - __builtin_prefetch(p0 + 32); - pp[0] = p0[0]; - pp[1] = p0[1]; - pp[2] = p0[2]; - pp[3] = p0[3]; - pp += 4; - p0 += 8; - } - } if (elempack == 4) { const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k * 4; @@ -152,63 +136,38 @@ static void pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, int k, in #endif // __loongarch_sx for (; ii + 1 < max_ii; ii += 2) { - const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k; - const unsigned short* p1 = (const unsigned short*)A + (i + ii + 1) * A_hstep + k; - -#if __loongarch_sx - if (elempack == 8) + // if (elempack == 1) { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; + const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k; + const unsigned short* p1 = (const unsigned short*)A + (i + ii + 1) * A_hstep + k; for (int kk = 0; kk < max_kk; kk++) { - pp[0] = p0[0]; - pp[1] = p0[1]; - pp += 2; - p0 += 8; - } - } -#endif // __loongarch_sx - for (int kk = 0; kk < max_kk; kk++) - { - if (elempack == 1) - { - pp[0] = p0[0]; - pp[1] = p1[0]; - pp += 2; - p0++; - p1++; + if (elempack == 1) + { + pp[0] = p0[0]; + pp[1] = p1[0]; + pp += 2; + p0++; + p1++; + } } } } for (; ii < max_ii; ii++) { - const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k; - -#if __loongarch_sx - if (elempack == 8) + // if (elempack == 1) { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; + const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k; for (int kk = 0; kk < max_kk; kk++) { - pp[0] = p0[0]; - pp += 1; - p0 += 8; - } - } -#endif // __loongarch_sx - for (int kk = 0; kk < max_kk; kk++) - { - if (elempack == 1) - { - pp[0] = p0[0]; - pp += 1; - p0++; + if (elempack == 1) + { + pp[0] = p0[0]; + pp += 1; + p0++; + } } } } @@ -227,49 +186,32 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, { if (elempack == 8) { - if ((k & 7) == 0) + const unsigned short* p0 = (const unsigned short*)A + k * A_hstep + (i + ii) * 8; + + for (int kk = 0; kk + 7 < max_kk; kk += 8) { - const unsigned short* p0 = (const unsigned short*)A + k * A_hstep + (i + ii) * 8; - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp[2] = p0[16]; - pp[3] = p0[24]; - pp[4] = p0[32]; - pp[5] = p0[40]; - pp[6] = p0[48]; - pp[7] = p0[56]; - pp += 8; - p0++; - if ((kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } - } - else - { - const int k_pack = k / 8; - const int k_lane = k % 8; - const unsigned short* p0 = (const unsigned short*)A + (size_t)k_pack * A_hstep * 8 + (i + ii) * 8 + k_lane; - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp[2] = p0[16]; - pp[3] = p0[24]; - pp[4] = p0[32]; - pp[5] = p0[40]; - pp[6] = p0[48]; - pp[7] = p0[56]; - pp += 8; - p0++; - if ((k_lane + kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } + __m128i _r0 = __lsx_vld(p0, 0); + __m128i _r1 = __lsx_vld(p0 + 8, 0); + __m128i _r2 = __lsx_vld(p0 + 16, 0); + __m128i _r3 = __lsx_vld(p0 + 24, 0); + __m128i _r4 = __lsx_vld(p0 + 32, 0); + __m128i _r5 = __lsx_vld(p0 + 40, 0); + __m128i _r6 = __lsx_vld(p0 + 48, 0); + __m128i _r7 = __lsx_vld(p0 + 56, 0); + + transpose8x8_epi16(_r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7); + + __lsx_vst(_r0, pp, 0); + __lsx_vst(_r1, pp + 8, 0); + __lsx_vst(_r2, pp + 16, 0); + __lsx_vst(_r3, pp + 24, 0); + __lsx_vst(_r4, pp + 32, 0); + __lsx_vst(_r5, pp + 40, 0); + __lsx_vst(_r6, pp + 48, 0); + __lsx_vst(_r7, pp + 56, 0); + + pp += 64; + p0 += A_hstep * 8; } } if (elempack == 4) @@ -317,56 +259,22 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, { if (elempack == 8) { - if ((k & 7) == 0) - { - const unsigned short* p0 = (const unsigned short*)A + k * A_hstep + (i + ii) * 8; + const unsigned short* p0 = (const unsigned short*)A + k * A_hstep + (i + ii) * 8; - int kk = 0; - for (; kk + 7 < max_kk; kk += 8) - { - for (int q = 0; q < 8; q++) - { - pp[q * 4] = p0[q]; - pp[q * 4 + 1] = p0[8 + q]; - pp[q * 4 + 2] = p0[16 + q]; - pp[q * 4 + 3] = p0[24 + q]; - } - pp += 32; - p0 += A_hstep * 8; - } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp[2] = p0[16]; - pp[3] = p0[24]; - pp += 4; - p0++; - if ((kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } - } - else + int kk = 0; + for (; kk + 7 < max_kk; kk += 8) { - const int k_pack = k / 8; - const int k_lane = k % 8; - const unsigned short* p0 = (const unsigned short*)A + (size_t)k_pack * A_hstep * 8 + (i + ii) * 8 + k_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp[2] = p0[16]; - pp[3] = p0[24]; - pp += 4; - p0++; - if ((k_lane + kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } + __m128i _r0 = __lsx_vld(p0, 0); + __m128i _r1 = __lsx_vld(p0 + 8, 0); + __m128i _r2 = __lsx_vld(p0 + 16, 0); + __m128i _r3 = __lsx_vld(p0 + 24, 0); + transpose8x4_epi16(_r0, _r1, _r2, _r3); + __lsx_vst(_r0, pp, 0); + __lsx_vst(_r1, pp + 8, 0); + __lsx_vst(_r2, pp + 16, 0); + __lsx_vst(_r3, pp + 24, 0); + pp += 32; + p0 += A_hstep * 8; } } if (elempack == 4) @@ -411,59 +319,27 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, #if __loongarch_sx if (elempack == 8) { - if ((k & 7) == 0) + int kk = 0; + for (; kk + 7 < max_kk; kk += 8) { - int kk = 0; - for (; kk + 7 < max_kk; kk += 8) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp[2] = p0[1]; - pp[3] = p0[9]; - pp[4] = p0[2]; - pp[5] = p0[10]; - pp[6] = p0[3]; - pp[7] = p0[11]; - pp[8] = p0[4]; - pp[9] = p0[12]; - pp[10] = p0[5]; - pp[11] = p0[13]; - pp[12] = p0[6]; - pp[13] = p0[14]; - pp[14] = p0[7]; - pp[15] = p0[15]; - pp += 16; - p0 += A_hstep * 8; - } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp += 2; - p0++; - if ((kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } - } - else - { - const int k_pack = k / 8; - const int k_lane = k % 8; - p0 = (const unsigned short*)A + (size_t)k_pack * A_hstep * 8 + (i + ii) * 8 + k_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp += 2; - p0++; - if ((k_lane + kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } + pp[0] = p0[0]; + pp[1] = p0[8]; + pp[2] = p0[1]; + pp[3] = p0[9]; + pp[4] = p0[2]; + pp[5] = p0[10]; + pp[6] = p0[3]; + pp[7] = p0[11]; + pp[8] = p0[4]; + pp[9] = p0[12]; + pp[10] = p0[5]; + pp[11] = p0[13]; + pp[12] = p0[6]; + pp[13] = p0[14]; + pp[14] = p0[7]; + pp[15] = p0[15]; + pp += 16; + p0 += A_hstep * 8; } } if (elempack == 4) @@ -504,49 +380,19 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, #if __loongarch_sx if (elempack == 8) { - if ((k & 7) == 0) + int kk = 0; + for (; kk + 7 < max_kk; kk += 8) { - int kk = 0; - for (; kk + 7 < max_kk; kk += 8) - { - pp[0] = p0[0]; - pp[1] = p0[1]; - pp[2] = p0[2]; - pp[3] = p0[3]; - pp[4] = p0[4]; - pp[5] = p0[5]; - pp[6] = p0[6]; - pp[7] = p0[7]; - pp += 8; - p0 += A_hstep * 8; - } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - if ((kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } - } - else - { - const int k_pack = k / 8; - const int k_lane = k % 8; - p0 = (const unsigned short*)A + (size_t)k_pack * A_hstep * 8 + (i + ii) * 8 + k_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - if ((k_lane + kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } - } + pp[0] = p0[0]; + pp[1] = p0[1]; + pp[2] = p0[2]; + pp[3] = p0[3]; + pp[4] = p0[4]; + pp[5] = p0[5]; + pp[6] = p0[6]; + pp[7] = p0[7]; + pp += 8; + p0 += A_hstep * 8; } } if (elempack == 4) @@ -616,7 +462,30 @@ static void pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, int k, in const unsigned short* p2 = (const unsigned short*)B + (j + jj + 8) * B_hstep + k * 4; const unsigned short* p3 = (const unsigned short*)B + (j + jj + 12) * B_hstep + k * 4; - for (int kk = 0; kk < max_kk; kk++) + int kk = 0; + for (; kk + 1 < max_kk; kk += 2) + { + __builtin_prefetch(p0 + 32); + __builtin_prefetch(p1 + 32); + __builtin_prefetch(p2 + 32); + __builtin_prefetch(p3 + 32); + __m128i _r0 = __lsx_vld(p0, 0); + __m128i _r1 = __lsx_vld(p1, 0); + __m128i _r2 = __lsx_vld(p2, 0); + __m128i _r3 = __lsx_vld(p3, 0); + __m128i _r01 = __lsx_vilvl_d(_r1, _r0); + __m128i _r23 = __lsx_vilvl_d(_r3, _r2); + __lasx_xvst(__lasx_concat_128(_r01, _r23), pp, 0); + _r01 = __lsx_vilvh_d(_r1, _r0); + _r23 = __lsx_vilvh_d(_r3, _r2); + __lasx_xvst(__lasx_concat_128(_r01, _r23), pp + 16, 0); + pp += 32; + p0 += 8; + p1 += 8; + p2 += 8; + p3 += 8; + } + for (; kk < max_kk; kk++) { __builtin_prefetch(p0 + 32); __builtin_prefetch(p1 + 32); @@ -793,27 +662,11 @@ static void pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, int k, in #endif // __loongarch_sx for (; jj + 3 < max_jj; jj += 4) { - const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k * elempack; #if __loongarch_sx - if (elempack == 8) - { - const int j_pack = (j + jj) / 8; - const int j_lane = (j + jj) % 8; - p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 8 + k * 8 + j_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - __builtin_prefetch(p0 + 32); - pp[0] = p0[0]; - pp[1] = p0[1]; - pp[2] = p0[2]; - pp[3] = p0[3]; - pp += 4; - p0 += 8; - } - } if (elempack == 4) { + const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k * 4; + for (int kk = 0; kk < max_kk; kk++) { __builtin_prefetch(p0 + 32); @@ -825,13 +678,13 @@ static void pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, int k, in p0 += 4; } } - if (elempack == 1) #endif // __loongarch_sx + if (elempack == 1) { + const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; const unsigned short* p1 = (const unsigned short*)B + (j + jj + 1) * B_hstep + k; const unsigned short* p2 = (const unsigned short*)B + (j + jj + 2) * B_hstep + k; const unsigned short* p3 = (const unsigned short*)B + (j + jj + 3) * B_hstep + k; - p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; for (int kk = 0; kk < max_kk; kk++) { @@ -853,60 +706,33 @@ static void pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, int k, in } for (; jj + 1 < max_jj; jj += 2) { - const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; - const unsigned short* p1 = (const unsigned short*)B + (j + jj + 1) * B_hstep + k; - -#if __loongarch_sx - if (elempack == 8) + // if (elempack == 1) { - const int j_pack = (j + jj) / 8; - const int j_lane = (j + jj) % 8; - p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 8 + k * 8 + j_lane; + const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; + const unsigned short* p1 = (const unsigned short*)B + (j + jj + 1) * B_hstep + k; for (int kk = 0; kk < max_kk; kk++) { pp[0] = p0[0]; - pp[1] = p0[1]; + pp[1] = p1[0]; pp += 2; - p0 += 8; + p0++; + p1++; } - continue; - } -#endif // __loongarch_sx - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p1[0]; - pp += 2; - p0++; - p1++; } } for (; jj < max_jj; jj++) { - const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; - -#if __loongarch_sx - if (elempack == 8) + // if (elempack == 1) { - const int j_pack = (j + jj) / 8; - const int j_lane = (j + jj) % 8; - p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 8 + k * 8 + j_lane; + const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; for (int kk = 0; kk < max_kk; kk++) { pp[0] = p0[0]; pp += 1; - p0 += 8; + p0++; } - continue; - } -#endif // __loongarch_sx - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; } } } @@ -1130,15 +956,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 32; p0 += B_hstep * 8; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp[2] = p0[16]; - pp[3] = p0[24]; - pp += 4; - p0++; - } } if (elempack == 4) { @@ -1164,15 +981,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 16; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp += 4; - p0++; - } } if (elempack == 1) #endif // __loongarch_sx @@ -1220,13 +1028,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 16; p0 += B_hstep * 8; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[8]; - pp += 2; - p0++; - } } if (elempack == 4) { @@ -1244,13 +1045,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 8; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp += 2; - p0++; - } } if (elempack == 1) #endif // __loongarch_sx @@ -1288,12 +1082,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 8; p0 += B_hstep * 8; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - } } if (elempack == 4) { @@ -1307,12 +1095,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 4; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - } } if (elempack == 1) #endif // __loongarch_sx diff --git a/src/layer/loongarch/gemm_int8.h b/src/layer/loongarch/gemm_int8.h index b1d2da320..cfd227d1c 100644 --- a/src/layer/loongarch/gemm_int8.h +++ b/src/layer/loongarch/gemm_int8.h @@ -1261,9 +1261,9 @@ static void pack_A_tile_fp32_to_int8(const Mat& A, Mat& AT, int i, int max_ii, i } #endif // __loongarch_sx - if (elempack == 1) + for (; ii + 1 < max_ii; ii += 2) { - for (; ii + 1 < max_ii; ii += 2) + // if (elempack == 1) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k; const float* p1 = (const float*)A + (i + ii + 1) * A_hstep + k; @@ -1294,7 +1294,10 @@ static void pack_A_tile_fp32_to_int8(const Mat& A, Mat& AT, int i, int max_ii, i p1++; } } - for (; ii < max_ii; ii++) + } + for (; ii < max_ii; ii++) + { + // if (elempack == 1) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k; const float scale0 = scales[i + ii]; @@ -2212,9 +2215,9 @@ static void pack_B_tile_fp32_to_int8(const Mat& B, Mat& BT, int j, int max_jj, i } #endif // __loongarch_sx - if (elempack == 1) + for (; jj + 1 < max_jj; jj += 2) { - for (; jj + 1 < max_jj; jj += 2) + // if (elempack == 1) { const float* p0 = (const float*)B + (j + jj) * B_hstep + k; const float* p1 = (const float*)B + (j + jj + 1) * B_hstep + k; @@ -2243,7 +2246,10 @@ static void pack_B_tile_fp32_to_int8(const Mat& B, Mat& BT, int j, int max_jj, i p1++; } } - for (; jj < max_jj; jj++) + } + for (; jj < max_jj; jj++) + { + // if (elempack == 1) { const float* p0 = (const float*)B + (j + jj) * B_hstep + k; diff --git a/src/layer/loongarch/gemm_loongarch.cpp b/src/layer/loongarch/gemm_loongarch.cpp index 04fdb6176..4f6f21ff0 100644 --- a/src/layer/loongarch/gemm_loongarch.cpp +++ b/src/layer/loongarch/gemm_loongarch.cpp @@ -169,20 +169,6 @@ static void pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int k, int max } for (; ii + 3 < max_ii; ii += 4) { - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const float* p0 = (const float*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - __builtin_prefetch(p0 + 32); - __lsx_vst(__lsx_vld(p0, 0), pp, 0); - pp += 4; - p0 += 8; - } - } if (elempack == 4) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k * 4; @@ -242,23 +228,6 @@ static void pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int k, int max for (; ii + 1 < max_ii; ii += 2) { - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const float* p0 = (const float*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[1]; - pp += 2; - p0 += 8; - } - - continue; - } - // if (elempack == 1) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k; @@ -277,22 +246,6 @@ static void pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int k, int max for (; ii < max_ii; ii += 1) { - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const float* p0 = (const float*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0 += 8; - } - - continue; - } - // if (elempack == 1) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k; @@ -2228,19 +2181,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - transpose8x4_ps(_sum0, _sum1, _sum2, _sum3); - float* p1f = p0f; - __lsx_vst(__lasx_extract_128_lo((__m256i)_sum0), p1f, 0); - __lsx_vst(__lasx_extract_128_hi((__m256i)_sum0), p1f + 8, 0); - __lsx_vst(__lasx_extract_128_lo((__m256i)_sum1), p1f + 16, 0); - __lsx_vst(__lasx_extract_128_hi((__m256i)_sum1), p1f + 24, 0); - __lsx_vst(__lasx_extract_128_lo((__m256i)_sum2), p1f + 32, 0); - __lsx_vst(__lasx_extract_128_hi((__m256i)_sum2), p1f + 40, 0); - __lsx_vst(__lasx_extract_128_lo((__m256i)_sum3), p1f + 48, 0); - __lsx_vst(__lasx_extract_128_hi((__m256i)_sum3), p1f + 56, 0); - } if (out_elempack == 4) { transpose8x4_ps(_sum0, _sum1, _sum2, _sum3); @@ -2305,19 +2245,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int if (output_transpose) { - if (out_elempack == 8) - { - transpose8x4_epi16(_bf0, _bf1, _bf2, _bf3); - unsigned short* p1 = p0; - __lsx_vstelm_d(_bf0, p1, 0, 0); - __lsx_vstelm_d(_bf0, p1 + 8, 0, 1); - __lsx_vstelm_d(_bf1, p1 + 16, 0, 0); - __lsx_vstelm_d(_bf1, p1 + 24, 0, 1); - __lsx_vstelm_d(_bf2, p1 + 32, 0, 0); - __lsx_vstelm_d(_bf2, p1 + 40, 0, 1); - __lsx_vstelm_d(_bf3, p1 + 48, 0, 0); - __lsx_vstelm_d(_bf3, p1 + 56, 0, 1); - } if (out_elempack == 4) { transpose8x4_epi16(_bf0, _bf1, _bf2, _bf3); @@ -2432,19 +2359,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - __lasx_xvst(_sum0, p0f, 0); - __lasx_xvst(_sum1, p0f + 8, 0); - } - if (out_elempack == 4) - { - __lsx_vst(__lasx_extract_128_lo((__m256i)_sum0), p0f, 0); - __lsx_vst(__lasx_extract_128_lo((__m256i)_sum1), p0f + 4, 0); - __lsx_vst(__lasx_extract_128_hi((__m256i)_sum0), p0f + out_hstep * 4, 0); - __lsx_vst(__lasx_extract_128_hi((__m256i)_sum1), p0f + out_hstep * 4 + 4, 0); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lasx_xvst(_sum0, p0f, 0); __lasx_xvst(_sum1, p0f + out_hstep, 0); @@ -2491,19 +2406,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int if (output_transpose) { - if (out_elempack == 8) - { - __lsx_vst(_bf0, p0, 0); - __lsx_vst(_bf1, p0 + 8, 0); - } - if (out_elempack == 4) - { - __lsx_vstelm_d(_bf0, p0, 0, 0); - __lsx_vstelm_d(_bf1, p0 + 4, 0, 0); - __lsx_vstelm_d(_bf0, p0 + out_hstep * 4, 0, 1); - __lsx_vstelm_d(_bf1, p0 + out_hstep * 4 + 4, 0, 1); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vst(_bf0, p0, 0); __lsx_vst(_bf1, p0 + out_hstep, 0); @@ -2585,16 +2488,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - __lasx_xvst(_sum, p0f, 0); - } - if (out_elempack == 4) - { - __lsx_vst(__lasx_extract_128_lo((__m256i)_sum), p0f, 0); - __lsx_vst(__lasx_extract_128_hi((__m256i)_sum), p0f + out_hstep * 4, 0); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lasx_xvst(_sum, p0f, 0); } @@ -2632,16 +2526,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int __m128i _bf = float2bfloat_lasx(_sum); if (output_transpose) { - if (out_elempack == 8) - { - __lsx_vst(_bf, p0, 0); - } - if (out_elempack == 4) - { - __lsx_vstelm_d(_bf, p0, 0, 0); - __lsx_vstelm_d(_bf, p0 + out_hstep * 4, 0, 1); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vst(_bf, p0, 0); } @@ -3330,22 +3215,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - __m128i _bf0 = float2bfloat_lsx(_sum00, _sum01); - __m128i _bf1 = float2bfloat_lsx(_sum10, _sum11); - __m128i _bf2 = float2bfloat_lsx(_sum20, _sum21); - __m128i _bf3 = float2bfloat_lsx(_sum30, _sum31); - transpose8x4_epi16(_bf0, _bf1, _bf2, _bf3); - __lsx_vstelm_d(_bf0, p0, 0, 0); - __lsx_vstelm_d(_bf0, p0 + 8, 0, 1); - __lsx_vstelm_d(_bf1, p0 + 16, 0, 0); - __lsx_vstelm_d(_bf1, p0 + 24, 0, 1); - __lsx_vstelm_d(_bf2, p0 + 32, 0, 0); - __lsx_vstelm_d(_bf2, p0 + 40, 0, 1); - __lsx_vstelm_d(_bf3, p0 + 48, 0, 0); - __lsx_vstelm_d(_bf3, p0 + 56, 0, 1); - } if (out_elempack == 4) { transpose4x4_ps(_sum00, _sum10, _sum20, _sum30); @@ -3498,26 +3367,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - __lsx_vstelm_w((__m128i)_sum00, p0f, 0, 0); - __lsx_vstelm_w((__m128i)_sum10, p0f + 1, 0, 0); - __lsx_vstelm_w((__m128i)_sum00, p0f + 4, 0, 1); - __lsx_vstelm_w((__m128i)_sum10, p0f + 5, 0, 1); - __lsx_vstelm_w((__m128i)_sum00, p0f + 8, 0, 2); - __lsx_vstelm_w((__m128i)_sum10, p0f + 9, 0, 2); - __lsx_vstelm_w((__m128i)_sum00, p0f + 12, 0, 3); - __lsx_vstelm_w((__m128i)_sum10, p0f + 13, 0, 3); - __lsx_vstelm_w((__m128i)_sum01, p0f + 16, 0, 0); - __lsx_vstelm_w((__m128i)_sum11, p0f + 17, 0, 0); - __lsx_vstelm_w((__m128i)_sum01, p0f + 20, 0, 1); - __lsx_vstelm_w((__m128i)_sum11, p0f + 21, 0, 1); - __lsx_vstelm_w((__m128i)_sum01, p0f + 24, 0, 2); - __lsx_vstelm_w((__m128i)_sum11, p0f + 25, 0, 2); - __lsx_vstelm_w((__m128i)_sum01, p0f + 28, 0, 3); - __lsx_vstelm_w((__m128i)_sum11, p0f + 29, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vst((__m128i)_sum00, p0f, 0); __lsx_vst((__m128i)_sum01, p0f + 4, 0); @@ -3564,45 +3414,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int if (output_transpose) { - if (out_elempack == 8) - { - __lsx_vstelm_h(_bf00, p0, 0, 0); - __lsx_vstelm_h(_bf10, p0 + 1, 0, 0); - __lsx_vstelm_h(_bf00, p0 + 8, 0, 1); - __lsx_vstelm_h(_bf10, p0 + 9, 0, 1); - __lsx_vstelm_h(_bf00, p0 + 16, 0, 2); - __lsx_vstelm_h(_bf10, p0 + 17, 0, 2); - __lsx_vstelm_h(_bf00, p0 + 24, 0, 3); - __lsx_vstelm_h(_bf10, p0 + 25, 0, 3); - __lsx_vstelm_h(_bf01, p0 + 32, 0, 0); - __lsx_vstelm_h(_bf11, p0 + 33, 0, 0); - __lsx_vstelm_h(_bf01, p0 + 40, 0, 1); - __lsx_vstelm_h(_bf11, p0 + 41, 0, 1); - __lsx_vstelm_h(_bf01, p0 + 48, 0, 2); - __lsx_vstelm_h(_bf11, p0 + 49, 0, 2); - __lsx_vstelm_h(_bf01, p0 + 56, 0, 3); - __lsx_vstelm_h(_bf11, p0 + 57, 0, 3); - } - if (out_elempack == 4) - { - __lsx_vstelm_h(_bf00, p0, 0, 0); - __lsx_vstelm_h(_bf10, p0 + 1, 0, 0); - __lsx_vstelm_h(_bf00, p0 + 4, 0, 1); - __lsx_vstelm_h(_bf10, p0 + 5, 0, 1); - __lsx_vstelm_h(_bf00, p0 + 8, 0, 2); - __lsx_vstelm_h(_bf10, p0 + 9, 0, 2); - __lsx_vstelm_h(_bf00, p0 + 12, 0, 3); - __lsx_vstelm_h(_bf10, p0 + 13, 0, 3); - __lsx_vstelm_h(_bf01, p0 + 16, 0, 0); - __lsx_vstelm_h(_bf11, p0 + 17, 0, 0); - __lsx_vstelm_h(_bf01, p0 + 20, 0, 1); - __lsx_vstelm_h(_bf11, p0 + 21, 0, 1); - __lsx_vstelm_h(_bf01, p0 + 24, 0, 2); - __lsx_vstelm_h(_bf11, p0 + 25, 0, 2); - __lsx_vstelm_h(_bf01, p0 + 28, 0, 3); - __lsx_vstelm_h(_bf11, p0 + 29, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vstelm_d(_bf00, p0, 0, 0); __lsx_vstelm_d(_bf01, p0 + 4, 0, 0); @@ -3690,18 +3502,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - __lsx_vstelm_w((__m128i)_sum0, p0f, 0, 0); - __lsx_vstelm_w((__m128i)_sum0, p0f + 4, 0, 1); - __lsx_vstelm_w((__m128i)_sum0, p0f + 8, 0, 2); - __lsx_vstelm_w((__m128i)_sum0, p0f + 12, 0, 3); - __lsx_vstelm_w((__m128i)_sum1, p0f + 16, 0, 0); - __lsx_vstelm_w((__m128i)_sum1, p0f + 20, 0, 1); - __lsx_vstelm_w((__m128i)_sum1, p0f + 24, 0, 2); - __lsx_vstelm_w((__m128i)_sum1, p0f + 28, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vst((__m128i)_sum0, p0f, 0); __lsx_vst((__m128i)_sum1, p0f + 4, 0); @@ -3738,29 +3539,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int if (output_transpose) { - if (out_elempack == 8) - { - __lsx_vstelm_h(_bf0, p0, 0, 0); - __lsx_vstelm_h(_bf0, p0 + 8, 0, 1); - __lsx_vstelm_h(_bf0, p0 + 16, 0, 2); - __lsx_vstelm_h(_bf0, p0 + 24, 0, 3); - __lsx_vstelm_h(_bf1, p0 + 32, 0, 0); - __lsx_vstelm_h(_bf1, p0 + 40, 0, 1); - __lsx_vstelm_h(_bf1, p0 + 48, 0, 2); - __lsx_vstelm_h(_bf1, p0 + 56, 0, 3); - } - if (out_elempack == 4) - { - __lsx_vstelm_h(_bf0, p0, 0, 0); - __lsx_vstelm_h(_bf0, p0 + 4, 0, 1); - __lsx_vstelm_h(_bf0, p0 + 8, 0, 2); - __lsx_vstelm_h(_bf0, p0 + 12, 0, 3); - __lsx_vstelm_h(_bf1, p0 + 16, 0, 0); - __lsx_vstelm_h(_bf1, p0 + 20, 0, 1); - __lsx_vstelm_h(_bf1, p0 + 24, 0, 2); - __lsx_vstelm_h(_bf1, p0 + 28, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vstelm_d(_bf0, p0, 0, 0); __lsx_vstelm_d(_bf1, p0 + 4, 0, 0); @@ -4164,18 +3943,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - __lsx_vstelm_d(float2bfloat_lsx(_sum0), p0, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum1), p0 + 8, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum2), p0 + 16, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum3), p0 + 24, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum4), p0 + 32, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum5), p0 + 40, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum6), p0 + 48, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum7), p0 + 56, 0, 0); - p0 += 64; - } if (out_elempack == 4) { __lsx_vst(float2bfloat_lsx(_sum0, _sum1), p0, 0); @@ -4438,14 +4205,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - __lsx_vstelm_d(float2bfloat_lsx(_sum0), p0, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum1), p0 + 8, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum2), p0 + 16, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum3), p0 + 24, 0, 0); - p0 += 32; - } if (out_elempack == 4) { __lsx_vst(float2bfloat_lsx(_sum0, _sum1), p0, 0); @@ -4544,14 +4303,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - transpose4x4_ps(_sum0, _sum1, _sum2, _sum3); - __lsx_vst((__m128i)_sum0, p0f, 0); - __lsx_vst((__m128i)_sum1, p0f + 8, 0); - __lsx_vst((__m128i)_sum2, p0f + 16, 0); - __lsx_vst((__m128i)_sum3, p0f + 24, 0); - } if (out_elempack == 4) { transpose4x4_ps(_sum0, _sum1, _sum2, _sum3); @@ -4594,14 +4345,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - transpose4x4_ps(_sum0, _sum1, _sum2, _sum3); - __lsx_vstelm_d(float2bfloat_lsx(_sum0), p0, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum1), p0 + 8, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum2), p0 + 16, 0, 0); - __lsx_vstelm_d(float2bfloat_lsx(_sum3), p0 + 24, 0, 0); - } if (out_elempack == 4) { transpose4x4_ps(_sum0, _sum1, _sum2, _sum3); @@ -4691,18 +4434,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - __lsx_vstelm_w((__m128i)_sum0, p0f, 0, 0); - __lsx_vstelm_w((__m128i)_sum1, p0f + 1, 0, 0); - __lsx_vstelm_w((__m128i)_sum0, p0f + 4, 0, 1); - __lsx_vstelm_w((__m128i)_sum1, p0f + 5, 0, 1); - __lsx_vstelm_w((__m128i)_sum0, p0f + 8, 0, 2); - __lsx_vstelm_w((__m128i)_sum1, p0f + 9, 0, 2); - __lsx_vstelm_w((__m128i)_sum0, p0f + 12, 0, 3); - __lsx_vstelm_w((__m128i)_sum1, p0f + 13, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vst((__m128i)_sum0, p0f, 0); __lsx_vst((__m128i)_sum1, p0f + out_hstep, 0); @@ -4736,29 +4468,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int if (output_transpose) { - if (out_elempack == 8) - { - __lsx_vstelm_h(_bf0, p0, 0, 0); - __lsx_vstelm_h(_bf1, p0 + 1, 0, 0); - __lsx_vstelm_h(_bf0, p0 + 8, 0, 1); - __lsx_vstelm_h(_bf1, p0 + 9, 0, 1); - __lsx_vstelm_h(_bf0, p0 + 16, 0, 2); - __lsx_vstelm_h(_bf1, p0 + 17, 0, 2); - __lsx_vstelm_h(_bf0, p0 + 24, 0, 3); - __lsx_vstelm_h(_bf1, p0 + 25, 0, 3); - } - if (out_elempack == 4) - { - __lsx_vstelm_h(_bf0, p0, 0, 0); - __lsx_vstelm_h(_bf1, p0 + 1, 0, 0); - __lsx_vstelm_h(_bf0, p0 + 4, 0, 1); - __lsx_vstelm_h(_bf1, p0 + 5, 0, 1); - __lsx_vstelm_h(_bf0, p0 + 8, 0, 2); - __lsx_vstelm_h(_bf1, p0 + 9, 0, 2); - __lsx_vstelm_h(_bf0, p0 + 12, 0, 3); - __lsx_vstelm_h(_bf1, p0 + 13, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vstelm_d(_bf0, p0, 0, 0); __lsx_vstelm_d(_bf1, p0 + out_hstep, 0, 0); @@ -4767,12 +4477,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - __lsx_vstelm_d(_bf0, p0, 0, 0); - __lsx_vstelm_d(_bf1, p0 + 8, 0, 0); - p0 += 16; - } if (out_elempack == 4) { __lsx_vstelm_d(_bf0, p0, 0, 0); @@ -4828,14 +4532,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - __lsx_vstelm_w((__m128i)_sum0, p0f, 0, 0); - __lsx_vstelm_w((__m128i)_sum0, p0f + 4, 0, 1); - __lsx_vstelm_w((__m128i)_sum0, p0f + 8, 0, 2); - __lsx_vstelm_w((__m128i)_sum0, p0f + 12, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vst((__m128i)_sum0, p0f, 0); } @@ -4864,21 +4561,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int if (output_transpose) { - if (out_elempack == 8) - { - __lsx_vstelm_h(_bf, p0, 0, 0); - __lsx_vstelm_h(_bf, p0 + 8, 0, 1); - __lsx_vstelm_h(_bf, p0 + 16, 0, 2); - __lsx_vstelm_h(_bf, p0 + 24, 0, 3); - } - if (out_elempack == 4) - { - __lsx_vstelm_h(_bf, p0, 0, 0); - __lsx_vstelm_h(_bf, p0 + 4, 0, 1); - __lsx_vstelm_h(_bf, p0 + 8, 0, 2); - __lsx_vstelm_h(_bf, p0 + 12, 0, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __lsx_vstelm_d(_bf, p0, 0, 0); } @@ -4886,11 +4569,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - __lsx_vstelm_d(_bf, p0, 0, 0); - p0 += 8; - } if (out_elempack == 4) { __lsx_vstelm_d(_bf, p0, 0, 0); @@ -5172,7 +4850,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_elemtype == 1) { - if (out_elempack == 1) + // if (out_elempack == 1) { __m128 _r0 = (__m128)__lsx_vpickev_w((__m128i)_f2, (__m128i)_f0); __m128 _r1 = (__m128)__lsx_vpickev_w((__m128i)_f6, (__m128i)_f4); @@ -5192,55 +4870,10 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int __lsx_vst((__m128i)_r7, p0f + out_hstep + 12, 0); p0f += 16; } - if (out_elempack == 4 || out_elempack == 8) - { - __lsx_vstelm_w((__m128i)_f0, p0f, 0, 0); - __lsx_vstelm_w((__m128i)_f2, p0f + 1, 0, 0); - __lsx_vstelm_w((__m128i)_f0, p0f + out_elempack, 0, 1); - __lsx_vstelm_w((__m128i)_f2, p0f + out_elempack + 1, 0, 1); - __lsx_vstelm_w((__m128i)_f0, p0f + out_elempack * 2, 0, 2); - __lsx_vstelm_w((__m128i)_f2, p0f + out_elempack * 2 + 1, 0, 2); - __lsx_vstelm_w((__m128i)_f0, p0f + out_elempack * 3, 0, 3); - __lsx_vstelm_w((__m128i)_f2, p0f + out_elempack * 3 + 1, 0, 3); - __lsx_vstelm_w((__m128i)_f1, p0f + out_elempack * 4, 0, 0); - __lsx_vstelm_w((__m128i)_f3, p0f + out_elempack * 4 + 1, 0, 0); - __lsx_vstelm_w((__m128i)_f1, p0f + out_elempack * 5, 0, 1); - __lsx_vstelm_w((__m128i)_f3, p0f + out_elempack * 5 + 1, 0, 1); - __lsx_vstelm_w((__m128i)_f1, p0f + out_elempack * 6, 0, 2); - __lsx_vstelm_w((__m128i)_f3, p0f + out_elempack * 6 + 1, 0, 2); - __lsx_vstelm_w((__m128i)_f1, p0f + out_elempack * 7, 0, 3); - __lsx_vstelm_w((__m128i)_f3, p0f + out_elempack * 7 + 1, 0, 3); - __lsx_vstelm_w((__m128i)_f4, p0f + out_elempack * 8, 0, 0); - __lsx_vstelm_w((__m128i)_f6, p0f + out_elempack * 8 + 1, 0, 0); - __lsx_vstelm_w((__m128i)_f4, p0f + out_elempack * 9, 0, 1); - __lsx_vstelm_w((__m128i)_f6, p0f + out_elempack * 9 + 1, 0, 1); - __lsx_vstelm_w((__m128i)_f4, p0f + out_elempack * 10, 0, 2); - __lsx_vstelm_w((__m128i)_f6, p0f + out_elempack * 10 + 1, 0, 2); - __lsx_vstelm_w((__m128i)_f4, p0f + out_elempack * 11, 0, 3); - __lsx_vstelm_w((__m128i)_f6, p0f + out_elempack * 11 + 1, 0, 3); - __lsx_vstelm_w((__m128i)_f5, p0f + out_elempack * 12, 0, 0); - __lsx_vstelm_w((__m128i)_f7, p0f + out_elempack * 12 + 1, 0, 0); - __lsx_vstelm_w((__m128i)_f5, p0f + out_elempack * 13, 0, 1); - __lsx_vstelm_w((__m128i)_f7, p0f + out_elempack * 13 + 1, 0, 1); - __lsx_vstelm_w((__m128i)_f5, p0f + out_elempack * 14, 0, 2); - __lsx_vstelm_w((__m128i)_f7, p0f + out_elempack * 14 + 1, 0, 2); - __lsx_vstelm_w((__m128i)_f5, p0f + out_elempack * 15, 0, 3); - __lsx_vstelm_w((__m128i)_f7, p0f + out_elempack * 15 + 1, 0, 3); - p0f += 16 * out_elempack; - } } else { - __m128i _bf0 = float2bfloat_lsx(_f0); - __m128i _bf1 = float2bfloat_lsx(_f1); - __m128i _bf2 = float2bfloat_lsx(_f2); - __m128i _bf3 = float2bfloat_lsx(_f3); - __m128i _bf4 = float2bfloat_lsx(_f4); - __m128i _bf5 = float2bfloat_lsx(_f5); - __m128i _bf6 = float2bfloat_lsx(_f6); - __m128i _bf7 = float2bfloat_lsx(_f7); - - if (out_elempack == 1) + // if (out_elempack == 1) { __m128 _r0 = (__m128)__lsx_vpickev_w((__m128i)_f2, (__m128i)_f0); __m128 _r1 = (__m128)__lsx_vpickev_w((__m128i)_f6, (__m128i)_f4); @@ -5260,42 +4893,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int __lsx_vstelm_d(float2bfloat_lsx(_r7), p0 + out_hstep + 12, 0, 0); p0 += 16; } - if (out_elempack == 4 || out_elempack == 8) - { - __lsx_vstelm_h(_bf0, p0, 0, 0); - __lsx_vstelm_h(_bf2, p0 + 1, 0, 0); - __lsx_vstelm_h(_bf0, p0 + out_elempack, 0, 1); - __lsx_vstelm_h(_bf2, p0 + out_elempack + 1, 0, 1); - __lsx_vstelm_h(_bf0, p0 + out_elempack * 2, 0, 2); - __lsx_vstelm_h(_bf2, p0 + out_elempack * 2 + 1, 0, 2); - __lsx_vstelm_h(_bf0, p0 + out_elempack * 3, 0, 3); - __lsx_vstelm_h(_bf2, p0 + out_elempack * 3 + 1, 0, 3); - __lsx_vstelm_h(_bf1, p0 + out_elempack * 4, 0, 0); - __lsx_vstelm_h(_bf3, p0 + out_elempack * 4 + 1, 0, 0); - __lsx_vstelm_h(_bf1, p0 + out_elempack * 5, 0, 1); - __lsx_vstelm_h(_bf3, p0 + out_elempack * 5 + 1, 0, 1); - __lsx_vstelm_h(_bf1, p0 + out_elempack * 6, 0, 2); - __lsx_vstelm_h(_bf3, p0 + out_elempack * 6 + 1, 0, 2); - __lsx_vstelm_h(_bf1, p0 + out_elempack * 7, 0, 3); - __lsx_vstelm_h(_bf3, p0 + out_elempack * 7 + 1, 0, 3); - __lsx_vstelm_h(_bf4, p0 + out_elempack * 8, 0, 0); - __lsx_vstelm_h(_bf6, p0 + out_elempack * 8 + 1, 0, 0); - __lsx_vstelm_h(_bf4, p0 + out_elempack * 9, 0, 1); - __lsx_vstelm_h(_bf6, p0 + out_elempack * 9 + 1, 0, 1); - __lsx_vstelm_h(_bf4, p0 + out_elempack * 10, 0, 2); - __lsx_vstelm_h(_bf6, p0 + out_elempack * 10 + 1, 0, 2); - __lsx_vstelm_h(_bf4, p0 + out_elempack * 11, 0, 3); - __lsx_vstelm_h(_bf6, p0 + out_elempack * 11 + 1, 0, 3); - __lsx_vstelm_h(_bf5, p0 + out_elempack * 12, 0, 0); - __lsx_vstelm_h(_bf7, p0 + out_elempack * 12 + 1, 0, 0); - __lsx_vstelm_h(_bf5, p0 + out_elempack * 13, 0, 1); - __lsx_vstelm_h(_bf7, p0 + out_elempack * 13 + 1, 0, 1); - __lsx_vstelm_h(_bf5, p0 + out_elempack * 14, 0, 2); - __lsx_vstelm_h(_bf7, p0 + out_elempack * 14 + 1, 0, 2); - __lsx_vstelm_h(_bf5, p0 + out_elempack * 15, 0, 3); - __lsx_vstelm_h(_bf7, p0 + out_elempack * 15 + 1, 0, 3); - p0 += 16 * out_elempack; - } } } } @@ -5361,17 +4958,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0f[0] = sum0; - p0f[out_hstep] = sum1; - } - if (out_elempack == 4) - { - p0f[0] = sum0; - p0f[out_hstep] = sum1; - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0f[0] = sum0; p0f[out_hstep] = sum1; @@ -5400,17 +4987,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0[0] = float32_to_bfloat16(sum0); - p0[out_hstep] = float32_to_bfloat16(sum1); - } - if (out_elempack == 4) - { - p0[0] = float32_to_bfloat16(sum0); - p0[out_hstep] = float32_to_bfloat16(sum1); - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0[0] = float32_to_bfloat16(sum0); p0[out_hstep] = float32_to_bfloat16(sum1); @@ -5532,15 +5109,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0f[0] = sum; - } - if (out_elempack == 4) - { - p0f[0] = sum; - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0f[0] = sum; } @@ -5565,15 +5134,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0[0] = float32_to_bfloat16(sum); - } - if (out_elempack == 4) - { - p0[0] = float32_to_bfloat16(sum); - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0[0] = float32_to_bfloat16(sum); } @@ -9046,21 +8607,6 @@ int Gemm_loongarch::forward_bf16s(const std::vector& bottom_blobs, std::vec { C = !CT_data.empty() ? CT_data : C_data; broadcast_type_C = constant_broadcast_type_C; - - if (!C.empty()) - { - if (C.elembits() == 16) - { - Option opt_cast = opt; - opt_cast.blob_allocator = opt.workspace_allocator; - - Mat C_fp32; - cast_bfloat16_to_float32(C, C_fp32, opt_cast); - if (C_fp32.empty()) - return -100; - C = C_fp32; - } - } } else { diff --git a/src/layer/mips/convolution_im2col_gemm_bf16s.h b/src/layer/mips/convolution_im2col_gemm_bf16s.h index 198ecf5e6..8f99da217 100644 --- a/src/layer/mips/convolution_im2col_gemm_bf16s.h +++ b/src/layer/mips/convolution_im2col_gemm_bf16s.h @@ -1945,25 +1945,19 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl { __builtin_prefetch(p0 + bottom_blob.cstep * 4); - v8i16 _zero_bf16 = __msa_fill_h(0); v8i16 _r01 = (v8i16)__msa_ld_h(p0, 0); - v4f32 _r0 = (v4f32)__msa_ilvr_h(_r01, _zero_bf16); - v4f32 _r1 = (v4f32)__msa_ilvl_h(_r01, _zero_bf16); - v8i16 _r23 = (v8i16)__msa_ld_h(p0 + 4 * 2, 0); - v8i16 _r45 = (v8i16)__msa_ld_h(p0 + 4 * 4, 0); - v8i16 _r67 = (v8i16)__msa_ld_h(p0 + 4 * 6, 0); - v4f32 _r2 = (v4f32)__msa_ilvr_h(_r23, _zero_bf16); - v4f32 _r3 = (v4f32)__msa_ilvl_h(_r23, _zero_bf16); - v4f32 _r4 = (v4f32)__msa_ilvr_h(_r45, _zero_bf16); - v4f32 _r5 = (v4f32)__msa_ilvl_h(_r45, _zero_bf16); - v4f32 _r6 = (v4f32)__msa_ilvr_h(_r67, _zero_bf16); - v4f32 _r7 = (v4f32)__msa_ilvl_h(_r67, _zero_bf16); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - __msa_st_w(float2bfloat_msa(_r0, _r4), pp, 0); - __msa_st_w(float2bfloat_msa(_r1, _r5), pp + 4 * 2, 0); - __msa_st_w(float2bfloat_msa(_r2, _r6), pp + 4 * 4, 0); - __msa_st_w(float2bfloat_msa(_r3, _r7), pp + 4 * 6, 0); + v8i16 _r23 = (v8i16)__msa_ld_h(p0 + 8, 0); + v8i16 _r45 = (v8i16)__msa_ld_h(p0 + 16, 0); + v8i16 _r67 = (v8i16)__msa_ld_h(p0 + 24, 0); + v8i16 _r0 = (v8i16)__msa_ilvr_d((v2i64)_r45, (v2i64)_r01); + v8i16 _r1 = (v8i16)__msa_ilvl_d((v2i64)_r45, (v2i64)_r01); + v8i16 _r2 = (v8i16)__msa_ilvr_d((v2i64)_r67, (v2i64)_r23); + v8i16 _r3 = (v8i16)__msa_ilvl_d((v2i64)_r67, (v2i64)_r23); + transpose8x4_epi16(_r0, _r1, _r2, _r3); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r2, (v2i64)_r0), pp, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r2, (v2i64)_r0), pp + 8, 0); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r3, (v2i64)_r1), pp + 16, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r3, (v2i64)_r1), pp + 24, 0); pp += 32; p0 += bottom_blob.cstep * 4; } @@ -2025,16 +2019,14 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl { __builtin_prefetch(p0 + bottom_blob.cstep * 4); - v8i16 _zero_bf16 = __msa_fill_h(0); v8i16 _r01 = (v8i16)__msa_ld_h(p0, 0); - v4f32 _r0 = (v4f32)__msa_ilvr_h(_r01, _zero_bf16); - v4f32 _r1 = (v4f32)__msa_ilvl_h(_r01, _zero_bf16); - v8i16 _r23 = (v8i16)__msa_ld_h(p0 + 4 * 2, 0); - v4f32 _r2 = (v4f32)__msa_ilvr_h(_r23, _zero_bf16); - v4f32 _r3 = (v4f32)__msa_ilvl_h(_r23, _zero_bf16); - transpose4x4_ps(_r0, _r1, _r2, _r3); - __msa_st_w(float2bfloat_msa(_r0, _r1), pp, 0); - __msa_st_w(float2bfloat_msa(_r2, _r3), pp + 4 * 2, 0); + v8i16 _r23 = (v8i16)__msa_ld_h(p0 + 8, 0); + v8i16 _r02 = (v8i16)__msa_ilvr_d((v2i64)_r23, (v2i64)_r01); + v8i16 _r13 = (v8i16)__msa_ilvl_d((v2i64)_r23, (v2i64)_r01); + v8i16 _tmp0 = (v8i16)__msa_ilvr_h(_r13, _r02); + v8i16 _tmp1 = (v8i16)__msa_ilvl_h(_r13, _r02); + __msa_st_h((v8i16)__msa_ilvr_w((v4i32)_tmp1, (v4i32)_tmp0), pp, 0); + __msa_st_h((v8i16)__msa_ilvl_w((v4i32)_tmp1, (v4i32)_tmp0), pp + 8, 0); pp += 16; p0 += bottom_blob.cstep * 4; } @@ -2092,13 +2084,11 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl // transpose4x2 __builtin_prefetch(p0 + bottom_blob.cstep * 4); - v8i16 _zero_bf16 = __msa_fill_h(0); v8i16 _r01 = (v8i16)__msa_ld_h(p0, 0); - v4f32 _r0 = (v4f32)__msa_ilvr_h(_r01, _zero_bf16); - v4f32 _r1 = (v4f32)__msa_ilvl_h(_r01, _zero_bf16); - v4f32 _tmp0 = (v4f32)__msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4f32 _tmp1 = (v4f32)__msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - __msa_st_w(float2bfloat_msa(_tmp0, _tmp1), pp, 0); + v8i16 _r0 = (v8i16)__msa_ilvr_d((v2i64)_r01, (v2i64)_r01); + v8i16 _r1 = (v8i16)__msa_ilvl_d((v2i64)_r01, (v2i64)_r01); + v8i16 _tmp0 = (v8i16)__msa_ilvr_h(_r1, _r0); + __msa_st_h(_tmp0, pp, 0); pp += 8; p0 += bottom_blob.cstep * 4; } @@ -2148,7 +2138,10 @@ static void convolution_im2col_input_tile_conv1x1s1d1_bf16s(const Mat& bottom_bl { __builtin_prefetch(p0 + bottom_blob.cstep * 4); - __msa_storel_d(float2bfloat_msa(bfloat2float_msa(p0)), pp); + pp[0] = p0[0]; + pp[1] = p0[1]; + pp[2] = p0[2]; + pp[3] = p0[3]; pp += 4; p0 += bottom_blob.cstep * 4; } @@ -2252,20 +2245,39 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - v4f32 _r0 = bfloat2float_msa(sptr); - v4f32 _r1 = bfloat2float_msa(sptr + stride_w * 4); - v4f32 _r2 = bfloat2float_msa(sptr + stride_w * 8); - v4f32 _r3 = bfloat2float_msa(sptr + stride_w * 12); - v4f32 _r4 = bfloat2float_msa(sptr + stride_w * 16); - v4f32 _r5 = bfloat2float_msa(sptr + stride_w * 20); - v4f32 _r6 = bfloat2float_msa(sptr + stride_w * 24); - v4f32 _r7 = bfloat2float_msa(sptr + stride_w * 28); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - __msa_st_w(float2bfloat_msa(_r0, _r4), pp, 0); - __msa_st_w(float2bfloat_msa(_r1, _r5), pp + 4 * 2, 0); - __msa_st_w(float2bfloat_msa(_r2, _r6), pp + 4 * 4, 0); - __msa_st_w(float2bfloat_msa(_r3, _r7), pp + 4 * 6, 0); + if (stride_w == 1) + { + v8i16 _r01 = (v8i16)__msa_ld_h(sptr, 0); + v8i16 _r23 = (v8i16)__msa_ld_h(sptr + 8, 0); + v8i16 _r45 = (v8i16)__msa_ld_h(sptr + 16, 0); + v8i16 _r67 = (v8i16)__msa_ld_h(sptr + 24, 0); + v8i16 _r0 = (v8i16)__msa_ilvr_d((v2i64)_r45, (v2i64)_r01); + v8i16 _r1 = (v8i16)__msa_ilvl_d((v2i64)_r45, (v2i64)_r01); + v8i16 _r2 = (v8i16)__msa_ilvr_d((v2i64)_r67, (v2i64)_r23); + v8i16 _r3 = (v8i16)__msa_ilvl_d((v2i64)_r67, (v2i64)_r23); + transpose8x4_epi16(_r0, _r1, _r2, _r3); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r2, (v2i64)_r0), pp, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r2, (v2i64)_r0), pp + 8, 0); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r3, (v2i64)_r1), pp + 16, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r3, (v2i64)_r1), pp + 24, 0); + } + else + { + v8i16 _r0 = (v8i16)__msa_loadl_d(sptr); + v8i16 _r1 = (v8i16)__msa_loadl_d(sptr + stride_w * 4); + v8i16 _r2 = (v8i16)__msa_loadl_d(sptr + stride_w * 8); + v8i16 _r3 = (v8i16)__msa_loadl_d(sptr + stride_w * 12); + v8i16 _r4 = (v8i16)__msa_loadl_d(sptr + stride_w * 16); + v8i16 _r5 = (v8i16)__msa_loadl_d(sptr + stride_w * 20); + v8i16 _r6 = (v8i16)__msa_loadl_d(sptr + stride_w * 24); + v8i16 _r7 = (v8i16)__msa_loadl_d(sptr + stride_w * 28); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + transpose4x4_epi16(_r4, _r5, _r6, _r7); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r4, (v2i64)_r0), pp, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r4, (v2i64)_r0), pp + 8, 0); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r5, (v2i64)_r1), pp + 16, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r5, (v2i64)_r1), pp + 24, 0); + } pp += 32; } if (elempack == 1) @@ -2343,20 +2355,20 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - v4f32 _r0 = bfloat2float_msa(sptr0); - v4f32 _r1 = bfloat2float_msa(sptr1); - v4f32 _r2 = bfloat2float_msa(sptr2); - v4f32 _r3 = bfloat2float_msa(sptr3); - v4f32 _r4 = bfloat2float_msa(sptr4); - v4f32 _r5 = bfloat2float_msa(sptr5); - v4f32 _r6 = bfloat2float_msa(sptr6); - v4f32 _r7 = bfloat2float_msa(sptr7); - transpose4x4_ps(_r0, _r1, _r2, _r3); - transpose4x4_ps(_r4, _r5, _r6, _r7); - __msa_st_w(float2bfloat_msa(_r0, _r4), pp, 0); - __msa_st_w(float2bfloat_msa(_r1, _r5), pp + 4 * 2, 0); - __msa_st_w(float2bfloat_msa(_r2, _r6), pp + 4 * 4, 0); - __msa_st_w(float2bfloat_msa(_r3, _r7), pp + 4 * 6, 0); + v8i16 _r0 = (v8i16)__msa_loadl_d(sptr0); + v8i16 _r1 = (v8i16)__msa_loadl_d(sptr1); + v8i16 _r2 = (v8i16)__msa_loadl_d(sptr2); + v8i16 _r3 = (v8i16)__msa_loadl_d(sptr3); + v8i16 _r4 = (v8i16)__msa_loadl_d(sptr4); + v8i16 _r5 = (v8i16)__msa_loadl_d(sptr5); + v8i16 _r6 = (v8i16)__msa_loadl_d(sptr6); + v8i16 _r7 = (v8i16)__msa_loadl_d(sptr7); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + transpose4x4_epi16(_r4, _r5, _r6, _r7); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r4, (v2i64)_r0), pp, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r4, (v2i64)_r0), pp + 8, 0); + __msa_st_h((v8i16)__msa_ilvr_d((v2i64)_r5, (v2i64)_r1), pp + 16, 0); + __msa_st_h((v8i16)__msa_ilvl_d((v2i64)_r5, (v2i64)_r1), pp + 24, 0); pp += 32; } if (elempack == 1) @@ -2417,13 +2429,27 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - v4f32 _r0 = bfloat2float_msa(sptr); - v4f32 _r1 = bfloat2float_msa(sptr + stride_w * 4); - v4f32 _r2 = bfloat2float_msa(sptr + stride_w * 8); - v4f32 _r3 = bfloat2float_msa(sptr + stride_w * 12); - transpose4x4_ps(_r0, _r1, _r2, _r3); - __msa_st_w(float2bfloat_msa(_r0, _r1), pp, 0); - __msa_st_w(float2bfloat_msa(_r2, _r3), pp + 4 * 2, 0); + if (stride_w == 1) + { + v8i16 _r01 = (v8i16)__msa_ld_h(sptr, 0); + v8i16 _r23 = (v8i16)__msa_ld_h(sptr + 8, 0); + v8i16 _r02 = (v8i16)__msa_ilvr_d((v2i64)_r23, (v2i64)_r01); + v8i16 _r13 = (v8i16)__msa_ilvl_d((v2i64)_r23, (v2i64)_r01); + v8i16 _tmp0 = (v8i16)__msa_ilvr_h(_r13, _r02); + v8i16 _tmp1 = (v8i16)__msa_ilvl_h(_r13, _r02); + __msa_st_h((v8i16)__msa_ilvr_w((v4i32)_tmp1, (v4i32)_tmp0), pp, 0); + __msa_st_h((v8i16)__msa_ilvl_w((v4i32)_tmp1, (v4i32)_tmp0), pp + 8, 0); + } + else + { + v8i16 _r0 = (v8i16)__msa_loadl_d(sptr); + v8i16 _r1 = (v8i16)__msa_loadl_d(sptr + stride_w * 4); + v8i16 _r2 = (v8i16)__msa_loadl_d(sptr + stride_w * 8); + v8i16 _r3 = (v8i16)__msa_loadl_d(sptr + stride_w * 12); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + __msa_st_h(_r0, pp, 0); + __msa_st_h(_r1, pp + 8, 0); + } pp += 16; } if (elempack == 1) @@ -2477,13 +2503,13 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - v4f32 _r0 = bfloat2float_msa(sptr0); - v4f32 _r1 = bfloat2float_msa(sptr1); - v4f32 _r2 = bfloat2float_msa(sptr2); - v4f32 _r3 = bfloat2float_msa(sptr3); - transpose4x4_ps(_r0, _r1, _r2, _r3); - __msa_st_w(float2bfloat_msa(_r0, _r1), pp, 0); - __msa_st_w(float2bfloat_msa(_r2, _r3), pp + 4 * 2, 0); + v8i16 _r0 = (v8i16)__msa_loadl_d(sptr0); + v8i16 _r1 = (v8i16)__msa_loadl_d(sptr1); + v8i16 _r2 = (v8i16)__msa_loadl_d(sptr2); + v8i16 _r3 = (v8i16)__msa_loadl_d(sptr3); + transpose4x4_epi16(_r0, _r1, _r2, _r3); + __msa_st_h(_r0, pp, 0); + __msa_st_h(_r1, pp + 8, 0); pp += 16; } if (elempack == 1) @@ -2535,11 +2561,21 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - v4f32 _r0 = bfloat2float_msa(sptr); - v4f32 _r1 = bfloat2float_msa(sptr + stride_w * 4); - v4f32 _tmp0 = (v4f32)__msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4f32 _tmp1 = (v4f32)__msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - __msa_st_w(float2bfloat_msa(_tmp0, _tmp1), pp, 0); + v8i16 _r0; + v8i16 _r1; + if (stride_w == 1) + { + v8i16 _r01 = (v8i16)__msa_ld_h(sptr, 0); + _r0 = (v8i16)__msa_ilvr_d((v2i64)_r01, (v2i64)_r01); + _r1 = (v8i16)__msa_ilvl_d((v2i64)_r01, (v2i64)_r01); + } + else + { + _r0 = (v8i16)__msa_loadl_d(sptr); + _r1 = (v8i16)__msa_loadl_d(sptr + stride_w * 4); + } + v8i16 _tmp0 = (v8i16)__msa_ilvr_h(_r1, _r0); + __msa_st_h(_tmp0, pp, 0); pp += 8; } #endif // __mips_msa @@ -2584,11 +2620,10 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - v4f32 _r0 = bfloat2float_msa(sptr0); - v4f32 _r1 = bfloat2float_msa(sptr1); - v4f32 _tmp0 = (v4f32)__msa_ilvr_w((v4i32)_r1, (v4i32)_r0); - v4f32 _tmp1 = (v4f32)__msa_ilvl_w((v4i32)_r1, (v4i32)_r0); - __msa_st_w(float2bfloat_msa(_tmp0, _tmp1), pp, 0); + v8i16 _r0 = (v8i16)__msa_loadl_d(sptr0); + v8i16 _r1 = (v8i16)__msa_loadl_d(sptr1); + v8i16 _tmp0 = (v8i16)__msa_ilvr_h(_r1, _r0); + __msa_st_h(_tmp0, pp, 0); pp += 8; } #endif // __mips_msa @@ -2629,7 +2664,10 @@ static inline void convolution_im2col_input_tile_impl_bf16s(const Mat& bottom_bl } if (elempack == 4) { - __msa_storel_d(float2bfloat_msa(bfloat2float_msa(sptr)), pp); + pp[0] = sptr[0]; + pp[1] = sptr[1]; + pp[2] = sptr[2]; + pp[3] = sptr[3]; pp += 4; } #endif // __mips_msa diff --git a/src/layer/mips/convolution_im2col_gemm_int8.h b/src/layer/mips/convolution_im2col_gemm_int8.h index a94204362..1f178ff4f 100644 --- a/src/layer/mips/convolution_im2col_gemm_int8.h +++ b/src/layer/mips/convolution_im2col_gemm_int8.h @@ -707,38 +707,13 @@ static void convolution_im2col_input_tile_conv1x1s1d1_int8(const Mat& bottom_blo const signed char* p3 = p2 + bottom_blob.cstep; __builtin_prefetch(p0 + bottom_blob.cstep * 4); - pp[0] = p0[0]; - pp[1] = p1[0]; - pp[2] = p2[0]; - pp[3] = p3[0]; - pp[4] = p0[1]; - pp[5] = p1[1]; - pp[6] = p2[1]; - pp[7] = p3[1]; - pp[8] = p0[2]; - pp[9] = p1[2]; - pp[10] = p2[2]; - pp[11] = p3[2]; - pp[12] = p0[3]; - pp[13] = p1[3]; - pp[14] = p2[3]; - pp[15] = p3[3]; - pp[16] = p0[4]; - pp[17] = p1[4]; - pp[18] = p2[4]; - pp[19] = p3[4]; - pp[20] = p0[5]; - pp[21] = p1[5]; - pp[22] = p2[5]; - pp[23] = p3[5]; - pp[24] = p0[6]; - pp[25] = p1[6]; - pp[26] = p2[6]; - pp[27] = p3[6]; - pp[28] = p0[7]; - pp[29] = p1[7]; - pp[30] = p2[7]; - pp[31] = p3[7]; + v16i8 _r0 = (v16i8)__msa_loadl_d(p0); + v16i8 _r1 = (v16i8)__msa_loadl_d(p1); + v16i8 _r2 = (v16i8)__msa_loadl_d(p2); + v16i8 _r3 = (v16i8)__msa_loadl_d(p3); + transpose16x4_epi8(_r0, _r1, _r2, _r3); + __msa_st_b(_r0, pp, 0); + __msa_st_b(_r1, pp + 16, 0); pp += 32; p0 += bottom_blob.cstep * 4; } @@ -805,22 +780,12 @@ static void convolution_im2col_input_tile_conv1x1s1d1_int8(const Mat& bottom_blo const signed char* p3 = p2 + bottom_blob.cstep; __builtin_prefetch(p0 + bottom_blob.cstep * 4); - pp[0] = p0[0]; - pp[1] = p1[0]; - pp[2] = p2[0]; - pp[3] = p3[0]; - pp[4] = p0[1]; - pp[5] = p1[1]; - pp[6] = p2[1]; - pp[7] = p3[1]; - pp[8] = p0[2]; - pp[9] = p1[2]; - pp[10] = p2[2]; - pp[11] = p3[2]; - pp[12] = p0[3]; - pp[13] = p1[3]; - pp[14] = p2[3]; - pp[15] = p3[3]; + v16i8 _r0 = (v16i8)__msa_fill_w_ptr(p0); + v16i8 _r1 = (v16i8)__msa_fill_w_ptr(p1); + v16i8 _r2 = (v16i8)__msa_fill_w_ptr(p2); + v16i8 _r3 = (v16i8)__msa_fill_w_ptr(p3); + transpose16x4_epi8(_r0, _r1, _r2, _r3); + __msa_st_b(_r0, pp, 0); pp += 16; p0 += bottom_blob.cstep * 4; } @@ -1053,12 +1018,134 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo pp += 8; } } - for (; kk + 3 < max_kk / elempack; kk += 4) + if (elempack == 1) { - for (int q = 0; q < 4; q++) + if (stride_w == 1) { - int p = (k / elempack + kk + q) / maxk; - int uv = (k / elempack + kk + q) % maxk; + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = dx0 + dilation_w * v0; + int x1 = dx0 + dilation_w * v1; + int x2 = dx0 + dilation_w * v2; + int x3 = dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + v16i8 _r0 = (v16i8)__msa_loadl_d(sptr0); + v16i8 _r1 = (v16i8)__msa_loadl_d(sptr1); + v16i8 _r2 = (v16i8)__msa_loadl_d(sptr2); + v16i8 _r3 = (v16i8)__msa_loadl_d(sptr3); + transpose16x4_epi8(_r0, _r1, _r2, _r3); + __msa_st_b(_r0, pp, 0); + __msa_st_b(_r1, pp + 16, 0); + pp += 32; + } + } + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = stride_w * dx0 + dilation_w * v0; + int x1 = stride_w * dx0 + dilation_w * v1; + int x2 = stride_w * dx0 + dilation_w * v2; + int x3 = stride_w * dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + pp[0] = sptr0[0]; + pp[1] = sptr1[0]; + pp[2] = sptr2[0]; + pp[3] = sptr3[0]; + pp[4] = sptr0[stride_w]; + pp[5] = sptr1[stride_w]; + pp[6] = sptr2[stride_w]; + pp[7] = sptr3[stride_w]; + pp[8] = sptr0[stride_w * 2]; + pp[9] = sptr1[stride_w * 2]; + pp[10] = sptr2[stride_w * 2]; + pp[11] = sptr3[stride_w * 2]; + pp[12] = sptr0[stride_w * 3]; + pp[13] = sptr1[stride_w * 3]; + pp[14] = sptr2[stride_w * 3]; + pp[15] = sptr3[stride_w * 3]; + pp[16] = sptr0[stride_w * 4]; + pp[17] = sptr1[stride_w * 4]; + pp[18] = sptr2[stride_w * 4]; + pp[19] = sptr3[stride_w * 4]; + pp[20] = sptr0[stride_w * 5]; + pp[21] = sptr1[stride_w * 5]; + pp[22] = sptr2[stride_w * 5]; + pp[23] = sptr3[stride_w * 5]; + pp[24] = sptr0[stride_w * 6]; + pp[25] = sptr1[stride_w * 6]; + pp[26] = sptr2[stride_w * 6]; + pp[27] = sptr3[stride_w * 6]; + pp[28] = sptr0[stride_w * 7]; + pp[29] = sptr1[stride_w * 7]; + pp[30] = sptr2[stride_w * 7]; + pp[31] = sptr3[stride_w * 7]; + pp += 32; + } + for (; kk < max_kk; kk++) + { + int p = (k + kk) / maxk; + int uv = (k + kk) % maxk; int u = uv / kernel_w; int v = uv % kernel_w; @@ -1067,38 +1154,8 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo int x0 = stride_w * dx0 + dilation_w * v; int y0 = stride_h * dy0 + dilation_h * u; - const signed char* sptr = img.row(y0) + x0 * elempack; + const signed char* sptr = img.row(y0) + x0; - if (elempack == 1) - { - pp[q] = sptr[0]; - pp[4 + q] = sptr[stride_w]; - pp[8 + q] = sptr[stride_w * 2]; - pp[12 + q] = sptr[stride_w * 3]; - pp[16 + q] = sptr[stride_w * 4]; - pp[20 + q] = sptr[stride_w * 5]; - pp[24 + q] = sptr[stride_w * 6]; - pp[28 + q] = sptr[stride_w * 7]; - } - } - pp += 32; - } - for (; kk < max_kk / elempack; kk++) - { - int p = (k / elempack + kk) / maxk; - int uv = (k / elempack + kk) % maxk; - int u = uv / kernel_w; - int v = uv % kernel_w; - - const Mat img = bottom_blob.channel(p); - - int x0 = stride_w * dx0 + dilation_w * v; - int y0 = stride_h * dy0 + dilation_h * u; - - const signed char* sptr = img.row(y0) + x0 * elempack; - - if (elempack == 1) - { pp[0] = sptr[0]; pp[1] = sptr[stride_w]; pp[2] = sptr[stride_w * 2]; @@ -1383,12 +1440,117 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo pp += 4; } } - for (; kk + 3 < max_kk / elempack; kk += 4) + if (elempack == 1) { - for (int q = 0; q < 4; q++) + if (stride_w == 1) { - int p = (k / elempack + kk + q) / maxk; - int uv = (k / elempack + kk + q) % maxk; + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = dx0 + dilation_w * v0; + int x1 = dx0 + dilation_w * v1; + int x2 = dx0 + dilation_w * v2; + int x3 = dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + v16i8 _r0 = (v16i8)__msa_fill_w_ptr(sptr0); + v16i8 _r1 = (v16i8)__msa_fill_w_ptr(sptr1); + v16i8 _r2 = (v16i8)__msa_fill_w_ptr(sptr2); + v16i8 _r3 = (v16i8)__msa_fill_w_ptr(sptr3); + transpose16x4_epi8(_r0, _r1, _r2, _r3); + __msa_st_b(_r0, pp, 0); + pp += 16; + } + } + for (; kk + 3 < max_kk; kk += 4) + { + int p0 = (k + kk) / maxk; + int p1 = (k + kk + 1) / maxk; + int p2 = (k + kk + 2) / maxk; + int p3 = (k + kk + 3) / maxk; + int uv0 = (k + kk) % maxk; + int uv1 = (k + kk + 1) % maxk; + int uv2 = (k + kk + 2) % maxk; + int uv3 = (k + kk + 3) % maxk; + int u0 = uv0 / kernel_w; + int u1 = uv1 / kernel_w; + int u2 = uv2 / kernel_w; + int u3 = uv3 / kernel_w; + int v0 = uv0 % kernel_w; + int v1 = uv1 % kernel_w; + int v2 = uv2 % kernel_w; + int v3 = uv3 % kernel_w; + + const Mat img0 = bottom_blob.channel(p0); + const Mat img1 = bottom_blob.channel(p1); + const Mat img2 = bottom_blob.channel(p2); + const Mat img3 = bottom_blob.channel(p3); + + int x0 = stride_w * dx0 + dilation_w * v0; + int x1 = stride_w * dx0 + dilation_w * v1; + int x2 = stride_w * dx0 + dilation_w * v2; + int x3 = stride_w * dx0 + dilation_w * v3; + int y0 = stride_h * dy0 + dilation_h * u0; + int y1 = stride_h * dy0 + dilation_h * u1; + int y2 = stride_h * dy0 + dilation_h * u2; + int y3 = stride_h * dy0 + dilation_h * u3; + + const signed char* sptr0 = img0.row(y0) + x0; + const signed char* sptr1 = img1.row(y1) + x1; + const signed char* sptr2 = img2.row(y2) + x2; + const signed char* sptr3 = img3.row(y3) + x3; + + pp[0] = sptr0[0]; + pp[1] = sptr1[0]; + pp[2] = sptr2[0]; + pp[3] = sptr3[0]; + pp[4] = sptr0[stride_w]; + pp[5] = sptr1[stride_w]; + pp[6] = sptr2[stride_w]; + pp[7] = sptr3[stride_w]; + pp[8] = sptr0[stride_w * 2]; + pp[9] = sptr1[stride_w * 2]; + pp[10] = sptr2[stride_w * 2]; + pp[11] = sptr3[stride_w * 2]; + pp[12] = sptr0[stride_w * 3]; + pp[13] = sptr1[stride_w * 3]; + pp[14] = sptr2[stride_w * 3]; + pp[15] = sptr3[stride_w * 3]; + pp += 16; + } + for (; kk < max_kk; kk++) + { + int p = (k + kk) / maxk; + int uv = (k + kk) % maxk; int u = uv / kernel_w; int v = uv % kernel_w; @@ -1397,34 +1559,8 @@ static inline void convolution_im2col_input_tile_impl_int8(const Mat& bottom_blo int x0 = stride_w * dx0 + dilation_w * v; int y0 = stride_h * dy0 + dilation_h * u; - const signed char* sptr = img.row(y0) + x0 * elempack; + const signed char* sptr = img.row(y0) + x0; - if (elempack == 1) - { - pp[q] = sptr[0]; - pp[4 + q] = sptr[stride_w]; - pp[8 + q] = sptr[stride_w * 2]; - pp[12 + q] = sptr[stride_w * 3]; - } - } - pp += 16; - } - for (; kk < max_kk / elempack; kk++) - { - int p = (k / elempack + kk) / maxk; - int uv = (k / elempack + kk) % maxk; - int u = uv / kernel_w; - int v = uv % kernel_w; - - const Mat img = bottom_blob.channel(p); - - int x0 = stride_w * dx0 + dilation_w * v; - int y0 = stride_h * dy0 + dilation_h * u; - - const signed char* sptr = img.row(y0) + x0 * elempack; - - if (elempack == 1) - { pp[0] = sptr[0]; pp[1] = sptr[stride_w]; pp[2] = sptr[stride_w * 2]; diff --git a/src/layer/mips/gemm_bf16s.h b/src/layer/mips/gemm_bf16s.h index ee0c9906d..8cf1f40b6 100644 --- a/src/layer/mips/gemm_bf16s.h +++ b/src/layer/mips/gemm_bf16s.h @@ -88,23 +88,6 @@ static void pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, int k, in } for (; ii + 3 < max_ii; ii += 4) { - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const unsigned short* p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - __builtin_prefetch(p0 + 32); - pp[0] = p0[0]; - pp[1] = p0[1]; - pp[2] = p0[2]; - pp[3] = p0[3]; - pp += 4; - p0 += 8; - } - } if (elempack == 4) { const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k * 4; @@ -148,37 +131,7 @@ static void pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, int k, in #endif // __mips_msa for (; ii + 1 < max_ii; ii += 2) { -#if __mips_msa - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const unsigned short* p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[1]; - pp += 2; - p0 += 8; - } - } - if (elempack == 4) - { - const int i_pack = (i + ii) / 4; - const int i_lane = (i + ii) % 4; - const unsigned short* p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 4 + k * 4 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[1]; - pp += 2; - p0 += 4; - } - } -#endif // __mips_msa - if (elempack == 1) + // if (elempack == 1) { const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k; const unsigned short* p1 = (const unsigned short*)A + (i + ii + 1) * A_hstep + k; @@ -195,35 +148,7 @@ static void pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, int k, in } for (; ii < max_ii; ii++) { -#if __mips_msa - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const unsigned short* p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0 += 8; - } - } - if (elempack == 4) - { - const int i_pack = (i + ii) / 4; - const int i_lane = (i + ii) % 4; - const unsigned short* p0 = (const unsigned short*)A + (size_t)i_pack * A_hstep * 4 + k * 4 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0 += 4; - } - } -#endif // __mips_msa - if (elempack == 1) + // if (elempack == 1) { const unsigned short* p0 = (const unsigned short*)A + (i + ii) * A_hstep + k; @@ -250,27 +175,32 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, { if (elempack == 8) { - const int k_pack = k / 8; - const int k_lane = k % 8; - const unsigned short* p0 = (const unsigned short*)A + (size_t)k_pack * A_hstep * 8 + (i + ii) * 8 + k_lane; + const unsigned short* p0 = (const unsigned short*)A + k * A_hstep + (i + ii) * 8; - for (int kk = 0; kk < max_kk; kk++) + for (int kk = 0; kk + 7 < max_kk; kk += 8) { - __builtin_prefetch(p0 + A_hstep); - pp[0] = p0[0]; - pp[1] = p0[8]; - pp[2] = p0[16]; - pp[3] = p0[24]; - pp[4] = p0[32]; - pp[5] = p0[40]; - pp[6] = p0[48]; - pp[7] = p0[56]; - pp += 8; - p0++; - if ((k_lane + kk + 1) % 8 == 0) - { - p0 += A_hstep * 8 - 8; - } + v8i16 _r0 = (v8i16)__msa_ld_h(p0, 0); + v8i16 _r1 = (v8i16)__msa_ld_h(p0 + 8, 0); + v8i16 _r2 = (v8i16)__msa_ld_h(p0 + 16, 0); + v8i16 _r3 = (v8i16)__msa_ld_h(p0 + 24, 0); + v8i16 _r4 = (v8i16)__msa_ld_h(p0 + 32, 0); + v8i16 _r5 = (v8i16)__msa_ld_h(p0 + 40, 0); + v8i16 _r6 = (v8i16)__msa_ld_h(p0 + 48, 0); + v8i16 _r7 = (v8i16)__msa_ld_h(p0 + 56, 0); + + transpose8x8_epi16(_r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7); + + __msa_st_h(_r0, pp, 0); + __msa_st_h(_r1, pp + 8, 0); + __msa_st_h(_r2, pp + 16, 0); + __msa_st_h(_r3, pp + 24, 0); + __msa_st_h(_r4, pp + 32, 0); + __msa_st_h(_r5, pp + 40, 0); + __msa_st_h(_r6, pp + 48, 0); + __msa_st_h(_r7, pp + 56, 0); + + pp += 64; + p0 += A_hstep * 8; } } if (elempack == 4) @@ -316,19 +246,6 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, pp += 32; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp[4] = p0[16]; - pp[5] = p0[20]; - pp[6] = p0[24]; - pp[7] = p0[28]; - pp += 8; - p0++; - } } if (elempack == 1) { @@ -400,15 +317,6 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, pp += 16; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp += 4; - p0++; - } } if (elempack == 1) { @@ -468,13 +376,6 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, pp += 8; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp += 2; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -527,12 +428,6 @@ static void transpose_pack_A_tile_bf16(const Mat& A, Mat& AT, int i, int max_ii, pp += 4; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -638,23 +533,6 @@ static void pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, int k, in for (; jj + 3 < max_jj; jj += 4) { #if __mips_msa - if (elempack == 8) - { - const int j_pack = (j + jj) / 8; - const int j_lane = (j + jj) % 8; - const unsigned short* p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 8 + k * 8 + j_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - __builtin_prefetch(p0 + 32); - pp[0] = p0[0]; - pp[1] = p0[1]; - pp[2] = p0[2]; - pp[3] = p0[3]; - pp += 4; - p0 += 8; - } - } if (elempack == 4) { const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k * 4; @@ -698,93 +576,33 @@ static void pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, int k, in } for (; jj + 1 < max_jj; jj += 2) { -#if __mips_msa - if (elempack == 8) + // if (elempack == 1) { - const int j_pack = (j + jj) / 8; - const int j_lane = (j + jj) % 8; - const unsigned short* p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 8 + k * 8 + j_lane; + const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; + const unsigned short* p1 = (const unsigned short*)B + (j + jj + 1) * B_hstep + k; for (int kk = 0; kk < max_kk; kk++) { pp[0] = p0[0]; - pp[1] = p0[1]; + pp[1] = p1[0]; pp += 2; - p0 += 8; + p0++; + p1++; } - - continue; - } - if (elempack == 4) - { - const int j_pack = (j + jj) / 4; - const int j_lane = (j + jj) % 4; - const unsigned short* p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 4 + k * 4 + j_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[1]; - pp += 2; - p0 += 4; - } - - continue; - } -#endif // __mips_msa - const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; - const unsigned short* p1 = (const unsigned short*)B + (j + jj + 1) * B_hstep + k; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p1[0]; - pp += 2; - p0++; - p1++; } } for (; jj < max_jj; jj++) { -#if __mips_msa - if (elempack == 8) + // if (elempack == 1) { - const int j_pack = (j + jj) / 8; - const int j_lane = (j + jj) % 8; - const unsigned short* p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 8 + k * 8 + j_lane; + const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; for (int kk = 0; kk < max_kk; kk++) { pp[0] = p0[0]; pp += 1; - p0 += 8; + p0++; } - - continue; - } - if (elempack == 4) - { - const int j_pack = (j + jj) / 4; - const int j_lane = (j + jj) % 4; - const unsigned short* p0 = (const unsigned short*)B + (size_t)j_pack * B_hstep * 4 + k * 4 + j_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0 += 4; - } - - continue; - } -#endif // __mips_msa - const unsigned short* p0 = (const unsigned short*)B + (j + jj) * B_hstep + k; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; } } } @@ -924,15 +742,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 16; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp += 4; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -992,13 +801,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 8; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp += 2; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -1051,12 +853,6 @@ static void transpose_pack_B_tile_bf16(const Mat& B, Mat& BT, int j, int max_jj, pp += 4; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - } } #endif // __mips_msa if (elempack == 1) diff --git a/src/layer/mips/gemm_int8.h b/src/layer/mips/gemm_int8.h index d079a4963..813623404 100644 --- a/src/layer/mips/gemm_int8.h +++ b/src/layer/mips/gemm_int8.h @@ -1148,9 +1148,9 @@ static void pack_A_tile_fp32_to_int8(const Mat& A, Mat& AT, int i, int max_ii, i } } #endif // __mips_msa - if (elempack == 1) + for (; ii + 1 < max_ii; ii += 2) { - for (; ii + 1 < max_ii; ii += 2) + // if (elempack == 1) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k; const float* p1 = (const float*)A + (i + ii + 1) * A_hstep + k; @@ -1181,7 +1181,10 @@ static void pack_A_tile_fp32_to_int8(const Mat& A, Mat& AT, int i, int max_ii, i p1++; } } - for (; ii < max_ii; ii++) + } + for (; ii < max_ii; ii++) + { + // if (elempack == 1) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k; const float scale0 = scales[i + ii]; @@ -1755,9 +1758,9 @@ static void pack_B_tile_fp32_to_int8(const Mat& B, Mat& BT, int j, int max_jj, i } } #endif // __mips_msa - if (elempack == 1) + for (; jj + 1 < max_jj; jj += 2) { - for (; jj + 1 < max_jj; jj += 2) + // if (elempack == 1) { const float* p0 = (const float*)B + (j + jj) * B_hstep + k; const float* p1 = (const float*)B + (j + jj + 1) * B_hstep + k; @@ -1786,7 +1789,10 @@ static void pack_B_tile_fp32_to_int8(const Mat& B, Mat& BT, int j, int max_jj, i p1++; } } - for (; jj < max_jj; jj++) + } + for (; jj < max_jj; jj++) + { + // if (elempack == 1) { const float* p0 = (const float*)B + (j + jj) * B_hstep + k; diff --git a/src/layer/mips/gemm_mips.cpp b/src/layer/mips/gemm_mips.cpp index 26ef641e7..77d80676b 100644 --- a/src/layer/mips/gemm_mips.cpp +++ b/src/layer/mips/gemm_mips.cpp @@ -176,20 +176,6 @@ static void pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int k, int max #if __mips_msa for (; ii + 3 < max_ii; ii += 4) { - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - const float* p0 = (const float*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - __builtin_prefetch(p0 + 32); - __msa_st_w(__msa_ld_w(p0, 0), pp, 0); - pp += 4; - p0 += 8; - } - } if (elempack == 4) { const float* p0 = (const float*)A + (i + ii) * A_hstep + k * 4; @@ -252,25 +238,6 @@ static void pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int k, int max const float* p0 = (const float*)A + (i + ii) * A_hstep + k; const float* p1 = (const float*)A + (i + ii + 1) * A_hstep + k; -#if __mips_msa - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - p0 = (const float*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[1]; - pp += 2; - p0 += 8; - } - - continue; - } -#endif // __mips_msa - for (int kk = 0; kk < max_kk; kk++) { pp[0] = p0[0]; @@ -285,24 +252,6 @@ static void pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int k, int max { const float* p0 = (const float*)A + (i + ii) * A_hstep + k; -#if __mips_msa - if (elempack == 8) - { - const int i_pack = (i + ii) / 8; - const int i_lane = (i + ii) % 8; - p0 = (const float*)A + (size_t)i_pack * A_hstep * 8 + k * 8 + i_lane; - - for (int kk = 0; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0 += 8; - } - - continue; - } -#endif // __mips_msa - int kk = 0; #if __mips_msa for (; kk + 3 < max_kk; kk += 4) @@ -361,19 +310,6 @@ static void transpose_pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int pp += 32; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp[4] = p0[16]; - pp[5] = p0[20]; - pp[6] = p0[24]; - pp[7] = p0[28]; - pp += 8; - p0++; - } } if (elempack == 1) { @@ -414,15 +350,6 @@ static void transpose_pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int pp += 16; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp += 4; - p0++; - } } if (elempack == 1) { @@ -470,13 +397,6 @@ static void transpose_pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int pp += 8; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp += 2; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -501,20 +421,13 @@ static void transpose_pack_A_tile(const Mat& A, Mat& AT, int i, int max_ii, int { const float* p0 = (const float*)A + k * A_hstep + (i + ii) * 4; - int kk = 0; - for (; kk + 3 < max_kk; kk += 4) + for (int kk = 0; kk + 3 < max_kk; kk += 4) { __builtin_prefetch(p0 + A_hstep * 4); __msa_st_w(__msa_ld_w(p0, 0), pp, 0); pp += 4; p0 += A_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -772,19 +685,6 @@ static void transpose_pack_B_tile(const Mat& B, Mat& BT, int j, int max_jj, int pp += 32; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp[4] = p0[16]; - pp[5] = p0[20]; - pp[6] = p0[24]; - pp[7] = p0[28]; - pp += 8; - p0++; - } } if (elempack == 1) { @@ -824,15 +724,6 @@ static void transpose_pack_B_tile(const Mat& B, Mat& BT, int j, int max_jj, int pp += 16; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp[2] = p0[8]; - pp[3] = p0[12]; - pp += 4; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -880,13 +771,6 @@ static void transpose_pack_B_tile(const Mat& B, Mat& BT, int j, int max_jj, int pp += 8; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp[1] = p0[4]; - pp += 2; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -919,12 +803,6 @@ static void transpose_pack_B_tile(const Mat& B, Mat& BT, int j, int max_jj, int pp += 4; p0 += B_hstep * 4; } - for (; kk < max_kk; kk++) - { - pp[0] = p0[0]; - pp += 1; - p0++; - } } #endif // __mips_msa if (elempack == 1) @@ -1811,27 +1689,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - float* p1f = p0f + out_hstep * 4; - uint64_t v0 = (uint32_t)__msa_copy_s_w((v4i32)_sum00, 0) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum10, 0) << 32); - uint64_t v1 = (uint32_t)__msa_copy_s_w((v4i32)_sum00, 1) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum10, 1) << 32); - uint64_t v2 = (uint32_t)__msa_copy_s_w((v4i32)_sum00, 2) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum10, 2) << 32); - uint64_t v3 = (uint32_t)__msa_copy_s_w((v4i32)_sum00, 3) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum10, 3) << 32); - uint64_t v4 = (uint32_t)__msa_copy_s_w((v4i32)_sum01, 0) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum11, 0) << 32); - uint64_t v5 = (uint32_t)__msa_copy_s_w((v4i32)_sum01, 1) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum11, 1) << 32); - uint64_t v6 = (uint32_t)__msa_copy_s_w((v4i32)_sum01, 2) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum11, 2) << 32); - uint64_t v7 = (uint32_t)__msa_copy_s_w((v4i32)_sum01, 3) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum11, 3) << 32); - memcpy(p0f, &v0, 8); - memcpy(p0f + 4, &v1, 8); - memcpy(p0f + 8, &v2, 8); - memcpy(p0f + 12, &v3, 8); - memcpy(p1f, &v4, 8); - memcpy(p1f + 4, &v5, 8); - memcpy(p1f + 8, &v6, 8); - memcpy(p1f + 12, &v7, 8); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __msa_st_w((v4i32)_sum00, p0f, 0); __msa_st_w((v4i32)_sum01, p0f + 4, 0); @@ -1877,31 +1735,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - unsigned short* p1 = p0 + out_hstep * 4; - v8i16 _bf0 = (v8i16)float2bfloat_msa(_sum00); - v8i16 _bf1 = (v8i16)float2bfloat_msa(_sum10); - v8i16 _bf2 = (v8i16)float2bfloat_msa(_sum01); - v8i16 _bf3 = (v8i16)float2bfloat_msa(_sum11); - uint32_t v0 = (unsigned short)__msa_copy_s_h(_bf0, 0) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 0) << 16); - uint32_t v1 = (unsigned short)__msa_copy_s_h(_bf0, 1) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 1) << 16); - uint32_t v2 = (unsigned short)__msa_copy_s_h(_bf0, 2) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 2) << 16); - uint32_t v3 = (unsigned short)__msa_copy_s_h(_bf0, 3) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 3) << 16); - uint32_t v4 = (unsigned short)__msa_copy_s_h(_bf2, 0) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf3, 0) << 16); - uint32_t v5 = (unsigned short)__msa_copy_s_h(_bf2, 1) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf3, 1) << 16); - uint32_t v6 = (unsigned short)__msa_copy_s_h(_bf2, 2) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf3, 2) << 16); - uint32_t v7 = (unsigned short)__msa_copy_s_h(_bf2, 3) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf3, 3) << 16); - memcpy(p0, &v0, 4); - memcpy(p0 + 4, &v1, 4); - memcpy(p0 + 8, &v2, 4); - memcpy(p0 + 12, &v3, 4); - memcpy(p1, &v4, 4); - memcpy(p1 + 4, &v5, 4); - memcpy(p1 + 8, &v6, 4); - memcpy(p1 + 12, &v7, 4); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __msa_st_w(float2bfloat_msa(_sum00, _sum01), p0, 0); __msa_st_w(float2bfloat_msa(_sum10, _sum11), p0 + out_hstep, 0); @@ -1992,27 +1826,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - float* p1f = p0f + out_hstep * 4; - uint32_t v0 = __msa_copy_s_w((v4i32)_sum0, 0); - uint32_t v1 = __msa_copy_s_w((v4i32)_sum0, 1); - uint32_t v2 = __msa_copy_s_w((v4i32)_sum0, 2); - uint32_t v3 = __msa_copy_s_w((v4i32)_sum0, 3); - uint32_t v4 = __msa_copy_s_w((v4i32)_sum1, 0); - uint32_t v5 = __msa_copy_s_w((v4i32)_sum1, 1); - uint32_t v6 = __msa_copy_s_w((v4i32)_sum1, 2); - uint32_t v7 = __msa_copy_s_w((v4i32)_sum1, 3); - memcpy(p0f, &v0, 4); - memcpy(p0f + 4, &v1, 4); - memcpy(p0f + 8, &v2, 4); - memcpy(p0f + 12, &v3, 4); - memcpy(p1f, &v4, 4); - memcpy(p1f + 4, &v5, 4); - memcpy(p1f + 8, &v6, 4); - memcpy(p1f + 12, &v7, 4); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __msa_st_w((v4i32)_sum0, p0f, 0); __msa_st_w((v4i32)_sum1, p0f + 4, 0); @@ -2054,21 +1868,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - unsigned short* p1 = p0 + out_hstep * 4; - v8i16 _bf0 = (v8i16)float2bfloat_msa(_sum0); - v8i16 _bf1 = (v8i16)float2bfloat_msa(_sum1); - p0[0] = (unsigned short)__msa_copy_s_h(_bf0, 0); - p0[4] = (unsigned short)__msa_copy_s_h(_bf0, 1); - p0[8] = (unsigned short)__msa_copy_s_h(_bf0, 2); - p0[12] = (unsigned short)__msa_copy_s_h(_bf0, 3); - p1[0] = (unsigned short)__msa_copy_s_h(_bf1, 0); - p1[4] = (unsigned short)__msa_copy_s_h(_bf1, 1); - p1[8] = (unsigned short)__msa_copy_s_h(_bf1, 2); - p1[12] = (unsigned short)__msa_copy_s_h(_bf1, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __msa_st_w(float2bfloat_msa(_sum0, _sum1), p0, 0); } @@ -2626,18 +2426,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - uint64_t v0 = (uint32_t)__msa_copy_s_w((v4i32)_sum0, 0) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum1, 0) << 32); - uint64_t v1 = (uint32_t)__msa_copy_s_w((v4i32)_sum0, 1) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum1, 1) << 32); - uint64_t v2 = (uint32_t)__msa_copy_s_w((v4i32)_sum0, 2) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum1, 2) << 32); - uint64_t v3 = (uint32_t)__msa_copy_s_w((v4i32)_sum0, 3) | ((uint64_t)(uint32_t)__msa_copy_s_w((v4i32)_sum1, 3) << 32); - memcpy(p0f, &v0, 8); - memcpy(p0f + 4, &v1, 8); - memcpy(p0f + 8, &v2, 8); - memcpy(p0f + 12, &v3, 8); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __msa_st_w((v4i32)_sum0, p0f, 0); __msa_st_w((v4i32)_sum1, p0f + out_hstep, 0); @@ -2670,20 +2459,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - v8i16 _bf0 = (v8i16)float2bfloat_msa(_sum0); - v8i16 _bf1 = (v8i16)float2bfloat_msa(_sum1); - uint32_t v0 = (unsigned short)__msa_copy_s_h(_bf0, 0) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 0) << 16); - uint32_t v1 = (unsigned short)__msa_copy_s_h(_bf0, 1) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 1) << 16); - uint32_t v2 = (unsigned short)__msa_copy_s_h(_bf0, 2) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 2) << 16); - uint32_t v3 = (unsigned short)__msa_copy_s_h(_bf0, 3) | ((uint32_t)(unsigned short)__msa_copy_s_h(_bf1, 3) << 16); - memcpy(p0, &v0, 4); - memcpy(p0 + 4, &v1, 4); - memcpy(p0 + 8, &v2, 4); - memcpy(p0 + 12, &v3, 4); - } - if (out_elempack == 1) + // if (out_elempack == 1) { v4i32 _bf = float2bfloat_msa(_sum0, _sum1); int64_t v0 = __msa_copy_s_d((v2i64)_bf, 0); @@ -2751,18 +2527,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - uint32_t v0 = __msa_copy_s_w((v4i32)_sum0, 0); - uint32_t v1 = __msa_copy_s_w((v4i32)_sum0, 1); - uint32_t v2 = __msa_copy_s_w((v4i32)_sum0, 2); - uint32_t v3 = __msa_copy_s_w((v4i32)_sum0, 3); - memcpy(p0f, &v0, 4); - memcpy(p0f + 4, &v1, 4); - memcpy(p0f + 8, &v2, 4); - memcpy(p0f + 12, &v3, 4); - } - if (out_elempack == 1) + // if (out_elempack == 1) { __msa_st_w((v4i32)_sum0, p0f, 0); } @@ -2793,15 +2558,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 4) - { - v8i16 _bf = (v8i16)float2bfloat_msa(_sum0); - p0[0] = (unsigned short)__msa_copy_s_h(_bf, 0); - p0[4] = (unsigned short)__msa_copy_s_h(_bf, 1); - p0[8] = (unsigned short)__msa_copy_s_h(_bf, 2); - p0[12] = (unsigned short)__msa_copy_s_h(_bf, 3); - } - if (out_elempack == 1) + // if (out_elempack == 1) { v4i32 _bf = float2bfloat_msa(_sum0); int64_t v = __msa_copy_s_d((v2i64)_bf, 0); @@ -2908,11 +2665,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - p0f[q] = sum0; - p0f[q + 8] = sum1; - } if (out_elempack == 4) { p0f[q] = sum0; @@ -2926,17 +2678,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0f[0] = sum0; - p0f[out_hstep] = sum1; - } - if (out_elempack == 4) - { - p0f[0] = sum0; - p0f[out_hstep] = sum1; - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0f[0] = sum0; p0f[out_hstep] = sum1; @@ -2965,17 +2707,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0[0] = float32_to_bfloat16(sum0); - p0[out_hstep] = float32_to_bfloat16(sum1); - } - if (out_elempack == 4) - { - p0[0] = float32_to_bfloat16(sum0); - p0[out_hstep] = float32_to_bfloat16(sum1); - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0[0] = float32_to_bfloat16(sum0); p0[out_hstep] = float32_to_bfloat16(sum1); @@ -3027,8 +2759,11 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - p0 = (unsigned short*)top_blob + (i + ii) * out_hstep + j * out_elempack; - p0f = (float*)top_blob + (i + ii) * out_hstep + j * out_elempack; + // if (out_elempack == 1) + { + p0 = (unsigned short*)top_blob + (i + ii) * out_hstep + j; + p0f = (float*)top_blob + (i + ii) * out_hstep + j; + } } const float* pC0 = pC; @@ -3082,10 +2817,6 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int { if (output_transpose) { - if (out_elempack == 8) - { - p0f[q] = sum; - } if (out_elempack == 4) { p0f[q] = sum; @@ -3097,15 +2828,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0f[0] = sum; - } - if (out_elempack == 4) - { - p0f[0] = sum; - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0f[0] = sum; } @@ -3130,15 +2853,7 @@ static void unpack_output_tile(const Mat& topT, const Mat& C, Mat& top_blob, int } else { - if (out_elempack == 8) - { - p0[0] = float32_to_bfloat16(sum); - } - if (out_elempack == 4) - { - p0[0] = float32_to_bfloat16(sum); - } - if (out_elempack == 1) + // if (out_elempack == 1) { p0[0] = float32_to_bfloat16(sum); } @@ -5942,21 +5657,6 @@ int Gemm_mips::forward_bf16s(const std::vector& bottom_blobs, std::vector 0.45f && vv < 0.55f) || (hvv > 0.45f && hvv < 0.55f) || (bvv > 0.45f && bvv < 0.55f)) + { + p[j] = RandomFloat(-randabsmax, randabsmax); + v = p[j] * (127.f / randabsmax); + vv = fabs(v - (int)v); + + hp = ncnn::float16_to_float32(ncnn::float32_to_float16(p[j])); + hv = hp * (127.f / randabsmax); + hvv = fabs(hv - (int)hv); + + bp = ncnn::bfloat16_to_float32(ncnn::float32_to_bfloat16(p[j])); + bv = bp * (127.f / randabsmax); + bvv = fabs(bv - (int)bv); + } + } + } + } + else // if (transA == 1) + { + std::vector randabsmaxes(m.w); + for (int j = 0; j < m.w; j++) + { + float randabsmax = RandomFloat(absmax * 0.5f, absmax); + randabsmax = ncnn::float16_to_float32(ncnn::float32_to_float16(randabsmax)); + randabsmax = ncnn::bfloat16_to_float32(ncnn::float32_to_bfloat16(randabsmax)); + randabsmaxes[j] = randabsmax; + } + + const int h = m.dims == 3 ? m.c : m.h; + for (int i = 0; i < h; i++) + { + float* p = m.dims == 3 ? m.channel(i) : m.row(i); + for (int j = 0; j < m.w; j++) + { + const float randabsmax = randabsmaxes[j]; + p[j] = RandomFloat(-randabsmax, randabsmax); + } + + // drop 0.45 ~ 0.55 + for (int j = 0; j < m.w; j++) + { + const float randabsmax = randabsmaxes[j]; + float v = p[j] * (127.f / randabsmax); + float vv = fabs(v - (int)v); + + float hp = ncnn::float16_to_float32(ncnn::float32_to_float16(p[j])); + float hv = hp * (127.f / randabsmax); + float hvv = fabs(hv - (int)hv); + + float bp = ncnn::bfloat16_to_float32(ncnn::float32_to_bfloat16(p[j])); + float bv = bp * (127.f / randabsmax); + float bvv = fabs(bv - (int)bv); + + while ((vv > 0.45f && vv < 0.55f) || (hvv > 0.45f && hvv < 0.55f) || (bvv > 0.45f && bvv < 0.55f)) + { + p[j] = RandomFloat(-randabsmax, randabsmax); + v = p[j] * (127.f / randabsmax); + vv = fabs(v - (int)v); + + hp = ncnn::float16_to_float32(ncnn::float32_to_float16(p[j])); + hv = hp * (127.f / randabsmax); + hvv = fabs(hv - (int)hv); + + bp = ncnn::bfloat16_to_float32(ncnn::float32_to_bfloat16(p[j])); + bv = bp * (127.f / randabsmax); + bvv = fabs(bv - (int)bv); + } + } + } + + for (int j = 0; j < m.w; j++) + { + const int randi0 = RandomInt(0, h - 1); + const int randi1 = RandomInt(0, h - 1); + float* p0 = m.dims == 3 ? m.channel(randi0) : m.row(randi0); + float* p1 = m.dims == 3 ? m.channel(randi1) : m.row(randi1); + + const float randabsmax = randabsmaxes[j]; + + // set random a and b + p0[j] = -randabsmax; + p1[j] = randabsmax; + } + } +} + +static void RandomizeB(ncnn::Mat& m, float absmax) +{ + absmax = ncnn::float16_to_float32(ncnn::float32_to_float16(absmax)); + absmax = ncnn::bfloat16_to_float32(ncnn::float32_to_bfloat16(absmax)); + + const int h = m.dims == 3 ? m.c : m.h; + float* p = m; + for (int i = 0; i < h; i++) + { + float* p = m.dims == 3 ? m.channel(i) : m.row(i); + for (int j = 0; j < m.w; j++) + { + p[j] = RandomFloat(-absmax, absmax); + + // drop 0.45 ~ 0.55 + float v = p[j] * (127.f / absmax); + float vv = fabs(v - (int)v); + + float hp = ncnn::float16_to_float32(ncnn::float32_to_float16(p[j])); + float hv = hp * (127.f / absmax); + float hvv = fabs(hv - (int)hv); + + float bp = ncnn::bfloat16_to_float32(ncnn::float32_to_bfloat16(p[j])); + float bv = bp * (127.f / absmax); + float bvv = fabs(bv - (int)bv); + + while ((vv > 0.45f && vv < 0.55f) || (hvv > 0.45f && hvv < 0.55f) || (bvv > 0.45f && bvv < 0.55f)) + { + p[j] = RandomFloat(-absmax, absmax); + v = p[j] * (127.f / absmax); + vv = fabs(v - (int)v); + + hp = ncnn::float16_to_float32(ncnn::float32_to_float16(p[j])); + hv = hp * (127.f / absmax); + hvv = fabs(hv - (int)hv); + + bp = ncnn::bfloat16_to_float32(ncnn::float32_to_bfloat16(p[j])); + bv = bp * (127.f / absmax); + bvv = fabs(bv - (int)bv); + } + } + } + + // set random a and b + if (m.dims == 3) + { + m.channel(RandomInt(0, h - 1))[RandomInt(0, m.w - 1)] = -absmax; + m.channel(RandomInt(0, h - 1))[RandomInt(0, m.w - 1)] = absmax; + } + else + { + m.row(RandomInt(0, h - 1))[RandomInt(0, m.w - 1)] = -absmax; + m.row(RandomInt(0, h - 1))[RandomInt(0, m.w - 1)] = absmax; + } +} + +static int test_gemm_int8_nt(int M, int N, int K, float alpha, int transA, int transB, int output_transpose, int constantA, int constantB) +{ + ncnn::ParamDict pd; + pd.set(0, alpha); + pd.set(1, 1.f); // beta + pd.set(2, transA); + pd.set(3, transB); + pd.set(4, constantA); + pd.set(5, constantB); + pd.set(6, 1); + pd.set(7, M); + pd.set(8, N); + pd.set(9, K); + pd.set(10, -1); + pd.set(13, 0); // output_elemtype = fp32 + pd.set(14, output_transpose); + pd.set(18, 2); // int8_scale_term + + std::vector weights; + if (constantA) weights.push_back(transA ? RandomS8Mat(M, K) : RandomS8Mat(K, M)); + if (constantB) weights.push_back(transB ? RandomS8Mat(K, N) : RandomS8Mat(N, K)); + if (constantA) weights.push_back(RandomMat(M, 10.f, 20.f)); + if (constantB) weights.push_back(RandomMat(1, 10.f, 20.f)); + + std::vector a; + if (!constantA) + { + a.push_back(transA ? ncnn::Mat(M, K) : ncnn::Mat(K, M)); + RandomizeA(a[a.size() - 1], transA, 10.f); + } + if (!constantB) + { + a.push_back(transB ? ncnn::Mat(K, N) : ncnn::Mat(N, K)); + RandomizeB(a[a.size() - 1], 10.f); + } + + float epsilon = 0.001; + + int ret = test_layer("Gemm", pd, weights, a, 1, epsilon, TEST_LAYER_ENABLE_THREADING); + if (ret != 0) + { + fprintf(stderr, "test_gemm_int8_nt failed M=%d N=%d K=%d alpha=%f transA=%d transB=%d output_transpose=%d constantA=%d constantB=%d\n", M, N, K, alpha, transA, transB, output_transpose, constantA, constantB); + } + + return ret; +} + +static int test_gemm_int8_bias_nt(int M, int N, int K, const ncnn::Mat& C, float alpha, float beta, int transA, int transB, int output_transpose, int constantA, int constantB, int constantC) +{ + int broadcast_type_C = 0; + if (C.dims == 1 && C.w == 1) + { + // scalar + broadcast_type_C = 0; + } + if (C.dims == 1 && C.w == M) + { + // M + // auto broadcast from h to w is the ncnn-style convention + broadcast_type_C = 1; + } + if (C.dims == 1 && C.w == N) + { + // N + broadcast_type_C = 4; + } + if (C.dims == 2 && C.w == 1 && C.h == M) + { + // Mx1 + broadcast_type_C = 2; + } + if (C.dims == 2 && C.w == N && C.h == M) + { + // MxN + broadcast_type_C = 3; + } + if (C.dims == 2 && C.w == N && C.h == 1) + { + // 1xN + broadcast_type_C = 4; + } + + ncnn::ParamDict pd; + pd.set(0, alpha); + pd.set(1, beta); + pd.set(2, transA); + pd.set(3, transB); + pd.set(4, constantA); + pd.set(5, constantB); + pd.set(6, constantC); + pd.set(7, M); + pd.set(8, N); + pd.set(9, K); + pd.set(10, broadcast_type_C); + pd.set(13, 0); // output_elemtype = fp32 + pd.set(14, output_transpose); + pd.set(18, 2); // int8_scale_term + + std::vector weights; + if (constantA) weights.push_back(transA ? RandomS8Mat(M, K) : RandomS8Mat(K, M)); + if (constantB) weights.push_back(transB ? RandomS8Mat(K, N) : RandomS8Mat(N, K)); + if (constantC) weights.push_back(C); + if (constantA) weights.push_back(RandomMat(M, 10.f, 20.f)); + if (constantB) weights.push_back(RandomMat(1, 10.f, 20.f)); + + std::vector a; + if (!constantA) + { + a.push_back(transA ? ncnn::Mat(M, K) : ncnn::Mat(K, M)); + RandomizeA(a[a.size() - 1], transA, 10.f); + } + if (!constantB) + { + a.push_back(transB ? ncnn::Mat(K, N) : ncnn::Mat(N, K)); + RandomizeB(a[a.size() - 1], 10.f); + } + if (!constantC) a.push_back(C); + + float epsilon = 0.001; + + int ret = test_layer("Gemm", pd, weights, a, 1, epsilon, TEST_LAYER_ENABLE_THREADING); + if (ret != 0) + { + fprintf(stderr, "test_gemm_int8_bias_nt failed M=%d N=%d K=%d C.dims=%d C=(%d %d %d) alpha=%f beta=%f transA=%d transB=%d output_transpose=%d constantA=%d constantB=%d constantC=%d\n", M, N, K, C.dims, C.w, C.h, C.c, alpha, beta, transA, transB, output_transpose, constantA, constantB, constantC); + } + + return ret; +} + +static int test_gemm_0(int M, int N, int K) +{ + return 0 + || test_gemm_int8_nt(M, N, K, 2.1f, 0, 0, 0, 0, 0) + || test_gemm_int8_nt(M, N, K, 3.1f, 0, 1, 0, 0, 0) + || test_gemm_int8_nt(M, N, K, 4.1f, 1, 0, 1, 0, 0) + || test_gemm_int8_nt(M, N, K, 5.1f, 1, 1, 1, 0, 0) + + || test_gemm_int8_nt(M, N, K, 0.2f, 0, 0, 1, 1, 0) + || test_gemm_int8_nt(M, N, K, 0.3f, 0, 1, 1, 1, 0) + || test_gemm_int8_nt(M, N, K, 0.4f, 1, 0, 0, 1, 0) + || test_gemm_int8_nt(M, N, K, 0.5f, 1, 1, 0, 1, 0) + + || test_gemm_int8_nt(M, N, K, 1.2f, 0, 0, 0, 0, 1) + || test_gemm_int8_nt(M, N, K, 1.3f, 0, 1, 1, 0, 1) + || test_gemm_int8_nt(M, N, K, 1.4f, 1, 0, 0, 0, 1) + || test_gemm_int8_nt(M, N, K, 1.5f, 1, 1, 1, 0, 1) + + || test_gemm_int8_nt(M, N, K, -1.2f, 0, 0, 1, 1, 1) + || test_gemm_int8_nt(M, N, K, -1.3f, 0, 1, 0, 1, 1) + || test_gemm_int8_nt(M, N, K, -1.4f, 1, 0, 1, 1, 1) + || test_gemm_int8_nt(M, N, K, -1.5f, 1, 1, 0, 1, 1); +} + +static int test_gemm_1(int M, int N, int K) +{ + return 0 + || test_gemm_int8_bias_nt(M, N, K, RandomMat(1), 2.1f, 0.5f, 0, 0, 0, 0, 0, 0) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(M), 3.1f, 0.6f, 0, 1, 0, 0, 0, 0) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(1, M), 4.1f, 0.7f, 1, 0, 1, 0, 0, 0) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(N, M), 5.1f, 0.8f, 1, 1, 1, 0, 0, 0) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(N, 1), 2.1f, 0.5f, 0, 0, 0, 0, 0, 0) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(N), 3.1f, 0.6f, 0, 1, 0, 0, 0, 0) + + || test_gemm_int8_bias_nt(M, N, K, RandomMat(1), -2.1f, 0.5f, 1, 1, 0, 1, 1, 1) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(M), -3.1f, 0.6f, 0, 0, 0, 1, 1, 1) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(1, M), -4.1f, 0.7f, 0, 1, 1, 1, 1, 1) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(N, M), -5.1f, 0.8f, 1, 0, 1, 1, 1, 1) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(N, 1), -2.1f, 0.5f, 1, 1, 0, 1, 1, 1) + || test_gemm_int8_bias_nt(M, N, K, RandomMat(N), -3.1f, 0.6f, 0, 0, 0, 1, 1, 1); +} +#endif // NCNN_INT8 + +int main() +{ + SRAND(7767517); + +#if NCNN_INT8 + int mnk[][3] = { + {1, 20, 40}, + {20, 2, 39}, + {3, 30, 13}, + {33, 1, 19} + }; + + int mnk_count = sizeof(mnk) / sizeof(int) / 3; + + for (int i = 0; i < mnk_count; i++) + { + int M = mnk[i][0]; + int N = mnk[i][1]; + int K = mnk[i][2]; + + int ret = test_gemm_0(M, N, K) || test_gemm_1(M, N, K); + + if (ret != 0) + return ret; + } +#else + // test nothing for non-int8 build +#endif + + return 0; +} diff --git a/tests/testutil.cpp b/tests/testutil.cpp index 1020a8e59..fc4ebd459 100644 --- a/tests/testutil.cpp +++ b/tests/testutil.cpp @@ -765,7 +765,7 @@ int test_layer_cpu(int typeindex, const ncnn::ParamDict& pd, const std::vectorvkdev = vkdev; if (flag & TEST_LAYER_ENABLE_THREADING) - opt.num_threads = ncnn::get_physical_big_cpu_count(); + opt.num_threads = std::min(ncnn::get_physical_big_cpu_count(), 4); else opt.num_threads = 1; @@ -1365,7 +1365,7 @@ int test_layer_cpu(int typeindex, const ncnn::ParamDict& pd, const std::vector