| /* |
| * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. |
| * |
| * SPDX-License-Identifier: Apache-2.0 |
| * |
| * Licensed under the Apache License, Version 2.0 (the License); you may |
| * not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an AS IS BASIS, WITHOUT |
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /* ---------------------------------------------------------------------- |
| * Project: CMSIS NN Library |
| * Title: arm_fully_connected_q7_opt.c |
| * Description: Q7 basic fully-connected layer function |
| * |
| * $Date: 17. January 2018 |
| * $Revision: V.1.0.0 |
| * |
| * Target Processor: Cortex-M cores |
| * |
| * -------------------------------------------------------------------- */ |
| |
| #include "arm_math.h" |
| #include "arm_nnfunctions.h" |
| |
| /** |
| * @ingroup groupNN |
| */ |
| |
| /** |
| * @addtogroup FC |
| * @{ |
| */ |
| |
| /** |
| * @brief Q7 opt fully-connected layer function |
| * @param[in] pV pointer to input vector |
| * @param[in] pM pointer to matrix weights |
| * @param[in] dim_vec length of the vector |
| * @param[in] num_of_rows number of rows in weight matrix |
| * @param[in] bias_shift amount of left-shift for bias |
| * @param[in] out_shift amount of right-shift for output |
| * @param[in] bias pointer to bias |
| * @param[in,out] pOut pointer to output vector |
| * @param[in,out] vec_buffer pointer to buffer space for input |
| * @return The function returns <code>ARM_MATH_SUCCESS</code> |
| * |
| * @details |
| * |
| * <b>Buffer size:</b> |
| * |
| * vec_buffer size: dim_vec |
| * |
| * This opt function is designed to work with interleaved weight |
| * matrix. The vector input is assumed in q7_t format, we call |
| * arm_q7_to_q15_no_shift_shuffle function to expand into |
| * q15_t format with certain weight re-ordering, refer to the function |
| * comments for more details. |
| * Here we use only one pointer to read 4 rows in the weight |
| * matrix. So if the original q7_t matrix looks like this: |
| * |
| * | a11 | a12 | a13 | a14 | a15 | a16 | a17 | |
| * |
| * | a21 | a22 | a23 | a24 | a25 | a26 | a27 | |
| * |
| * | a31 | a32 | a33 | a34 | a35 | a36 | a37 | |
| * |
| * | a41 | a42 | a43 | a44 | a45 | a46 | a47 | |
| * |
| * | a51 | a52 | a53 | a54 | a55 | a56 | a57 | |
| * |
| * | a61 | a62 | a63 | a64 | a65 | a66 | a67 | |
| * |
| * |
| * We operates on multiple-of-4 rows, so the first four rows becomes |
| * |
| * | a11 | a21 | a13 | a23 | a31 | a41 | a33 | a43 | |
| * |
| * | a12 | a22 | a14 | a24 | a32 | a42 | a34 | a44 | |
| * |
| * | a15 | a25 | a35 | a45 | a16 | a26 | a36 | a46 | |
| * |
| * So within the kernel, we first read the re-ordered vector in as: |
| * |
| * | b1 | b3 | and | b2 | b4 | |
| * |
| * the four q31_t weights will look like |
| * |
| * | a11 | a13 |, | a21 | a23 |, | a31 | a33 |, | a41 | a43 | |
| * |
| * | a12 | a14 |, | a22 | a24 |, | a32 | a34 |, | a42 | a44 | |
| * |
| * The column left over will be in-order. |
| * which is: |
| * |
| * | a17 | a27 | a37 | a47 | |
| * |
| * For the left-over rows, we do 1x1 computation, so the data remains |
| * as its original order. |
| * |
| * So the stored weight matrix looks like this: |
| * |
| * | a11 | a21 | a13 | a23 | a31 | a41 | |
| * |
| * | a33 | a43 | a12 | a22 | a14 | a24 | |
| * |
| * | a32 | a42 | a34 | a44 | a15 | a25 | |
| * |
| * | a35 | a45 | a16 | a26 | a36 | a46 | |
| * |
| * | a17 | a27 | a37 | a47 | a51 | a52 | |
| * |
| * | a53 | a54 | a55 | a56 | a57 | a61 | |
| * |
| * | a62 | a63 | a64 | a65 | a66 | a67 | |
| * |
| * |
| */ |
| |
| arm_status |
| arm_fully_connected_q7_opt(const q7_t * pV, |
| const q7_t * pM, |
| const uint16_t dim_vec, |
| const uint16_t num_of_rows, |
| const uint16_t bias_shift, |
| const uint16_t out_shift, |
| const q7_t * bias, |
| q7_t * pOut, |
| q15_t * vec_buffer) |
| { |
| |
| #if defined (ARM_MATH_DSP) |
| /* Run the following code for Cortex-M4 and Cortex-M7 */ |
| |
| const q7_t *pB = pM; |
| q7_t *pO = pOut; |
| const q7_t *pBias = bias; |
| q15_t *pA; |
| uint16_t rowCnt = num_of_rows >> 2; |
| |
| arm_q7_to_q15_reordered_no_shift(pV, vec_buffer, dim_vec); |
| |
| while (rowCnt) |
| { |
| |
| q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| |
| uint16_t colCnt = dim_vec >> 2; |
| |
| pA = vec_buffer; |
| |
| #ifdef USE_INTRINSIC |
| |
| #ifndef ARM_MATH_BIG_ENDIAN |
| while (colCnt) |
| { |
| q31_t inM11, inM12, inM13, inM14; |
| q31_t inV; |
| |
| inV = *__SIMD32(pA)++; |
| inM11 = *__SIMD32(pB)++; |
| inM12 = __SXTB16(__ROR(inM11, 8)); |
| inM11 = __SXTB16(inM11); |
| sum = __SMLAD(inM11, inV, sum); |
| sum2 = __SMLAD(inM12, inV, sum2); |
| inM13 = *__SIMD32(pB)++; |
| inM14 = __SXTB16(__ROR(inM13, 8)); |
| inM13 = __SXTB16(inM13); |
| sum3 = __SMLAD(inM13, inV, sum3); |
| sum4 = __SMLAD(inM14, inV, sum4); |
| |
| inV = *__SIMD32(pA)++; |
| inM11 = *__SIMD32(pB)++; |
| inM12 = __SXTB16(__ROR(inM11, 8)); |
| inM11 = __SXTB16(inM11); |
| sum = __SMLAD(inM11, inV, sum); |
| sum2 = __SMLAD(inM12, inV, sum2); |
| inM13 = *__SIMD32(pB)++; |
| inM14 = __SXTB16(__ROR(inM13, 8)); |
| inM13 = __SXTB16(inM13); |
| sum3 = __SMLAD(inM13, inV, sum3); |
| sum4 = __SMLAD(inM14, inV, sum4); |
| colCnt--; |
| } |
| #else |
| while (colCnt) |
| { |
| q31_t inM11, inM12, inM13, inM14; |
| q31_t inV; |
| |
| inV = *__SIMD32(pA)++; |
| inM11 = *__SIMD32(pB)++; |
| inM12 = __SXTB16(__ROR(inM11, 8)); |
| inM11 = __SXTB16(inM11); |
| sum = __SMLAD(inM12, inV, sum); |
| sum2 = __SMLAD(inM11, inV, sum2); |
| inM13 = *__SIMD32(pB)++; |
| inM14 = __SXTB16(__ROR(inM13, 8)); |
| inM13 = __SXTB16(inM13); |
| sum3 = __SMLAD(inM14, inV, sum3); |
| sum4 = __SMLAD(inM13, inV, sum4); |
| |
| inV = *__SIMD32(pA)++; |
| inM11 = *__SIMD32(pB)++; |
| inM12 = __SXTB16(__ROR(inM11, 8)); |
| inM11 = __SXTB16(inM11); |
| sum = __SMLAD(inM12, inV, sum); |
| sum2 = __SMLAD(inM11, inV, sum2); |
| inM13 = *__SIMD32(pB)++; |
| inM14 = __SXTB16(__ROR(inM13, 8)); |
| inM13 = __SXTB16(inM13); |
| sum3 = __SMLAD(inM14, inV, sum3); |
| sum4 = __SMLAD(inM13, inV, sum4); |
| colCnt--; |
| } |
| #endif /* ARM_MATH_BIG_ENDIAN */ |
| |
| #else |
| |
| /* |
| * register needed: |
| * loop counter: colCnt |
| * accumulators: sum, sum2, sum3, sum4 |
| * pointers: pB, pA |
| * weight data: inM11, inM12, inM13, inM14 |
| * activation data: inV |
| */ |
| |
| #ifndef ARM_MATH_BIG_ENDIAN |
| asm volatile ("COL_LOOP_%=:\n" |
| "ldr.w r4, [%[pA]], #8\n" |
| "ldr.w r1, [%[pB]], #16\n" |
| "mov.w r0, r1, ror #8\n" |
| "sxtb16 r0, r0\n" |
| "sxtb16 r1, r1\n" |
| "smlad %[sum], r4, r1, %[sum]\n" |
| "smlad %[sum2], r4, r0, %[sum2]\n" |
| "ldr.w r3, [%[pB], #-12]\n" |
| "mov.w r2, r3, ror #8\n" |
| "sxtb16 r2, r2\n" |
| "sxtb16 r3, r3\n" |
| "smlad %[sum3], r4, r3, %[sum3]\n" |
| "smlad %[sum4], r4, r2, %[sum4]\n" |
| "ldr.w r4, [%[pA], #-4]\n" |
| "ldr.w r1, [%[pB], #-8]\n" |
| "mov.w r0, r1, ror #8\n" |
| "sxtb16 r0, r0\n" |
| "sxtb16 r1, r1\n" |
| "smlad %[sum], r4, r1, %[sum]\n" |
| "smlad %[sum2], r4, r0, %[sum2]\n" |
| "ldr.w r3, [%[pB], #-4]\n" |
| "mov.w r2, r3, ror #8\n" |
| "sxtb16 r2, r2\n" |
| "sxtb16 r3, r3\n" |
| "smlad %[sum3], r4, r3, %[sum3]\n" |
| "smlad %[sum4], r4, r2, %[sum4]\n" |
| "subs %[colCnt], #1\n" |
| "bne COL_LOOP_%=\n":[sum] "+r"(sum), |
| [sum2] "+r"(sum2),[sum3] "+r"(sum3), |
| [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4"); |
| #else |
| asm volatile ("COL_LOOP_%=:\n" |
| "ldr.w r4, [%[pA]], #8\n" |
| "ldr.w r1, [%[pB]], #16\n" |
| "mov.w r0, r1, ror #8\n" |
| "sxtb16 r0, r0\n" |
| "sxtb16 r1, r1\n" |
| "smlad %[sum], r4, r0, %[sum]\n" |
| "smlad %[sum2], r4, r1, %[sum2]\n" |
| "ldr.w r3, [%[pB], #-12]\n" |
| "mov.w r2, r3, ror #8\n" |
| "sxtb16 r2, r2\n" |
| "sxtb16 r3, r3\n" |
| "smlad %[sum3], r4, r2, %[sum3]\n" |
| "smlad %[sum4], r4, r3, %[sum4]\n" |
| "ldr.w r4, [%[pA], #-4]\n" |
| "ldr.w r1, [%[pB], #-8]\n" |
| "mov.w r0, r1, ror #8\n" |
| "sxtb16 r0, r0\n" |
| "sxtb16 r1, r1\n" |
| "smlad %[sum], r4, r0, %[sum]\n" |
| "smlad %[sum2], r4, r1, %[sum2]\n" |
| "ldr.w r3, [%[pB], #-4]\n" |
| "mov.w r2, r3, ror #8\n" |
| "sxtb16 r2, r2\n" |
| "sxtb16 r3, r3\n" |
| "smlad %[sum3], r4, r2, %[sum3]\n" |
| "smlad %[sum4], r4, r3, %[sum4]\n" |
| "subs %[colCnt], #1\n" |
| "bne COL_LOOP_%=\n":[sum] "+r"(sum), |
| [sum2] "+r"(sum2),[sum3] "+r"(sum3), |
| [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4"); |
| #endif /* ARM_MATH_BIG_ENDIAN */ |
| |
| #endif /* USE_INTRINSIC */ |
| |
| colCnt = dim_vec & 0x3; |
| while (colCnt) |
| { |
| q15_t inV = *pA++; |
| q7_t inM = *pB++; |
| q7_t inM2 = *pB++; |
| q7_t inM3 = *pB++; |
| q7_t inM4 = *pB++; |
| |
| sum += inV * inM; |
| sum2 += inV * inM2; |
| sum3 += inV * inM3; |
| sum4 += inV * inM4; |
| colCnt--; |
| } /* while over colCnt */ |
| *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8)); |
| *pO++ = (q7_t) (__SSAT((sum2 >> out_shift), 8)); |
| *pO++ = (q7_t) (__SSAT((sum3 >> out_shift), 8)); |
| *pO++ = (q7_t) (__SSAT((sum4 >> out_shift), 8)); |
| |
| /* adjust the pointers and counters */ |
| rowCnt--; |
| } |
| |
| /* left-over part of the rows */ |
| rowCnt = num_of_rows & 0x3; |
| |
| while (rowCnt) |
| { |
| q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| uint16_t colCnt = dim_vec >> 2; |
| |
| pA = vec_buffer; |
| |
| while (colCnt) |
| { |
| q31_t inV1, inV2, inM11, inM12; |
| |
| pB = (q7_t *) read_and_pad_reordered((void *)pB, &inM11, &inM12); |
| |
| inV1 = *__SIMD32(pA)++; |
| sum = __SMLAD(inV1, inM11, sum); |
| |
| inV2 = *__SIMD32(pA)++; |
| sum = __SMLAD(inV2, inM12, sum); |
| |
| colCnt--; |
| } |
| |
| /* left-over of the vector */ |
| colCnt = dim_vec & 0x3; |
| while (colCnt) |
| { |
| q15_t inV = *pA++; |
| q7_t inM = *pB++; |
| sum += inV * inM; |
| colCnt--; |
| } |
| |
| *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8)); |
| |
| rowCnt--; |
| } |
| |
| #else |
| /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ |
| uint16_t rowCnt = num_of_rows >> 2; |
| const q7_t *pB = pM; |
| const q7_t *pA; |
| q7_t *pO = pOut; |
| const q7_t *pBias = bias; |
| |
| while (rowCnt) |
| { |
| q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| |
| uint16_t colCnt = dim_vec >> 2; |
| |
| pA = pV; |
| |
| while (colCnt) |
| { |
| q7_t inA1 = *pA++; |
| q7_t inA3 = *pA++; |
| q7_t inA2 = *pA++; |
| q7_t inA4 = *pA++; |
| |
| q7_t inB1 = *pB++; |
| q7_t inB3 = *pB++; |
| q7_t inB2 = *pB++; |
| q7_t inB4 = *pB++; |
| |
| sum += inA1 * inB1 + inA2 * inB2; |
| sum2 += inA1 * inB3 + inA2 * inB4; |
| |
| inB1 = *pB++; |
| inB3 = *pB++; |
| inB2 = *pB++; |
| inB4 = *pB++; |
| |
| sum3 += inA1 * inB1 + inA2 * inB2; |
| sum4 += inA1 * inB3 + inA2 * inB4; |
| |
| inB1 = *pB++; |
| inB3 = *pB++; |
| inB2 = *pB++; |
| inB4 = *pB++; |
| |
| sum += inA3 * inB1 + inA4 * inB2; |
| sum2 += inA3 * inB3 + inA4 * inB4; |
| |
| inB1 = *pB++; |
| inB3 = *pB++; |
| inB2 = *pB++; |
| inB4 = *pB++; |
| |
| sum3 += inA3 * inB1 + inA4 * inB2; |
| sum4 += inA3 * inB3 + inA4 * inB4; |
| |
| colCnt--; |
| } |
| colCnt = dim_vec & 0x3; |
| while (colCnt) |
| { |
| q7_t inA = *pA++; |
| q7_t inB = *pB++; |
| sum += inA * inB; |
| inB = *pB++; |
| sum2 += inA * inB; |
| inB = *pB++; |
| sum3 += inA * inB; |
| inB = *pB++; |
| sum4 += inA * inB; |
| |
| colCnt--; |
| } |
| *pO++ = (q7_t) __SSAT((sum >> out_shift), 8); |
| *pO++ = (q7_t) __SSAT((sum2 >> out_shift), 8); |
| *pO++ = (q7_t) __SSAT((sum3 >> out_shift), 8); |
| *pO++ = (q7_t) __SSAT((sum4 >> out_shift), 8); |
| |
| rowCnt--; |
| } |
| |
| rowCnt = num_of_rows & 0x3; |
| |
| while (rowCnt) |
| { |
| int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); |
| |
| int j; |
| |
| pA = pV; |
| for (j = 0; j < dim_vec; j++) |
| { |
| q7_t inA = *pA++; |
| q7_t inB = *pB++; |
| ip_out += inA * inB; |
| } |
| *pO++ = (q7_t) __SSAT((ip_out >> out_shift), 8); |
| |
| rowCnt--; |
| } |
| |
| #endif /* ARM_MATH_DSP */ |
| |
| /* Return to ARM_MATH_SUCCESS */ |
| return (ARM_MATH_SUCCESS); |
| |
| } |
| |
| /** |
| * @} end of FC group |
| */ |