/* * Copyright (c) 2022 Samsung Electronics Co., Ltd. * All Rights Reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * - Neither the name of the copyright owner, nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "oapv_def.h" #include "oapv_tq_neon.h" #if ARM_NEON const s32 oapv_coeff[8][4] = { {64, 64, 64, 64}, // 0th row coeff {89, 75, 50, 18}, // 2nd row coeff {84, 35, 84, 35}, // 3rd row coeff {75,-18,-89,-50}, // 4th row coeff {64,-64, 64,-64}, // 5th row coeff {50,-89, 18, 75}, // 6th row coeff {35,-84, 35,-84}, // 7th row coeff {18,-50, 75,-89} // 8th row coeff }; #define multiply_s32(part1, part2, coeff, res) \ low = vmulq_s32(part1, coeff); \ high = vmulq_s32(part2, coeff); \ res = vcombine_s32(vpadd_s32(vget_low_s32(low), vget_high_s32(low)), vpadd_s32(vget_low_s32(high), vget_high_s32(high))); \ static void oapv_tx_pb8b_neon(s16 *src, s16 *dst, const int shift, int line) { s16 i; s16 *tempSrc = src; int16x4_t src_part1, src_part2; int32x4_t coeff0, coeff1, coeff2, coeff3, coeff4, coeff5, coeff6, coeff7; int32x4_t add = vdupq_n_s32(1 << (shift - 1)); int32x4_t sh = vdupq_n_s32(-shift); int32x4_t EE_part1, EE_part2, EO_part1, EO_part2, low, high, result0, result1, result2, result3, result4, result5, result6, result7, E1, O1, E2, O2, res1, res2, res3, res4; for(i = 0; i < 8; i += 4) { // Loading src[0 - 3] and src[4 - 7] src_part1 = vld1_s16(tempSrc); tempSrc += 4; src_part2 = vld1_s16(tempSrc); tempSrc += 4; //reverse src_part2 src_part2 = vrev64_s16(src_part2); E1 = vaddl_s16(src_part1, src_part2); O1 = vsubl_s16(src_part1, src_part2); // Loading src[8 - 11] and src[12 - 15] src_part1 = vld1_s16(tempSrc); tempSrc += 4; src_part2 = vld1_s16(tempSrc); tempSrc += 4; //reverse src_part2 src_part2 = vrev64_s16(src_part2); E2 = vaddl_s16(src_part1, src_part2); O2 = vsubl_s16(src_part1, src_part2); int32x4_t tmp1 = vcombine_s32(vget_low_s32(E1), vget_low_s32(E2)); int32x4_t tmp2 = vcombine_s32(vget_high_s32(E1), vget_high_s32(E2)); tmp2 = vrev64q_s32(tmp2); EE_part1 = vaddq_s32(tmp1, tmp2); EO_part1 = vsubq_s32(tmp1, tmp2); coeff1 = vld1q_s32(oapv_coeff[1]); coeff3 = vld1q_s32(oapv_coeff[3]); coeff5 = vld1q_s32(oapv_coeff[5]); coeff7 = vld1q_s32(oapv_coeff[7]); multiply_s32(O1, O2, coeff1, result1); multiply_s32(O1, O2, coeff3, result3); multiply_s32(O1, O2, coeff5, result5); multiply_s32(O1, O2, coeff7, result7); res1 = vpaddq_s32(result1, result3); res2 = vpaddq_s32(result5, result7); // add and shift res1 = vshlq_s32(vaddq_s32(res1, add), sh); res2 = vshlq_s32(vaddq_s32(res2, add), sh); // Loading src[16 - 19] and src[20 - 23] src_part1 = vld1_s16(tempSrc); tempSrc += 4; src_part2 = vld1_s16(tempSrc); tempSrc += 4; //reverse src_part2 src_part2 = vrev64_s16(src_part2); E1 = vaddl_s16(src_part1, src_part2); O1 = vsubl_s16(src_part1, src_part2); // Loading src[24 - 27] and src[28 - 31] src_part1 = vld1_s16(tempSrc); tempSrc += 4; src_part2 = vld1_s16(tempSrc); tempSrc += 4; //reverse src_part2 src_part2 = vrev64_s16(src_part2); E2 = vaddl_s16(src_part1, src_part2); O2 = vsubl_s16(src_part1, src_part2); multiply_s32(O1, O2, coeff1, result1); multiply_s32(O1, O2, coeff3, result3); multiply_s32(O1, O2, coeff5, result5); multiply_s32(O1, O2, coeff7, result7); res3 = vpaddq_s32(result1, result3); res4 = vpaddq_s32(result5, result7); // add and shift res3 = vshlq_s32(vaddq_s32(res3, add), sh); res4 = vshlq_s32(vaddq_s32(res4, add), sh); // store result in destination vst1_s16(dst + 1 * line + i, vmovn_s32(vcombine_s32(vget_low_s32(res1), vget_low_s32(res3)))); vst1_s16(dst + 3 * line + i, vmovn_s32(vcombine_s32(vget_high_s32(res1), vget_high_s32(res3)))); vst1_s16(dst + 5 * line + i, vmovn_s32(vcombine_s32(vget_low_s32(res2), vget_low_s32(res4)))); vst1_s16(dst + 7 * line + i, vmovn_s32(vcombine_s32(vget_high_s32(res2), vget_high_s32(res4)))); coeff0 = vld1q_s32(oapv_coeff[0]); coeff2 = vld1q_s32(oapv_coeff[2]); coeff4 = vld1q_s32(oapv_coeff[4]); coeff6 = vld1q_s32(oapv_coeff[6]); tmp1 = vcombine_s32(vget_low_s32(E1), vget_low_s32(E2)); tmp2 = vcombine_s32(vget_high_s32(E1), vget_high_s32(E2)); tmp2 = vrev64q_s32(tmp2); EE_part2 = vaddq_s32(tmp1, tmp2); EO_part2 = vsubq_s32(tmp1, tmp2); multiply_s32(EE_part1, EE_part2, coeff0, result0); multiply_s32(EE_part1, EE_part2, coeff4, result4); multiply_s32(EO_part1, EO_part2, coeff2, result2); multiply_s32(EO_part1, EO_part2, coeff6, result6); // add and shift result0 = vshlq_s32(vaddq_s32(result0, add), sh); result2 = vshlq_s32(vaddq_s32(result2, add), sh); result4 = vshlq_s32(vaddq_s32(result4, add), sh); result6 = vshlq_s32(vaddq_s32(result6, add), sh); // store result in destination vst1_s16(dst + 0 * line + i, vmovn_s32(result0)); vst1_s16(dst + 2 * line + i, vmovn_s32(result2)); vst1_s16(dst + 4 * line + i, vmovn_s32(result4)); vst1_s16(dst + 6 * line + i, vmovn_s32(result6)); } } const oapv_fn_tx_t oapv_tbl_fn_txb_neon[2] = { oapv_tx_pb8b_neon, NULL }; /////////////////////////////////////////////////////////////////////////////// // end of encoder code // ENABLE_ENCODER /////////////////////////////////////////////////////////////////////////////// // Required coefficients from oapv_tbl_tm8 # define OAPV_INVTX_COEF_0 89 // coef10, -coef32, -coef51, coef73 # define OAPV_INVTX_COEF_1 75 // coef11, coef30, coef53, coef72 # define OAPV_INVTX_COEF_2 50 // coef12, -coef33, coef50, -coef71 # define OAPV_INVTX_COEF_3 18 // coef13, -coef31, coef52, coef70 # define OAPV_INVTX_COEF_5 84 // coef20, -coef61 # define OAPV_INVTX_COEF_6 35 // coef21, coef60 # define OAPV_INVTX_COEF_4_LOG2 6 // log2(coef00), log2(coef01), log2(coef40), log2(-coef41) void oapv_itx_pb8b_opt_neon(s16* src, int shift1, int shift2, int line) { int32x4_t add1 = vdupq_n_s32(1 << (shift1 - 1)); int32x4_t add2 = vdupq_n_s32(1 << (shift2 - 1)); int32x4_t sh1 = vdupq_n_s32(-shift1); int32x4_t sh2 = vdupq_n_s32(-shift2); int16x4_t dest0, dest1, dest2, dest3, dest4, dest5, dest6, dest7, dest8, dest9, dest10, dest11, dest12, dest13, dest14, dest15; //DCT Pass 1 { int16x8_t v_src_0_8 = vld1q_s16(src); int16x8_t v_src_1_9 = vld1q_s16(src + line); int16x8_t v_src_2_10 = vld1q_s16(src + 2 * line); int16x8_t v_src_3_11 = vld1q_s16(src + 3 * line); int16x8_t v_src_4_12 = vld1q_s16(src + 4 * line); int16x8_t v_src_5_13 = vld1q_s16(src + 5 * line); int16x8_t v_src_6_14 = vld1q_s16(src + 6 * line); int16x8_t v_src_7_15 = vld1q_s16(src + 7 * line); int16x4_t v_src_0 = vget_low_s16(v_src_0_8); int16x4_t v_src_1 = vget_low_s16(v_src_1_9); int16x4_t v_src_2 = vget_low_s16(v_src_2_10); int16x4_t v_src_3 = vget_low_s16(v_src_3_11); int16x4_t v_src_4 = vget_low_s16(v_src_4_12); int16x4_t v_src_5 = vget_low_s16(v_src_5_13); int16x4_t v_src_6 = vget_low_s16(v_src_6_14); int16x4_t v_src_7 = vget_low_s16(v_src_7_15); int16x4_t v_src_8 = vget_high_s16(v_src_0_8); int16x4_t v_src_9 = vget_high_s16(v_src_1_9); int16x4_t v_src_10 = vget_high_s16(v_src_2_10); int16x4_t v_src_11 = vget_high_s16(v_src_3_11); int16x4_t v_src_12 = vget_high_s16(v_src_4_12); int16x4_t v_src_13 = vget_high_s16(v_src_5_13); int16x4_t v_src_14 = vget_high_s16(v_src_6_14); int16x4_t v_src_15 = vget_high_s16(v_src_7_15); int32x4_t temp1 = vaddq_s32(vmull_n_s16(v_src_1, OAPV_INVTX_COEF_0), vmull_n_s16(v_src_3, OAPV_INVTX_COEF_1)); int32x4_t temp2 = vsubq_s32(vmull_n_s16(v_src_1, OAPV_INVTX_COEF_1), vmull_n_s16(v_src_3, OAPV_INVTX_COEF_3)); int32x4_t temp3 = vsubq_s32(vmull_n_s16(v_src_1, OAPV_INVTX_COEF_2), vmull_n_s16(v_src_3, OAPV_INVTX_COEF_0)); int32x4_t temp4 = vsubq_s32(vmull_n_s16(v_src_1, OAPV_INVTX_COEF_3), vmull_n_s16(v_src_3, OAPV_INVTX_COEF_2)); int32x4_t temp5 = vaddq_s32(vmull_n_s16(v_src_5, OAPV_INVTX_COEF_2), vmull_n_s16(v_src_7, OAPV_INVTX_COEF_3)); int32x4_t temp6 = vaddq_s32(vmull_n_s16(v_src_5, OAPV_INVTX_COEF_0), vmull_n_s16(v_src_7, OAPV_INVTX_COEF_2)); temp6 = vnegq_s32(temp6); int32x4_t temp7 = vaddq_s32(vmull_n_s16(v_src_5, OAPV_INVTX_COEF_3), vmull_n_s16(v_src_7, OAPV_INVTX_COEF_1)); int32x4_t temp8 = vsubq_s32(vmull_n_s16(v_src_5, OAPV_INVTX_COEF_1), vmull_n_s16(v_src_7, OAPV_INVTX_COEF_0)); int32x4_t temp9 = vaddq_s32(vmull_n_s16(v_src_9, OAPV_INVTX_COEF_0), vmull_n_s16(v_src_11, OAPV_INVTX_COEF_1)); int32x4_t temp10 = vsubq_s32(vmull_n_s16(v_src_9, OAPV_INVTX_COEF_1), vmull_n_s16(v_src_11, OAPV_INVTX_COEF_3)); int32x4_t temp11 = vsubq_s32(vmull_n_s16(v_src_9, OAPV_INVTX_COEF_2), vmull_n_s16(v_src_11, OAPV_INVTX_COEF_0)); int32x4_t temp12 = vsubq_s32(vmull_n_s16(v_src_9, OAPV_INVTX_COEF_3), vmull_n_s16(v_src_11, OAPV_INVTX_COEF_2)); int32x4_t temp13 = vaddq_s32(vmull_n_s16(v_src_13, OAPV_INVTX_COEF_2), vmull_n_s16(v_src_15, OAPV_INVTX_COEF_3)); int32x4_t temp14 = vaddq_s32(vmull_n_s16(v_src_13, OAPV_INVTX_COEF_0), vmull_n_s16(v_src_15, OAPV_INVTX_COEF_2)); temp14 = vnegq_s32(temp14); int32x4_t temp15 = vaddq_s32(vmull_n_s16(v_src_13, OAPV_INVTX_COEF_3), vmull_n_s16(v_src_15, OAPV_INVTX_COEF_1)); int32x4_t temp16 = vsubq_s32(vmull_n_s16(v_src_13, OAPV_INVTX_COEF_1), vmull_n_s16(v_src_15, OAPV_INVTX_COEF_0)); int32x4_t O0 = vaddq_s32(temp1, temp5); int32x4_t O1 = vaddq_s32(temp2, temp6); int32x4_t O2 = vaddq_s32(temp3, temp7); int32x4_t O3 = vaddq_s32(temp4, temp8); int32x4_t O4 = vaddq_s32(temp9, temp13); int32x4_t O5 = vaddq_s32(temp10, temp14); int32x4_t O6 = vaddq_s32(temp11, temp15); int32x4_t O7 = vaddq_s32(temp12, temp16); int32x4_t EO0 = vaddq_s32(vmull_n_s16(v_src_2, OAPV_INVTX_COEF_5), vmull_n_s16(v_src_6, OAPV_INVTX_COEF_6)); int32x4_t EO1 = vsubq_s32(vmull_n_s16(v_src_2, OAPV_INVTX_COEF_6), vmull_n_s16(v_src_6, OAPV_INVTX_COEF_5)); int32x4_t EE0 = vaddq_s32(vshll_n_s16(v_src_0, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(v_src_4, OAPV_INVTX_COEF_4_LOG2)); int32x4_t EE1 = vsubq_s32(vshll_n_s16(v_src_0, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(v_src_4, OAPV_INVTX_COEF_4_LOG2)); int32x4_t EO2 = vaddq_s32(vmull_n_s16(v_src_10, OAPV_INVTX_COEF_5), vmull_n_s16(v_src_14, OAPV_INVTX_COEF_6)); int32x4_t EO3 = vsubq_s32(vmull_n_s16(v_src_10, OAPV_INVTX_COEF_6), vmull_n_s16(v_src_14, OAPV_INVTX_COEF_5)); int32x4_t EE2 = vaddq_s32(vshll_n_s16(v_src_8, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(v_src_12, OAPV_INVTX_COEF_4_LOG2)); int32x4_t EE3 = vsubq_s32(vshll_n_s16(v_src_8, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(v_src_12, OAPV_INVTX_COEF_4_LOG2)); int32x4_t E0 = vaddq_s32(EE0, EO0); int32x4_t E1 = vaddq_s32(EE1, EO1); int32x4_t E2 = vsubq_s32(EE1, EO1); int32x4_t E3 = vsubq_s32(EE0, EO0); int32x4_t E4 = vaddq_s32(EE2, EO2); int32x4_t E5 = vaddq_s32(EE3, EO3); int32x4_t E6 = vsubq_s32(EE3, EO3); int32x4_t E7 = vsubq_s32(EE2, EO2); dest0 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E0, O0), add1), sh1)); dest1 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E1, O1), add1), sh1)); dest2 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E2, O2), add1), sh1)); dest3 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E3, O3), add1), sh1)); dest4 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E0, O0), add1), sh1)); dest5 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E1, O1), add1), sh1)); dest6 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E2, O2), add1), sh1)); dest7 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E3, O3), add1), sh1)); dest8 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E4, O4), add1), sh1)); dest9 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E5, O5), add1), sh1)); dest10 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E6, O6), add1), sh1)); dest11 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E7, O7), add1), sh1)); dest12 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E4, O4), add1), sh1)); dest13 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E5, O5), add1), sh1)); dest14 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E6, O6), add1), sh1)); dest15 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E7, O7), add1), sh1)); int16x4_t t0 = vzip1_s16(dest0, dest1); int16x4_t t1 = vzip1_s16(dest2, dest3); int16x4_t t2 = vzip2_s16(dest0, dest1); int16x4_t t3 = vzip2_s16(dest2, dest3); int16x4_t t4 = vzip1_s16(dest8, dest9); int16x4_t t5 = vzip1_s16(dest10, dest11); int16x4_t t6 = vzip2_s16(dest8, dest9); int16x4_t t7 = vzip2_s16(dest10, dest11); dest0 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t0), vreinterpret_s32_s16(t1))); dest1 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t0), vreinterpret_s32_s16(t1))); dest2 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t2), vreinterpret_s32_s16(t3))); dest3 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t2), vreinterpret_s32_s16(t3))); dest8 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t4), vreinterpret_s32_s16(t5))); dest9 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t4), vreinterpret_s32_s16(t5))); dest10 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t6), vreinterpret_s32_s16(t7))); dest11 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t6), vreinterpret_s32_s16(t7))); int16x4_t t8 = vzip1_s16(dest5, dest4); int16x4_t t9 = vzip1_s16(dest7, dest6); int16x4_t t10 = vzip2_s16(dest5, dest4); int16x4_t t11 = vzip2_s16(dest7, dest6); int16x4_t t12 = vzip1_s16(dest13, dest12); int16x4_t t13 = vzip1_s16(dest15, dest14); int16x4_t t14 = vzip2_s16(dest13, dest12); int16x4_t t15 = vzip2_s16(dest15, dest14); dest4 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t9), vreinterpret_s32_s16(t8))); dest5 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t9), vreinterpret_s32_s16(t8))); dest6 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t11), vreinterpret_s32_s16(t10))); dest7 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t11), vreinterpret_s32_s16(t10))); dest12 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t13), vreinterpret_s32_s16(t12))); dest13 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t13), vreinterpret_s32_s16(t12))); dest14 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t15), vreinterpret_s32_s16(t14))); dest15 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t15), vreinterpret_s32_s16(t14))); } //DCT Pass 2 { int32x4_t temp1 = vaddq_s32(vmull_n_s16(dest1, OAPV_INVTX_COEF_0), vmull_n_s16(dest3, OAPV_INVTX_COEF_1)); int32x4_t temp2 = vsubq_s32(vmull_n_s16(dest1, OAPV_INVTX_COEF_1), vmull_n_s16(dest3, OAPV_INVTX_COEF_3)); int32x4_t temp3 = vsubq_s32(vmull_n_s16(dest1, OAPV_INVTX_COEF_2), vmull_n_s16(dest3, OAPV_INVTX_COEF_0)); int32x4_t temp4 = vsubq_s32(vmull_n_s16(dest1, OAPV_INVTX_COEF_3), vmull_n_s16(dest3, OAPV_INVTX_COEF_2)); int32x4_t temp5 = vaddq_s32(vmull_n_s16(dest9, OAPV_INVTX_COEF_2), vmull_n_s16(dest11, OAPV_INVTX_COEF_3)); int32x4_t temp6 = vaddq_s32(vmull_n_s16(dest9, OAPV_INVTX_COEF_0), vmull_n_s16(dest11, OAPV_INVTX_COEF_2)); temp6 = vnegq_s32(temp6); int32x4_t temp7 = vaddq_s32(vmull_n_s16(dest9, OAPV_INVTX_COEF_3), vmull_n_s16(dest11, OAPV_INVTX_COEF_1)); int32x4_t temp8 = vsubq_s32(vmull_n_s16(dest9, OAPV_INVTX_COEF_1), vmull_n_s16(dest11, OAPV_INVTX_COEF_0)); int32x4_t temp9 = vaddq_s32(vmull_n_s16(dest5, OAPV_INVTX_COEF_0), vmull_n_s16(dest7, OAPV_INVTX_COEF_1)); int32x4_t temp10 = vsubq_s32(vmull_n_s16(dest5, OAPV_INVTX_COEF_1), vmull_n_s16(dest7, OAPV_INVTX_COEF_3)); int32x4_t temp11 = vsubq_s32(vmull_n_s16(dest5, OAPV_INVTX_COEF_2), vmull_n_s16(dest7, OAPV_INVTX_COEF_0)); int32x4_t temp12 = vsubq_s32(vmull_n_s16(dest5, OAPV_INVTX_COEF_3), vmull_n_s16(dest7, OAPV_INVTX_COEF_2)); int32x4_t temp13 = vaddq_s32(vmull_n_s16(dest13, OAPV_INVTX_COEF_2), vmull_n_s16(dest15, OAPV_INVTX_COEF_3)); int32x4_t temp14 = vaddq_s32(vmull_n_s16(dest13, OAPV_INVTX_COEF_0), vmull_n_s16(dest15, OAPV_INVTX_COEF_2)); temp14 = vnegq_s32(temp14); int32x4_t temp15 = vaddq_s32(vmull_n_s16(dest13, OAPV_INVTX_COEF_3), vmull_n_s16(dest15, OAPV_INVTX_COEF_1)); int32x4_t temp16 = vsubq_s32(vmull_n_s16(dest13, OAPV_INVTX_COEF_1), vmull_n_s16(dest15, OAPV_INVTX_COEF_0)); int32x4_t O0 = vaddq_s32(temp1, temp5); int32x4_t O1 = vaddq_s32(temp2, temp6); int32x4_t O2 = vaddq_s32(temp3, temp7); int32x4_t O3 = vaddq_s32(temp4, temp8); int32x4_t O4 = vaddq_s32(temp9, temp13); int32x4_t O5 = vaddq_s32(temp10, temp14); int32x4_t O6 = vaddq_s32(temp11, temp15); int32x4_t O7 = vaddq_s32(temp12, temp16); int32x4_t EO0 = vaddq_s32(vmull_n_s16(dest2, OAPV_INVTX_COEF_5), vmull_n_s16(dest10, OAPV_INVTX_COEF_6)); int32x4_t EO1 = vsubq_s32(vmull_n_s16(dest2, OAPV_INVTX_COEF_6), vmull_n_s16(dest10, OAPV_INVTX_COEF_5)); int32x4_t EE0 = vaddq_s32(vshll_n_s16(dest0, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(dest8, OAPV_INVTX_COEF_4_LOG2)); int32x4_t EE1 = vsubq_s32(vshll_n_s16(dest0, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(dest8, OAPV_INVTX_COEF_4_LOG2)); int32x4_t EO2 = vaddq_s32(vmull_n_s16(dest6, OAPV_INVTX_COEF_5), vmull_n_s16(dest14, OAPV_INVTX_COEF_6)); int32x4_t EO3 = vsubq_s32(vmull_n_s16(dest6, OAPV_INVTX_COEF_6), vmull_n_s16(dest14, OAPV_INVTX_COEF_5)); int32x4_t EE2 = vaddq_s32(vshll_n_s16(dest4, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(dest12, OAPV_INVTX_COEF_4_LOG2)); int32x4_t EE3 = vsubq_s32(vshll_n_s16(dest4, OAPV_INVTX_COEF_4_LOG2), vshll_n_s16(dest12, OAPV_INVTX_COEF_4_LOG2)); int32x4_t E0 = vaddq_s32(EE0, EO0); int32x4_t E1 = vaddq_s32(EE1, EO1); int32x4_t E2 = vsubq_s32(EE1, EO1); int32x4_t E3 = vsubq_s32(EE0, EO0); int32x4_t E4 = vaddq_s32(EE2, EO2); int32x4_t E5 = vaddq_s32(EE3, EO3); int32x4_t E6 = vsubq_s32(EE3, EO3); int32x4_t E7 = vsubq_s32(EE2, EO2); int16x4_t v_src_0 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E0, O0), add2), sh2)); int16x4_t v_src_1 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E1, O1), add2), sh2)); int16x4_t v_src_2 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E2, O2), add2), sh2)); int16x4_t v_src_3 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E3, O3), add2), sh2)); int16x4_t v_src_4 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E0, O0), add2), sh2)); int16x4_t v_src_5 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E1, O1), add2), sh2)); int16x4_t v_src_6 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E2, O2), add2), sh2)); int16x4_t v_src_7 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E3, O3), add2), sh2)); int16x4_t v_src_8 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E4, O4), add2), sh2)); int16x4_t v_src_9 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E5, O5), add2), sh2)); int16x4_t v_src_10 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E6, O6), add2), sh2)); int16x4_t v_src_11 = vmovn_s32(vshlq_s32(vaddq_s32(vaddq_s32(E7, O7), add2), sh2)); int16x4_t v_src_12 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E4, O4), add2), sh2)); int16x4_t v_src_13 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E5, O5), add2), sh2)); int16x4_t v_src_14 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E6, O6), add2), sh2)); int16x4_t v_src_15 = vmovn_s32(vshlq_s32(vaddq_s32(vsubq_s32(E7, O7), add2), sh2)); int16x4_t t0 = vzip1_s16(v_src_0, v_src_1); int16x4_t t1 = vzip1_s16(v_src_2, v_src_3); int16x4_t t2 = vzip2_s16(v_src_0, v_src_1); int16x4_t t3 = vzip2_s16(v_src_2, v_src_3); int16x4_t t4 = vzip1_s16(v_src_8, v_src_9); int16x4_t t5 = vzip1_s16(v_src_10, v_src_11); int16x4_t t6 = vzip2_s16(v_src_8, v_src_9); int16x4_t t7 = vzip2_s16(v_src_10, v_src_11); v_src_0 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t0), vreinterpret_s32_s16(t1))); v_src_1 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t0), vreinterpret_s32_s16(t1))); v_src_2 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t2), vreinterpret_s32_s16(t3))); v_src_3 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t2), vreinterpret_s32_s16(t3))); v_src_8 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t4), vreinterpret_s32_s16(t5))); v_src_9 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t4), vreinterpret_s32_s16(t5))); v_src_10 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t6), vreinterpret_s32_s16(t7))); v_src_11 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t6), vreinterpret_s32_s16(t7))); int16x4_t t8 = vzip1_s16(v_src_5, v_src_4); int16x4_t t9 = vzip1_s16(v_src_7, v_src_6); int16x4_t t10 = vzip2_s16(v_src_5, v_src_4); int16x4_t t11 = vzip2_s16(v_src_7, v_src_6); int16x4_t t12 = vzip1_s16(v_src_13, v_src_12); int16x4_t t13 = vzip1_s16(v_src_15, v_src_14); int16x4_t t14 = vzip2_s16(v_src_13, v_src_12); int16x4_t t15 = vzip2_s16(v_src_15, v_src_14); v_src_4 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t9), vreinterpret_s32_s16(t8))); v_src_5 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t9), vreinterpret_s32_s16(t8))); v_src_6 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t11), vreinterpret_s32_s16(t10))); v_src_7 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t11), vreinterpret_s32_s16(t10))); v_src_12 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t13), vreinterpret_s32_s16(t12))); v_src_13 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t13), vreinterpret_s32_s16(t12))); v_src_14 = vreinterpret_s16_s32(vzip1_s32(vreinterpret_s32_s16(t15), vreinterpret_s32_s16(t14))); v_src_15 = vreinterpret_s16_s32(vzip2_s32(vreinterpret_s32_s16(t15), vreinterpret_s32_s16(t14))); int16x8_t v_src_0_4 = vcombine_s16(v_src_0, v_src_4); int16x8_t v_src_1_5 = vcombine_s16(v_src_1, v_src_5); int16x8_t v_src_2_6 = vcombine_s16(v_src_2, v_src_6); int16x8_t v_src_3_7 = vcombine_s16(v_src_3, v_src_7); int16x8_t v_src_8_12 = vcombine_s16(v_src_8, v_src_12); int16x8_t v_src_9_13 = vcombine_s16(v_src_9, v_src_13); int16x8_t v_src_10_14 = vcombine_s16(v_src_10, v_src_14); int16x8_t v_src_11_15 = vcombine_s16(v_src_11, v_src_15); vst1q_s16(src, v_src_0_4); vst1q_s16(src + 8, v_src_1_5); vst1q_s16(src + 16, v_src_2_6); vst1q_s16(src + 24, v_src_3_7); vst1q_s16(src + 32, v_src_8_12); vst1q_s16(src + 40, v_src_9_13); vst1q_s16(src + 48, v_src_10_14); vst1q_s16(src + 56, v_src_11_15); } } const oapv_fn_itx_t oapv_tbl_fn_itx_neon[2] = { oapv_itx_pb8b_opt_neon, NULL }; static int oapv_quant_neon(s16* coef, u8 qp, int q_matrix[OAPV_BLK_D], int log2_w, int log2_h, int bit_depth, int deadzone_offset) { s64 offset; int shift; int tr_shift; int log2_size = (log2_w + log2_h) >> 1; tr_shift = MAX_TX_DYNAMIC_RANGE - bit_depth - log2_size; shift = QUANT_SHIFT + tr_shift + (qp / 6); offset = (s64)deadzone_offset << (shift - 9); int pixels=(1 << (log2_w + log2_h)); int i; int16x8_t coef_row; int64x2_t offset_vector = vdupq_n_s64(offset); int64x2_t shift_vector = vdupq_n_s64(-shift); uint16x8_t zero_vector = vdupq_n_s16(0); for (i = 0; i < pixels; i+=8) { // Load one coef row coef_row = vld1q_s16(coef+i); // Extract coef signs and construct abs coef-vec uint16x8_t sign_mask = vcltq_s16(coef_row, zero_vector); int16x8_t coef_row_abs = vabsq_s16(coef_row); // Split abs coef-vec and unpack to s32 int32x4_t coef_low_32b = vmovl_s16(vget_low_s16(coef_row_abs)); int32x4_t coef_high_32b = vmovl_high_s16(coef_row_abs); // Load q_matrix elements int32x4_t quant_matrix_low = vld1q_s32(q_matrix + i); int32x4_t quant_matrix_high = vld1q_s32(q_matrix + i + 4); // Multiply 2X: 32-bit coef with 32-bit q_matrix and add 64-bit offset_vector to store result as 64-bit int64x2_t coef_low_32b_first_half = vmlal_s32(offset_vector, vget_low_s32 (coef_low_32b), vget_low_s32 (quant_matrix_low)); int64x2_t coef_low_32b_second_half = vmlal_s32(offset_vector, vget_high_s32(coef_low_32b), vget_high_s32(quant_matrix_low)); int64x2_t coef_high_32b_first_half = vmlal_s32(offset_vector, vget_low_s32 (coef_high_32b), vget_low_s32 (quant_matrix_high)); int64x2_t coef_high_32b_second_half = vmlal_s32(offset_vector, vget_high_s32(coef_high_32b), vget_high_s32(quant_matrix_high)); // Shift 64-bit results coef_low_32b_first_half = vshlq_s64(coef_low_32b_first_half, shift_vector); coef_low_32b_second_half = vshlq_s64(coef_low_32b_second_half, shift_vector); coef_high_32b_first_half = vshlq_s64(coef_high_32b_first_half, shift_vector); coef_high_32b_second_half = vshlq_s64(coef_high_32b_second_half, shift_vector); // Combine 2X: 64x2 registers into one 32x4 register coef_low_32b = vcombine_u32(vmovn_s64(coef_low_32b_first_half), vmovn_s64(coef_low_32b_second_half)); coef_high_32b = vcombine_u32(vmovn_s64(coef_high_32b_first_half), vmovn_s64(coef_high_32b_second_half)); // Combine 2X: 32x4 registers into one 16x8 register int16x8_t output_vector = vcombine_u16(vmovn_s32(coef_low_32b), vmovn_s32(coef_high_32b)); // Apply extracted coef sign to result output_vector = vbslq_s16(sign_mask, vnegq_s16(output_vector), output_vector); // Store result row into buffer vst1q_s16(coef + i, output_vector); } return OAPV_OK; } const oapv_fn_quant_t oapv_tbl_fn_quant_neon[2] = { oapv_quant_neon, NULL }; #endif /* ARM_NEON */