/* * Copyright (c) 2023, Alliance for Open Media. All rights reserved. * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License * was not distributed with this source code in the LICENSE file, you can * obtain it at www.aomedia.org/license/software. If the Alliance for Open * Media Patent License 1.0 was not distributed with this source code in the * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ #include #include "config/aom_config.h" #include "config/aom_dsp_rtcd.h" #include "aom/aom_integer.h" #include "aom_dsp/arm/mem_neon.h" #include "aom_dsp/arm/sum_neon.h" static inline void highbd_obmc_variance_8x1_s16_neon(uint16x8_t pre, const int32_t *wsrc, const int32_t *mask, uint32x4_t *sse, int32x4_t *sum) { int16x8_t pre_s16 = vreinterpretq_s16_u16(pre); int32x4_t wsrc_lo = vld1q_s32(&wsrc[0]); int32x4_t wsrc_hi = vld1q_s32(&wsrc[4]); int32x4_t mask_lo = vld1q_s32(&mask[0]); int32x4_t mask_hi = vld1q_s32(&mask[4]); int16x8_t mask_s16 = vcombine_s16(vmovn_s32(mask_lo), vmovn_s32(mask_hi)); int32x4_t diff_lo = vmull_s16(vget_low_s16(pre_s16), vget_low_s16(mask_s16)); int32x4_t diff_hi = vmull_s16(vget_high_s16(pre_s16), vget_high_s16(mask_s16)); diff_lo = vsubq_s32(wsrc_lo, diff_lo); diff_hi = vsubq_s32(wsrc_hi, diff_hi); // ROUND_POWER_OF_TWO_SIGNED(value, 12) rounds to nearest with ties away // from zero, however vrshrq_n_s32 rounds to nearest with ties rounded up. // This difference only affects the bit patterns at the rounding breakpoints // exactly, so we can add -1 to all negative numbers to move the breakpoint // one value across and into the correct rounding region. diff_lo = vsraq_n_s32(diff_lo, diff_lo, 31); diff_hi = vsraq_n_s32(diff_hi, diff_hi, 31); int32x4_t round_lo = vrshrq_n_s32(diff_lo, 12); int32x4_t round_hi = vrshrq_n_s32(diff_hi, 12); *sum = vaddq_s32(*sum, round_lo); *sum = vaddq_s32(*sum, round_hi); *sse = vmlaq_u32(*sse, vreinterpretq_u32_s32(round_lo), vreinterpretq_u32_s32(round_lo)); *sse = vmlaq_u32(*sse, vreinterpretq_u32_s32(round_hi), vreinterpretq_u32_s32(round_hi)); } // For 12-bit data, we can only accumulate up to 256 elements in the unsigned // 32-bit elements (4095*4095*256 = 4292870400) before we have to accumulate // into 64-bit elements. Therefore blocks of size 32x64, 64x32, 64x64, 64x128, // 128x64, 128x128 are processed in a different helper function. static inline void highbd_obmc_variance_xlarge_neon( const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int width, int h, int h_limit, uint64_t *sse, int64_t *sum) { uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre); int32x4_t sum_s32 = vdupq_n_s32(0); uint64x2_t sse_u64 = vdupq_n_u64(0); // 'h_limit' is the number of 'w'-width rows we can process before our 32-bit // accumulator overflows. After hitting this limit we accumulate into 64-bit // elements. int h_tmp = h > h_limit ? h_limit : h; do { uint32x4_t sse_u32[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; int j = 0; do { int i = 0; do { uint16x8_t pre0 = vld1q_u16(pre_ptr + i); highbd_obmc_variance_8x1_s16_neon(pre0, wsrc, mask, &sse_u32[0], &sum_s32); uint16x8_t pre1 = vld1q_u16(pre_ptr + i + 8); highbd_obmc_variance_8x1_s16_neon(pre1, wsrc + 8, mask + 8, &sse_u32[1], &sum_s32); i += 16; wsrc += 16; mask += 16; } while (i < width); pre_ptr += pre_stride; j++; } while (j < h_tmp); sse_u64 = vpadalq_u32(sse_u64, sse_u32[0]); sse_u64 = vpadalq_u32(sse_u64, sse_u32[1]); h -= h_tmp; } while (h != 0); *sse = horizontal_add_u64x2(sse_u64); *sum = horizontal_long_add_s32x4(sum_s32); } static inline void highbd_obmc_variance_xlarge_neon_128xh( const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 128, h, 16, sse, sum); } static inline void highbd_obmc_variance_xlarge_neon_64xh( const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 64, h, 32, sse, sum); } static inline void highbd_obmc_variance_xlarge_neon_32xh( const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { highbd_obmc_variance_xlarge_neon(pre, pre_stride, wsrc, mask, 32, h, 64, sse, sum); } static inline void highbd_obmc_variance_large_neon( const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int width, int h, uint64_t *sse, int64_t *sum) { uint16_t *pre_ptr = CONVERT_TO_SHORTPTR(pre); uint32x4_t sse_u32 = vdupq_n_u32(0); int32x4_t sum_s32 = vdupq_n_s32(0); do { int i = 0; do { uint16x8_t pre0 = vld1q_u16(pre_ptr + i); highbd_obmc_variance_8x1_s16_neon(pre0, wsrc, mask, &sse_u32, &sum_s32); uint16x8_t pre1 = vld1q_u16(pre_ptr + i + 8); highbd_obmc_variance_8x1_s16_neon(pre1, wsrc + 8, mask + 8, &sse_u32, &sum_s32); i += 16; wsrc += 16; mask += 16; } while (i < width); pre_ptr += pre_stride; } while (--h != 0); *sse = horizontal_long_add_u32x4(sse_u32); *sum = horizontal_long_add_s32x4(sum_s32); } static inline void highbd_obmc_variance_neon_128xh( const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 128, h, sse, sum); } static inline void highbd_obmc_variance_neon_64xh(const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 64, h, sse, sum); } static inline void highbd_obmc_variance_neon_32xh(const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 32, h, sse, sum); } static inline void highbd_obmc_variance_neon_16xh(const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { highbd_obmc_variance_large_neon(pre, pre_stride, wsrc, mask, 16, h, sse, sum); } static inline void highbd_obmc_variance_neon_8xh(const uint8_t *pre8, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); uint32x4_t sse_u32 = vdupq_n_u32(0); int32x4_t sum_s32 = vdupq_n_s32(0); do { uint16x8_t pre_u16 = vld1q_u16(pre); highbd_obmc_variance_8x1_s16_neon(pre_u16, wsrc, mask, &sse_u32, &sum_s32); pre += pre_stride; wsrc += 8; mask += 8; } while (--h != 0); *sse = horizontal_long_add_u32x4(sse_u32); *sum = horizontal_long_add_s32x4(sum_s32); } static inline void highbd_obmc_variance_neon_4xh(const uint8_t *pre8, int pre_stride, const int32_t *wsrc, const int32_t *mask, int h, uint64_t *sse, int64_t *sum) { assert(h % 2 == 0); uint16_t *pre = CONVERT_TO_SHORTPTR(pre8); uint32x4_t sse_u32 = vdupq_n_u32(0); int32x4_t sum_s32 = vdupq_n_s32(0); do { uint16x8_t pre_u16 = load_unaligned_u16_4x2(pre, pre_stride); highbd_obmc_variance_8x1_s16_neon(pre_u16, wsrc, mask, &sse_u32, &sum_s32); pre += 2 * pre_stride; wsrc += 8; mask += 8; h -= 2; } while (h != 0); *sse = horizontal_long_add_u32x4(sse_u32); *sum = horizontal_long_add_s32x4(sum_s32); } static inline void highbd_8_obmc_variance_cast(int64_t sum64, uint64_t sse64, int *sum, unsigned int *sse) { *sum = (int)sum64; *sse = (unsigned int)sse64; } static inline void highbd_10_obmc_variance_cast(int64_t sum64, uint64_t sse64, int *sum, unsigned int *sse) { *sum = (int)ROUND_POWER_OF_TWO(sum64, 2); *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 4); } static inline void highbd_12_obmc_variance_cast(int64_t sum64, uint64_t sse64, int *sum, unsigned int *sse) { *sum = (int)ROUND_POWER_OF_TWO(sum64, 4); *sse = (unsigned int)ROUND_POWER_OF_TWO(sse64, 8); } #define HIGHBD_OBMC_VARIANCE_WXH_NEON(w, h, bitdepth) \ unsigned int aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ const int32_t *mask, unsigned int *sse) { \ int sum; \ int64_t sum64; \ uint64_t sse64; \ highbd_obmc_variance_neon_##w##xh(pre, pre_stride, wsrc, mask, h, &sse64, \ &sum64); \ highbd_##bitdepth##_obmc_variance_cast(sum64, sse64, &sum, sse); \ return *sse - (unsigned int)(((int64_t)sum * sum) / (w * h)); \ } #define HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(w, h, bitdepth) \ unsigned int aom_highbd_##bitdepth##_obmc_variance##w##x##h##_neon( \ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \ const int32_t *mask, unsigned int *sse) { \ int sum; \ int64_t sum64; \ uint64_t sse64; \ highbd_obmc_variance_xlarge_neon_##w##xh(pre, pre_stride, wsrc, mask, h, \ &sse64, &sum64); \ highbd_##bitdepth##_obmc_variance_cast(sum64, sse64, &sum, sse); \ return *sse - (unsigned int)(((int64_t)sum * sum) / (w * h)); \ } // 8-bit HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 64, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 32, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 64, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 128, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 64, 8) HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 128, 8) // 10-bit HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 64, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 32, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 64, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 128, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 64, 10) HIGHBD_OBMC_VARIANCE_WXH_NEON(128, 128, 10) // 12-bit HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 4, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 8, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(4, 16, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 4, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 8, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 16, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(8, 32, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 4, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 8, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 16, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 32, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(16, 64, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 8, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 16, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(32, 32, 12) HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(32, 64, 12) HIGHBD_OBMC_VARIANCE_WXH_NEON(64, 16, 12) HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 32, 12) HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 64, 12) HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(64, 128, 12) HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(128, 64, 12) HIGHBD_OBMC_VARIANCE_WXH_XLARGE_NEON(128, 128, 12)