/* * * Copyright (c) 2018, Alliance for Open Media. All rights reserved. * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License * was not distributed with this source code in the LICENSE file, you can * obtain it at www.aomedia.org/license/software. If the Alliance for Open * Media Patent License 1.0 was not distributed with this source code in the * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ #include #include #include #include "aom/aom_integer.h" #include "aom_dsp/blend.h" #include "aom_dsp/arm/mem_neon.h" #include "aom_ports/mem.h" #include "av1/common/blockd.h" #include "config/av1_rtcd.h" static inline void diffwtd_mask_d16_neon(uint8_t *mask, const bool inverse, const CONV_BUF_TYPE *src0, int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w, ConvolveParams *conv_params, int bd) { const int round = 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1 + (bd - 8); const int16x8_t round_vec = vdupq_n_s16((int16_t)(-round)); if (w >= 16) { int i = 0; do { int j = 0; do { uint16x8_t s0_lo = vld1q_u16(src0 + j); uint16x8_t s1_lo = vld1q_u16(src1 + j); uint16x8_t s0_hi = vld1q_u16(src0 + j + 8); uint16x8_t s1_hi = vld1q_u16(src1 + j + 8); uint16x8_t diff_lo_u16 = vrshlq_u16(vabdq_u16(s0_lo, s1_lo), round_vec); uint16x8_t diff_hi_u16 = vrshlq_u16(vabdq_u16(s0_hi, s1_hi), round_vec); uint8x8_t diff_lo_u8 = vshrn_n_u16(diff_lo_u16, DIFF_FACTOR_LOG2); uint8x8_t diff_hi_u8 = vshrn_n_u16(diff_hi_u16, DIFF_FACTOR_LOG2); uint8x16_t diff = vcombine_u8(diff_lo_u8, diff_hi_u8); uint8x16_t m; if (inverse) { m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0 } else { m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64)); } vst1q_u8(mask, m); mask += 16; j += 16; } while (j < w); src0 += src0_stride; src1 += src1_stride; } while (++i < h); } else if (w == 8) { int i = 0; do { uint16x8_t s0 = vld1q_u16(src0); uint16x8_t s1 = vld1q_u16(src1); uint16x8_t diff_u16 = vrshlq_u16(vabdq_u16(s0, s1), round_vec); uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, DIFF_FACTOR_LOG2); uint8x8_t m; if (inverse) { m = vqsub_u8(vdup_n_u8(64 - 38), diff_u8); // Saturating to 0 } else { m = vmin_u8(vadd_u8(diff_u8, vdup_n_u8(38)), vdup_n_u8(64)); } vst1_u8(mask, m); mask += 8; src0 += src0_stride; src1 += src1_stride; } while (++i < h); } else if (w == 4) { int i = 0; do { uint16x8_t s0 = vcombine_u16(vld1_u16(src0), vld1_u16(src0 + src0_stride)); uint16x8_t s1 = vcombine_u16(vld1_u16(src1), vld1_u16(src1 + src1_stride)); uint16x8_t diff_u16 = vrshlq_u16(vabdq_u16(s0, s1), round_vec); uint8x8_t diff_u8 = vshrn_n_u16(diff_u16, DIFF_FACTOR_LOG2); uint8x8_t m; if (inverse) { m = vqsub_u8(vdup_n_u8(64 - 38), diff_u8); // Saturating to 0 } else { m = vmin_u8(vadd_u8(diff_u8, vdup_n_u8(38)), vdup_n_u8(64)); } vst1_u8(mask, m); mask += 8; src0 += 2 * src0_stride; src1 += 2 * src1_stride; i += 2; } while (i < h); } } void av1_build_compound_diffwtd_mask_d16_neon( uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0, int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w, ConvolveParams *conv_params, int bd) { assert(h >= 4); assert(w >= 4); assert((mask_type == DIFFWTD_38_INV) || (mask_type == DIFFWTD_38)); if (mask_type == DIFFWTD_38) { diffwtd_mask_d16_neon(mask, /*inverse=*/false, src0, src0_stride, src1, src1_stride, h, w, conv_params, bd); } else { // mask_type == DIFFWTD_38_INV diffwtd_mask_d16_neon(mask, /*inverse=*/true, src0, src0_stride, src1, src1_stride, h, w, conv_params, bd); } } static inline void diffwtd_mask_neon(uint8_t *mask, const bool inverse, const uint8_t *src0, int src0_stride, const uint8_t *src1, int src1_stride, int h, int w) { if (w >= 16) { int i = 0; do { int j = 0; do { uint8x16_t s0 = vld1q_u8(src0 + j); uint8x16_t s1 = vld1q_u8(src1 + j); uint8x16_t diff = vshrq_n_u8(vabdq_u8(s0, s1), DIFF_FACTOR_LOG2); uint8x16_t m; if (inverse) { m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0 } else { m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64)); } vst1q_u8(mask, m); mask += 16; j += 16; } while (j < w); src0 += src0_stride; src1 += src1_stride; } while (++i < h); } else if (w == 8) { int i = 0; do { uint8x16_t s0 = vcombine_u8(vld1_u8(src0), vld1_u8(src0 + src0_stride)); uint8x16_t s1 = vcombine_u8(vld1_u8(src1), vld1_u8(src1 + src0_stride)); uint8x16_t diff = vshrq_n_u8(vabdq_u8(s0, s1), DIFF_FACTOR_LOG2); uint8x16_t m; if (inverse) { m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0 } else { m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64)); } vst1q_u8(mask, m); mask += 16; src0 += 2 * src0_stride; src1 += 2 * src1_stride; i += 2; } while (i < h); } else if (w == 4) { int i = 0; do { uint8x16_t s0 = load_unaligned_u8q(src0, src0_stride); uint8x16_t s1 = load_unaligned_u8q(src1, src1_stride); uint8x16_t diff = vshrq_n_u8(vabdq_u8(s0, s1), DIFF_FACTOR_LOG2); uint8x16_t m; if (inverse) { m = vqsubq_u8(vdupq_n_u8(64 - 38), diff); // Saturating to 0 } else { m = vminq_u8(vaddq_u8(diff, vdupq_n_u8(38)), vdupq_n_u8(64)); } vst1q_u8(mask, m); mask += 16; src0 += 4 * src0_stride; src1 += 4 * src1_stride; i += 4; } while (i < h); } } void av1_build_compound_diffwtd_mask_neon(uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const uint8_t *src0, int src0_stride, const uint8_t *src1, int src1_stride, int h, int w) { assert(h % 4 == 0); assert(w % 4 == 0); assert(mask_type == DIFFWTD_38_INV || mask_type == DIFFWTD_38); if (mask_type == DIFFWTD_38) { diffwtd_mask_neon(mask, /*inverse=*/false, src0, src0_stride, src1, src1_stride, h, w); } else { // mask_type == DIFFWTD_38_INV diffwtd_mask_neon(mask, /*inverse=*/true, src0, src0_stride, src1, src1_stride, h, w); } }