/* * Copyright (c) 2012-2017 Etnaviv Project * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sub license, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. * * Authors: * Wladimir J. van der Laan */ #include "etnaviv_rs.h" #include "etnaviv_clear_blit.h" #include "etnaviv_context.h" #include "etnaviv_emit.h" #include "etnaviv_format.h" #include "etnaviv_resource.h" #include "etnaviv_screen.h" #include "etnaviv_surface.h" #include "etnaviv_tiling.h" #include "etnaviv_translate.h" #include "etnaviv_util.h" #include "pipe/p_defines.h" #include "pipe/p_state.h" #include "util/compiler.h" #include "util/u_blitter.h" #include "util/u_inlines.h" #include "util/u_memory.h" #include "util/u_surface.h" #include "hw/state.xml.h" #include "hw/state_3d.xml.h" #include /* return a RS "compatible" format for use when copying */ static uint32_t etna_compatible_rs_format(enum pipe_format fmt) { /* YUYV and UYVY are blocksize 4, but 2 bytes per pixel */ if (fmt == PIPE_FORMAT_YUYV || fmt == PIPE_FORMAT_UYVY) return RS_FORMAT_A4R4G4B4; switch (util_format_get_blocksize(fmt)) { case 2: return RS_FORMAT_A4R4G4B4; case 4: return RS_FORMAT_A8R8G8B8; default: return ETNA_NO_MATCH; } } void etna_compile_rs_state(struct etna_context *ctx, struct compiled_rs_state *cs, const struct rs_state *rs) { struct etna_screen *screen = ctx->screen; memset(cs, 0, sizeof(*cs)); /* TILED and SUPERTILED layout have their strides multiplied with 4 in RS */ unsigned source_stride_shift = COND(rs->source_tiling != ETNA_LAYOUT_LINEAR, 2); unsigned dest_stride_shift = COND(rs->dest_tiling != ETNA_LAYOUT_LINEAR, 2); bool src_tiled = rs->source_tiling & ETNA_LAYOUT_BIT_TILE; bool dst_tiled = rs->dest_tiling & ETNA_LAYOUT_BIT_TILE; bool src_super = rs->source_tiling & ETNA_LAYOUT_BIT_SUPER; bool dst_super = rs->dest_tiling & ETNA_LAYOUT_BIT_SUPER; bool src_multi = rs->source_tiling & ETNA_LAYOUT_BIT_MULTI; bool dst_multi = rs->dest_tiling & ETNA_LAYOUT_BIT_MULTI; /* Vivante RS needs widths to be a multiple of 16 or bad things * happen, such as scribbing over memory, or the GPU hanging, * even for non-tiled formats. As this is serious, use abort(). */ if (rs->width & ETNA_RS_WIDTH_MASK) abort(); /* TODO could just pre-generate command buffer, would simply submit to one memcpy */ cs->RS_CONFIG = VIVS_RS_CONFIG_SOURCE_FORMAT(rs->source_format) | COND(rs->downsample_x, VIVS_RS_CONFIG_DOWNSAMPLE_X) | COND(rs->downsample_y, VIVS_RS_CONFIG_DOWNSAMPLE_Y) | COND(src_tiled, VIVS_RS_CONFIG_SOURCE_TILED) | VIVS_RS_CONFIG_DEST_FORMAT(rs->dest_format) | COND(dst_tiled, VIVS_RS_CONFIG_DEST_TILED) | COND(rs->swap_rb, VIVS_RS_CONFIG_SWAP_RB) | COND(rs->flip, VIVS_RS_CONFIG_FLIP); cs->RS_SOURCE_STRIDE = (rs->source_stride << source_stride_shift) | COND(src_super, VIVS_RS_SOURCE_STRIDE_TILING) | COND(src_multi, VIVS_RS_SOURCE_STRIDE_MULTI); if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_CACHE128B256BPERLINE)) cs->RS_SOURCE_STRIDE |= VIVS_RS_SOURCE_STRIDE_TS_MODE(rs->source_ts_mode) | COND(src_super, VIVS_RS_SOURCE_STRIDE_SUPER_TILED_NEW); else if ((rs->downsample_x || rs->downsample_y) && VIV_FEATURE(screen, ETNA_FEATURE_SMALL_MSAA)) cs->RS_SOURCE_STRIDE |= VIVS_RS_SOURCE_STRIDE_TS_MODE(TS_MODE_256B); /* Initially all pipes are set to the base address of the source and * destination buffer respectively. This will be overridden below as * necessary for the multi-pipe, multi-tiled case. */ for (unsigned pipe = 0; pipe < screen->specs.pixel_pipes; ++pipe) { cs->source[pipe].bo = rs->source; cs->source[pipe].offset = rs->source_offset; cs->source[pipe].flags = ETNA_RELOC_READ; cs->dest[pipe].bo = rs->dest; cs->dest[pipe].offset = rs->dest_offset; cs->dest[pipe].flags = ETNA_RELOC_WRITE; cs->RS_PIPE_OFFSET[pipe] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(0); } cs->RS_DEST_STRIDE = (rs->dest_stride << dest_stride_shift) | COND(dst_super, VIVS_RS_DEST_STRIDE_TILING) | COND(dst_multi, VIVS_RS_DEST_STRIDE_MULTI); if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_CACHE128B256BPERLINE)) cs->RS_DEST_STRIDE |= COND(dst_super, VIVS_RS_DEST_STRIDE_SUPER_TILED_NEW); if (src_multi) cs->source[1].offset = rs->source_offset + rs->source_stride * rs->source_padded_height / 2; if (dst_multi) cs->dest[1].offset = rs->dest_offset + rs->dest_stride * rs->dest_padded_height / 2; cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) | VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height); /* use dual pipe mode when required */ if (!screen->specs.single_buffer && screen->specs.pixel_pipes == 2 && !(rs->height & (rs->downsample_y ? 0xf : 0x7))) { cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) | VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height / 2); cs->RS_PIPE_OFFSET[1] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(rs->height / 2); } cs->RS_DITHER[0] = rs->dither[0]; cs->RS_DITHER[1] = rs->dither[1]; cs->RS_CLEAR_CONTROL = VIVS_RS_CLEAR_CONTROL_BITS(rs->clear_bits) | rs->clear_mode; cs->RS_FILL_VALUE[0] = rs->clear_value[0]; cs->RS_FILL_VALUE[1] = rs->clear_value[1]; cs->RS_FILL_VALUE[2] = rs->clear_value[2]; cs->RS_FILL_VALUE[3] = rs->clear_value[3]; cs->RS_EXTRA_CONFIG = VIVS_RS_EXTRA_CONFIG_AA(rs->aa) | VIVS_RS_EXTRA_CONFIG_ENDIAN(rs->endian_mode); /* If source the same as destination, and the hardware supports this, * do an in-place resolve to fill in unrendered tiles. */ if (screen->specs.single_buffer && rs->source == rs->dest && rs->source_offset == rs->dest_offset && rs->source_format == rs->dest_format && rs->source_tiling == rs->dest_tiling && src_super && rs->source_stride == rs->dest_stride && !rs->downsample_x && !rs->downsample_y && !rs->swap_rb && !rs->flip && !rs->clear_mode && rs->source_padded_width && !rs->source_ts_compressed) { if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_CACHE128B256BPERLINE)) cs->RS_EXTRA_CONFIG |= VIVS_RS_EXTRA_CONFIG_TS_MODE(rs->source_ts_mode); /* Total number of tiles (same as for autodisable) */ cs->RS_KICKER_INPLACE = rs->tile_count; } cs->source_ts_valid = rs->source_ts_valid; cs->valid = true; } /* modify the clear bits value in the compiled RS state */ static void etna_modify_rs_clearbits(struct compiled_rs_state *cs, uint32_t clear_bits) { cs->RS_CLEAR_CONTROL &= ~VIVS_RS_CLEAR_CONTROL_BITS__MASK; cs->RS_CLEAR_CONTROL |= VIVS_RS_CLEAR_CONTROL_BITS(clear_bits); } #define EMIT_STATE(state_name, src_value) \ etna_coalsence_emit(stream, &coalesce, VIVS_##state_name, src_value) #define EMIT_STATE_FIXP(state_name, src_value) \ etna_coalsence_emit_fixp(stream, &coalesce, VIVS_##state_name, src_value) #define EMIT_STATE_RELOC(state_name, src_value) \ etna_coalsence_emit_reloc(stream, &coalesce, VIVS_##state_name, src_value) /* submit RS state, without any processing and no dependence on context * except TS if this is a source-to-destination blit. */ static void etna_submit_rs_state(struct etna_context *ctx, const struct compiled_rs_state *cs) { struct etna_screen *screen = etna_screen(ctx->base.screen); struct etna_cmd_stream *stream = ctx->stream; struct etna_coalesce coalesce; if (cs->RS_KICKER_INPLACE && !cs->source_ts_valid) /* Inplace resolve is no-op if TS is not configured */ return; ctx->stats.rs_operations++; if (cs->RS_KICKER_INPLACE) { etna_cmd_stream_reserve(stream, 6); etna_coalesce_start(stream, &coalesce); /* 0/1 */ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG); /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE); /* 4/5 */ EMIT_STATE(RS_KICKER_INPLACE, cs->RS_KICKER_INPLACE); etna_coalesce_end(stream, &coalesce); } else if (screen->specs.pixel_pipes > 1 || VIV_FEATURE(screen, ETNA_FEATURE_RS_NEW_BASEADDR)) { etna_cmd_stream_reserve(stream, 34); /* worst case - both pipes multi=1 */ etna_coalesce_start(stream, &coalesce); /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG); /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE); /* 4/5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE); /* 6/7 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(0), &cs->source[0]); if (cs->RS_SOURCE_STRIDE & VIVS_RS_SOURCE_STRIDE_MULTI) { /*8 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(1), &cs->source[1]); /*9 - pad */ } /*10/11*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(0), &cs->dest[0]); if (cs->RS_DEST_STRIDE & VIVS_RS_DEST_STRIDE_MULTI) { /*12*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(1), &cs->dest[1]); /*13 - pad */ } /*14/15*/ EMIT_STATE(RS_PIPE_OFFSET(0), cs->RS_PIPE_OFFSET[0]); /*16 */ EMIT_STATE(RS_PIPE_OFFSET(1), cs->RS_PIPE_OFFSET[1]); /*17 - pad */ /*18/19*/ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE); /*20/21*/ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]); /*22 */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]); /*23 - pad */ /*24/25*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL); /*26 */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]); /*27 */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]); /*28 */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]); /*29 */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]); /*30/31*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG); /*32/33*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb); etna_coalesce_end(stream, &coalesce); } else { etna_cmd_stream_reserve(stream, 22); etna_coalesce_start(stream, &coalesce); /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG); /* 2 */ EMIT_STATE_RELOC(RS_SOURCE_ADDR, &cs->source[0]); /* 3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE); /* 4 */ EMIT_STATE_RELOC(RS_DEST_ADDR, &cs->dest[0]); /* 5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE); /* 6/7 */ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE); /* 8/9 */ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]); /*10 */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]); /*11 - pad */ /*12/13*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL); /*14 */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]); /*15 */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]); /*16 */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]); /*17 */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]); /*18/19*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG); /*20/21*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb); etna_coalesce_end(stream, &coalesce); } } /* Generate clear command for a surface (non-fast clear case) */ static void etna_rs_gen_clear_surface(struct etna_context *ctx, struct etna_surface *surf, uint64_t clear_value) { ASSERTED struct etna_screen *screen = ctx->screen; struct etna_resource *dst = etna_resource(surf->base.texture); uint32_t format; switch (util_format_get_blocksizebits(surf->base.format)) { case 16: format = RS_FORMAT_A4R4G4B4; break; case 32: format = RS_FORMAT_A8R8G8B8; break; case 64: assert(screen->info->halti >= 2); format = RS_FORMAT_64BPP_CLEAR; break; default: unreachable("bpp not supported for clear by RS"); break; } /* use tiled clear if width is multiple of 16 */ bool tiled_clear = (surf->level->padded_width & ETNA_RS_WIDTH_MASK) == 0 && (surf->level->padded_height & ETNA_RS_HEIGHT_MASK) == 0; etna_compile_rs_state( ctx, &surf->clear_command, &(struct rs_state) { .source_format = format, .dest_format = format, .dest = dst->bo, .dest_offset = surf->offset, .dest_stride = surf->level->stride, .dest_padded_height = surf->level->padded_height, .dest_tiling = tiled_clear ? dst->layout : ETNA_LAYOUT_LINEAR, .dither = {0xffffffff, 0xffffffff}, .width = surf->level->padded_width, /* These must be padded to 16x4 if !LINEAR, otherwise RS will hang */ .height = surf->level->padded_height, .clear_value = {clear_value, clear_value >> 32, clear_value, clear_value >> 32}, .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_ENABLED1, .clear_bits = 0xffff }); } static void etna_blit_clear_color_rs(struct pipe_context *pctx, struct pipe_surface *dst, const union pipe_color_union *color) { struct etna_context *ctx = etna_context(pctx); struct etna_surface *surf = etna_surface(dst); uint64_t new_clear_value = etna_clear_blit_pack_rgba(surf->base.format, color); if (surf->level->ts_size) { /* TS: use precompiled clear command */ ctx->framebuffer.TS_COLOR_CLEAR_VALUE = new_clear_value; ctx->framebuffer.TS_COLOR_CLEAR_VALUE_EXT = new_clear_value >> 32; if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_AUTO_DISABLE)) { /* Set number of color tiles to be filled */ etna_set_state(ctx->stream, VIVS_TS_COLOR_AUTO_DISABLE_COUNT, surf->level->padded_width * surf->level->padded_height / 16); ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_COLOR_AUTO_DISABLE; } /* update clear color in SW meta area of the buffer if TS is exported */ if (unlikely(new_clear_value != surf->level->clear_value && etna_resource_ext_ts(etna_resource(dst->texture)))) surf->level->ts_meta->v0.clear_value = new_clear_value; assert(surf->ts_clear_command.valid); etna_submit_rs_state(ctx, &surf->ts_clear_command); etna_resource_level_ts_mark_valid(surf->level); ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS; } else { /* Queue normal RS clear for non-TS surfaces */ /* If clear color changed or no valid command yet (re-)generate * stored command */ if (unlikely(new_clear_value != surf->level->clear_value || !surf->clear_command.valid)) etna_rs_gen_clear_surface(ctx, surf, new_clear_value); etna_submit_rs_state(ctx, &surf->clear_command); } surf->level->clear_value = new_clear_value; resource_written(ctx, surf->base.texture); etna_resource_level_mark_changed(surf->level); } static void etna_blit_clear_zs_rs(struct pipe_context *pctx, struct pipe_surface *dst, unsigned buffers, double depth, unsigned stencil) { struct etna_context *ctx = etna_context(pctx); struct etna_surface *surf = etna_surface(dst); uint32_t new_clear_value = translate_clear_depth_stencil(surf->base.format, depth, stencil); uint32_t new_clear_bits = 0, clear_bits_depth, clear_bits_stencil; /* Get the channels to clear */ switch (surf->base.format) { case PIPE_FORMAT_Z16_UNORM: case PIPE_FORMAT_X8Z24_UNORM: clear_bits_depth = 0xffff; clear_bits_stencil = 0; break; case PIPE_FORMAT_S8_UINT_Z24_UNORM: clear_bits_depth = 0xeeee; clear_bits_stencil = 0x1111; break; default: clear_bits_depth = clear_bits_stencil = 0xffff; break; } if (buffers & PIPE_CLEAR_DEPTH) new_clear_bits |= clear_bits_depth; if (buffers & PIPE_CLEAR_STENCIL) new_clear_bits |= clear_bits_stencil; if (surf->level->ts_size && new_clear_bits == 0xffff) { /* Set new clear depth value */ ctx->framebuffer.TS_DEPTH_CLEAR_VALUE = new_clear_value; if (VIV_FEATURE(ctx->screen, ETNA_FEATURE_AUTO_DISABLE)) { /* Set number of depth tiles to be filled */ etna_set_state(ctx->stream, VIVS_TS_DEPTH_AUTO_DISABLE_COUNT, surf->level->padded_width * surf->level->padded_height / 16); ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_DEPTH_AUTO_DISABLE; } assert(surf->ts_clear_command.valid); etna_submit_rs_state(ctx, &surf->ts_clear_command); etna_resource_level_ts_mark_valid(surf->level); ctx->dirty |= ETNA_DIRTY_TS; } else { /* Queue normal RS clear for non-TS surfaces */ /* If the level has valid TS state we need to flush it, as the regular * clear will not update the state and we must therefore invalidate it. */ etna_copy_resource(pctx, surf->base.texture, surf->base.texture, surf->base.u.tex.level, surf->base.u.tex.level); if (unlikely(new_clear_value != surf->level->clear_value || !surf->clear_command.valid)) { /* If clear depth/stencil value changed or no valid command yet * (re)-generate stored command */ etna_rs_gen_clear_surface(ctx, surf, new_clear_value); } /* Update the channels to be cleared */ etna_modify_rs_clearbits(&surf->clear_command, new_clear_bits); etna_submit_rs_state(ctx, &surf->clear_command); etna_resource_level_ts_mark_invalid(surf->level); } surf->level->clear_value = new_clear_value; resource_written(ctx, surf->base.texture); etna_resource_level_mark_changed(surf->level); ctx->dirty |= ETNA_DIRTY_DERIVE_TS; } static void etna_clear_rs(struct pipe_context *pctx, unsigned buffers, const struct pipe_scissor_state *scissor_state, const union pipe_color_union *color, double depth, unsigned stencil) { struct etna_context *ctx = etna_context(pctx); if (!etna_render_condition_check(pctx)) return; /* Flush color and depth cache before clearing anything. * This is especially important when coming from another surface, as * otherwise it may clear part of the old surface instead. */ etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH); etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE); /* Preparation: Flush the TS if needed. This must be done after flushing * color and depth, otherwise it can result in crashes */ bool need_ts_flush = false; if ((buffers & PIPE_CLEAR_COLOR) && ctx->framebuffer_s.nr_cbufs) { struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[0]); if (surf->level->ts_size) need_ts_flush = true; } if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL) { struct etna_surface *surf = etna_surface(ctx->framebuffer_s.zsbuf); if (surf->level->ts_size) need_ts_flush = true; } if (need_ts_flush) etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH); /* No need to set up the TS here as RS clear operations (in contrast to * resolve and copy) do not require the TS state. */ if (buffers & PIPE_CLEAR_COLOR) { for (int idx = 0; idx < ctx->framebuffer_s.nr_cbufs; ++idx) { struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[idx]); etna_blit_clear_color_rs(pctx, ctx->framebuffer_s.cbufs[idx], &color[idx]); if (!etna_resource(surf->prsc)->explicit_flush) etna_context_add_flush_resource(ctx, surf->prsc); } } /* Flush the color and depth caches before each RS clear operation * This fixes a hang on GC600. */ if (buffers & PIPE_CLEAR_DEPTHSTENCIL && buffers & PIPE_CLEAR_COLOR) etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH); if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL) etna_blit_clear_zs_rs(pctx, ctx->framebuffer_s.zsbuf, buffers, depth, stencil); etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE); } static bool etna_manual_blit(struct etna_resource *dst, struct etna_resource_level *dst_lev, unsigned int dst_offset, struct etna_resource *src, struct etna_resource_level *src_lev, unsigned int src_offset, const struct pipe_blit_info *blit_info) { void *smap, *srow, *dmap, *drow; size_t tile_size; assert(src->layout == ETNA_LAYOUT_TILED); assert(dst->layout == ETNA_LAYOUT_TILED); assert(src->base.nr_samples == 0); assert(dst->base.nr_samples == 0); tile_size = util_format_get_blocksize(blit_info->src.format) * 4 * 4; smap = etna_bo_map(src->bo); if (!smap) return false; dmap = etna_bo_map(dst->bo); if (!dmap) return false; srow = smap + src_offset; drow = dmap + dst_offset; etna_bo_cpu_prep(src->bo, DRM_ETNA_PREP_READ); etna_bo_cpu_prep(dst->bo, DRM_ETNA_PREP_WRITE); for (int y = 0; y < blit_info->src.box.height; y += 4) { memcpy(drow, srow, tile_size * blit_info->src.box.width); srow += src_lev->stride * 4; drow += dst_lev->stride * 4; } etna_bo_cpu_fini(dst->bo); etna_bo_cpu_fini(src->bo); return true; } static inline size_t etna_compute_tileoffset(const struct pipe_box *box, enum pipe_format format, size_t stride, enum etna_surface_layout layout) { size_t offset; unsigned int x = box->x, y = box->y; unsigned int blocksize = util_format_get_blocksize(format); switch (layout) { case ETNA_LAYOUT_LINEAR: offset = y * stride + x * blocksize; break; case ETNA_LAYOUT_MULTI_TILED: y >>= 1; FALLTHROUGH; case ETNA_LAYOUT_TILED: assert(!(x & 0x03) && !(y & 0x03)); offset = (y & ~0x03) * stride + blocksize * ((x & ~0x03) << 2); break; case ETNA_LAYOUT_MULTI_SUPERTILED: y >>= 1; FALLTHROUGH; case ETNA_LAYOUT_SUPER_TILED: assert(!(x & 0x3f) && !(y & 0x3f)); offset = (y & ~0x3f) * stride + blocksize * ((x & ~0x3f) << 6); break; default: unreachable("invalid resource layout"); } return offset; } static inline void etna_get_rs_alignment_mask(const struct etna_context *ctx, const enum etna_surface_layout layout, unsigned int *width_mask, unsigned int *height_mask) { struct etna_screen *screen = ctx->screen; unsigned int h_align, w_align; if (layout & ETNA_LAYOUT_BIT_SUPER) { w_align = 64; h_align = 64 * screen->specs.pixel_pipes; } else { w_align = ETNA_RS_WIDTH_MASK + 1; h_align = ETNA_RS_HEIGHT_MASK + 1; } *width_mask = w_align - 1; *height_mask = h_align -1; } static bool etna_try_rs_blit(struct pipe_context *pctx, const struct pipe_blit_info *blit_info) { struct etna_context *ctx = etna_context(pctx); struct etna_resource *src = etna_resource(blit_info->src.resource); struct etna_resource *dst = etna_resource(blit_info->dst.resource); struct compiled_rs_state copy_to_screen; int src_xscale, src_yscale, dst_xscale, dst_yscale; bool downsample_x = false, downsample_y = false; /* Ensure that the level is valid */ assert(blit_info->src.level <= src->base.last_level); assert(blit_info->dst.level <= dst->base.last_level); if (!translate_samples_to_xyscale(src->base.nr_samples, &src_xscale, &src_yscale)) return false; if (!translate_samples_to_xyscale(dst->base.nr_samples, &dst_xscale, &dst_yscale)) return false; /* RS does not support upscaling */ if ((src_xscale < dst_xscale) || (src_yscale < dst_yscale)) return false; if (src_xscale > dst_xscale) downsample_x = true; if (src_yscale > dst_yscale) downsample_y = true; /* The width/height are in pixels; they do not change as a result of * multi-sampling. So, when blitting from a 4x multisampled surface * to a non-multisampled surface, the width and height will be * identical. As we do not support scaling, reject different sizes. */ if (blit_info->dst.box.width != blit_info->src.box.width || blit_info->dst.box.height != blit_info->src.box.height) { DBG("scaling requested: source %dx%d destination %dx%d", blit_info->src.box.width, blit_info->src.box.height, blit_info->dst.box.width, blit_info->dst.box.height); return false; } /* No masks - RS can't copy specific channels */ unsigned mask = util_format_get_mask(blit_info->dst.format); if ((blit_info->mask & mask) != mask) { DBG("sub-mask requested: 0x%02x vs format mask 0x%02x", blit_info->mask, mask); return false; } /* Only support same format (used tiling/detiling) blits for now. * TODO: figure out which different-format blits are possible and test them * - fail if swizzle needed * - avoid trying to convert between float/int formats? */ if (blit_info->src.format != blit_info->dst.format) return false; /* try to find a exact format match first */ uint32_t format = translate_rs_format(blit_info->dst.format); /* When not resolving MSAA, but only doing a layout conversion, we can get * away with a fallback format of matching size. */ if (format == ETNA_NO_MATCH && !downsample_x && !downsample_y) format = etna_compatible_rs_format(blit_info->dst.format); if (format == ETNA_NO_MATCH) return false; if (blit_info->scissor_enable || blit_info->dst.box.depth != blit_info->src.box.depth || blit_info->dst.box.depth != 1) { return false; } unsigned w_mask, h_mask; etna_get_rs_alignment_mask(ctx, src->layout, &w_mask, &h_mask); if ((blit_info->src.box.x & w_mask) || (blit_info->src.box.y & h_mask)) return false; etna_get_rs_alignment_mask(ctx, dst->layout, &w_mask, &h_mask); if ((blit_info->dst.box.x & w_mask) || (blit_info->dst.box.y & h_mask)) return false; struct etna_resource_level *src_lev = &src->levels[blit_info->src.level]; struct etna_resource_level *dst_lev = &dst->levels[blit_info->dst.level]; /* we may be given coordinates up to the padded width to avoid * any alignment issues with different tiling formats */ assert((blit_info->src.box.x + blit_info->src.box.width) * src_xscale <= src_lev->padded_width); assert((blit_info->src.box.y + blit_info->src.box.height) * src_yscale <= src_lev->padded_height); assert(blit_info->dst.box.x + blit_info->dst.box.width <= dst_lev->padded_width); assert(blit_info->dst.box.y + blit_info->dst.box.height <= dst_lev->padded_height); unsigned src_offset = src_lev->offset + blit_info->src.box.z * src_lev->layer_stride + etna_compute_tileoffset(&blit_info->src.box, blit_info->src.format, src_lev->stride, src->layout); unsigned dst_offset = dst_lev->offset + blit_info->dst.box.z * dst_lev->layer_stride + etna_compute_tileoffset(&blit_info->dst.box, blit_info->dst.format, dst_lev->stride, dst->layout); if (src_lev->padded_width <= ETNA_RS_WIDTH_MASK || dst_lev->padded_width <= ETNA_RS_WIDTH_MASK || src_lev->padded_height <= ETNA_RS_HEIGHT_MASK || dst_lev->padded_height <= ETNA_RS_HEIGHT_MASK) goto manual; /* If the width is not aligned to the RS width, but is within our * padding, adjust the width to suite the RS width restriction. * Note: the RS width/height are converted to source samples here. */ unsigned int width = blit_info->src.box.width * src_xscale; unsigned int height = blit_info->src.box.height * src_yscale; unsigned int w_align = (ETNA_RS_WIDTH_MASK + 1) * src_xscale; unsigned int h_align = (ETNA_RS_HEIGHT_MASK + 1) * src_yscale; if (width & (w_align - 1) && width >= src_lev->width * src_xscale && width >= dst_lev->width) width = align(width, w_align); if (height & (h_align - 1) && height >= src_lev->height * src_yscale && height >= dst_lev->height) { height = align(height, h_align); /* Try to increase alignment to multi-pipe requirements to unlock * multi-pipe resolve for increased performance. */ if (!ctx->screen->specs.single_buffer) { unsigned int pipe_align = align(height, h_align * ctx->screen->specs.pixel_pipes); if (pipe_align <= src_lev->padded_height && pipe_align <= dst_lev->padded_height * src_yscale) height = pipe_align; } } /* The padded dimensions are in samples */ if (width > src_lev->padded_width || width > dst_lev->padded_width * src_xscale || height > src_lev->padded_height || height > dst_lev->padded_height * src_yscale || width & (w_align - 1) || height & (h_align - 1)) goto manual; /* Flush destination, as the blit will invalidate any pending TS changes. */ if (dst != src && etna_resource_level_needs_flush(dst_lev)) etna_copy_resource(pctx, &dst->base, &dst->base, blit_info->dst.level, blit_info->dst.level); /* Always flush color and depth cache together before resolving. This makes * sure that all previous cache content written by the PE is flushed out * before RS uses the pixel pipes, which invalidates those caches. */ etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH); etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE); /* Set up color TS to source surface before blit, if needed */ bool source_ts_valid = false; if (etna_resource_level_ts_valid(src_lev)) { struct etna_reloc reloc; unsigned ts_offset = src_lev->ts_offset + blit_info->src.box.z * src_lev->ts_layer_stride; uint32_t ts_mem_config = 0; /* flush TS cache before changing to another TS configuration */ etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH); if (src_lev->ts_compress_fmt >= 0) { ts_mem_config |= VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION | VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION_FORMAT(src_lev->ts_compress_fmt); } etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG, VIVS_TS_MEM_CONFIG_COLOR_FAST_CLEAR | ts_mem_config); memset(&reloc, 0, sizeof(struct etna_reloc)); reloc.bo = src->ts_bo; reloc.offset = ts_offset; reloc.flags = ETNA_RELOC_READ; etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_STATUS_BASE, &reloc); memset(&reloc, 0, sizeof(struct etna_reloc)); reloc.bo = src->bo; reloc.offset = src_lev->offset + blit_info->src.box.z * src_lev->layer_stride; reloc.flags = ETNA_RELOC_READ; etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_SURFACE_BASE, &reloc); etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE, src_lev->clear_value); etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE_EXT, src_lev->clear_value >> 32); source_ts_valid = true; } else { etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG, 0); } ctx->dirty |= ETNA_DIRTY_TS; /* Kick off RS here */ etna_compile_rs_state(ctx, ©_to_screen, &(struct rs_state) { .source_format = format, .source_tiling = src->layout, .source = src->bo, .source_offset = src_offset, .source_stride = src_lev->stride, .source_padded_width = src_lev->padded_width, .source_padded_height = src_lev->padded_height, .source_ts_valid = source_ts_valid, .source_ts_mode = src_lev->ts_mode, .source_ts_compressed = src_lev->ts_compress_fmt >= 0, .dest_format = format, .dest_tiling = dst->layout, .dest = dst->bo, .dest_offset = dst_offset, .dest_stride = dst_lev->stride, .dest_padded_height = dst_lev->padded_height, .downsample_x = downsample_x, .downsample_y = downsample_y, .swap_rb = translate_rb_src_dst_swap(src->base.format, dst->base.format), .dither = {0xffffffff, 0xffffffff}, // XXX dither when going from 24 to 16 bit? .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_DISABLED, .width = width, .height = height, .tile_count = src_lev->layer_stride / etna_screen_get_tile_size(ctx->screen, src_lev->ts_mode, src->base.nr_samples > 1), }); etna_submit_rs_state(ctx, ©_to_screen); resource_read(ctx, &src->base); resource_written(ctx, &dst->base); etna_resource_level_mark_changed(dst_lev); /* We don't need to mark the TS as invalid if this was just a flush without * compression, as in that case only clear tiles are filled and the tile * status still matches the blit target buffer. For compressed formats the * tiles are decompressed, so tile status doesn't match anymore. */ if (src != dst || src_lev->ts_compress_fmt >= 0) etna_resource_level_ts_mark_invalid(dst_lev); ctx->dirty |= ETNA_DIRTY_DERIVE_TS; return true; manual: if (src->layout == ETNA_LAYOUT_TILED && dst->layout == ETNA_LAYOUT_TILED) { if ((etna_resource_status(ctx, src) & ETNA_PENDING_WRITE) || (etna_resource_status(ctx, dst) & ETNA_PENDING_WRITE)) etna_flush(pctx, NULL, 0, true); perf_debug_ctx(ctx, "RS blit falls back to sw"); return etna_manual_blit(dst, dst_lev, dst_offset, src, src_lev, src_offset, blit_info); } return false; } void etna_clear_blit_rs_init(struct pipe_context *pctx) { struct etna_context *ctx = etna_context(pctx); DBG("etnaviv: Using RS blit engine"); pctx->clear = etna_clear_rs; ctx->blit = etna_try_rs_blit; }