/* * Copyright © 2013 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include "util/ralloc.h" #include "util/macros.h" /* Needed for MAX3 and MAX2 for format_rgb9e5 */ #include "util/format_rgb9e5.h" #include "util/format_srgb.h" #include "util/u_math.h" #include "blorp_priv.h" #include "dev/intel_debug.h" #include "dev/intel_device_info.h" #include "blorp_nir_builder.h" #define FILE_DEBUG_FLAG DEBUG_BLORP #pragma pack(push, 1) struct blorp_const_color_prog_key { struct blorp_base_key base; bool use_simd16_replicated_data; bool clear_rgb_as_red; uint8_t local_y; }; #pragma pack(pop) static bool blorp_params_get_clear_kernel_fs(struct blorp_batch *batch, struct blorp_params *params, bool want_replicated_data, bool clear_rgb_as_red) { const bool use_replicated_data = want_replicated_data && batch->blorp->isl_dev->info->ver < 20; struct blorp_context *blorp = batch->blorp; const struct blorp_const_color_prog_key blorp_key = { .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_CLEAR), .base.shader_pipeline = BLORP_SHADER_PIPELINE_RENDER, .use_simd16_replicated_data = use_replicated_data, .clear_rgb_as_red = clear_rgb_as_red, .local_y = 0, }; params->shader_type = blorp_key.base.shader_type; params->shader_pipeline = blorp_key.base.shader_pipeline; if (blorp->lookup_shader(batch, &blorp_key, sizeof(blorp_key), ¶ms->wm_prog_kernel, ¶ms->wm_prog_data)) return true; void *mem_ctx = ralloc_context(NULL); nir_builder b; blorp_nir_init_shader(&b, blorp, mem_ctx, MESA_SHADER_FRAGMENT, blorp_shader_type_to_name(blorp_key.base.shader_type)); nir_variable *v_color = BLORP_CREATE_NIR_INPUT(b.shader, clear_color, glsl_vec4_type()); nir_def *color = nir_load_var(&b, v_color); if (clear_rgb_as_red) { nir_def *pos = nir_f2i32(&b, nir_load_frag_coord(&b)); nir_def *comp = nir_umod_imm(&b, nir_channel(&b, pos, 0), 3); color = nir_pad_vec4(&b, nir_vector_extract(&b, color, comp)); } nir_variable *frag_color = nir_variable_create(b.shader, nir_var_shader_out, glsl_vec4_type(), "gl_FragColor"); frag_color->data.location = FRAG_RESULT_COLOR; nir_store_var(&b, frag_color, color, 0xf); const bool multisample_fbo = false; struct blorp_program p = blorp_compile_fs(blorp, mem_ctx, b.shader, multisample_fbo, use_replicated_data); bool result = blorp->upload_shader(batch, MESA_SHADER_FRAGMENT, &blorp_key, sizeof(blorp_key), p.kernel, p.kernel_size, p.prog_data, p.prog_data_size, ¶ms->wm_prog_kernel, ¶ms->wm_prog_data); ralloc_free(mem_ctx); return result; } static bool blorp_params_get_clear_kernel_cs(struct blorp_batch *batch, struct blorp_params *params, bool clear_rgb_as_red) { struct blorp_context *blorp = batch->blorp; const struct blorp_const_color_prog_key blorp_key = { .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_CLEAR), .base.shader_pipeline = BLORP_SHADER_PIPELINE_COMPUTE, .use_simd16_replicated_data = false, .clear_rgb_as_red = clear_rgb_as_red, .local_y = blorp_get_cs_local_y(params), }; params->shader_type = blorp_key.base.shader_type; params->shader_pipeline = blorp_key.base.shader_pipeline; if (blorp->lookup_shader(batch, &blorp_key, sizeof(blorp_key), ¶ms->cs_prog_kernel, ¶ms->cs_prog_data)) return true; void *mem_ctx = ralloc_context(NULL); nir_builder b; blorp_nir_init_shader(&b, blorp, mem_ctx, MESA_SHADER_COMPUTE, "BLORP-gpgpu-clear"); blorp_set_cs_dims(b.shader, blorp_key.local_y); nir_def *dst_pos = nir_load_global_invocation_id(&b, 32); nir_variable *v_color = BLORP_CREATE_NIR_INPUT(b.shader, clear_color, glsl_vec4_type()); nir_def *color = nir_load_var(&b, v_color); nir_variable *v_bounds_rect = BLORP_CREATE_NIR_INPUT(b.shader, bounds_rect, glsl_vec4_type()); nir_def *bounds_rect = nir_load_var(&b, v_bounds_rect); nir_def *in_bounds = blorp_check_in_bounds(&b, bounds_rect, dst_pos); if (clear_rgb_as_red) { nir_def *comp = nir_umod_imm(&b, nir_channel(&b, dst_pos, 0), 3); color = nir_pad_vec4(&b, nir_vector_extract(&b, color, comp)); } nir_push_if(&b, in_bounds); nir_image_store(&b, nir_imm_int(&b, 0), nir_pad_vector_imm_int(&b, dst_pos, 0, 4), nir_imm_int(&b, 0), nir_pad_vector_imm_int(&b, color, 0, 4), nir_imm_int(&b, 0), .image_dim = GLSL_SAMPLER_DIM_2D, .image_array = true, .access = ACCESS_NON_READABLE); nir_pop_if(&b, NULL); const struct blorp_program p = blorp_compile_cs(blorp, mem_ctx, b.shader); bool result = blorp->upload_shader(batch, MESA_SHADER_COMPUTE, &blorp_key, sizeof(blorp_key), p.kernel, p.kernel_size, p.prog_data, p.prog_data_size, ¶ms->cs_prog_kernel, ¶ms->cs_prog_data); ralloc_free(mem_ctx); return result; } static bool blorp_params_get_clear_kernel(struct blorp_batch *batch, struct blorp_params *params, bool use_replicated_data, bool clear_rgb_as_red) { if (batch->flags & BLORP_BATCH_USE_COMPUTE) { assert(!use_replicated_data); return blorp_params_get_clear_kernel_cs(batch, params, clear_rgb_as_red); } else { return blorp_params_get_clear_kernel_fs(batch, params, use_replicated_data, clear_rgb_as_red); } } /* The x0, y0, x1, and y1 parameters must already be populated with the render * area of the framebuffer to be cleared. */ static void get_fast_clear_rect(const struct isl_device *dev, const struct isl_surf *surf, const struct isl_surf *aux_surf, unsigned *x0, unsigned *y0, unsigned *x1, unsigned *y1) { unsigned int x_align, y_align; unsigned int x_scaledown, y_scaledown; /* Only single sampled surfaces need to (and actually can) be resolved. */ if (surf->samples == 1) { const uint32_t bs = isl_format_get_layout(surf->format)->bpb / 8; if (dev->info->ver >= 20) { /* From Bspec 57340, "MCS/CCS Buffers, Fast Clear for Render Target(s)": * * Table "Tile4/Tile64 2D/2D Array/Cube Surface" * Table "Tile64 3D/Volumetric" * * The below calculation is derived from these tables. */ assert(surf->tiling == ISL_TILING_4 || surf->tiling == ISL_TILING_64_XE2); x_align = x_scaledown = 64 / bs; y_align = y_scaledown = 4; } else if (dev->info->verx10 >= 125) { /* From Bspec 47709, "MCS/CCS Buffer for Render Target(s)": * * SW must ensure that clearing rectangle dimensions cover the * entire area desired, to accomplish this task initial X/Y * dimensions need to be rounded up to next multiple of scaledown * factor before dividing by scale down factor: * * The X and Y scale down factors in the table that follows are used * for both alignment and scaling down. */ if (surf->tiling == ISL_TILING_4) { x_align = x_scaledown = 1024 / bs; y_align = y_scaledown = 16; } else if (surf->tiling == ISL_TILING_64) { switch (bs) { case 1: x_align = x_scaledown = 128; y_align = y_scaledown = 128; break; case 2: x_align = x_scaledown = 128; y_align = y_scaledown = 64; break; case 4: x_align = x_scaledown = 64; y_align = y_scaledown = 64; break; case 8: x_align = x_scaledown = 64; y_align = y_scaledown = 32; break; case 16: x_align = x_scaledown = 32; y_align = y_scaledown = 32; break; default: unreachable("unsupported bpp"); } } else { unreachable("Unsupported tiling format"); } } else { /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render * Target(s)", beneath the "Fast Color Clear" bullet (p327): * * Clear pass must have a clear rectangle that must follow * alignment rules in terms of pixels and lines as shown in the * table below. Further, the clear-rectangle height and width * must be multiple of the following dimensions. If the height * and width of the render target being cleared do not meet these * requirements, an MCS buffer can be created such that it * follows the requirement and covers the RT. * * The alignment size in the table that follows is a multiple of the * alignment size that is baked into the CCS surface format. */ enum isl_format ccs_format; if (ISL_GFX_VERX10(dev) == 120) { assert(surf->tiling == ISL_TILING_Y0); switch (isl_format_get_layout(surf->format)->bpb) { case 8: ccs_format = ISL_FORMAT_GFX12_CCS_8BPP_Y0; break; case 16: ccs_format = ISL_FORMAT_GFX12_CCS_16BPP_Y0; break; case 32: ccs_format = ISL_FORMAT_GFX12_CCS_32BPP_Y0; break; case 64: ccs_format = ISL_FORMAT_GFX12_CCS_64BPP_Y0; break; case 128: ccs_format = ISL_FORMAT_GFX12_CCS_128BPP_Y0; break; default: unreachable("Invalid surface bpb for fast clearing"); } } else { assert(aux_surf->usage == ISL_SURF_USAGE_CCS_BIT); ccs_format = aux_surf->format; } x_align = isl_format_get_layout(ccs_format)->bw * 16; y_align = isl_format_get_layout(ccs_format)->bh * 32 / isl_format_get_layout(ccs_format)->bpb; /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render * Target(s)", beneath the "Fast Color Clear" bullet (p327): * * In order to optimize the performance MCS buffer (when bound to * 1X RT) clear similarly to MCS buffer clear for MSRT case, * clear rect is required to be scaled by the following factors * in the horizontal and vertical directions: * * The X and Y scale down factors in the table that follows are each * equal to half the alignment value computed above. */ x_scaledown = x_align / 2; y_scaledown = y_align / 2; } if (ISL_DEV_IS_HASWELL(dev)) { /* From BSpec: 3D-Media-GPGPU Engine > 3D Pipeline > Pixel > Pixel * Backend > MCS Buffer for Render Target(s) [DevIVB+] > Table "Color * Clear of Non-MultiSampled Render Target Restrictions": * * Clear rectangle must be aligned to two times the number of * pixels in the table shown below due to 16x16 hashing across the * slice. * * This restriction is only documented to exist on HSW GT3 but * empirical evidence suggests that it's also needed GT2. */ x_align *= 2; y_align *= 2; } } else { assert(aux_surf->usage == ISL_SURF_USAGE_MCS_BIT); /* From the Ivy Bridge PRM, Vol2 Part1 11.7 "MCS Buffer for Render * Target(s)", beneath the "MSAA Compression" bullet (p326): * * Clear pass for this case requires that scaled down primitive * is sent down with upper left coordinate to coincide with * actual rectangle being cleared. For MSAA, clear rectangle’s * height and width need to as show in the following table in * terms of (width,height) of the RT. * * MSAA Width of Clear Rect Height of Clear Rect * 2X Ceil(1/8*width) Ceil(1/2*height) * 4X Ceil(1/8*width) Ceil(1/2*height) * 8X Ceil(1/2*width) Ceil(1/2*height) * 16X width Ceil(1/2*height) * * The text "with upper left coordinate to coincide with actual * rectangle being cleared" is a little confusing--it seems to imply * that to clear a rectangle from (x,y) to (x+w,y+h), one needs to * feed the pipeline using the rectangle (x,y) to * (x+Ceil(w/N),y+Ceil(h/2)), where N is either 2 or 8 depending on * the number of samples. Experiments indicate that this is not * quite correct; actually, what the hardware appears to do is to * align whatever rectangle is sent down the pipeline to the nearest * multiple of 2x2 blocks, and then scale it up by a factor of N * horizontally and 2 vertically. So the resulting alignment is 4 * vertically and either 4 or 16 horizontally, and the scaledown * factor is 2 vertically and either 2 or 8 horizontally. * * On Xe2+: * Bspec 57340 (r59562): * * Fast Clear MCS Surface * (Table) * * The scaled down values in the Xe2 table are different from what's in * the previous platforms. */ switch (aux_surf->format) { case ISL_FORMAT_MCS_2X: case ISL_FORMAT_MCS_4X: x_scaledown = dev->info->ver >= 20 ? 64 : 8; break; case ISL_FORMAT_MCS_8X: x_scaledown = dev->info->ver >= 20 ? 16 : 2; break; case ISL_FORMAT_MCS_16X: x_scaledown = dev->info->ver >= 20 ? 8 : 1; break; default: unreachable("Unexpected MCS format for fast clear"); } y_scaledown = dev->info->ver >= 20 ? 4 : 2; x_align = x_scaledown * 2; y_align = y_scaledown * 2; } *x0 = ROUND_DOWN_TO(*x0, x_align) / x_scaledown; *y0 = ROUND_DOWN_TO(*y0, y_align) / y_scaledown; *x1 = ALIGN(*x1, x_align) / x_scaledown; *y1 = ALIGN(*y1, y_align) / y_scaledown; } void blorp_fast_clear(struct blorp_batch *batch, const struct blorp_surf *surf, enum isl_format format, struct isl_swizzle swizzle, uint32_t level, uint32_t start_layer, uint32_t num_layers, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1) { struct blorp_params params; blorp_params_init(¶ms); params.num_layers = num_layers; assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0); params.x0 = x0; params.y0 = y0; params.x1 = x1; params.y1 = y1; if (batch->blorp->isl_dev->info->ver >= 20) { /* Bspec 57340 (r59562): * * Overview of Fast Clear: * Pixel shader's color output is treated as Clear Value, value * should be a constant. */ memcpy(¶ms.wm_inputs.clear_color, &surf->clear_color, 4 * sizeof(float)); } else { /* BSpec: 2423 (r153658): * * The pixel shader kernel requires no attributes, and delivers a * value of 0xFFFFFFFF in all channels of the render target write * message The replicated color message should be used. */ memset(¶ms.wm_inputs.clear_color, 0xff, 4 * sizeof(float)); } params.fast_clear_op = ISL_AUX_OP_FAST_CLEAR; get_fast_clear_rect(batch->blorp->isl_dev, surf->surf, surf->aux_surf, ¶ms.x0, ¶ms.y0, ¶ms.x1, ¶ms.y1); if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false)) return; blorp_surface_info_init(batch, ¶ms.dst, surf, level, start_layer, format, true); params.num_samples = params.dst.surf.samples; assert(params.num_samples != 0); if (params.num_samples == 1) params.op = BLORP_OP_CCS_COLOR_CLEAR; else params.op = BLORP_OP_MCS_COLOR_CLEAR; /* If a swizzle was provided, we need to swizzle the clear color so that * the hardware color format conversion will work properly. */ params.dst.clear_color = isl_color_value_swizzle_inv(params.dst.clear_color, swizzle); batch->blorp->exec(batch, ¶ms); } bool blorp_clear_supports_blitter(struct blorp_context *blorp, const struct blorp_surf *surf, uint8_t color_write_disable, bool blend_enabled) { const struct intel_device_info *devinfo = blorp->isl_dev->info; if (devinfo->ver < 12) return false; if (surf->surf->samples > 1) return false; if (color_write_disable != 0 || blend_enabled) return false; if (!blorp_blitter_supports_aux(devinfo, surf->aux_usage)) return false; const struct isl_format_layout *fmtl = isl_format_get_layout(surf->surf->format); /* We can only support linear mode for 96bpp. */ if (fmtl->bpb == 96 && surf->surf->tiling != ISL_TILING_LINEAR) return false; return true; } bool blorp_clear_supports_compute(struct blorp_context *blorp, uint8_t color_write_disable, bool blend_enabled, enum isl_aux_usage aux_usage) { if (blorp->isl_dev->info->ver < 7) return false; if (color_write_disable != 0 || blend_enabled) return false; if (blorp->isl_dev->info->ver >= 12) { return aux_usage == ISL_AUX_USAGE_FCV_CCS_E || aux_usage == ISL_AUX_USAGE_CCS_E || aux_usage == ISL_AUX_USAGE_NONE; } else { return aux_usage == ISL_AUX_USAGE_NONE; } } void blorp_clear(struct blorp_batch *batch, const struct blorp_surf *surf, enum isl_format format, struct isl_swizzle swizzle, uint32_t level, uint32_t start_layer, uint32_t num_layers, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, union isl_color_value clear_color, uint8_t color_write_disable) { struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_SLOW_COLOR_CLEAR; const bool compute = batch->flags & BLORP_BATCH_USE_COMPUTE; if (compute) { assert(blorp_clear_supports_compute(batch->blorp, color_write_disable, false, surf->aux_usage)); } else if (batch->flags & BLORP_BATCH_USE_BLITTER) { assert(blorp_clear_supports_blitter(batch->blorp, surf, color_write_disable, false)); } /* Manually apply the clear destination swizzle. This way swizzled clears * will work for swizzles which we can't normally use for rendering and it * also ensures that they work on pre-Haswell hardware which can't swizlle * at all. */ clear_color = isl_color_value_swizzle_inv(clear_color, swizzle); swizzle = ISL_SWIZZLE_IDENTITY; bool clear_rgb_as_red = false; if (format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) { clear_color.u32[0] = float3_to_rgb9e5(clear_color.f32); format = ISL_FORMAT_R32_UINT; } else if (format == ISL_FORMAT_L8_UNORM_SRGB) { clear_color.f32[0] = util_format_linear_to_srgb_float(clear_color.f32[0]); format = ISL_FORMAT_R8_UNORM; } else if (format == ISL_FORMAT_A4B4G4R4_UNORM) { /* Broadwell and earlier cannot render to this format so we need to work * around it by swapping the colors around and using B4G4R4A4 instead. */ const struct isl_swizzle ARGB = ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE); clear_color = isl_color_value_swizzle_inv(clear_color, ARGB); format = ISL_FORMAT_B4G4R4A4_UNORM; } else if (isl_format_get_layout(format)->bpb % 3 == 0) { clear_rgb_as_red = true; if (format == ISL_FORMAT_R8G8B8_UNORM_SRGB) { clear_color.f32[0] = util_format_linear_to_srgb_float(clear_color.f32[0]); clear_color.f32[1] = util_format_linear_to_srgb_float(clear_color.f32[1]); clear_color.f32[2] = util_format_linear_to_srgb_float(clear_color.f32[2]); } } memcpy(¶ms.wm_inputs.clear_color, clear_color.f32, sizeof(float) * 4); bool use_simd16_replicated_data = true; /* From the SNB PRM (Vol4_Part1): * * "Replicated data (Message Type = 111) is only supported when * accessing tiled memory. Using this Message Type to access linear * (untiled) memory is UNDEFINED." */ if (surf->surf->tiling == ISL_TILING_LINEAR) use_simd16_replicated_data = false; /* Replicated clears don't work before gfx6 */ if (batch->blorp->isl_dev->info->ver < 6) use_simd16_replicated_data = false; /* From the BSpec: 47719 (TGL/DG2/MTL) Replicate Data: * * "Replicate Data Render Target Write message should not be used * on all projects TGL+." * * Xe2 spec (57350) does not mention this restriction. * * See 14017879046, 14017880152 for additional information. */ if (batch->blorp->isl_dev->info->ver >= 12 && batch->blorp->isl_dev->info->ver < 20) use_simd16_replicated_data = false; if (compute) use_simd16_replicated_data = false; /* Constant color writes ignore everything in blend and color calculator * state. This is not documented. */ params.color_write_disable = color_write_disable & BITFIELD_MASK(4); if (color_write_disable) use_simd16_replicated_data = false; if (!blorp_params_get_clear_kernel(batch, ¶ms, use_simd16_replicated_data, clear_rgb_as_red)) return; if (!compute && !blorp_ensure_sf_program(batch, ¶ms)) return; while (num_layers > 0) { blorp_surface_info_init(batch, ¶ms.dst, surf, level, start_layer, format, true); params.dst.view.swizzle = swizzle; params.x0 = x0; params.y0 = y0; params.x1 = x1; params.y1 = y1; if (compute) { params.wm_inputs.bounds_rect.x0 = x0; params.wm_inputs.bounds_rect.y0 = y0; params.wm_inputs.bounds_rect.x1 = x1; params.wm_inputs.bounds_rect.y1 = y1; } if (params.dst.tile_x_sa || params.dst.tile_y_sa) { assert(params.dst.surf.samples == 1); assert(num_layers == 1); params.x0 += params.dst.tile_x_sa; params.y0 += params.dst.tile_y_sa; params.x1 += params.dst.tile_x_sa; params.y1 += params.dst.tile_y_sa; } /* The MinLOD and MinimumArrayElement don't work properly for cube maps. * Convert them to a single slice on gfx4. */ if (batch->blorp->isl_dev->info->ver == 4 && (params.dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT)) { blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms.dst); } if (clear_rgb_as_red) { surf_fake_rgb_with_red(batch->blorp->isl_dev, ¶ms.dst); params.x0 *= 3; params.x1 *= 3; } if (isl_format_is_compressed(params.dst.surf.format)) { blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.dst, NULL, NULL, NULL, NULL); //&dst_x, &dst_y, &dst_w, &dst_h); } if (params.dst.tile_x_sa || params.dst.tile_y_sa) { /* Either we're on gfx4 where there is no multisampling or the * surface is compressed which also implies no multisampling. * Therefore, sa == px and we don't need to do a conversion. */ assert(params.dst.surf.samples == 1); params.x0 += params.dst.tile_x_sa; params.y0 += params.dst.tile_y_sa; params.x1 += params.dst.tile_x_sa; params.y1 += params.dst.tile_y_sa; } params.num_samples = params.dst.surf.samples; /* We may be restricted on the number of layers we can bind at any one * time. In particular, Sandy Bridge has a maximum number of layers of * 512 but a maximum 3D texture size is much larger. */ params.num_layers = MIN2(params.dst.view.array_len, num_layers); const unsigned max_image_width = 16 * 1024; if (params.dst.surf.logical_level0_px.width > max_image_width) { /* Clearing an RGB image as red multiplies the surface width by 3 * so it may now be too wide for the hardware surface limits. We * have to break the clear up into pieces in order to clear wide * images. */ assert(clear_rgb_as_red); assert(params.dst.surf.dim == ISL_SURF_DIM_2D); assert(params.dst.surf.tiling == ISL_TILING_LINEAR); assert(params.dst.surf.logical_level0_px.depth == 1); assert(params.dst.surf.logical_level0_px.array_len == 1); assert(params.dst.surf.levels == 1); assert(params.dst.surf.samples == 1); assert(params.dst.tile_x_sa == 0 || params.dst.tile_y_sa == 0); assert(params.dst.aux_usage == ISL_AUX_USAGE_NONE); /* max_image_width rounded down to a multiple of 3 */ const unsigned max_fake_rgb_width = (max_image_width / 3) * 3; const unsigned cpp = isl_format_get_layout(params.dst.surf.format)->bpb / 8; params.dst.surf.logical_level0_px.width = max_fake_rgb_width; params.dst.surf.phys_level0_sa.width = max_fake_rgb_width; uint32_t orig_x0 = params.x0, orig_x1 = params.x1; uint64_t orig_offset = params.dst.addr.offset; for (uint32_t x = orig_x0; x < orig_x1; x += max_fake_rgb_width) { /* Offset to the surface. It's easy because we're linear */ params.dst.addr.offset = orig_offset + x * cpp; params.x0 = 0; params.x1 = MIN2(orig_x1 - x, max_image_width); batch->blorp->exec(batch, ¶ms); } } else { batch->blorp->exec(batch, ¶ms); } start_layer += params.num_layers; num_layers -= params.num_layers; } } static bool blorp_clear_stencil_as_rgba(struct blorp_batch *batch, const struct blorp_surf *surf, uint32_t level, uint32_t start_layer, uint32_t num_layers, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, uint8_t stencil_mask, uint8_t stencil_value) { assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0); /* Stencil mask support would require piles of shader magic */ if (stencil_mask != 0xff) return false; /* We only support separate W-tiled stencil for now */ if (surf->surf->format != ISL_FORMAT_R8_UINT || surf->surf->tiling != ISL_TILING_W) return false; if (surf->surf->samples > 1) { /* Adjust x0, y0, x1, and y1 to be in units of samples */ assert(surf->surf->msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED); struct isl_extent2d msaa_px_size_sa = isl_get_interleaved_msaa_px_size_sa(surf->surf->samples); x0 *= msaa_px_size_sa.w; y0 *= msaa_px_size_sa.h; x1 *= msaa_px_size_sa.w; y1 *= msaa_px_size_sa.h; } /* W-tiles and Y-tiles have the same layout as far as cache lines are * concerned: both are 8x8 cache lines laid out Y-major. The difference is * entirely in how the data is arranged within the cache line. W-tiling * is 8x8 pixels in a swizzled pattern while Y-tiling is 16B by 4 rows * regardless of image format size. As long as everything is aligned to 8, * we can just treat the W-tiled image as Y-tiled, ignore the layout * difference within a cache line, and blast out data. */ if (x0 % 8 != 0 || y0 % 8 != 0 || x1 % 8 != 0 || y1 % 8 != 0) return false; struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_SLOW_DEPTH_CLEAR; if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false)) return false; memset(¶ms.wm_inputs.clear_color, stencil_value, sizeof(params.wm_inputs.clear_color)); /* The Sandy Bridge PRM Vol. 4 Pt. 2, section 2.11.2.1.1 has the * following footnote to the format table: * * 128 BPE Formats cannot be Tiled Y when used as render targets * * We have to use RGBA16_UINT on SNB. */ enum isl_format wide_format; if (ISL_GFX_VER(batch->blorp->isl_dev) <= 6) { wide_format = ISL_FORMAT_R16G16B16A16_UINT; /* For RGBA16_UINT, we need to mask the stencil value otherwise, we risk * clamping giving us the wrong values */ for (unsigned i = 0; i < 4; i++) params.wm_inputs.clear_color[i] &= 0xffff; } else { wide_format = ISL_FORMAT_R32G32B32A32_UINT; } for (uint32_t a = 0; a < num_layers; a++) { uint32_t layer = start_layer + a; blorp_surface_info_init(batch, ¶ms.dst, surf, level, layer, ISL_FORMAT_UNSUPPORTED, true); if (surf->surf->samples > 1) blorp_surf_fake_interleaved_msaa(batch->blorp->isl_dev, ¶ms.dst); /* Make it Y-tiled */ blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms.dst); unsigned wide_Bpp = isl_format_get_layout(wide_format)->bpb / 8; params.dst.view.format = params.dst.surf.format = wide_format; assert(params.dst.surf.logical_level0_px.width % wide_Bpp == 0); params.dst.surf.logical_level0_px.width /= wide_Bpp; assert(params.dst.tile_x_sa % wide_Bpp == 0); params.dst.tile_x_sa /= wide_Bpp; params.x0 = params.dst.tile_x_sa + x0 / (wide_Bpp / 2); params.y0 = params.dst.tile_y_sa + y0 / 2; params.x1 = params.dst.tile_x_sa + x1 / (wide_Bpp / 2); params.y1 = params.dst.tile_y_sa + y1 / 2; batch->blorp->exec(batch, ¶ms); } return true; } void blorp_clear_depth_stencil(struct blorp_batch *batch, const struct blorp_surf *depth, const struct blorp_surf *stencil, uint32_t level, uint32_t start_layer, uint32_t num_layers, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, bool clear_depth, float depth_value, uint8_t stencil_mask, uint8_t stencil_value) { assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0); if (!clear_depth && blorp_clear_stencil_as_rgba(batch, stencil, level, start_layer, num_layers, x0, y0, x1, y1, stencil_mask, stencil_value)) return; struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_SLOW_DEPTH_CLEAR; params.x0 = x0; params.y0 = y0; params.x1 = x1; params.y1 = y1; if (ISL_GFX_VER(batch->blorp->isl_dev) == 6) { /* For some reason, Sandy Bridge gets occlusion queries wrong if we * don't have a shader. In particular, it records samples even though * we disable statistics in 3DSTATE_WM. Give it the usual clear shader * to work around the issue. */ if (!blorp_params_get_clear_kernel(batch, ¶ms, false, false)) return; } while (num_layers > 0) { params.num_layers = num_layers; if (stencil_mask) { blorp_surface_info_init(batch, ¶ms.stencil, stencil, level, start_layer, ISL_FORMAT_UNSUPPORTED, true); params.stencil_mask = stencil_mask; params.stencil_ref = stencil_value; params.dst.surf.samples = params.stencil.surf.samples; params.dst.surf.logical_level0_px = params.stencil.surf.logical_level0_px; params.dst.view = params.stencil.view; params.num_samples = params.stencil.surf.samples; /* We may be restricted on the number of layers we can bind at any * one time. In particular, Sandy Bridge has a maximum number of * layers of 512 but a maximum 3D texture size is much larger. */ if (params.stencil.view.array_len < params.num_layers) params.num_layers = params.stencil.view.array_len; } if (clear_depth) { blorp_surface_info_init(batch, ¶ms.depth, depth, level, start_layer, ISL_FORMAT_UNSUPPORTED, true); params.z = depth_value; params.depth_format = isl_format_get_depth_format(depth->surf->format, false); params.dst.surf.samples = params.depth.surf.samples; params.dst.surf.logical_level0_px = params.depth.surf.logical_level0_px; params.dst.view = params.depth.view; params.num_samples = params.depth.surf.samples; /* We may be restricted on the number of layers we can bind at any * one time. In particular, Sandy Bridge has a maximum number of * layers of 512 but a maximum 3D texture size is much larger. */ if (params.depth.view.array_len < params.num_layers) params.num_layers = params.depth.view.array_len; } batch->blorp->exec(batch, ¶ms); start_layer += params.num_layers; num_layers -= params.num_layers; } } static bool blorp_can_clear_full_surface(const struct blorp_surf *depth, const struct blorp_surf *stencil, uint32_t level, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, bool clear_depth, bool clear_stencil) { uint32_t width = 0, height = 0; if (clear_stencil) { width = u_minify(stencil->surf->logical_level0_px.width, level); height = u_minify(stencil->surf->logical_level0_px.height, level); } if (clear_depth && !(width || height)) { width = u_minify(depth->surf->logical_level0_px.width, level); height = u_minify(depth->surf->logical_level0_px.height, level); } return x0 == 0 && y0 == 0 && width == x1 && height == y1; } void blorp_hiz_clear_depth_stencil(struct blorp_batch *batch, const struct blorp_surf *depth, const struct blorp_surf *stencil, uint32_t level, uint32_t start_layer, uint32_t num_layers, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, bool clear_depth, float depth_value, bool clear_stencil, uint8_t stencil_value) { struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_HIZ_CLEAR; /* This requires WM_HZ_OP which only exists on gfx8+ */ assert(ISL_GFX_VER(batch->blorp->isl_dev) >= 8); params.hiz_op = ISL_AUX_OP_FAST_CLEAR; /* From BSpec: 3DSTATE_WM_HZ_OP_BODY >> Full Surface Depth and Stencil Clear * * "Software must set this only when the APP requires the entire Depth * surface to be cleared." */ params.full_surface_hiz_op = blorp_can_clear_full_surface(depth, stencil, level, x0, y0, x1, y1, clear_depth, clear_stencil); params.num_layers = 1; params.x0 = x0; params.y0 = y0; params.x1 = x1; params.y1 = y1; for (uint32_t l = 0; l < num_layers; l++) { const uint32_t layer = start_layer + l; if (clear_stencil) { blorp_surface_info_init(batch, ¶ms.stencil, stencil, level, layer, ISL_FORMAT_UNSUPPORTED, true); params.stencil_mask = 0xff; params.stencil_ref = stencil_value; params.num_samples = params.stencil.surf.samples; } if (clear_depth) { /* If we're clearing depth, we must have HiZ */ assert(depth && isl_aux_usage_has_hiz(depth->aux_usage)); blorp_surface_info_init(batch, ¶ms.depth, depth, level, layer, ISL_FORMAT_UNSUPPORTED, true); params.depth.clear_color.f32[0] = depth_value; params.depth_format = isl_format_get_depth_format(depth->surf->format, false); params.num_samples = params.depth.surf.samples; } batch->blorp->exec(batch, ¶ms); } } /* Given a depth stencil attachment, this function performs a fast depth clear * on a depth portion and a regular clear on the stencil portion. When * performing a fast depth clear on the depth portion, the HiZ buffer is simply * tagged as cleared so the depth clear value is not actually needed. */ void blorp_gfx8_hiz_clear_attachments(struct blorp_batch *batch, uint32_t num_samples, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, bool clear_depth, bool clear_stencil, uint8_t stencil_value) { assert(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL); struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_HIZ_CLEAR; params.num_layers = 1; params.hiz_op = ISL_AUX_OP_FAST_CLEAR; params.x0 = x0; params.y0 = y0; params.x1 = x1; params.y1 = y1; params.num_samples = num_samples; params.depth.enabled = clear_depth; params.stencil.enabled = clear_stencil; params.stencil_ref = stencil_value; batch->blorp->exec(batch, ¶ms); } /** Clear active color/depth/stencili attachments * * This function performs a clear operation on the currently bound * color/depth/stencil attachments. It is assumed that any information passed * in here is valid, consistent, and in-bounds relative to the currently * attached depth/stencil. The binding_table_offset parameter is the 32-bit * offset relative to surface state base address where pre-baked binding table * that we are to use lives. If clear_color is false, binding_table_offset * must point to a binding table with one entry which is a valid null surface * that matches the currently bound depth and stencil. */ void blorp_clear_attachments(struct blorp_batch *batch, uint32_t binding_table_offset, enum isl_format depth_format, uint32_t num_samples, uint32_t start_layer, uint32_t num_layers, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, bool clear_color, union isl_color_value color_value, bool clear_depth, float depth_value, uint8_t stencil_mask, uint8_t stencil_value) { struct blorp_params params; blorp_params_init(¶ms); assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0); assert(batch->flags & BLORP_BATCH_NO_EMIT_DEPTH_STENCIL); params.x0 = x0; params.y0 = y0; params.x1 = x1; params.y1 = y1; params.use_pre_baked_binding_table = true; params.pre_baked_binding_table_offset = binding_table_offset; params.num_layers = num_layers; params.num_samples = num_samples; if (clear_color) { params.dst.enabled = true; params.op = BLORP_OP_SLOW_COLOR_CLEAR; memcpy(¶ms.wm_inputs.clear_color, color_value.f32, sizeof(float) * 4); /* Unfortunately, without knowing whether or not our destination surface * is tiled or not, we have to assume it may be linear. This means no * SIMD16_REPDATA for us. :-( */ if (!blorp_params_get_clear_kernel(batch, ¶ms, false, false)) return; } if (clear_depth) { params.depth.enabled = true; params.op = BLORP_OP_SLOW_DEPTH_CLEAR; params.z = depth_value; params.depth_format = isl_format_get_depth_format(depth_format, false); } if (stencil_mask) { params.stencil.enabled = true; params.op = BLORP_OP_SLOW_DEPTH_CLEAR; params.stencil_mask = stencil_mask; params.stencil_ref = stencil_value; } if (!blorp_params_get_layer_offset_vs(batch, ¶ms)) return; params.vs_inputs.base_layer = start_layer; batch->blorp->exec(batch, ¶ms); } void blorp_ccs_resolve(struct blorp_batch *batch, struct blorp_surf *surf, uint32_t level, uint32_t start_layer, uint32_t num_layers, enum isl_format format, enum isl_aux_op resolve_op) { assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0); struct blorp_params params; blorp_params_init(¶ms); switch(resolve_op) { case ISL_AUX_OP_AMBIGUATE: params.op = BLORP_OP_CCS_AMBIGUATE; break; case ISL_AUX_OP_FULL_RESOLVE: params.op = BLORP_OP_CCS_RESOLVE; break; case ISL_AUX_OP_PARTIAL_RESOLVE: params.op = BLORP_OP_CCS_PARTIAL_RESOLVE; break; default: assert(false); } blorp_surface_info_init(batch, ¶ms.dst, surf, level, start_layer, format, true); params.x0 = params.y0 = 0; params.x1 = u_minify(params.dst.surf.logical_level0_px.width, level); params.y1 = u_minify(params.dst.surf.logical_level0_px.height, level); if (ISL_GFX_VER(batch->blorp->isl_dev) >= 9) { /* From Bspec 2424, "Render Target Resolve": * * The Resolve Rectangle size is same as Clear Rectangle size from * SKL+. * * Note that this differs from Vol7 of the Sky Lake PRM, which only * specifies aligning by the scaledown factors. */ get_fast_clear_rect(batch->blorp->isl_dev, surf->surf, surf->aux_surf, ¶ms.x0, ¶ms.y0, ¶ms.x1, ¶ms.y1); } else { /* From the Ivy Bridge PRM, Vol2 Part1 11.9 "Render Target Resolve": * * A rectangle primitive must be scaled down by the following factors * with respect to render target being resolved. * * The scaledown factors in the table that follows are related to the * block size of the CCS format. For IVB and HSW, we divide by two, for * BDW we multiply by 8 and 16. */ const struct isl_format_layout *aux_fmtl = isl_format_get_layout(params.dst.aux_surf.format); assert(aux_fmtl->txc == ISL_TXC_CCS); unsigned x_scaledown, y_scaledown; if (ISL_GFX_VER(batch->blorp->isl_dev) >= 8) { x_scaledown = aux_fmtl->bw * 8; y_scaledown = aux_fmtl->bh * 16; } else { x_scaledown = aux_fmtl->bw / 2; y_scaledown = aux_fmtl->bh / 2; } params.x1 = ALIGN(params.x1, x_scaledown) / x_scaledown; params.y1 = ALIGN(params.y1, y_scaledown) / y_scaledown; } if (batch->blorp->isl_dev->info->ver >= 10) { assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE || resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE || resolve_op == ISL_AUX_OP_AMBIGUATE); } else if (batch->blorp->isl_dev->info->ver >= 9) { assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE || resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE); } else { /* Broadwell and earlier do not have a partial resolve */ assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE); } params.fast_clear_op = resolve_op; params.num_layers = num_layers; /* Note: there is no need to initialize push constants because it doesn't * matter what data gets dispatched to the render target. However, we must * ensure that the fragment shader delivers the data using the "replicated * color" message. */ if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false)) return; batch->blorp->exec(batch, ¶ms); if (batch->blorp->isl_dev->info->ver <= 8) { assert(surf->aux_usage == ISL_AUX_USAGE_CCS_D); assert(resolve_op == ISL_AUX_OP_FULL_RESOLVE); /* ISL's state-machine of CCS_D describes full resolves as leaving the * aux buffer in the pass-through state. Hardware doesn't behave this * way on Broadwell however. On that platform, full resolves transition * the aux buffer to the resolved state. We assume that gfx7 behaves the * same. Use an ambiguate to match driver expectations. */ for (int l = 0; l < num_layers; l++) blorp_ccs_ambiguate(batch, surf, level, start_layer + l); } } static nir_def * blorp_nir_bit(nir_builder *b, nir_def *src, unsigned bit) { return nir_iand_imm(b, nir_ushr_imm(b, src, bit), 1); } #pragma pack(push, 1) struct blorp_mcs_partial_resolve_key { struct blorp_base_key base; bool indirect_clear_color; bool int_format; uint32_t num_samples; }; #pragma pack(pop) static bool blorp_params_get_mcs_partial_resolve_kernel(struct blorp_batch *batch, struct blorp_params *params) { struct blorp_context *blorp = batch->blorp; const struct blorp_mcs_partial_resolve_key blorp_key = { .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_MCS_PARTIAL_RESOLVE), .indirect_clear_color = params->dst.clear_color_addr.buffer != NULL, .int_format = isl_format_has_int_channel(params->dst.view.format), .num_samples = params->num_samples, }; if (blorp->lookup_shader(batch, &blorp_key, sizeof(blorp_key), ¶ms->wm_prog_kernel, ¶ms->wm_prog_data)) return true; void *mem_ctx = ralloc_context(NULL); nir_builder b; blorp_nir_init_shader(&b, blorp, mem_ctx, MESA_SHADER_FRAGMENT, blorp_shader_type_to_name(blorp_key.base.shader_type)); nir_variable *v_color = BLORP_CREATE_NIR_INPUT(b.shader, clear_color, glsl_vec4_type()); nir_variable *frag_color = nir_variable_create(b.shader, nir_var_shader_out, glsl_vec4_type(), "gl_FragColor"); frag_color->data.location = FRAG_RESULT_COLOR; /* Do an MCS fetch and check if it is equal to the magic clear value */ nir_def *mcs = blorp_nir_txf_ms_mcs(&b, nir_f2i32(&b, nir_load_frag_coord(&b)), nir_load_layer_id(&b)); nir_def *is_clear = blorp_nir_mcs_is_clear_color(&b, mcs, blorp_key.num_samples); /* If we aren't the clear value, discard. */ nir_discard_if(&b, nir_inot(&b, is_clear)); nir_def *clear_color = nir_load_var(&b, v_color); if (blorp_key.indirect_clear_color && blorp->isl_dev->info->ver <= 8) { /* Gfx7-8 clear colors are stored as single 0/1 bits */ clear_color = nir_vec4(&b, blorp_nir_bit(&b, clear_color, 31), blorp_nir_bit(&b, clear_color, 30), blorp_nir_bit(&b, clear_color, 29), blorp_nir_bit(&b, clear_color, 28)); if (!blorp_key.int_format) clear_color = nir_i2f32(&b, clear_color); } nir_store_var(&b, frag_color, clear_color, 0xf); const bool multisample_fbo = true; const struct blorp_program p = blorp_compile_fs(blorp, mem_ctx, b.shader, multisample_fbo, false); bool result = blorp->upload_shader(batch, MESA_SHADER_FRAGMENT, &blorp_key, sizeof(blorp_key), p.kernel, p.kernel_size, p.prog_data, p.prog_data_size, ¶ms->wm_prog_kernel, ¶ms->wm_prog_data); ralloc_free(mem_ctx); return result; } void blorp_mcs_partial_resolve(struct blorp_batch *batch, struct blorp_surf *surf, enum isl_format format, uint32_t start_layer, uint32_t num_layers) { struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_MCS_PARTIAL_RESOLVE; assert(batch->blorp->isl_dev->info->ver >= 7); params.x0 = 0; params.y0 = 0; params.x1 = surf->surf->logical_level0_px.width; params.y1 = surf->surf->logical_level0_px.height; blorp_surface_info_init(batch, ¶ms.src, surf, 0, start_layer, format, false); blorp_surface_info_init(batch, ¶ms.dst, surf, 0, start_layer, format, true); params.num_samples = params.dst.surf.samples; params.num_layers = num_layers; params.dst_clear_color_as_input = surf->clear_color_addr.buffer != NULL; memcpy(¶ms.wm_inputs.clear_color, surf->clear_color.f32, sizeof(float) * 4); if (!blorp_params_get_mcs_partial_resolve_kernel(batch, ¶ms)) return; batch->blorp->exec(batch, ¶ms); } static uint64_t get_mcs_ambiguate_pixel(int sample_count) { /* See the Broadwell PRM, Volume 5 "Memory Views", Section "Compressed * Multisample Surfaces". */ assert(sample_count >= 2); assert(sample_count <= 16); /* Each MCS element contains an array of sample slice (SS) elements. The * size of this array matches the sample count. */ const int num_ss_entries = sample_count; /* The width of each SS entry is just large enough to index every slice. */ const int ss_entry_size_b = util_logbase2(num_ss_entries); /* The encoding for "ambiguated" has each sample slice value storing its * index (e.g., SS[0] = 0, SS[1] = 1, etc.). The values are stored in * little endian order. The unused bits are defined as either Reserved or * Reserved (MBZ). We choose to interpret both as MBZ. */ uint64_t ambiguate_pixel = 0; for (uint64_t entry = 0; entry < num_ss_entries; entry++) ambiguate_pixel |= entry << (entry * ss_entry_size_b); return ambiguate_pixel; } /** Clear an MCS to the "uncompressed" state * * This pass is the MCS equivalent of a "HiZ resolve". It sets the MCS values * for a given layer of a surface to a sample-count dependent value which is * the "uncompressed" state which tells the sampler to go look at the main * surface. */ void blorp_mcs_ambiguate(struct blorp_batch *batch, struct blorp_surf *surf, uint32_t start_layer, uint32_t num_layers) { assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0); struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_MCS_AMBIGUATE; assert(ISL_GFX_VER(batch->blorp->isl_dev) >= 7); enum isl_format renderable_format; switch (isl_format_get_layout(surf->aux_surf->format)->bpb) { case 8: renderable_format = ISL_FORMAT_R8_UINT; break; case 32: renderable_format = ISL_FORMAT_R32_UINT; break; case 64: renderable_format = ISL_FORMAT_R32G32_UINT; break; default: unreachable("Unexpected MCS format size for ambiguate"); } /* From Bspec 57340 (r59562): * * To the calculated MCS size we add 4kb page to be used as clear value * storage. * * and * * When allocating memory, MCS buffer size is extended by 4KB over its * original calculated size. First 4KB page of the MCS is reserved for * internal HW usage. * * We shift aux buffer's start address by 4KB, accordingly. */ struct blorp_address aux_addr = surf->aux_addr; if (ISL_GFX_VER(batch->blorp->isl_dev) >= 20) aux_addr.offset += 4096; params.dst = (struct blorp_surface_info) { .enabled = true, .surf = *surf->aux_surf, .addr = aux_addr, .view = { .usage = ISL_SURF_USAGE_RENDER_TARGET_BIT, .format = renderable_format, .base_level = 0, .base_array_layer = start_layer, .levels = 1, .array_len = num_layers, .swizzle = ISL_SWIZZLE_IDENTITY, }, }; params.x0 = 0; params.y0 = 0; params.x1 = params.dst.surf.logical_level0_px.width; params.y1 = params.dst.surf.logical_level0_px.height; params.num_layers = params.dst.view.array_len; const uint64_t pixel = get_mcs_ambiguate_pixel(surf->surf->samples); params.wm_inputs.clear_color[0] = pixel & 0xFFFFFFFF; params.wm_inputs.clear_color[1] = pixel >> 32; if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false)) return; batch->blorp->exec(batch, ¶ms); } /** Clear a CCS to the "uncompressed" state * * This pass is the CCS equivalent of a "HiZ resolve". It sets the CCS values * for a given layer/level of a surface to 0x0 which is the "uncompressed" * state which tells the sampler to go look at the main surface. */ void blorp_ccs_ambiguate(struct blorp_batch *batch, struct blorp_surf *surf, uint32_t level, uint32_t layer) { assert((batch->flags & BLORP_BATCH_USE_COMPUTE) == 0); if (ISL_GFX_VER(batch->blorp->isl_dev) >= 10) { /* On gfx10 and above, we have a hardware resolve op for this */ return blorp_ccs_resolve(batch, surf, level, layer, 1, surf->surf->format, ISL_AUX_OP_AMBIGUATE); } struct blorp_params params; blorp_params_init(¶ms); params.op = BLORP_OP_CCS_AMBIGUATE; assert(ISL_GFX_VER(batch->blorp->isl_dev) >= 7); const struct isl_format_layout *aux_fmtl = isl_format_get_layout(surf->aux_surf->format); assert(aux_fmtl->txc == ISL_TXC_CCS); params.dst = (struct blorp_surface_info) { .enabled = true, .addr = surf->aux_addr, .view = { .usage = ISL_SURF_USAGE_RENDER_TARGET_BIT, .format = ISL_FORMAT_R32G32B32A32_UINT, .base_level = 0, .base_array_layer = 0, .levels = 1, .array_len = 1, .swizzle = ISL_SWIZZLE_IDENTITY, }, }; uint32_t z = 0; if (surf->surf->dim == ISL_SURF_DIM_3D) { z = layer; layer = 0; } uint64_t offset_B; uint32_t x_offset_el, y_offset_el; isl_surf_get_image_offset_B_tile_el(surf->aux_surf, level, layer, z, &offset_B, &x_offset_el, &y_offset_el); params.dst.addr.offset += offset_B; const uint32_t width_px = u_minify(surf->aux_surf->logical_level0_px.width, level); const uint32_t height_px = u_minify(surf->aux_surf->logical_level0_px.height, level); const uint32_t width_el = DIV_ROUND_UP(width_px, aux_fmtl->bw); const uint32_t height_el = DIV_ROUND_UP(height_px, aux_fmtl->bh); struct isl_tile_info ccs_tile_info; isl_surf_get_tile_info(surf->aux_surf, &ccs_tile_info); /* We're going to map it as a regular RGBA32_UINT surface. We need to * downscale a good deal. We start by computing the area on the CCS to * clear in units of Y-tiled cache lines. */ uint32_t x_offset_cl, y_offset_cl, width_cl, height_cl; if (ISL_GFX_VER(batch->blorp->isl_dev) >= 8) { /* From the Sky Lake PRM Vol. 12 in the section on planes: * * "The Color Control Surface (CCS) contains the compression status * of the cache-line pairs. The compression state of the cache-line * pair is specified by 2 bits in the CCS. Each CCS cache-line * represents an area on the main surface of 16x16 sets of 128 byte * Y-tiled cache-line-pairs. CCS is always Y tiled." * * Each 2-bit surface element in the CCS corresponds to a single * cache-line pair in the main surface. This means that 16x16 el block * in the CCS maps to a Y-tiled cache line. Fortunately, CCS layouts * are calculated with a very large alignment so we can round up to a * whole cache line without worrying about overdraw. */ /* On Broadwell and above, a CCS tile is the same as a Y tile when * viewed at the cache-line granularity. Fortunately, the horizontal * and vertical alignment requirements of the CCS are such that we can * align to an entire cache line without worrying about crossing over * from one LOD to another. */ const uint32_t x_el_per_cl = ccs_tile_info.logical_extent_el.w / 8; const uint32_t y_el_per_cl = ccs_tile_info.logical_extent_el.h / 8; assert(surf->aux_surf->image_alignment_el.w % x_el_per_cl == 0); assert(surf->aux_surf->image_alignment_el.h % y_el_per_cl == 0); assert(x_offset_el % x_el_per_cl == 0); assert(y_offset_el % y_el_per_cl == 0); x_offset_cl = x_offset_el / x_el_per_cl; y_offset_cl = y_offset_el / y_el_per_cl; width_cl = DIV_ROUND_UP(width_el, x_el_per_cl); height_cl = DIV_ROUND_UP(height_el, y_el_per_cl); } else { /* On gfx7, the CCS tiling is not so nice. However, there we are * guaranteed that we only have a single level and slice so we don't * have to worry about it and can just align to a whole tile. */ assert(surf->aux_surf->logical_level0_px.depth == 1); assert(surf->aux_surf->logical_level0_px.array_len == 1); assert(x_offset_el == 0 && y_offset_el == 0); const uint32_t width_tl = DIV_ROUND_UP(width_el, ccs_tile_info.logical_extent_el.w); const uint32_t height_tl = DIV_ROUND_UP(height_el, ccs_tile_info.logical_extent_el.h); x_offset_cl = 0; y_offset_cl = 0; width_cl = width_tl * 8; height_cl = height_tl * 8; } /* We're going to use a RGBA32 format so as to write data as quickly as * possible. A y-tiled cache line will then be 1x4 px. */ const uint32_t x_offset_rgba_px = x_offset_cl; const uint32_t y_offset_rgba_px = y_offset_cl * 4; const uint32_t width_rgba_px = width_cl; const uint32_t height_rgba_px = height_cl * 4; ASSERTED bool ok = isl_surf_init(batch->blorp->isl_dev, ¶ms.dst.surf, .dim = ISL_SURF_DIM_2D, .format = ISL_FORMAT_R32G32B32A32_UINT, .width = width_rgba_px + x_offset_rgba_px, .height = height_rgba_px + y_offset_rgba_px, .depth = 1, .levels = 1, .array_len = 1, .samples = 1, .row_pitch_B = surf->aux_surf->row_pitch_B, .usage = ISL_SURF_USAGE_RENDER_TARGET_BIT, .tiling_flags = ISL_TILING_Y0_BIT); assert(ok); params.x0 = x_offset_rgba_px; params.y0 = y_offset_rgba_px; params.x1 = x_offset_rgba_px + width_rgba_px; params.y1 = y_offset_rgba_px + height_rgba_px; /* A CCS value of 0 means "uncompressed." */ memset(¶ms.wm_inputs.clear_color, 0, sizeof(params.wm_inputs.clear_color)); if (!blorp_params_get_clear_kernel(batch, ¶ms, true, false)) return; batch->blorp->exec(batch, ¶ms); }