/* * Copyright © 2021 Raspberry Pi Ltd * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include "v3dv_private.h" #include "broadcom/common/v3d_macros.h" #include "broadcom/common/v3d_util.h" #include "broadcom/cle/v3dx_pack.h" #include "broadcom/compiler/v3d_compiler.h" #include "util/half_float.h" #include "util/u_pack_color.h" #include "vk_format.h" void v3dX(job_emit_binning_flush)(struct v3dv_job *job) { assert(job); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(FLUSH)); v3dv_return_if_oom(NULL, job); cl_emit(&job->bcl, FLUSH, flush); } void v3dX(job_emit_enable_double_buffer)(struct v3dv_job *job) { assert(job->can_use_double_buffer); assert(job->frame_tiling.double_buffer); assert(!job->frame_tiling.msaa); assert(job->bcl_tile_binning_mode_ptr); const struct v3dv_frame_tiling *tiling = &job->frame_tiling; struct cl_packet_struct(TILE_BINNING_MODE_CFG) config = { cl_packet_header(TILE_BINNING_MODE_CFG), }; config.width_in_pixels = tiling->width; config.height_in_pixels = tiling->height; #if V3D_VERSION == 42 config.number_of_render_targets = MAX2(tiling->render_target_count, 1); config.multisample_mode_4x = tiling->msaa; config.double_buffer_in_non_ms_mode = tiling->double_buffer; config.maximum_bpp_of_all_render_targets = tiling->internal_bpp; #endif #if V3D_VERSION >= 71 unreachable("HW generation 71 not supported yet."); #endif uint8_t *rewrite_addr = (uint8_t *)job->bcl_tile_binning_mode_ptr; cl_packet_pack(TILE_BINNING_MODE_CFG)(NULL, rewrite_addr, &config); } void v3dX(job_emit_binning_prolog)(struct v3dv_job *job, const struct v3dv_frame_tiling *tiling, uint32_t layers) { /* This must go before the binning mode configuration. It is * required for layered framebuffers to work. */ cl_emit(&job->bcl, NUMBER_OF_LAYERS, config) { config.number_of_layers = layers; } assert(!tiling->double_buffer || !tiling->msaa); job->bcl_tile_binning_mode_ptr = cl_start(&job->bcl); cl_emit(&job->bcl, TILE_BINNING_MODE_CFG, config) { config.width_in_pixels = tiling->width; config.height_in_pixels = tiling->height; #if V3D_VERSION == 42 config.number_of_render_targets = MAX2(tiling->render_target_count, 1); config.multisample_mode_4x = tiling->msaa; config.double_buffer_in_non_ms_mode = tiling->double_buffer; config.maximum_bpp_of_all_render_targets = tiling->internal_bpp; #endif #if V3D_VERSION >= 71 config.log2_tile_width = log2_tile_size(tiling->tile_width); config.log2_tile_height = log2_tile_size(tiling->tile_height); /* FIXME: ideally we would like next assert on the packet header (as is * general, so also applies to GL). We would need to expand * gen_pack_header for that. */ assert(config.log2_tile_width == config.log2_tile_height || config.log2_tile_width == config.log2_tile_height + 1); #endif } /* There's definitely nothing in the VCD cache we want. */ cl_emit(&job->bcl, FLUSH_VCD_CACHE, bin); /* "Binning mode lists must have a Start Tile Binning item (6) after * any prefix state data before the binning list proper starts." */ cl_emit(&job->bcl, START_TILE_BINNING, bin); } void v3dX(cmd_buffer_end_render_pass_secondary)(struct v3dv_cmd_buffer *cmd_buffer) { assert(cmd_buffer->state.job); v3dv_cl_ensure_space_with_branch(&cmd_buffer->state.job->bcl, cl_packet_length(RETURN_FROM_SUB_LIST)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&cmd_buffer->state.job->bcl, RETURN_FROM_SUB_LIST, ret); } void v3dX(job_emit_clip_window)(struct v3dv_job *job, const VkRect2D *rect) { assert(job); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(CLIP_WINDOW)); v3dv_return_if_oom(NULL, job); cl_emit(&job->bcl, CLIP_WINDOW, clip) { clip.clip_window_left_pixel_coordinate = rect->offset.x; clip.clip_window_bottom_pixel_coordinate = rect->offset.y; clip.clip_window_width_in_pixels = rect->extent.width; clip.clip_window_height_in_pixels = rect->extent.height; } } static void cmd_buffer_render_pass_emit_load(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_cl *cl, struct v3dv_image_view *iview, uint32_t layer, uint32_t buffer) { const struct v3dv_image *image = (struct v3dv_image *) iview->vk.image; /* We don't support rendering to ycbcr images, so the image view should be * single-plane, and using a single-plane format. But note that the underlying * image can be a ycbcr format, as we support rendering to a specific plane * of an image. This is used for example on some meta_copy code paths, in * order to copy from/to a plane of a ycbcr image. */ assert(iview->plane_count == 1); assert(iview->format->plane_count == 1); uint8_t image_plane = v3dv_plane_from_aspect(iview->vk.aspects); const struct v3d_resource_slice *slice = &image->planes[image_plane].slices[iview->vk.base_mip_level]; uint32_t layer_offset = v3dv_layer_offset(image, iview->vk.base_mip_level, iview->vk.base_array_layer + layer, image_plane); cl_emit(cl, LOAD_TILE_BUFFER_GENERAL, load) { load.buffer_to_load = buffer; load.address = v3dv_cl_address(image->planes[image_plane].mem->bo, layer_offset); load.input_image_format = iview->format->planes[0].rt_type; /* If we create an image view with only the stencil format, we * re-interpret the format as RGBA8_UINT, as it is want we want in * general (see CreateImageView). * * However, when we are loading/storing tiles from the ZSTENCIL tile * buffer, we need to use the underlying DS format. */ if (buffer == ZSTENCIL && iview->format->planes[0].rt_type == V3D_OUTPUT_IMAGE_FORMAT_RGBA8UI) { assert(image->format->planes[image_plane].rt_type == V3D_OUTPUT_IMAGE_FORMAT_D24S8); load.input_image_format = image->format->planes[image_plane].rt_type; } load.r_b_swap = iview->planes[0].swap_rb; load.channel_reverse = iview->planes[0].channel_reverse; load.memory_format = slice->tiling; if (slice->tiling == V3D_TILING_UIF_NO_XOR || slice->tiling == V3D_TILING_UIF_XOR) { load.height_in_ub_or_stride = slice->padded_height_of_output_image_in_uif_blocks; } else if (slice->tiling == V3D_TILING_RASTER) { load.height_in_ub_or_stride = slice->stride; } if (image->vk.samples > VK_SAMPLE_COUNT_1_BIT) load.decimate_mode = V3D_DECIMATE_MODE_ALL_SAMPLES; else load.decimate_mode = V3D_DECIMATE_MODE_SAMPLE_0; } } static inline uint32_t v3dv_zs_buffer(bool depth, bool stencil) { if (depth && stencil) return ZSTENCIL; else if (depth) return Z; else if (stencil) return STENCIL; return NONE; } static void cmd_buffer_render_pass_emit_loads(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_cl *cl, uint32_t layer) { const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; const struct v3dv_render_pass *pass = state->pass; const struct v3dv_subpass *subpass = &pass->subpasses[state->subpass_idx]; assert(!pass->multiview_enabled || layer < MAX_MULTIVIEW_VIEW_COUNT); for (uint32_t i = 0; i < subpass->color_count; i++) { uint32_t attachment_idx = subpass->color_attachments[i].attachment; if (attachment_idx == VK_ATTACHMENT_UNUSED) continue; const struct v3dv_render_pass_attachment *attachment = &state->pass->attachments[attachment_idx]; /* According to the Vulkan spec: * * "The load operation for each sample in an attachment happens before * any recorded command which accesses the sample in the first subpass * where the attachment is used." * * If the load operation is CLEAR, we must only clear once on the first * subpass that uses the attachment (and in that case we don't LOAD). * After that, we always want to load so we don't lose any rendering done * by a previous subpass to the same attachment. We also want to load * if the current job is continuing subpass work started by a previous * job, for the same reason. * * If the render area is not aligned to tile boundaries then we have * tiles which are partially covered by it. In this case, we need to * load the tiles so we can preserve the pixels that are outside the * render area for any such tiles. */ uint32_t first_subpass = !pass->multiview_enabled ? attachment->first_subpass : attachment->views[layer].first_subpass; uint32_t last_subpass = !pass->multiview_enabled ? attachment->last_subpass : attachment->views[layer].last_subpass; bool needs_load = v3dv_cmd_buffer_check_needs_load(state, VK_IMAGE_ASPECT_COLOR_BIT, first_subpass, attachment->desc.loadOp, last_subpass, attachment->desc.storeOp); if (needs_load) { struct v3dv_image_view *iview = state->attachments[attachment_idx].image_view; cmd_buffer_render_pass_emit_load(cmd_buffer, cl, iview, layer, RENDER_TARGET_0 + i); } } uint32_t ds_attachment_idx = subpass->ds_attachment.attachment; if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { const struct v3dv_render_pass_attachment *ds_attachment = &state->pass->attachments[ds_attachment_idx]; const VkImageAspectFlags ds_aspects = vk_format_aspects(ds_attachment->desc.format); uint32_t ds_first_subpass = !pass->multiview_enabled ? ds_attachment->first_subpass : ds_attachment->views[layer].first_subpass; uint32_t ds_last_subpass = !pass->multiview_enabled ? ds_attachment->last_subpass : ds_attachment->views[layer].last_subpass; const bool needs_depth_load = v3dv_cmd_buffer_check_needs_load(state, ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT, ds_first_subpass, ds_attachment->desc.loadOp, ds_last_subpass, ds_attachment->desc.storeOp); const bool needs_stencil_load = v3dv_cmd_buffer_check_needs_load(state, ds_aspects & VK_IMAGE_ASPECT_STENCIL_BIT, ds_first_subpass, ds_attachment->desc.stencilLoadOp, ds_last_subpass, ds_attachment->desc.stencilStoreOp); if (needs_depth_load || needs_stencil_load) { struct v3dv_image_view *iview = state->attachments[ds_attachment_idx].image_view; /* From the Vulkan spec: * * "When an image view of a depth/stencil image is used as a * depth/stencil framebuffer attachment, the aspectMask is ignored * and both depth and stencil image subresources are used." * * So we ignore the aspects from the subresource range of the image * view for the depth/stencil attachment, but we still need to restrict * the to aspects compatible with the render pass and the image. */ const uint32_t zs_buffer = v3dv_zs_buffer(needs_depth_load, needs_stencil_load); cmd_buffer_render_pass_emit_load(cmd_buffer, cl, iview, layer, zs_buffer); } } cl_emit(cl, END_OF_LOADS, end); } static void cmd_buffer_render_pass_emit_store(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_cl *cl, uint32_t attachment_idx, uint32_t layer, uint32_t buffer, bool clear, bool is_multisample_resolve) { const struct v3dv_image_view *iview = cmd_buffer->state.attachments[attachment_idx].image_view; const struct v3dv_image *image = (struct v3dv_image *) iview->vk.image; /* We don't support rendering to ycbcr images, so the image view should be * one-plane, and using a single-plane format. But note that the underlying * image can be a ycbcr format, as we support rendering to a specific plane * of an image. This is used for example on some meta_copy code paths, in * order to copy from/to a plane of a ycbcr image. */ assert(iview->plane_count == 1); assert(iview->format->plane_count == 1); uint8_t image_plane = v3dv_plane_from_aspect(iview->vk.aspects); const struct v3d_resource_slice *slice = &image->planes[image_plane].slices[iview->vk.base_mip_level]; uint32_t layer_offset = v3dv_layer_offset(image, iview->vk.base_mip_level, iview->vk.base_array_layer + layer, image_plane); /* The Clear Buffer bit is not supported for Z/Stencil stores in 7.x and it * is broken in earlier V3D versions. */ assert((buffer != Z && buffer != STENCIL && buffer != ZSTENCIL) || !clear); cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) { store.buffer_to_store = buffer; store.address = v3dv_cl_address(image->planes[image_plane].mem->bo, layer_offset); store.clear_buffer_being_stored = clear; store.output_image_format = iview->format->planes[0].rt_type; /* If we create an image view with only the stencil format, we * re-interpret the format as RGBA8_UINT, as it is want we want in * general (see CreateImageView). * * However, when we are loading/storing tiles from the ZSTENCIL tile * buffer, we need to use the underlying DS format. */ if (buffer == ZSTENCIL && iview->format->planes[0].rt_type == V3D_OUTPUT_IMAGE_FORMAT_RGBA8UI) { assert(image->format->planes[image_plane].rt_type == V3D_OUTPUT_IMAGE_FORMAT_D24S8); store.output_image_format = image->format->planes[image_plane].rt_type; } store.r_b_swap = iview->planes[0].swap_rb; store.channel_reverse = iview->planes[0].channel_reverse; store.memory_format = slice->tiling; if (slice->tiling == V3D_TILING_UIF_NO_XOR || slice->tiling == V3D_TILING_UIF_XOR) { store.height_in_ub_or_stride = slice->padded_height_of_output_image_in_uif_blocks; } else if (slice->tiling == V3D_TILING_RASTER) { store.height_in_ub_or_stride = slice->stride; } if (image->vk.samples > VK_SAMPLE_COUNT_1_BIT) store.decimate_mode = V3D_DECIMATE_MODE_ALL_SAMPLES; else if (is_multisample_resolve) store.decimate_mode = V3D_DECIMATE_MODE_4X; else store.decimate_mode = V3D_DECIMATE_MODE_SAMPLE_0; } } static bool check_needs_clear(const struct v3dv_cmd_buffer_state *state, VkImageAspectFlags aspect, uint32_t first_subpass_idx, VkAttachmentLoadOp load_op, bool do_clear_with_draw) { /* We call this with image->vk.aspects & aspect, so 0 means the aspect we are * testing does not exist in the image. */ if (!aspect) return false; /* If the aspect needs to be cleared with a draw call then we won't emit * the clear here. */ if (do_clear_with_draw) return false; /* If this is resuming a subpass started with another job, then attachment * load operations don't apply. */ if (state->job->is_subpass_continue) return false; /* If the render area is not aligned to tile boundaries we can't use the * TLB for a clear. */ if (!state->tile_aligned_render_area) return false; /* If this job is running in a subpass other than the first subpass in * which this attachment (or view) is used then attachment load operations * don't apply. */ if (state->job->first_subpass != first_subpass_idx) return false; /* The attachment load operation must be CLEAR */ return load_op == VK_ATTACHMENT_LOAD_OP_CLEAR; } static void cmd_buffer_render_pass_emit_stores(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_cl *cl, uint32_t layer) { struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; struct v3dv_render_pass *pass = state->pass; const struct v3dv_subpass *subpass = &pass->subpasses[state->subpass_idx]; bool has_stores = false; bool use_global_zs_clear = false; bool use_global_rt_clear = false; assert(!pass->multiview_enabled || layer < MAX_MULTIVIEW_VIEW_COUNT); /* FIXME: separate stencil */ uint32_t ds_attachment_idx = subpass->ds_attachment.attachment; if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { const struct v3dv_render_pass_attachment *ds_attachment = &state->pass->attachments[ds_attachment_idx]; assert(state->job->first_subpass >= ds_attachment->first_subpass); assert(state->subpass_idx >= ds_attachment->first_subpass); assert(state->subpass_idx <= ds_attachment->last_subpass); /* From the Vulkan spec, VkImageSubresourceRange: * * "When an image view of a depth/stencil image is used as a * depth/stencil framebuffer attachment, the aspectMask is ignored * and both depth and stencil image subresources are used." * * So we ignore the aspects from the subresource range of the image * view for the depth/stencil attachment, but we still need to restrict * the to aspects compatible with the render pass and the image. */ const VkImageAspectFlags aspects = vk_format_aspects(ds_attachment->desc.format); #if V3D_VERSION <= 42 /* GFXH-1689: The per-buffer store command's clear buffer bit is broken * for depth/stencil. * * There used to be some confusion regarding the Clear Tile Buffers * Z/S bit also being broken, but we confirmed with Broadcom that this * is not the case, it was just that some other hardware bugs (that we * need to work around, such as GFXH-1461) could cause this bit to behave * incorrectly. * * There used to be another issue where the RTs bit in the Clear Tile * Buffers packet also cleared Z/S, but Broadcom confirmed this is * fixed since V3D 4.1. * * So if we have to emit a clear of depth or stencil we don't use * the per-buffer store clear bit, even if we need to store the buffers, * instead we always have to use the Clear Tile Buffers Z/S bit. * If we have configured the job to do early Z/S clearing, then we * don't want to emit any Clear Tile Buffers command at all here. * * Note that GFXH-1689 is not reproduced in the simulator, where * using the clear buffer bit in depth/stencil stores works fine. */ /* Only clear once on the first subpass that uses the attachment */ uint32_t ds_first_subpass = !state->pass->multiview_enabled ? ds_attachment->first_subpass : ds_attachment->views[layer].first_subpass; bool needs_depth_clear = check_needs_clear(state, aspects & VK_IMAGE_ASPECT_DEPTH_BIT, ds_first_subpass, ds_attachment->desc.loadOp, subpass->do_depth_clear_with_draw); bool needs_stencil_clear = check_needs_clear(state, aspects & VK_IMAGE_ASPECT_STENCIL_BIT, ds_first_subpass, ds_attachment->desc.stencilLoadOp, subpass->do_stencil_clear_with_draw); use_global_zs_clear = !state->job->early_zs_clear && (needs_depth_clear || needs_stencil_clear); #endif #if V3D_VERSION >= 71 /* The store command's clear buffer bit cannot be used for Z/S stencil: * since V3D 4.5.6 Z/S buffers are automatically cleared between tiles, * so we don't want to emit redundant clears here. */ use_global_zs_clear = false; #endif /* Skip the last store if it is not required */ uint32_t ds_last_subpass = !pass->multiview_enabled ? ds_attachment->last_subpass : ds_attachment->views[layer].last_subpass; bool needs_depth_store = v3dv_cmd_buffer_check_needs_store(state, aspects & VK_IMAGE_ASPECT_DEPTH_BIT, ds_last_subpass, ds_attachment->desc.storeOp); bool needs_stencil_store = v3dv_cmd_buffer_check_needs_store(state, aspects & VK_IMAGE_ASPECT_STENCIL_BIT, ds_last_subpass, ds_attachment->desc.stencilStoreOp); /* If we have a resolve, handle it before storing the tile */ const struct v3dv_cmd_buffer_attachment_state *ds_att_state = &state->attachments[ds_attachment_idx]; if (ds_att_state->use_tlb_resolve) { assert(ds_att_state->has_resolve); assert(subpass->resolve_depth || subpass->resolve_stencil); const uint32_t resolve_attachment_idx = subpass->ds_resolve_attachment.attachment; assert(resolve_attachment_idx != VK_ATTACHMENT_UNUSED); const uint32_t zs_buffer = v3dv_zs_buffer(subpass->resolve_depth, subpass->resolve_stencil); cmd_buffer_render_pass_emit_store(cmd_buffer, cl, resolve_attachment_idx, layer, zs_buffer, false, false); has_stores = true; } else if (ds_att_state->has_resolve) { /* If we can't use the TLB to implement the resolve we will need to * store the attachment so we can implement it later using a blit. */ needs_depth_store = subpass->resolve_depth; needs_stencil_store = subpass->resolve_stencil; } if (needs_depth_store || needs_stencil_store) { const uint32_t zs_buffer = v3dv_zs_buffer(needs_depth_store, needs_stencil_store); cmd_buffer_render_pass_emit_store(cmd_buffer, cl, ds_attachment_idx, layer, zs_buffer, false, false); has_stores = true; } } for (uint32_t i = 0; i < subpass->color_count; i++) { uint32_t attachment_idx = subpass->color_attachments[i].attachment; if (attachment_idx == VK_ATTACHMENT_UNUSED) continue; const struct v3dv_render_pass_attachment *attachment = &state->pass->attachments[attachment_idx]; assert(state->job->first_subpass >= attachment->first_subpass); assert(state->subpass_idx >= attachment->first_subpass); assert(state->subpass_idx <= attachment->last_subpass); /* Only clear once on the first subpass that uses the attachment */ uint32_t first_subpass = !pass->multiview_enabled ? attachment->first_subpass : attachment->views[layer].first_subpass; bool needs_clear = check_needs_clear(state, VK_IMAGE_ASPECT_COLOR_BIT, first_subpass, attachment->desc.loadOp, false); /* Skip the last store if it is not required */ uint32_t last_subpass = !pass->multiview_enabled ? attachment->last_subpass : attachment->views[layer].last_subpass; bool needs_store = v3dv_cmd_buffer_check_needs_store(state, VK_IMAGE_ASPECT_COLOR_BIT, last_subpass, attachment->desc.storeOp); /* If we need to resolve this attachment emit that store first. Notice * that we must not request a tile buffer clear here in that case, since * that would clear the tile buffer before we get to emit the actual * color attachment store below, since the clear happens after the * store is completed. * * If the attachment doesn't support TLB resolves (or the render area * is not aligned to tile boundaries) then we will have to fallback to * doing the resolve in a shader separately after this job, so we will * need to store the multisampled attachment even if that wasn't * requested by the client. */ const struct v3dv_cmd_buffer_attachment_state *att_state = &state->attachments[attachment_idx]; if (att_state->use_tlb_resolve) { assert(att_state->has_resolve); const uint32_t resolve_attachment_idx = subpass->resolve_attachments[i].attachment; cmd_buffer_render_pass_emit_store(cmd_buffer, cl, resolve_attachment_idx, layer, RENDER_TARGET_0 + i, false, true); has_stores = true; } else if (att_state->has_resolve) { needs_store = true; } /* Emit the color attachment store if needed */ if (needs_store) { cmd_buffer_render_pass_emit_store(cmd_buffer, cl, attachment_idx, layer, RENDER_TARGET_0 + i, needs_clear && !use_global_rt_clear, false); has_stores = true; } else if (needs_clear) { use_global_rt_clear = true; } } /* We always need to emit at least one dummy store */ if (!has_stores) { cl_emit(cl, STORE_TILE_BUFFER_GENERAL, store) { store.buffer_to_store = NONE; } } /* If we have any depth/stencil clears we can't use the per-buffer clear * bit and instead we have to emit a single clear of all tile buffers. */ if (use_global_zs_clear || use_global_rt_clear) { #if V3D_VERSION == 42 cl_emit(cl, CLEAR_TILE_BUFFERS, clear) { clear.clear_z_stencil_buffer = use_global_zs_clear; clear.clear_all_render_targets = use_global_rt_clear; } #endif #if V3D_VERSION >= 71 cl_emit(cl, CLEAR_RENDER_TARGETS, clear); #endif } } static void cmd_buffer_render_pass_emit_per_tile_rcl(struct v3dv_cmd_buffer *cmd_buffer, uint32_t layer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); /* Emit the generic list in our indirect state -- the rcl will just * have pointers into it. */ struct v3dv_cl *cl = &job->indirect; v3dv_cl_ensure_space(cl, 200, 1); v3dv_return_if_oom(cmd_buffer, NULL); struct v3dv_cl_reloc tile_list_start = v3dv_cl_get_address(cl); cl_emit(cl, TILE_COORDINATES_IMPLICIT, coords); cmd_buffer_render_pass_emit_loads(cmd_buffer, cl, layer); /* The binner starts out writing tiles assuming that the initial mode * is triangles, so make sure that's the case. */ cl_emit(cl, PRIM_LIST_FORMAT, fmt) { fmt.primitive_type = LIST_TRIANGLES; } /* PTB assumes that value to be 0, but hw will not set it. */ cl_emit(cl, SET_INSTANCEID, set) { set.instance_id = 0; } cl_emit(cl, BRANCH_TO_IMPLICIT_TILE_LIST, branch); cmd_buffer_render_pass_emit_stores(cmd_buffer, cl, layer); cl_emit(cl, END_OF_TILE_MARKER, end); cl_emit(cl, RETURN_FROM_SUB_LIST, ret); cl_emit(&job->rcl, START_ADDRESS_OF_GENERIC_TILE_LIST, branch) { branch.start = tile_list_start; branch.end = v3dv_cl_get_address(cl); } } static void cmd_buffer_emit_render_pass_layer_rcl(struct v3dv_cmd_buffer *cmd_buffer, uint32_t layer) { const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; struct v3dv_job *job = cmd_buffer->state.job; struct v3dv_cl *rcl = &job->rcl; /* If doing multicore binning, we would need to initialize each * core's tile list here. */ const struct v3dv_frame_tiling *tiling = &job->frame_tiling; const uint32_t tile_alloc_offset = 64 * layer * tiling->draw_tiles_x * tiling->draw_tiles_y; cl_emit(rcl, MULTICORE_RENDERING_TILE_LIST_SET_BASE, list) { list.address = v3dv_cl_address(job->tile_alloc, tile_alloc_offset); } cmd_buffer_render_pass_emit_per_tile_rcl(cmd_buffer, layer); uint32_t supertile_w_in_pixels = tiling->tile_width * tiling->supertile_width; uint32_t supertile_h_in_pixels = tiling->tile_height * tiling->supertile_height; const uint32_t min_x_supertile = state->render_area.offset.x / supertile_w_in_pixels; const uint32_t min_y_supertile = state->render_area.offset.y / supertile_h_in_pixels; uint32_t max_render_x = state->render_area.offset.x; if (state->render_area.extent.width > 0) max_render_x += state->render_area.extent.width - 1; uint32_t max_render_y = state->render_area.offset.y; if (state->render_area.extent.height > 0) max_render_y += state->render_area.extent.height - 1; const uint32_t max_x_supertile = max_render_x / supertile_w_in_pixels; const uint32_t max_y_supertile = max_render_y / supertile_h_in_pixels; for (int y = min_y_supertile; y <= max_y_supertile; y++) { for (int x = min_x_supertile; x <= max_x_supertile; x++) { cl_emit(rcl, SUPERTILE_COORDINATES, coords) { coords.column_number_in_supertiles = x; coords.row_number_in_supertiles = y; } } } } static void set_rcl_early_z_config(struct v3dv_job *job, bool *early_z_disable, uint32_t *early_z_test_and_update_direction) { /* Disable if none of the draw calls in this job enabled EZ */ if (!job->has_ez_draws) { *early_z_disable = true; return; } switch (job->first_ez_state) { case V3D_EZ_UNDECIDED: case V3D_EZ_LT_LE: *early_z_disable = false; *early_z_test_and_update_direction = EARLY_Z_DIRECTION_LT_LE; break; case V3D_EZ_GT_GE: *early_z_disable = false; *early_z_test_and_update_direction = EARLY_Z_DIRECTION_GT_GE; break; case V3D_EZ_DISABLED: *early_z_disable = true; break; } } /* Note that for v71, render target cfg packets has just one field that * combined the internal type and clamp mode. For simplicity we keep just one * helper. * * Note: rt_type is in fact a "enum V3DX(Internal_Type)". * * FIXME: for v71 we are not returning all the possible combinations for * render target internal type and clamp. For example for int types we are * always using clamp int, and for 16f we are using clamp none or pos (that * seems to be the equivalent for no-clamp on 4.2), but not pq or hlg. In * summary right now we are just porting what we were doing on 4.2 */ uint32_t v3dX(clamp_for_format_and_type)(uint32_t rt_type, VkFormat vk_format) { #if V3D_VERSION == 42 if (vk_format_is_int(vk_format)) return V3D_RENDER_TARGET_CLAMP_INT; else if (vk_format_is_srgb(vk_format)) return V3D_RENDER_TARGET_CLAMP_NORM; else return V3D_RENDER_TARGET_CLAMP_NONE; #endif #if V3D_VERSION >= 71 switch (rt_type) { case V3D_INTERNAL_TYPE_8I: return V3D_RENDER_TARGET_TYPE_CLAMP_8I_CLAMPED; case V3D_INTERNAL_TYPE_8UI: return V3D_RENDER_TARGET_TYPE_CLAMP_8UI_CLAMPED; case V3D_INTERNAL_TYPE_8: return V3D_RENDER_TARGET_TYPE_CLAMP_8; case V3D_INTERNAL_TYPE_16I: return V3D_RENDER_TARGET_TYPE_CLAMP_16I_CLAMPED; case V3D_INTERNAL_TYPE_16UI: return V3D_RENDER_TARGET_TYPE_CLAMP_16UI_CLAMPED; case V3D_INTERNAL_TYPE_16F: return vk_format_is_srgb(vk_format) ? V3D_RENDER_TARGET_TYPE_CLAMP_16F_CLAMP_NORM : V3D_RENDER_TARGET_TYPE_CLAMP_16F; case V3D_INTERNAL_TYPE_32I: return V3D_RENDER_TARGET_TYPE_CLAMP_32I_CLAMPED; case V3D_INTERNAL_TYPE_32UI: return V3D_RENDER_TARGET_TYPE_CLAMP_32UI_CLAMPED; case V3D_INTERNAL_TYPE_32F: return V3D_RENDER_TARGET_TYPE_CLAMP_32F; default: unreachable("Unknown internal render target type"); } return V3D_RENDER_TARGET_TYPE_CLAMP_INVALID; #endif } static void cmd_buffer_render_pass_setup_render_target(struct v3dv_cmd_buffer *cmd_buffer, int rt, uint32_t *rt_bpp, #if V3D_VERSION == 42 uint32_t *rt_type, uint32_t *rt_clamp) #else uint32_t *rt_type_clamp) #endif { const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; assert(state->subpass_idx < state->pass->subpass_count); const struct v3dv_subpass *subpass = &state->pass->subpasses[state->subpass_idx]; if (rt >= subpass->color_count) return; struct v3dv_subpass_attachment *attachment = &subpass->color_attachments[rt]; const uint32_t attachment_idx = attachment->attachment; if (attachment_idx == VK_ATTACHMENT_UNUSED) return; assert(attachment_idx < state->framebuffer->attachment_count && attachment_idx < state->attachment_alloc_count); struct v3dv_image_view *iview = state->attachments[attachment_idx].image_view; assert(vk_format_is_color(iview->vk.format)); assert(iview->plane_count == 1); *rt_bpp = iview->planes[0].internal_bpp; #if V3D_VERSION == 42 *rt_type = iview->planes[0].internal_type; *rt_clamp = v3dX(clamp_for_format_and_type)(iview->planes[0].internal_type, iview->vk.format); #endif #if V3D_VERSION >= 71 *rt_type_clamp = v3dX(clamp_for_format_and_type)(iview->planes[0].internal_type, iview->vk.format); #endif } void v3dX(cmd_buffer_emit_render_pass_rcl)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); const struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; const struct v3dv_framebuffer *framebuffer = state->framebuffer; /* We can't emit the RCL until we have a framebuffer, which we may not have * if we are recording a secondary command buffer. In that case, we will * have to wait until vkCmdExecuteCommands is called from a primary command * buffer. */ if (!framebuffer) { assert(cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_SECONDARY); return; } const struct v3dv_frame_tiling *tiling = &job->frame_tiling; const uint32_t fb_layers = job->frame_tiling.layers; v3dv_cl_ensure_space_with_branch(&job->rcl, 200 + MAX2(fb_layers, 1) * 256 * cl_packet_length(SUPERTILE_COORDINATES)); v3dv_return_if_oom(cmd_buffer, NULL); assert(state->subpass_idx < state->pass->subpass_count); const struct v3dv_render_pass *pass = state->pass; const struct v3dv_subpass *subpass = &pass->subpasses[state->subpass_idx]; struct v3dv_cl *rcl = &job->rcl; /* Common config must be the first TILE_RENDERING_MODE_CFG and * Z_STENCIL_CLEAR_VALUES must be last. The ones in between are optional * updates to the previous HW state. */ bool do_early_zs_clear = false; const uint32_t ds_attachment_idx = subpass->ds_attachment.attachment; assert(!tiling->msaa || !tiling->double_buffer); cl_emit(rcl, TILE_RENDERING_MODE_CFG_COMMON, config) { config.image_width_pixels = framebuffer->width; config.image_height_pixels = framebuffer->height; config.number_of_render_targets = MAX2(subpass->color_count, 1); config.multisample_mode_4x = tiling->msaa; config.double_buffer_in_non_ms_mode = tiling->double_buffer; #if V3D_VERSION == 42 config.maximum_bpp_of_all_render_targets = tiling->internal_bpp; #endif #if V3D_VERSION >= 71 config.log2_tile_width = log2_tile_size(tiling->tile_width); config.log2_tile_height = log2_tile_size(tiling->tile_height); /* FIXME: ideallly we would like next assert on the packet header (as is * general, so also applies to GL). We would need to expand * gen_pack_header for that. */ assert(config.log2_tile_width == config.log2_tile_height || config.log2_tile_width == config.log2_tile_height + 1); #endif if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { const struct v3dv_image_view *iview = state->attachments[ds_attachment_idx].image_view; /* At this point the image view should be single-plane. But note that * the underlying image can be multi-plane, and the image view refer * to one specific plane. */ assert(iview->plane_count == 1); assert(iview->format->plane_count == 1); config.internal_depth_type = iview->planes[0].internal_type; set_rcl_early_z_config(job, &config.early_z_disable, &config.early_z_test_and_update_direction); /* Early-Z/S clear can be enabled if the job is clearing and not * storing (or loading) depth. If a stencil aspect is also present * we have the same requirements for it, however, in this case we * can accept stencil loadOp DONT_CARE as well, so instead of * checking that stencil is cleared we check that is not loaded. * * Early-Z/S clearing is independent of Early Z/S testing, so it is * possible to enable one but not the other so long as their * respective requirements are met. * * From V3D 4.5.6, Z/S buffers are always cleared automatically * between tiles, but we still want to enable early ZS clears * when Z/S are not loaded or stored. */ struct v3dv_render_pass_attachment *ds_attachment = &pass->attachments[ds_attachment_idx]; const VkImageAspectFlags ds_aspects = vk_format_aspects(ds_attachment->desc.format); bool needs_depth_store = v3dv_cmd_buffer_check_needs_store(state, ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT, ds_attachment->last_subpass, ds_attachment->desc.storeOp) || subpass->resolve_depth; #if V3D_VERSION <= 42 bool needs_depth_clear = check_needs_clear(state, ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT, ds_attachment->first_subpass, ds_attachment->desc.loadOp, subpass->do_depth_clear_with_draw); do_early_zs_clear = needs_depth_clear && !needs_depth_store; #endif #if V3D_VERSION >= 71 bool needs_depth_load = v3dv_cmd_buffer_check_needs_load(state, ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT, ds_attachment->first_subpass, ds_attachment->desc.loadOp, ds_attachment->last_subpass, ds_attachment->desc.storeOp); do_early_zs_clear = !needs_depth_load && !needs_depth_store; #endif if (do_early_zs_clear && vk_format_has_stencil(ds_attachment->desc.format)) { bool needs_stencil_load = v3dv_cmd_buffer_check_needs_load(state, ds_aspects & VK_IMAGE_ASPECT_STENCIL_BIT, ds_attachment->first_subpass, ds_attachment->desc.stencilLoadOp, ds_attachment->last_subpass, ds_attachment->desc.stencilStoreOp); bool needs_stencil_store = v3dv_cmd_buffer_check_needs_store(state, ds_aspects & VK_IMAGE_ASPECT_STENCIL_BIT, ds_attachment->last_subpass, ds_attachment->desc.stencilStoreOp) || subpass->resolve_stencil; do_early_zs_clear = !needs_stencil_load && !needs_stencil_store; } config.early_depth_stencil_clear = do_early_zs_clear; } else { config.early_z_disable = true; } } /* If we enabled early Z/S clear, then we can't emit any "Clear Tile Buffers" * commands with the Z/S bit set, so keep track of whether we enabled this * in the job so we can skip these later. */ job->early_zs_clear = do_early_zs_clear; #if V3D_VERSION >= 71 uint32_t base_addr = 0; #endif for (uint32_t i = 0; i < subpass->color_count; i++) { uint32_t attachment_idx = subpass->color_attachments[i].attachment; if (attachment_idx == VK_ATTACHMENT_UNUSED) { #if V3D_VERSION >= 71 cl_emit(rcl, TILE_RENDERING_MODE_CFG_RENDER_TARGET_PART1, rt) { rt.render_target_number = i; rt.stride = 1; /* Unused */ } #endif continue; } struct v3dv_image_view *iview = state->attachments[attachment_idx].image_view; assert(iview->plane_count == 1); const struct v3dv_image *image = (struct v3dv_image *) iview->vk.image; uint8_t plane = v3dv_plane_from_aspect(iview->vk.aspects); const struct v3d_resource_slice *slice = &image->planes[plane].slices[iview->vk.base_mip_level]; UNUSED const uint32_t *clear_color = &state->attachments[attachment_idx].clear_value.color[0]; UNUSED uint32_t clear_pad = 0; if (slice->tiling == V3D_TILING_UIF_NO_XOR || slice->tiling == V3D_TILING_UIF_XOR) { int uif_block_height = v3d_utile_height(image->planes[plane].cpp) * 2; uint32_t implicit_padded_height = align(framebuffer->height, uif_block_height) / uif_block_height; if (slice->padded_height_of_output_image_in_uif_blocks - implicit_padded_height >= 15) { clear_pad = slice->padded_height_of_output_image_in_uif_blocks; } } #if V3D_VERSION == 42 cl_emit(rcl, TILE_RENDERING_MODE_CFG_CLEAR_COLORS_PART1, clear) { clear.clear_color_low_32_bits = clear_color[0]; clear.clear_color_next_24_bits = clear_color[1] & 0xffffff; clear.render_target_number = i; }; if (iview->planes[0].internal_bpp >= V3D_INTERNAL_BPP_64) { cl_emit(rcl, TILE_RENDERING_MODE_CFG_CLEAR_COLORS_PART2, clear) { clear.clear_color_mid_low_32_bits = ((clear_color[1] >> 24) | (clear_color[2] << 8)); clear.clear_color_mid_high_24_bits = ((clear_color[2] >> 24) | ((clear_color[3] & 0xffff) << 8)); clear.render_target_number = i; }; } if (iview->planes[0].internal_bpp >= V3D_INTERNAL_BPP_128 || clear_pad) { cl_emit(rcl, TILE_RENDERING_MODE_CFG_CLEAR_COLORS_PART3, clear) { clear.uif_padded_height_in_uif_blocks = clear_pad; clear.clear_color_high_16_bits = clear_color[3] >> 16; clear.render_target_number = i; }; } #endif #if V3D_VERSION >= 71 cl_emit(rcl, TILE_RENDERING_MODE_CFG_RENDER_TARGET_PART1, rt) { rt.clear_color_low_bits = clear_color[0]; cmd_buffer_render_pass_setup_render_target(cmd_buffer, i, &rt.internal_bpp, &rt.internal_type_and_clamping); rt.stride = v3d_compute_rt_row_row_stride_128_bits(tiling->tile_width, v3d_internal_bpp_words(rt.internal_bpp)); rt.base_address = base_addr; rt.render_target_number = i; /* base_addr in multiples of 512 bits. We divide by 8 because stride * is in 128-bit units, but it is packing 2 rows worth of data, so we * need to divide it by 2 so it is only 1 row, and then again by 4 so * it is in 512-bit units. */ base_addr += (tiling->tile_height * rt.stride) / 8; } if (iview->planes[0].internal_bpp >= V3D_INTERNAL_BPP_64) { cl_emit(rcl, TILE_RENDERING_MODE_CFG_RENDER_TARGET_PART2, rt) { rt.clear_color_mid_bits = /* 40 bits (32 + 8) */ ((uint64_t) clear_color[1]) | (((uint64_t) (clear_color[2] & 0xff)) << 32); rt.render_target_number = i; } } if (iview->planes[0].internal_bpp >= V3D_INTERNAL_BPP_128) { cl_emit(rcl, TILE_RENDERING_MODE_CFG_RENDER_TARGET_PART3, rt) { rt.clear_color_top_bits = /* 56 bits (24 + 32) */ (((uint64_t) (clear_color[2] & 0xffffff00)) >> 8) | (((uint64_t) (clear_color[3])) << 24); rt.render_target_number = i; } } #endif } #if V3D_VERSION >= 71 /* If we don't have any color RTs, we still need to emit one and flag * it as not used using stride = 1. */ if (subpass->color_count == 0) { cl_emit(rcl, TILE_RENDERING_MODE_CFG_RENDER_TARGET_PART1, rt) { rt.stride = 1; } } #endif #if V3D_VERSION == 42 cl_emit(rcl, TILE_RENDERING_MODE_CFG_COLOR, rt) { cmd_buffer_render_pass_setup_render_target (cmd_buffer, 0, &rt.render_target_0_internal_bpp, &rt.render_target_0_internal_type, &rt.render_target_0_clamp); cmd_buffer_render_pass_setup_render_target (cmd_buffer, 1, &rt.render_target_1_internal_bpp, &rt.render_target_1_internal_type, &rt.render_target_1_clamp); cmd_buffer_render_pass_setup_render_target (cmd_buffer, 2, &rt.render_target_2_internal_bpp, &rt.render_target_2_internal_type, &rt.render_target_2_clamp); cmd_buffer_render_pass_setup_render_target (cmd_buffer, 3, &rt.render_target_3_internal_bpp, &rt.render_target_3_internal_type, &rt.render_target_3_clamp); } #endif /* Ends rendering mode config. */ if (ds_attachment_idx != VK_ATTACHMENT_UNUSED) { cl_emit(rcl, TILE_RENDERING_MODE_CFG_ZS_CLEAR_VALUES, clear) { clear.z_clear_value = state->attachments[ds_attachment_idx].clear_value.z; clear.stencil_clear_value = state->attachments[ds_attachment_idx].clear_value.s; }; } else { cl_emit(rcl, TILE_RENDERING_MODE_CFG_ZS_CLEAR_VALUES, clear) { clear.z_clear_value = 1.0f; clear.stencil_clear_value = 0; }; } /* Always set initial block size before the first branch, which needs * to match the value from binning mode config. */ cl_emit(rcl, TILE_LIST_INITIAL_BLOCK_SIZE, init) { init.use_auto_chained_tile_lists = true; init.size_of_first_block_in_chained_tile_lists = TILE_ALLOCATION_BLOCK_SIZE_64B; } cl_emit(rcl, MULTICORE_RENDERING_SUPERTILE_CFG, config) { config.number_of_bin_tile_lists = 1; config.total_frame_width_in_tiles = tiling->draw_tiles_x; config.total_frame_height_in_tiles = tiling->draw_tiles_y; config.supertile_width_in_tiles = tiling->supertile_width; config.supertile_height_in_tiles = tiling->supertile_height; config.total_frame_width_in_supertiles = tiling->frame_width_in_supertiles; config.total_frame_height_in_supertiles = tiling->frame_height_in_supertiles; } /* Emit an initial clear of the tile buffers. This is necessary * for any buffers that should be cleared (since clearing * normally happens at the *end* of the generic tile list), but * it's also nice to clear everything so the first tile doesn't * inherit any contents from some previous frame. * * Also, implement the GFXH-1742 workaround. There's a race in * the HW between the RCL updating the TLB's internal type/size * and the spawning of the QPU instances using the TLB's current * internal type/size. To make sure the QPUs get the right * state, we need 1 dummy store in between internal type/size * changes on V3D 3.x, and 2 dummy stores on 4.x. */ for (int i = 0; i < 2; i++) { cl_emit(rcl, TILE_COORDINATES, coords); cl_emit(rcl, END_OF_LOADS, end); cl_emit(rcl, STORE_TILE_BUFFER_GENERAL, store) { store.buffer_to_store = NONE; } if (cmd_buffer->state.tile_aligned_render_area && (i == 0 || v3dv_do_double_initial_tile_clear(tiling))) { #if V3D_VERSION == 42 cl_emit(rcl, CLEAR_TILE_BUFFERS, clear) { clear.clear_z_stencil_buffer = !job->early_zs_clear; clear.clear_all_render_targets = true; } #endif #if V3D_VERSION >= 71 cl_emit(rcl, CLEAR_RENDER_TARGETS, clear_rt); #endif } cl_emit(rcl, END_OF_TILE_MARKER, end); } cl_emit(rcl, FLUSH_VCD_CACHE, flush); for (int layer = 0; layer < MAX2(1, fb_layers); layer++) { if (subpass->view_mask == 0 || (subpass->view_mask & (1u << layer))) cmd_buffer_emit_render_pass_layer_rcl(cmd_buffer, layer); } cl_emit(rcl, END_OF_RENDERING, end); } void v3dX(viewport_compute_xform)(const VkViewport *viewport, float scale[3], float translate[3]) { float x = viewport->x; float y = viewport->y; float half_width = 0.5f * viewport->width; float half_height = 0.5f * viewport->height; double n = viewport->minDepth; double f = viewport->maxDepth; scale[0] = half_width; translate[0] = half_width + x; scale[1] = half_height; translate[1] = half_height + y; scale[2] = (f - n); translate[2] = n; /* It seems that if the scale is small enough the hardware won't clip * correctly so we work around this my choosing the smallest scale that * seems to work. * * This case is exercised by CTS: * dEQP-VK.draw.renderpass.inverted_depth_ranges.nodepthclamp_deltazero * * V3D 7.x fixes this by using the new * CLIPPER_Z_SCALE_AND_OFFSET_NO_GUARDBAND. */ #if V3D_VERSION <= 42 const float min_abs_scale = 0.0005f; if (fabs(scale[2]) < min_abs_scale) scale[2] = scale[2] < 0 ? -min_abs_scale : min_abs_scale; #endif } void v3dX(cmd_buffer_emit_viewport)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_dynamic_state *dynamic = &cmd_buffer->state.dynamic; struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; assert(pipeline); /* FIXME: right now we don't support multiViewport so viewports[0] would * work now, but would need to change if we allow multiple viewports. */ float *vptranslate = dynamic->viewport.translate[0]; float *vpscale = dynamic->viewport.scale[0]; struct v3dv_job *job = cmd_buffer->state.job; assert(job); const uint32_t required_cl_size = cl_packet_length(CLIPPER_XY_SCALING) + cl_packet_length(CLIPPER_Z_SCALE_AND_OFFSET) + cl_packet_length(CLIPPER_Z_MIN_MAX_CLIPPING_PLANES) + cl_packet_length(VIEWPORT_OFFSET); v3dv_cl_ensure_space_with_branch(&job->bcl, required_cl_size); v3dv_return_if_oom(cmd_buffer, NULL); #if V3D_VERSION == 42 cl_emit(&job->bcl, CLIPPER_XY_SCALING, clip) { clip.viewport_half_width_in_1_256th_of_pixel = vpscale[0] * 256.0f; clip.viewport_half_height_in_1_256th_of_pixel = vpscale[1] * 256.0f; } #endif #if V3D_VERSION >= 71 cl_emit(&job->bcl, CLIPPER_XY_SCALING, clip) { clip.viewport_half_width_in_1_64th_of_pixel = vpscale[0] * 64.0f; clip.viewport_half_height_in_1_64th_of_pixel = vpscale[1] * 64.0f; } #endif float translate_z, scale_z; v3dv_cmd_buffer_state_get_viewport_z_xform(cmd_buffer, 0, &translate_z, &scale_z); #if V3D_VERSION == 42 cl_emit(&job->bcl, CLIPPER_Z_SCALE_AND_OFFSET, clip) { clip.viewport_z_offset_zc_to_zs = translate_z; clip.viewport_z_scale_zc_to_zs = scale_z; } #endif #if V3D_VERSION >= 71 /* If the Z scale is too small guardband clipping may not clip correctly */ if (fabsf(scale_z) < 0.01f) { cl_emit(&job->bcl, CLIPPER_Z_SCALE_AND_OFFSET_NO_GUARDBAND, clip) { clip.viewport_z_offset_zc_to_zs = translate_z; clip.viewport_z_scale_zc_to_zs = scale_z; } } else { cl_emit(&job->bcl, CLIPPER_Z_SCALE_AND_OFFSET, clip) { clip.viewport_z_offset_zc_to_zs = translate_z; clip.viewport_z_scale_zc_to_zs = scale_z; } } #endif cl_emit(&job->bcl, CLIPPER_Z_MIN_MAX_CLIPPING_PLANES, clip) { /* Vulkan's default Z NDC is [0..1]. If 'negative_one_to_one' is enabled, * we are using OpenGL's [-1, 1] instead. */ float z1 = pipeline->negative_one_to_one ? translate_z - scale_z : translate_z; float z2 = translate_z + scale_z; clip.minimum_zw = MIN2(z1, z2); clip.maximum_zw = MAX2(z1, z2); } cl_emit(&job->bcl, VIEWPORT_OFFSET, vp) { float vp_fine_x = vptranslate[0]; float vp_fine_y = vptranslate[1]; int32_t vp_coarse_x = 0; int32_t vp_coarse_y = 0; /* The fine coordinates must be unsigned, but coarse can be signed */ if (unlikely(vp_fine_x < 0)) { int32_t blocks_64 = DIV_ROUND_UP(fabsf(vp_fine_x), 64); vp_fine_x += 64.0f * blocks_64; vp_coarse_x -= blocks_64; } if (unlikely(vp_fine_y < 0)) { int32_t blocks_64 = DIV_ROUND_UP(fabsf(vp_fine_y), 64); vp_fine_y += 64.0f * blocks_64; vp_coarse_y -= blocks_64; } vp.fine_x = vp_fine_x; vp.fine_y = vp_fine_y; vp.coarse_x = vp_coarse_x; vp.coarse_y = vp_coarse_y; } BITSET_CLEAR(cmd_buffer->vk.dynamic_graphics_state.dirty, MESA_VK_DYNAMIC_VP_VIEWPORTS); } void v3dX(cmd_buffer_emit_stencil)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; bool has_stencil = pipeline->rendering_info.stencil_attachment_format != VK_FORMAT_UNDEFINED; if (!(dyn->ds.stencil.test_enable && has_stencil)) return; v3dv_cl_ensure_space_with_branch(&job->bcl, 2 * cl_packet_length(STENCIL_CFG)); v3dv_return_if_oom(cmd_buffer, NULL); bool any_dynamic_stencil_state = BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_STENCIL_COMPARE_MASK) || BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_STENCIL_WRITE_MASK) || BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_STENCIL_REFERENCE) || BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_STENCIL_OP); bool emitted_stencil = false; const struct vk_stencil_test_face_state *front = &dyn->ds.stencil.front; const struct vk_stencil_test_face_state *back = &dyn->ds.stencil.back; const bool needs_front_and_back = any_dynamic_stencil_state ? memcmp(front, back, sizeof(*front)) != 0 : pipeline->emit_stencil_cfg[1] == true; for (uint32_t i = 0; i < 2; i++) { if (any_dynamic_stencil_state) { const struct vk_stencil_test_face_state *stencil_state = i == 0 ? front : back; /* If we have any dynamic stencil state we just emit the entire * packet since for simplicity */ cl_emit(&job->bcl, STENCIL_CFG, config) { config.front_config = !needs_front_and_back || i == 0; config.back_config = !needs_front_and_back || i == 1; config.stencil_test_mask = stencil_state->compare_mask & 0xff; config.stencil_write_mask = stencil_state->write_mask & 0xff; config.stencil_ref_value = stencil_state->reference & 0xff; config.stencil_test_function = stencil_state->op.compare; config.stencil_pass_op = v3dX(translate_stencil_op)(stencil_state->op.pass); config.depth_test_fail_op = v3dX(translate_stencil_op)(stencil_state->op.depth_fail); config.stencil_test_fail_op = v3dX(translate_stencil_op)(stencil_state->op.fail); } } else { assert(pipeline->emit_stencil_cfg[i]); cl_emit_prepacked(&job->bcl, &pipeline->stencil_cfg[i]); } emitted_stencil = true; if (!needs_front_and_back) break; } if (emitted_stencil) { BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_DS_STENCIL_COMPARE_MASK); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_DS_STENCIL_REFERENCE); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_DS_STENCIL_WRITE_MASK); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_DS_STENCIL_OP); } } void v3dX(cmd_buffer_emit_depth_bias)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; assert(pipeline); struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; if (!dyn->rs.depth_bias.enable) return; struct v3dv_job *job = cmd_buffer->state.job; assert(job); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(DEPTH_OFFSET)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, DEPTH_OFFSET, bias) { bias.depth_offset_factor = dyn->rs.depth_bias.slope; bias.depth_offset_units = dyn->rs.depth_bias.constant; #if V3D_VERSION <= 42 if (pipeline->rendering_info.depth_attachment_format == VK_FORMAT_D16_UNORM) bias.depth_offset_units *= 256.0f; #endif bias.limit = dyn->rs.depth_bias.clamp; } BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_RS_DEPTH_BIAS_FACTORS); } void v3dX(cmd_buffer_emit_depth_bounds)(struct v3dv_cmd_buffer *cmd_buffer) { /* No depthBounds support for v42, so this method is empty in that case. * * Note that this method is being called as v3dv_job_init flags all state * as dirty. See FIXME note in v3dv_job_init. */ #if V3D_VERSION >= 71 struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; if (!dyn->ds.depth.bounds_test.enable) return; struct v3dv_job *job = cmd_buffer->state.job; assert(job); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(DEPTH_BOUNDS_TEST_LIMITS)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, DEPTH_BOUNDS_TEST_LIMITS, bounds) { bounds.lower_test_limit = dyn->ds.depth.bounds_test.min; bounds.upper_test_limit = dyn->ds.depth.bounds_test.max; } BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_DS_DEPTH_BOUNDS_TEST_BOUNDS); #endif } void v3dX(cmd_buffer_emit_line_width)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(LINE_WIDTH)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, LINE_WIDTH, line) { line.line_width = v3dv_get_aa_line_width(cmd_buffer->state.gfx.pipeline, cmd_buffer); } BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_RS_LINE_WIDTH); } void v3dX(cmd_buffer_emit_default_point_size)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(POINT_SIZE)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, POINT_SIZE, point) { point.point_size = 1.0f; } job->emitted_default_point_size = true; } void v3dX(cmd_buffer_emit_sample_state)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; assert(pipeline); struct v3dv_job *job = cmd_buffer->state.job; assert(job); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(SAMPLE_STATE)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, SAMPLE_STATE, state) { state.coverage = 1.0f; state.mask = pipeline->sample_mask; } } void v3dX(cmd_buffer_emit_blend)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; assert(pipeline); const struct v3d_device_info *devinfo = &cmd_buffer->device->devinfo; const uint32_t max_color_rts = V3D_MAX_RENDER_TARGETS(devinfo->ver); const uint32_t blend_packets_size = cl_packet_length(BLEND_ENABLES) + cl_packet_length(BLEND_CONSTANT_COLOR) + cl_packet_length(BLEND_CFG) * max_color_rts; v3dv_cl_ensure_space_with_branch(&job->bcl, blend_packets_size); v3dv_return_if_oom(cmd_buffer, NULL); if (cmd_buffer->state.dirty & V3DV_CMD_DIRTY_PIPELINE) { if (pipeline->blend.enables) { cl_emit(&job->bcl, BLEND_ENABLES, enables) { enables.mask = pipeline->blend.enables; } } for (uint32_t i = 0; i < max_color_rts; i++) { if (pipeline->blend.enables & (1 << i)) cl_emit_prepacked(&job->bcl, &pipeline->blend.cfg[i]); } } if (pipeline->blend.needs_color_constants) { const struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; cl_emit(&job->bcl, BLEND_CONSTANT_COLOR, color) { color.red_f16 = _mesa_float_to_half(dyn->cb.blend_constants[0]); color.green_f16 = _mesa_float_to_half(dyn->cb.blend_constants[1]); color.blue_f16 = _mesa_float_to_half(dyn->cb.blend_constants[2]); color.alpha_f16 = _mesa_float_to_half(dyn->cb.blend_constants[3]); } } BITSET_CLEAR(cmd_buffer->vk.dynamic_graphics_state.dirty, MESA_VK_DYNAMIC_CB_BLEND_CONSTANTS); } void v3dX(cmd_buffer_emit_color_write_mask)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(COLOR_WRITE_MASKS)); struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; struct v3dv_dynamic_state *v3dv_dyn = &cmd_buffer->state.dynamic; uint32_t color_write_mask = ~v3dv_dyn->color_write_enable | pipeline->blend.color_write_masks; #if V3D_VERSION <= 42 /* Only 4 RTs */ color_write_mask &= 0xffff; #endif cl_emit(&job->bcl, COLOR_WRITE_MASKS, mask) { mask.mask = color_write_mask; } BITSET_CLEAR(cmd_buffer->vk.dynamic_graphics_state.dirty, MESA_VK_DYNAMIC_CB_COLOR_WRITE_ENABLES); } static void emit_flat_shade_flags(struct v3dv_job *job, int varying_offset, uint32_t varyings, enum V3DX(Varying_Flags_Action) lower, enum V3DX(Varying_Flags_Action) higher) { v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(FLAT_SHADE_FLAGS)); v3dv_return_if_oom(NULL, job); cl_emit(&job->bcl, FLAT_SHADE_FLAGS, flags) { flags.varying_offset_v0 = varying_offset; flags.flat_shade_flags_for_varyings_v024 = varyings; flags.action_for_flat_shade_flags_of_lower_numbered_varyings = lower; flags.action_for_flat_shade_flags_of_higher_numbered_varyings = higher; } } static void emit_noperspective_flags(struct v3dv_job *job, int varying_offset, uint32_t varyings, enum V3DX(Varying_Flags_Action) lower, enum V3DX(Varying_Flags_Action) higher) { v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(NON_PERSPECTIVE_FLAGS)); v3dv_return_if_oom(NULL, job); cl_emit(&job->bcl, NON_PERSPECTIVE_FLAGS, flags) { flags.varying_offset_v0 = varying_offset; flags.non_perspective_flags_for_varyings_v024 = varyings; flags.action_for_non_perspective_flags_of_lower_numbered_varyings = lower; flags.action_for_non_perspective_flags_of_higher_numbered_varyings = higher; } } static void emit_centroid_flags(struct v3dv_job *job, int varying_offset, uint32_t varyings, enum V3DX(Varying_Flags_Action) lower, enum V3DX(Varying_Flags_Action) higher) { v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(CENTROID_FLAGS)); v3dv_return_if_oom(NULL, job); cl_emit(&job->bcl, CENTROID_FLAGS, flags) { flags.varying_offset_v0 = varying_offset; flags.centroid_flags_for_varyings_v024 = varyings; flags.action_for_centroid_flags_of_lower_numbered_varyings = lower; flags.action_for_centroid_flags_of_higher_numbered_varyings = higher; } } static bool emit_varying_flags(struct v3dv_job *job, uint32_t num_flags, const uint32_t *flags, void (*flag_emit_callback)(struct v3dv_job *job, int varying_offset, uint32_t flags, enum V3DX(Varying_Flags_Action) lower, enum V3DX(Varying_Flags_Action) higher)) { bool emitted_any = false; for (int i = 0; i < num_flags; i++) { if (!flags[i]) continue; if (emitted_any) { flag_emit_callback(job, i, flags[i], V3D_VARYING_FLAGS_ACTION_UNCHANGED, V3D_VARYING_FLAGS_ACTION_UNCHANGED); } else if (i == 0) { flag_emit_callback(job, i, flags[i], V3D_VARYING_FLAGS_ACTION_UNCHANGED, V3D_VARYING_FLAGS_ACTION_ZEROED); } else { flag_emit_callback(job, i, flags[i], V3D_VARYING_FLAGS_ACTION_ZEROED, V3D_VARYING_FLAGS_ACTION_ZEROED); } emitted_any = true; } return emitted_any; } void v3dX(cmd_buffer_emit_varyings_state)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; struct v3d_fs_prog_data *prog_data_fs = pipeline->shared_data->variants[BROADCOM_SHADER_FRAGMENT]->prog_data.fs; const uint32_t num_flags = ARRAY_SIZE(prog_data_fs->flat_shade_flags); const uint32_t *flat_shade_flags = prog_data_fs->flat_shade_flags; const uint32_t *noperspective_flags = prog_data_fs->noperspective_flags; const uint32_t *centroid_flags = prog_data_fs->centroid_flags; if (!emit_varying_flags(job, num_flags, flat_shade_flags, emit_flat_shade_flags)) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(ZERO_ALL_FLAT_SHADE_FLAGS)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, ZERO_ALL_FLAT_SHADE_FLAGS, flags); } if (!emit_varying_flags(job, num_flags, noperspective_flags, emit_noperspective_flags)) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(ZERO_ALL_NON_PERSPECTIVE_FLAGS)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, ZERO_ALL_NON_PERSPECTIVE_FLAGS, flags); } if (!emit_varying_flags(job, num_flags, centroid_flags, emit_centroid_flags)) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(ZERO_ALL_CENTROID_FLAGS)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, ZERO_ALL_CENTROID_FLAGS, flags); } } #if V3D_VERSION == 42 /* Updates cmd_buffer, and their job, early z state tracking. Returns false if * EZ must be disabled for the current draw call. */ static bool cmd_buffer_update_ez_state(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_pipeline *pipeline) { struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; /* Update first cmd_buffer ez_state tracking. If possible we reuse the * values from the pipeline */ if (!BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_STENCIL_OP) && !BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_STENCIL_TEST_ENABLE) && !BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_DEPTH_TEST_ENABLE) && !BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_DS_DEPTH_COMPARE_OP)) { cmd_buffer->state.ez_state = pipeline->ez_state; cmd_buffer->state.incompatible_ez_test = pipeline->incompatible_ez_test; } else { v3dv_compute_ez_state(dyn, pipeline, &cmd_buffer->state.ez_state, &cmd_buffer->state.incompatible_ez_test); } struct v3dv_job *job = cmd_buffer->state.job; assert(job); /* If first_ez_state is V3D_EZ_DISABLED it means that we have already * determined that we should disable EZ completely for all draw calls in * this job. This will cause us to disable EZ for the entire job in the * Tile Rendering Mode RCL packet and when we do that we need to make sure * we never emit a draw call in the job with EZ enabled in the CFG_BITS * packet, so ez_state must also be V3D_EZ_DISABLED; */ if (job->first_ez_state == V3D_EZ_DISABLED) { assert(job->ez_state == V3D_EZ_DISABLED); return false; } /* If ez_state is V3D_EZ_DISABLED it means that we have already decided * that EZ must be disabled for the remaining of the frame. */ if (job->ez_state == V3D_EZ_DISABLED) return false; /* This is part of the pre draw call handling, so we should be inside a * render pass. */ assert(cmd_buffer->state.pass); /* If this is the first time we update EZ state for this job we first check * if there is anything that requires disabling it completely for the entire * job (based on state that is not related to the current draw call and * pipeline/cmd_buffer state). */ if (!job->decided_global_ez_enable) { job->decided_global_ez_enable = true; struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; assert(state->subpass_idx < state->pass->subpass_count); struct v3dv_subpass *subpass = &state->pass->subpasses[state->subpass_idx]; if (subpass->ds_attachment.attachment == VK_ATTACHMENT_UNUSED) { job->first_ez_state = V3D_EZ_DISABLED; job->ez_state = V3D_EZ_DISABLED; return false; } /* GFXH-1918: the early-z buffer may load incorrect depth values if the * frame has odd width or height, or if the buffer is 16-bit and * multisampled. * * So we need to disable EZ in these cases. */ const struct v3dv_render_pass_attachment *ds_attachment = &state->pass->attachments[subpass->ds_attachment.attachment]; const VkImageAspectFlags ds_aspects = vk_format_aspects(ds_attachment->desc.format); bool needs_depth_load = v3dv_cmd_buffer_check_needs_load(state, ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT, ds_attachment->first_subpass, ds_attachment->desc.loadOp, ds_attachment->last_subpass, ds_attachment->desc.storeOp); if (needs_depth_load) { if (ds_attachment->desc.format == VK_FORMAT_D16_UNORM && ds_attachment->desc.samples != VK_SAMPLE_COUNT_1_BIT) { perf_debug("Loading depth aspect from a multisampled 16-bit " "depth buffer disables early-Z tests.\n"); job->first_ez_state = V3D_EZ_DISABLED; job->ez_state = V3D_EZ_DISABLED; return false; } struct v3dv_framebuffer *fb = state->framebuffer; if (!fb) { assert(cmd_buffer->vk.level == VK_COMMAND_BUFFER_LEVEL_SECONDARY); perf_debug("Loading depth aspect in a secondary command buffer " "without framebuffer info disables early-z tests.\n"); job->first_ez_state = V3D_EZ_DISABLED; job->ez_state = V3D_EZ_DISABLED; return false; } if (((fb->width % 2) != 0 || (fb->height % 2) != 0)) { perf_debug("Loading depth aspect for framebuffer with odd width " "or height disables early-Z tests.\n"); job->first_ez_state = V3D_EZ_DISABLED; job->ez_state = V3D_EZ_DISABLED; return false; } } } /* Otherwise, we can decide to selectively enable or disable EZ for draw * calls using the CFG_BITS packet based on the bound pipeline state, or * cmd_buffer state if some stencil/depth flags were dynamic. */ bool disable_ez = false; bool incompatible_test = false; switch (cmd_buffer->state.ez_state) { case V3D_EZ_UNDECIDED: /* If the pipeline didn't pick a direction but didn't disable, then go * along with the current EZ state. This allows EZ optimization for Z * func == EQUAL or NEVER. */ break; case V3D_EZ_LT_LE: case V3D_EZ_GT_GE: /* If the pipeline picked a direction, then it needs to match the current * direction if we've decided on one. */ if (job->ez_state == V3D_EZ_UNDECIDED) { job->ez_state = cmd_buffer->state.ez_state; } else if (job->ez_state != pipeline->ez_state) { disable_ez = true; incompatible_test = true; } break; case V3D_EZ_DISABLED: disable_ez = true; incompatible_test = cmd_buffer->state.incompatible_ez_test; break; } if (job->first_ez_state == V3D_EZ_UNDECIDED && !disable_ez) { assert(job->ez_state != V3D_EZ_DISABLED); job->first_ez_state = job->ez_state; } /* If we had to disable EZ because of an incompatible test direction and * and the cmd buffer writes depth then we need to disable EZ for the rest * of the frame. */ if (incompatible_test && cmd_buffer->state.z_updates_enable) { assert(disable_ez); job->ez_state = V3D_EZ_DISABLED; } if (!disable_ez) job->has_ez_draws = true; return !disable_ez; } #endif void v3dX(cmd_buffer_emit_configuration_bits)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); struct v3dv_pipeline *pipeline = cmd_buffer->state.gfx.pipeline; assert(pipeline); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(CFG_BITS)); v3dv_return_if_oom(cmd_buffer, NULL); struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; /* Disable depth/stencil if we don't have a D/S attachment */ bool has_depth = pipeline->rendering_info.depth_attachment_format != VK_FORMAT_UNDEFINED; bool has_stencil = pipeline->rendering_info.stencil_attachment_format != VK_FORMAT_UNDEFINED; cl_emit_with_prepacked(&job->bcl, CFG_BITS, pipeline->cfg_bits, config) { if (dyn->ds.depth.test_enable && has_depth) { config.z_updates_enable = dyn->ds.depth.write_enable; config.depth_test_function = dyn->ds.depth.compare_op; } else { config.depth_test_function = VK_COMPARE_OP_ALWAYS; } config.stencil_enable = dyn->ds.stencil.test_enable && has_stencil; cmd_buffer->state.z_updates_enable = config.z_updates_enable; #if V3D_VERSION == 42 bool enable_ez = cmd_buffer_update_ez_state(cmd_buffer, pipeline); config.early_z_enable = enable_ez; config.early_z_updates_enable = config.early_z_enable && cmd_buffer->state.z_updates_enable; #endif if (!dyn->rs.rasterizer_discard_enable) { assert(BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_RS_CULL_MODE)); assert(BITSET_TEST(dyn->set, MESA_VK_DYNAMIC_RS_FRONT_FACE)); config.enable_forward_facing_primitive = !(dyn->rs.cull_mode & VK_CULL_MODE_FRONT_BIT); config.enable_reverse_facing_primitive = !(dyn->rs.cull_mode & VK_CULL_MODE_BACK_BIT); /* Seems like the hardware is backwards regarding this setting... */ config.clockwise_primitives = dyn->rs.front_face == VK_FRONT_FACE_COUNTER_CLOCKWISE; } /* V3D 4.2 doesn't support depth bounds testing so we don't advertise that * feature and it shouldn't be used by any pipeline. */ assert(cmd_buffer->device->devinfo.ver >= 71 || !dyn->ds.depth.bounds_test.enable); #if V3D_VERSION >= 71 config.depth_bounds_test_enable = dyn->ds.depth.bounds_test.enable && has_depth; #endif config.enable_depth_offset = dyn->rs.depth_bias.enable; } BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_RS_CULL_MODE); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_RS_FRONT_FACE); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_DS_DEPTH_BOUNDS_TEST_ENABLE); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_DS_STENCIL_TEST_ENABLE); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_RS_DEPTH_BIAS_ENABLE); BITSET_CLEAR(dyn->dirty, MESA_VK_DYNAMIC_RS_RASTERIZER_DISCARD_ENABLE); } void v3dX(cmd_buffer_emit_occlusion_query)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(OCCLUSION_QUERY_COUNTER)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, OCCLUSION_QUERY_COUNTER, counter) { if (cmd_buffer->state.query.active_query.bo) { counter.address = v3dv_cl_address(cmd_buffer->state.query.active_query.bo, cmd_buffer->state.query.active_query.offset); } } cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_OCCLUSION_QUERY; } static struct v3dv_job * cmd_buffer_subpass_split_for_barrier(struct v3dv_cmd_buffer *cmd_buffer, bool is_bcl_barrier) { assert(cmd_buffer->state.subpass_idx != -1); v3dv_cmd_buffer_finish_job(cmd_buffer); struct v3dv_job *job = v3dv_cmd_buffer_subpass_resume(cmd_buffer, cmd_buffer->state.subpass_idx); if (!job) return NULL; /* FIXME: we can do better than all barriers */ job->serialize = V3DV_BARRIER_ALL; job->needs_bcl_sync = is_bcl_barrier; return job; } static void cmd_buffer_copy_secondary_end_query_state(struct v3dv_cmd_buffer *primary, struct v3dv_cmd_buffer *secondary) { struct v3dv_cmd_buffer_state *p_state = &primary->state; struct v3dv_cmd_buffer_state *s_state = &secondary->state; const uint32_t total_state_count = p_state->query.end.used_count + s_state->query.end.used_count; v3dv_cmd_buffer_ensure_array_state(primary, sizeof(struct v3dv_end_query_info), total_state_count, &p_state->query.end.alloc_count, (void **) &p_state->query.end.states); v3dv_return_if_oom(primary, NULL); for (uint32_t i = 0; i < s_state->query.end.used_count; i++) { const struct v3dv_end_query_info *s_qstate = &secondary->state.query.end.states[i]; struct v3dv_end_query_info *p_qstate = &p_state->query.end.states[p_state->query.end.used_count++]; memcpy(p_qstate, s_qstate, sizeof(struct v3dv_end_query_info)); } } void v3dX(cmd_buffer_execute_inside_pass)(struct v3dv_cmd_buffer *primary, uint32_t cmd_buffer_count, const VkCommandBuffer *cmd_buffers) { assert(primary->state.job); /* Typically we postpone applying binning syncs until we see a draw call * that may actually access proteted resources in the binning stage. However, * if the draw calls are recorded in a secondary command buffer and the * barriers were recorded in a primary command buffer, that won't work * and we will have to check if we need a binning sync when executing the * secondary. */ struct v3dv_job *primary_job = primary->state.job; if (primary_job->serialize && (primary->state.barrier.bcl_buffer_access || primary->state.barrier.bcl_image_access)) { v3dv_cmd_buffer_consume_bcl_sync(primary, primary_job); } /* Emit occlusion query state if needed so the draw calls inside our * secondaries update the counters. */ bool has_occlusion_query = primary->state.dirty & V3DV_CMD_DIRTY_OCCLUSION_QUERY; if (has_occlusion_query) v3dX(cmd_buffer_emit_occlusion_query)(primary); /* FIXME: if our primary job tiling doesn't enable MSSA but any of the * pipelines used by the secondaries do, we need to re-start the primary * job to enable MSAA. See cmd_buffer_restart_job_for_msaa_if_needed. */ struct v3dv_barrier_state pending_barrier = { 0 }; for (uint32_t i = 0; i < cmd_buffer_count; i++) { V3DV_FROM_HANDLE(v3dv_cmd_buffer, secondary, cmd_buffers[i]); assert(secondary->usage_flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT); list_for_each_entry(struct v3dv_job, secondary_job, &secondary->jobs, list_link) { if (secondary_job->type == V3DV_JOB_TYPE_GPU_CL_INCOMPLETE) { /* If the job is a CL, then we branch to it from the primary BCL. * In this case the secondary's BCL is finished with a * RETURN_FROM_SUB_LIST command to return back to the primary BCL * once we are done executing it. */ assert(v3dv_cl_offset(&secondary_job->rcl) == 0); assert(secondary_job->bcl.bo); /* Sanity check that secondary BCL ends with RETURN_FROM_SUB_LIST */ STATIC_ASSERT(cl_packet_length(RETURN_FROM_SUB_LIST) == 1); assert(v3dv_cl_offset(&secondary_job->bcl) >= 1); assert(*(((uint8_t *)secondary_job->bcl.next) - 1) == V3DX(RETURN_FROM_SUB_LIST_opcode)); /* If this secondary has any barriers (or we had any pending barrier * to apply), then we can't just branch to it from the primary, we * need to split the primary to create a new job that can consume * the barriers first. * * FIXME: in this case, maybe just copy the secondary BCL without * the RETURN_FROM_SUB_LIST into the primary job to skip the * branch? */ primary_job = primary->state.job; if (!primary_job || secondary_job->serialize || pending_barrier.dst_mask) { const bool needs_bcl_barrier = secondary_job->needs_bcl_sync || pending_barrier.bcl_buffer_access || pending_barrier.bcl_image_access; primary_job = cmd_buffer_subpass_split_for_barrier(primary, needs_bcl_barrier); v3dv_return_if_oom(primary, NULL); /* Since we have created a new primary we need to re-emit * occlusion query state. */ if (has_occlusion_query) v3dX(cmd_buffer_emit_occlusion_query)(primary); } /* Make sure our primary job has all required BO references */ set_foreach(secondary_job->bos, entry) { struct v3dv_bo *bo = (struct v3dv_bo *)entry->key; v3dv_job_add_bo(primary_job, bo); } /* Emit required branch instructions. We expect each of these * to end with a corresponding 'return from sub list' item. */ list_for_each_entry(struct v3dv_bo, bcl_bo, &secondary_job->bcl.bo_list, list_link) { v3dv_cl_ensure_space_with_branch(&primary_job->bcl, cl_packet_length(BRANCH_TO_SUB_LIST)); v3dv_return_if_oom(primary, NULL); cl_emit(&primary_job->bcl, BRANCH_TO_SUB_LIST, branch) { branch.address = v3dv_cl_address(bcl_bo, 0); } } if (!secondary_job->can_use_double_buffer) { primary_job->can_use_double_buffer = false; } else { primary_job->double_buffer_score.geom += secondary_job->double_buffer_score.geom; primary_job->double_buffer_score.render += secondary_job->double_buffer_score.render; } primary_job->tmu_dirty_rcl |= secondary_job->tmu_dirty_rcl; } else { /* This is a regular job (CPU or GPU), so just finish the current * primary job (if any) and then add the secondary job to the * primary's job list right after it. */ v3dv_cmd_buffer_finish_job(primary); v3dv_job_clone_in_cmd_buffer(secondary_job, primary); if (pending_barrier.dst_mask) { /* FIXME: do the same we do for primaries and only choose the * relevant src masks. */ secondary_job->serialize = pending_barrier.src_mask_graphics | pending_barrier.src_mask_transfer | pending_barrier.src_mask_compute; if (pending_barrier.bcl_buffer_access || pending_barrier.bcl_image_access) { secondary_job->needs_bcl_sync = true; } } } memset(&pending_barrier, 0, sizeof(pending_barrier)); } /* If the secondary has recorded any vkCmdEndQuery commands, we need to * copy this state to the primary so it is processed properly when the * current primary job is finished. */ cmd_buffer_copy_secondary_end_query_state(primary, secondary); /* If this secondary had any pending barrier state we will need that * barrier state consumed with whatever comes next in the primary. */ assert(secondary->state.barrier.dst_mask || (!secondary->state.barrier.bcl_buffer_access && !secondary->state.barrier.bcl_image_access)); pending_barrier = secondary->state.barrier; } if (pending_barrier.dst_mask) { v3dv_cmd_buffer_merge_barrier_state(&primary->state.barrier, &pending_barrier); } } static void emit_gs_shader_state_record(struct v3dv_job *job, struct v3dv_bo *assembly_bo, struct v3dv_shader_variant *gs_bin, struct v3dv_cl_reloc gs_bin_uniforms, struct v3dv_shader_variant *gs, struct v3dv_cl_reloc gs_render_uniforms) { cl_emit(&job->indirect, GEOMETRY_SHADER_STATE_RECORD, shader) { shader.geometry_bin_mode_shader_code_address = v3dv_cl_address(assembly_bo, gs_bin->assembly_offset); shader.geometry_bin_mode_shader_4_way_threadable = gs_bin->prog_data.gs->base.threads == 4; shader.geometry_bin_mode_shader_start_in_final_thread_section = gs_bin->prog_data.gs->base.single_seg; #if V3D_VERSION <= 42 shader.geometry_bin_mode_shader_propagate_nans = true; #endif shader.geometry_bin_mode_shader_uniforms_address = gs_bin_uniforms; shader.geometry_render_mode_shader_code_address = v3dv_cl_address(assembly_bo, gs->assembly_offset); shader.geometry_render_mode_shader_4_way_threadable = gs->prog_data.gs->base.threads == 4; shader.geometry_render_mode_shader_start_in_final_thread_section = gs->prog_data.gs->base.single_seg; #if V3D_VERSION <= 42 shader.geometry_render_mode_shader_propagate_nans = true; #endif shader.geometry_render_mode_shader_uniforms_address = gs_render_uniforms; } } static uint8_t v3d_gs_output_primitive(enum mesa_prim prim_type) { switch (prim_type) { case MESA_PRIM_POINTS: return GEOMETRY_SHADER_POINTS; case MESA_PRIM_LINE_STRIP: return GEOMETRY_SHADER_LINE_STRIP; case MESA_PRIM_TRIANGLE_STRIP: return GEOMETRY_SHADER_TRI_STRIP; default: unreachable("Unsupported primitive type"); } } static void emit_tes_gs_common_params(struct v3dv_job *job, uint8_t gs_out_prim_type, uint8_t gs_num_invocations) { cl_emit(&job->indirect, TESSELLATION_GEOMETRY_COMMON_PARAMS, shader) { shader.tessellation_type = TESSELLATION_TYPE_TRIANGLE; shader.tessellation_point_mode = false; shader.tessellation_edge_spacing = TESSELLATION_EDGE_SPACING_EVEN; shader.tessellation_clockwise = true; shader.tessellation_invocations = 1; shader.geometry_shader_output_format = v3d_gs_output_primitive(gs_out_prim_type); shader.geometry_shader_instances = gs_num_invocations & 0x1F; } } static uint8_t simd_width_to_gs_pack_mode(uint32_t width) { switch (width) { case 16: return V3D_PACK_MODE_16_WAY; case 8: return V3D_PACK_MODE_8_WAY; case 4: return V3D_PACK_MODE_4_WAY; case 1: return V3D_PACK_MODE_1_WAY; default: unreachable("Invalid SIMD width"); }; } static void emit_tes_gs_shader_params(struct v3dv_job *job, uint32_t gs_simd, uint32_t gs_vpm_output_size, uint32_t gs_max_vpm_input_size_per_batch) { cl_emit(&job->indirect, TESSELLATION_GEOMETRY_SHADER_PARAMS, shader) { shader.tcs_batch_flush_mode = V3D_TCS_FLUSH_MODE_FULLY_PACKED; shader.per_patch_data_column_depth = 1; shader.tcs_output_segment_size_in_sectors = 1; shader.tcs_output_segment_pack_mode = V3D_PACK_MODE_16_WAY; shader.tes_output_segment_size_in_sectors = 1; shader.tes_output_segment_pack_mode = V3D_PACK_MODE_16_WAY; shader.gs_output_segment_size_in_sectors = gs_vpm_output_size; shader.gs_output_segment_pack_mode = simd_width_to_gs_pack_mode(gs_simd); shader.tbg_max_patches_per_tcs_batch = 1; shader.tbg_max_extra_vertex_segs_for_patches_after_first = 0; shader.tbg_min_tcs_output_segments_required_in_play = 1; shader.tbg_min_per_patch_data_segments_required_in_play = 1; shader.tpg_max_patches_per_tes_batch = 1; shader.tpg_max_vertex_segments_per_tes_batch = 0; shader.tpg_max_tcs_output_segments_per_tes_batch = 1; shader.tpg_min_tes_output_segments_required_in_play = 1; shader.gbg_max_tes_output_vertex_segments_per_gs_batch = gs_max_vpm_input_size_per_batch; shader.gbg_min_gs_output_segments_required_in_play = 1; } } void v3dX(cmd_buffer_emit_gl_shader_state)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); struct v3dv_cmd_buffer_state *state = &cmd_buffer->state; struct v3dv_pipeline *pipeline = state->gfx.pipeline; assert(pipeline); struct v3dv_shader_variant *vs_variant = pipeline->shared_data->variants[BROADCOM_SHADER_VERTEX]; struct v3d_vs_prog_data *prog_data_vs = vs_variant->prog_data.vs; struct v3dv_shader_variant *vs_bin_variant = pipeline->shared_data->variants[BROADCOM_SHADER_VERTEX_BIN]; struct v3d_vs_prog_data *prog_data_vs_bin = vs_bin_variant->prog_data.vs; struct v3dv_shader_variant *fs_variant = pipeline->shared_data->variants[BROADCOM_SHADER_FRAGMENT]; struct v3d_fs_prog_data *prog_data_fs = fs_variant->prog_data.fs; struct v3dv_shader_variant *gs_variant = NULL; struct v3dv_shader_variant *gs_bin_variant = NULL; struct v3d_gs_prog_data *prog_data_gs = NULL; struct v3d_gs_prog_data *prog_data_gs_bin = NULL; if (pipeline->has_gs) { gs_variant = pipeline->shared_data->variants[BROADCOM_SHADER_GEOMETRY]; prog_data_gs = gs_variant->prog_data.gs; gs_bin_variant = pipeline->shared_data->variants[BROADCOM_SHADER_GEOMETRY_BIN]; prog_data_gs_bin = gs_bin_variant->prog_data.gs; } /* Update the cache dirty flag based on the shader progs data */ job->tmu_dirty_rcl |= prog_data_vs_bin->base.tmu_dirty_rcl; job->tmu_dirty_rcl |= prog_data_vs->base.tmu_dirty_rcl; job->tmu_dirty_rcl |= prog_data_fs->base.tmu_dirty_rcl; if (pipeline->has_gs) { job->tmu_dirty_rcl |= prog_data_gs_bin->base.tmu_dirty_rcl; job->tmu_dirty_rcl |= prog_data_gs->base.tmu_dirty_rcl; } /* See GFXH-930 workaround below */ uint32_t num_elements_to_emit = MAX2(pipeline->va_count, 1); uint32_t shader_state_record_length = cl_packet_length(GL_SHADER_STATE_RECORD); #if V3D_VERSION >= 71 if (v3d_device_has_draw_index(&pipeline->device->devinfo)) { shader_state_record_length = cl_packet_length(GL_SHADER_STATE_RECORD_DRAW_INDEX); } #endif if (pipeline->has_gs) { shader_state_record_length += cl_packet_length(GEOMETRY_SHADER_STATE_RECORD) + cl_packet_length(TESSELLATION_GEOMETRY_COMMON_PARAMS) + 2 * cl_packet_length(TESSELLATION_GEOMETRY_SHADER_PARAMS); } uint32_t shader_rec_offset = v3dv_cl_ensure_space(&job->indirect, shader_state_record_length + num_elements_to_emit * cl_packet_length(GL_SHADER_STATE_ATTRIBUTE_RECORD), 32); v3dv_return_if_oom(cmd_buffer, NULL); struct v3dv_bo *assembly_bo = pipeline->shared_data->assembly_bo; if (pipeline->has_gs) { emit_gs_shader_state_record(job, assembly_bo, gs_bin_variant, cmd_buffer->state.uniforms.gs_bin, gs_variant, cmd_buffer->state.uniforms.gs); emit_tes_gs_common_params(job, prog_data_gs->out_prim_type, prog_data_gs->num_invocations); emit_tes_gs_shader_params(job, pipeline->vpm_cfg_bin.gs_width, pipeline->vpm_cfg_bin.Gd, pipeline->vpm_cfg_bin.Gv); emit_tes_gs_shader_params(job, pipeline->vpm_cfg.gs_width, pipeline->vpm_cfg.Gd, pipeline->vpm_cfg.Gv); } #if V3D_VERSION == 42 struct v3dv_bo *default_attribute_values = pipeline->default_attribute_values != NULL ? pipeline->default_attribute_values : pipeline->device->default_attribute_float; #endif #if V3D_VERSION >= 71 if (v3d_device_has_draw_index(&pipeline->device->devinfo)) { cl_emit_with_prepacked(&job->indirect, GL_SHADER_STATE_RECORD_DRAW_INDEX, pipeline->shader_state_record, shader) { shader.min_coord_shader_input_segments_required_in_play = pipeline->vpm_cfg_bin.As; shader.min_vertex_shader_input_segments_required_in_play = pipeline->vpm_cfg.As; shader.coordinate_shader_code_address = v3dv_cl_address(assembly_bo, vs_bin_variant->assembly_offset); shader.vertex_shader_code_address = v3dv_cl_address(assembly_bo, vs_variant->assembly_offset); shader.fragment_shader_code_address = v3dv_cl_address(assembly_bo, fs_variant->assembly_offset); shader.coordinate_shader_uniforms_address = cmd_buffer->state.uniforms.vs_bin; shader.vertex_shader_uniforms_address = cmd_buffer->state.uniforms.vs; shader.fragment_shader_uniforms_address = cmd_buffer->state.uniforms.fs; shader.any_shader_reads_hardware_written_primitive_id = (pipeline->has_gs && prog_data_gs->uses_pid) || prog_data_fs->uses_pid; shader.insert_primitive_id_as_first_varying_to_fragment_shader = !pipeline->has_gs && prog_data_fs->uses_pid; } } else #endif { cl_emit_with_prepacked(&job->indirect, GL_SHADER_STATE_RECORD, pipeline->shader_state_record, shader) { /* FIXME: we are setting this values here and during the * prepacking. This is because both cl_emit_with_prepacked and v3dvx_pack * asserts for minimum values of these. It would be good to get * v3dvx_pack to assert on the final value if possible */ shader.min_coord_shader_input_segments_required_in_play = pipeline->vpm_cfg_bin.As; shader.min_vertex_shader_input_segments_required_in_play = pipeline->vpm_cfg.As; shader.coordinate_shader_code_address = v3dv_cl_address(assembly_bo, vs_bin_variant->assembly_offset); shader.vertex_shader_code_address = v3dv_cl_address(assembly_bo, vs_variant->assembly_offset); shader.fragment_shader_code_address = v3dv_cl_address(assembly_bo, fs_variant->assembly_offset); shader.coordinate_shader_uniforms_address = cmd_buffer->state.uniforms.vs_bin; shader.vertex_shader_uniforms_address = cmd_buffer->state.uniforms.vs; shader.fragment_shader_uniforms_address = cmd_buffer->state.uniforms.fs; #if V3D_VERSION == 42 shader.address_of_default_attribute_values = v3dv_cl_address(default_attribute_values, 0); #endif shader.any_shader_reads_hardware_written_primitive_id = (pipeline->has_gs && prog_data_gs->uses_pid) || prog_data_fs->uses_pid; shader.insert_primitive_id_as_first_varying_to_fragment_shader = !pipeline->has_gs && prog_data_fs->uses_pid; } } /* Upload vertex element attributes (SHADER_STATE_ATTRIBUTE_RECORD) */ bool cs_loaded_any = false; const bool cs_uses_builtins = prog_data_vs_bin->uses_iid || prog_data_vs_bin->uses_biid || prog_data_vs_bin->uses_vid; const uint32_t packet_length = cl_packet_length(GL_SHADER_STATE_ATTRIBUTE_RECORD); uint32_t emitted_va_count = 0; for (uint32_t i = 0; emitted_va_count < pipeline->va_count; i++) { assert(i < MAX_VERTEX_ATTRIBS); if (pipeline->va[i].vk_format == VK_FORMAT_UNDEFINED) continue; const uint32_t binding = pipeline->va[i].binding; /* We store each vertex attribute in the array using its driver location * as index. */ const uint32_t location = i; struct v3dv_vertex_binding *c_vb = &cmd_buffer->state.vertex_bindings[binding]; cl_emit_with_prepacked(&job->indirect, GL_SHADER_STATE_ATTRIBUTE_RECORD, &pipeline->vertex_attrs[i * packet_length], attr) { assert(c_vb->buffer->mem->bo); attr.address = v3dv_cl_address(c_vb->buffer->mem->bo, c_vb->buffer->mem_offset + pipeline->va[i].offset + c_vb->offset); attr.number_of_values_read_by_coordinate_shader = prog_data_vs_bin->vattr_sizes[location]; attr.number_of_values_read_by_vertex_shader = prog_data_vs->vattr_sizes[location]; /* GFXH-930: At least one attribute must be enabled and read by CS * and VS. If we have attributes being consumed by the VS but not * the CS, then set up a dummy load of the last attribute into the * CS's VPM inputs. (Since CS is just dead-code-elimination compared * to VS, we can't have CS loading but not VS). * * GFXH-1602: first attribute must be active if using builtins. */ if (prog_data_vs_bin->vattr_sizes[location]) cs_loaded_any = true; if (i == 0 && cs_uses_builtins && !cs_loaded_any) { attr.number_of_values_read_by_coordinate_shader = 1; cs_loaded_any = true; } else if (i == pipeline->va_count - 1 && !cs_loaded_any) { attr.number_of_values_read_by_coordinate_shader = 1; cs_loaded_any = true; } attr.stride = cmd_buffer->vk.dynamic_graphics_state.vi_binding_strides[binding]; attr.maximum_index = attr.stride == 0 ? 1u : MIN2(0xffffffu, c_vb->size / attr.stride); } emitted_va_count++; } if (pipeline->va_count == 0) { /* GFXH-930: At least one attribute must be enabled and read * by CS and VS. If we have no attributes being consumed by * the shader, set up a dummy to be loaded into the VPM. */ cl_emit(&job->indirect, GL_SHADER_STATE_ATTRIBUTE_RECORD, attr) { /* Valid address of data whose value will be unused. */ attr.address = v3dv_cl_address(job->indirect.bo, 0); attr.type = ATTRIBUTE_FLOAT; attr.stride = 0; attr.vec_size = 1; attr.number_of_values_read_by_coordinate_shader = 1; attr.number_of_values_read_by_vertex_shader = 1; } } if (cmd_buffer->state.dirty & V3DV_CMD_DIRTY_PIPELINE) { v3dv_cl_ensure_space_with_branch(&job->bcl, sizeof(pipeline->vcm_cache_size)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit_prepacked(&job->bcl, &pipeline->vcm_cache_size); } v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(GL_SHADER_STATE)); v3dv_return_if_oom(cmd_buffer, NULL); if (pipeline->has_gs) { cl_emit(&job->bcl, GL_SHADER_STATE_INCLUDING_GS, state) { state.address = v3dv_cl_address(job->indirect.bo, shader_rec_offset); state.number_of_attribute_arrays = num_elements_to_emit; } } else { cl_emit(&job->bcl, GL_SHADER_STATE, state) { state.address = v3dv_cl_address(job->indirect.bo, shader_rec_offset); state.number_of_attribute_arrays = num_elements_to_emit; } } /* Clearing push constants and descriptor sets for all stages is not quite * correct (some shader stages may not be used at all or they may not be * consuming push constants), however this is not relevant because if we * bind a different pipeline we always have to rebuild the uniform streams. */ cmd_buffer->state.dirty &= ~(V3DV_CMD_DIRTY_VERTEX_BUFFER | V3DV_CMD_DIRTY_DESCRIPTOR_SETS | V3DV_CMD_DIRTY_PUSH_CONSTANTS); cmd_buffer->state.dirty_descriptor_stages &= ~VK_SHADER_STAGE_ALL_GRAPHICS; cmd_buffer->state.dirty_push_constants_stages &= ~VK_SHADER_STAGE_ALL_GRAPHICS; } void v3dX(cmd_buffer_emit_draw)(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_draw_info *info) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); const struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; uint32_t hw_prim_type = v3dv_pipeline_primitive(dyn->ia.primitive_topology); if (info->first_instance > 0) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(BASE_VERTEX_BASE_INSTANCE)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, BASE_VERTEX_BASE_INSTANCE, base) { base.base_instance = info->first_instance; base.base_vertex = 0; } } if (info->instance_count > 1) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(VERTEX_ARRAY_INSTANCED_PRIMS)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, VERTEX_ARRAY_INSTANCED_PRIMS, prim) { prim.mode = hw_prim_type; prim.index_of_first_vertex = info->first_vertex; prim.number_of_instances = info->instance_count; prim.instance_length = info->vertex_count; } } else { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(VERTEX_ARRAY_PRIMS)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, VERTEX_ARRAY_PRIMS, prim) { prim.mode = hw_prim_type; prim.length = info->vertex_count; prim.index_of_first_vertex = info->first_vertex; } } } void v3dX(cmd_buffer_emit_index_buffer)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); /* We flag all state as dirty when we create a new job so make sure we * have a valid index buffer before attempting to emit state for it. */ struct v3dv_buffer *ibuffer = v3dv_buffer_from_handle(cmd_buffer->state.index_buffer.buffer); if (ibuffer) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(INDEX_BUFFER_SETUP)); v3dv_return_if_oom(cmd_buffer, NULL); const uint32_t offset = ibuffer->mem_offset + cmd_buffer->state.index_buffer.offset; assert(ibuffer->mem->bo->size >= offset); cl_emit(&job->bcl, INDEX_BUFFER_SETUP, ib) { ib.address = v3dv_cl_address(ibuffer->mem->bo, offset); ib.size = cmd_buffer->state.index_buffer.size; } } cmd_buffer->state.dirty &= ~V3DV_CMD_DIRTY_INDEX_BUFFER; } void v3dX(cmd_buffer_emit_draw_indexed)(struct v3dv_cmd_buffer *cmd_buffer, uint32_t indexCount, uint32_t instanceCount, uint32_t firstIndex, int32_t vertexOffset, uint32_t firstInstance) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); const struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; uint32_t hw_prim_type = v3dv_pipeline_primitive(dyn->ia.primitive_topology); uint8_t index_type = ffs(cmd_buffer->state.index_buffer.index_size) - 1; uint32_t index_offset = firstIndex * cmd_buffer->state.index_buffer.index_size; if (vertexOffset != 0 || firstInstance != 0) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(BASE_VERTEX_BASE_INSTANCE)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, BASE_VERTEX_BASE_INSTANCE, base) { base.base_instance = firstInstance; base.base_vertex = vertexOffset; } } if (instanceCount == 1) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(INDEXED_PRIM_LIST)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, INDEXED_PRIM_LIST, prim) { prim.index_type = index_type; prim.length = indexCount; prim.index_offset = index_offset; prim.mode = hw_prim_type; prim.enable_primitive_restarts = dyn->ia.primitive_restart_enable; } } else if (instanceCount > 1) { v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(INDEXED_INSTANCED_PRIM_LIST)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, INDEXED_INSTANCED_PRIM_LIST, prim) { prim.index_type = index_type; prim.index_offset = index_offset; prim.mode = hw_prim_type; prim.enable_primitive_restarts = dyn->ia.primitive_restart_enable; prim.number_of_instances = instanceCount; prim.instance_length = indexCount; } } } void v3dX(cmd_buffer_emit_draw_indirect)(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_buffer *buffer, VkDeviceSize offset, uint32_t drawCount, uint32_t stride) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); const struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; uint32_t hw_prim_type = v3dv_pipeline_primitive(dyn->ia.primitive_topology); v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(INDIRECT_VERTEX_ARRAY_INSTANCED_PRIMS)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, INDIRECT_VERTEX_ARRAY_INSTANCED_PRIMS, prim) { prim.mode = hw_prim_type; prim.number_of_draw_indirect_array_records = drawCount; prim.stride_in_multiples_of_4_bytes = stride >> 2; prim.address = v3dv_cl_address(buffer->mem->bo, buffer->mem_offset + offset); } } void v3dX(cmd_buffer_emit_indexed_indirect)(struct v3dv_cmd_buffer *cmd_buffer, struct v3dv_buffer *buffer, VkDeviceSize offset, uint32_t drawCount, uint32_t stride) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); const struct vk_dynamic_graphics_state *dyn = &cmd_buffer->vk.dynamic_graphics_state; uint32_t hw_prim_type = v3dv_pipeline_primitive(dyn->ia.primitive_topology); uint8_t index_type = ffs(cmd_buffer->state.index_buffer.index_size) - 1; v3dv_cl_ensure_space_with_branch( &job->bcl, cl_packet_length(INDIRECT_INDEXED_INSTANCED_PRIM_LIST)); v3dv_return_if_oom(cmd_buffer, NULL); cl_emit(&job->bcl, INDIRECT_INDEXED_INSTANCED_PRIM_LIST, prim) { prim.index_type = index_type; prim.mode = hw_prim_type; prim.enable_primitive_restarts = dyn->ia.primitive_restart_enable; prim.number_of_draw_indirect_indexed_records = drawCount; prim.stride_in_multiples_of_4_bytes = stride >> 2; prim.address = v3dv_cl_address(buffer->mem->bo, buffer->mem_offset + offset); } } void v3dX(cmd_buffer_suspend)(struct v3dv_cmd_buffer *cmd_buffer) { struct v3dv_job *job = cmd_buffer->state.job; assert(job); job->suspending = true; v3dv_cl_ensure_space_with_branch(&job->bcl, cl_packet_length(BRANCH)); job->suspend_branch_inst_ptr = cl_start(&job->bcl); cl_emit(&job->bcl, BRANCH, branch) { branch.address = v3dv_cl_address(NULL, 0); } /* The sim complains if the command list ends with a branch */ cl_emit(&job->bcl, NOP, nop); } void v3dX(job_patch_resume_address)(struct v3dv_job *first_suspend, struct v3dv_job *suspend, struct v3dv_job *resume) { assert(resume && resume->resuming); assert(first_suspend && first_suspend->suspending); assert(suspend && suspend->suspending); assert(suspend->suspend_branch_inst_ptr != NULL); struct v3dv_bo *resume_bo = list_first_entry(&resume->bcl.bo_list, struct v3dv_bo, list_link); struct cl_packet_struct(BRANCH) branch = { cl_packet_header(BRANCH), }; branch.address = v3dv_cl_address(NULL, resume_bo->offset); uint8_t *rewrite_addr = (uint8_t *) suspend->suspend_branch_inst_ptr; cl_packet_pack(BRANCH)(NULL, rewrite_addr, &branch); if (resume != first_suspend) { set_foreach(resume->bos, entry) { struct v3dv_bo *bo = (void *)entry->key; v3dv_job_add_bo(first_suspend, bo); } } first_suspend->suspended_bcl_end = resume->bcl.bo->offset + v3dv_cl_offset(&resume->bcl); } static void job_destroy_cb(VkDevice device, uint64_t pobj, VkAllocationCallbacks *allocb) { struct v3dv_job *clone = (struct v3dv_job *) (uintptr_t) pobj; v3dv_job_destroy(clone); } /** * This checks if the command buffer has been created with * VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT, in which case we won't be * able to safely patch the resume address into the job (since we could have * another instance of this job running in the GPU, potentially resuming in a * different address). In that case, we clone the job and make the clone have * its own BCL copied from the original job so we can later patch the resume * address into it safely. */ struct v3dv_job * v3dX(cmd_buffer_prepare_suspend_job_for_submit)(struct v3dv_job *job) { assert(job->suspending); assert(job->cmd_buffer); assert(job->type == V3DV_JOB_TYPE_GPU_CL); if (!(job->cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT)) return job; /* Create the clone job, but skip the BCL since we are going to create * our own below. */ struct v3dv_job *clone = v3dv_job_clone(job, true); if (!clone) return NULL; /* Compute total size of BCL we need to copy */ uint32_t bcl_size = 0; list_for_each_entry(struct v3dv_bo, bo, &job->bcl.bo_list, list_link) bcl_size += bo->size; /* Prepare the BCL for the cloned job. For this we go over the BOs in the * BCL of the original job and we copy their contents into the single BO * in the BCL of the cloned job. */ clone->clone_owns_bcl = true; v3dv_cl_init(clone, &clone->bcl); v3dv_cl_ensure_space(&clone->bcl, bcl_size, 4); if (!clone->bcl.bo) return NULL; assert(clone->bcl.base); assert(clone->bcl.base == clone->bcl.next); /* Unlink this job from the command buffer's execution list */ list_inithead(&clone->list_link); /* Copy the contents of each BO in the original job's BCL into the single * BO we have in the clone's BCL. * * If the BO is the last in the BCL (which we can tell because it wouldn't * have emitted a BRANCH instruction to link to another BO) we need to copy * up to the current BCL offset, otherwise we need to copy up to the BRANCH * instruction (excluded, since we are putting everything together into a * single BO here). */ list_for_each_entry(struct v3dv_bo, bo, &job->bcl.bo_list, list_link) { assert(bo->map); uint32_t copy_size; if (bo->cl_branch_offset == 0xffffffff) { /* Last BO in BCL */ assert(bo == list_last_entry(&job->bcl.bo_list, struct v3dv_bo, list_link)); copy_size = v3dv_cl_offset(&job->bcl); } else { assert(bo->cl_branch_offset >= cl_packet_length(BRANCH)); copy_size = bo->cl_branch_offset - cl_packet_length(BRANCH); } assert(v3dv_cl_offset(&job->bcl) + copy_size < bcl_size); memcpy(cl_start(&clone->bcl), bo->map, copy_size); cl_advance_and_end(&clone->bcl, copy_size); } /* Now we need to fixup the pointer to the suspend BRANCH instruction at the * end of the BCL so it points to the address in the new BCL. We know that * to suspend a command buffer we always emit a BRANCH+NOP combo, so we just * need to go back that many bytes in to the BCL to find the instruction. */ uint32_t suspend_terminator_size = cl_packet_length(BRANCH) + cl_packet_length(NOP); clone->suspend_branch_inst_ptr = (struct v3dv_cl_out *) (((uint8_t *)cl_start(&clone->bcl)) - suspend_terminator_size); assert(*(((uint8_t *)clone->suspend_branch_inst_ptr)) == V3DX(BRANCH_opcode)); /* This job is not in the execution list of the command buffer so it * won't be destroyed with it; add it as a private object to get it freed. * * FIXME: every time this job is submitted we clone the job and we only * destroy it when the command buffer is destroyed. If the user keeps the * command buffer for the entire lifetime of the application, this command * buffer could grow significantly, so maybe we want to do something smarter * like having a syncobj bound to these jobs and every time we submit the * command buffer again we first check these sncobjs to see if we can free * some of these clones so we avoid blowing up memory. */ v3dv_cmd_buffer_add_private_obj( job->cmd_buffer, (uintptr_t)clone, (v3dv_cmd_buffer_private_obj_destroy_cb)job_destroy_cb); return clone; }