/* * Copyright 2019 Google LLC * SPDX-License-Identifier: MIT * * based in part on anv and radv which are: * Copyright © 2015 Intel Corporation * Copyright © 2016 Red Hat. * Copyright © 2016 Bas Nieuwenhuizen */ #include "vn_queue.h" #include "util/libsync.h" #include "venus-protocol/vn_protocol_driver_event.h" #include "venus-protocol/vn_protocol_driver_fence.h" #include "venus-protocol/vn_protocol_driver_queue.h" #include "venus-protocol/vn_protocol_driver_semaphore.h" #include "venus-protocol/vn_protocol_driver_transport.h" #include "vn_command_buffer.h" #include "vn_device.h" #include "vn_device_memory.h" #include "vn_feedback.h" #include "vn_instance.h" #include "vn_physical_device.h" #include "vn_query_pool.h" #include "vn_renderer.h" #include "vn_wsi.h" /* queue commands */ struct vn_submit_info_pnext_fix { VkDeviceGroupSubmitInfo group; VkProtectedSubmitInfo protected; VkTimelineSemaphoreSubmitInfo timeline; }; struct vn_queue_submission { VkStructureType batch_type; VkQueue queue_handle; uint32_t batch_count; union { const void *batches; const VkSubmitInfo *submit_batches; const VkSubmitInfo2 *submit2_batches; const VkBindSparseInfo *sparse_batches; }; VkFence fence_handle; uint32_t cmd_count; uint32_t feedback_types; uint32_t pnext_count; uint32_t dev_mask_count; bool has_zink_sync_batch; const struct vn_device_memory *wsi_mem; struct vn_sync_payload_external external_payload; /* Temporary storage allocation for submission * * A single alloc for storage is performed and the offsets inside storage * are set as below: * * batches * - non-empty submission: copy of original batches * - empty submission: a single batch for fence feedback (ffb) * cmds * - for each batch: * - copy of original batch cmds * - a single cmd for query feedback (qfb) * - one cmd for each signal semaphore that has feedback (sfb) * - if last batch, a single cmd for ffb */ struct { void *storage; union { void *batches; VkSubmitInfo *submit_batches; VkSubmitInfo2 *submit2_batches; }; union { void *cmds; VkCommandBuffer *cmd_handles; VkCommandBufferSubmitInfo *cmd_infos; }; struct vn_submit_info_pnext_fix *pnexts; uint32_t *dev_masks; } temp; }; static inline uint32_t vn_get_wait_semaphore_count(struct vn_queue_submission *submit, uint32_t batch_index) { switch (submit->batch_type) { case VK_STRUCTURE_TYPE_SUBMIT_INFO: return submit->submit_batches[batch_index].waitSemaphoreCount; case VK_STRUCTURE_TYPE_SUBMIT_INFO_2: return submit->submit2_batches[batch_index].waitSemaphoreInfoCount; case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO: return submit->sparse_batches[batch_index].waitSemaphoreCount; default: unreachable("unexpected batch type"); } } static inline uint32_t vn_get_signal_semaphore_count(struct vn_queue_submission *submit, uint32_t batch_index) { switch (submit->batch_type) { case VK_STRUCTURE_TYPE_SUBMIT_INFO: return submit->submit_batches[batch_index].signalSemaphoreCount; case VK_STRUCTURE_TYPE_SUBMIT_INFO_2: return submit->submit2_batches[batch_index].signalSemaphoreInfoCount; case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO: return submit->sparse_batches[batch_index].signalSemaphoreCount; default: unreachable("unexpected batch type"); } } static inline VkSemaphore vn_get_wait_semaphore(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t semaphore_index) { switch (submit->batch_type) { case VK_STRUCTURE_TYPE_SUBMIT_INFO: return submit->submit_batches[batch_index] .pWaitSemaphores[semaphore_index]; case VK_STRUCTURE_TYPE_SUBMIT_INFO_2: return submit->submit2_batches[batch_index] .pWaitSemaphoreInfos[semaphore_index] .semaphore; case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO: return submit->sparse_batches[batch_index] .pWaitSemaphores[semaphore_index]; default: unreachable("unexpected batch type"); } } static inline VkSemaphore vn_get_signal_semaphore(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t semaphore_index) { switch (submit->batch_type) { case VK_STRUCTURE_TYPE_SUBMIT_INFO: return submit->submit_batches[batch_index] .pSignalSemaphores[semaphore_index]; case VK_STRUCTURE_TYPE_SUBMIT_INFO_2: return submit->submit2_batches[batch_index] .pSignalSemaphoreInfos[semaphore_index] .semaphore; case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO: return submit->sparse_batches[batch_index] .pSignalSemaphores[semaphore_index]; default: unreachable("unexpected batch type"); } } static inline size_t vn_get_batch_size(struct vn_queue_submission *submit) { assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) || (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2)); return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO ? sizeof(VkSubmitInfo) : sizeof(VkSubmitInfo2); } static inline size_t vn_get_cmd_size(struct vn_queue_submission *submit) { assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) || (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2)); return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO ? sizeof(VkCommandBuffer) : sizeof(VkCommandBufferSubmitInfo); } static inline uint32_t vn_get_cmd_count(struct vn_queue_submission *submit, uint32_t batch_index) { assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) || (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2)); return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO ? submit->submit_batches[batch_index].commandBufferCount : submit->submit2_batches[batch_index].commandBufferInfoCount; } static inline const void * vn_get_cmds(struct vn_queue_submission *submit, uint32_t batch_index) { assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) || (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2)); return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO ? (const void *)submit->submit_batches[batch_index] .pCommandBuffers : (const void *)submit->submit2_batches[batch_index] .pCommandBufferInfos; } static inline struct vn_command_buffer * vn_get_cmd(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t cmd_index) { assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) || (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2)); return vn_command_buffer_from_handle( submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO ? submit->submit_batches[batch_index].pCommandBuffers[cmd_index] : submit->submit2_batches[batch_index] .pCommandBufferInfos[cmd_index] .commandBuffer); } static inline void vn_set_temp_cmd(struct vn_queue_submission *submit, uint32_t cmd_index, VkCommandBuffer cmd_handle) { assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) || (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2)); if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) { submit->temp.cmd_infos[cmd_index] = (VkCommandBufferSubmitInfo){ .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cmd_handle, }; } else { submit->temp.cmd_handles[cmd_index] = cmd_handle; } } static uint64_t vn_get_signal_semaphore_counter(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t sem_index) { switch (submit->batch_type) { case VK_STRUCTURE_TYPE_SUBMIT_INFO: { const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info = vk_find_struct_const(submit->submit_batches[batch_index].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO); return timeline_sem_info->pSignalSemaphoreValues[sem_index]; } case VK_STRUCTURE_TYPE_SUBMIT_INFO_2: return submit->submit2_batches[batch_index] .pSignalSemaphoreInfos[sem_index] .value; default: unreachable("unexpected batch type"); } } static bool vn_has_zink_sync_batch(struct vn_queue_submission *submit) { struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle); struct vn_device *dev = (void *)queue->base.base.base.device; struct vn_instance *instance = dev->instance; const uint32_t last_batch_index = submit->batch_count - 1; if (!instance->engine_is_zink) return false; if (!submit->batch_count || !last_batch_index || vn_get_cmd_count(submit, last_batch_index)) return false; if (vn_get_wait_semaphore_count(submit, last_batch_index)) return false; const uint32_t signal_count = vn_get_signal_semaphore_count(submit, last_batch_index); for (uint32_t i = 0; i < signal_count; i++) { struct vn_semaphore *sem = vn_semaphore_from_handle( vn_get_signal_semaphore(submit, last_batch_index, i)); if (sem->feedback.slot) { return true; } } return false; } static bool vn_fix_batch_cmd_count_for_zink_sync(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t new_cmd_count) { /* If the last batch is a zink sync batch which is empty but contains * feedback, append the feedback to the previous batch instead so that * the last batch remains empty for perf. */ if (batch_index == submit->batch_count - 1 && submit->has_zink_sync_batch) { if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) { VkSubmitInfo2 *batch = &submit->temp.submit2_batches[batch_index - 1]; assert(batch->pCommandBufferInfos); batch->commandBufferInfoCount += new_cmd_count; } else { VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index - 1]; assert(batch->pCommandBuffers); batch->commandBufferCount += new_cmd_count; } return true; } return false; } static void vn_fix_device_group_cmd_count(struct vn_queue_submission *submit, uint32_t batch_index) { struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle); struct vn_device *dev = (void *)queue_vk->base.device; const VkSubmitInfo *src_batch = &submit->submit_batches[batch_index]; struct vn_submit_info_pnext_fix *pnext_fix = submit->temp.pnexts; VkBaseOutStructure *dst = (void *)&submit->temp.submit_batches[batch_index]; uint32_t new_cmd_count = submit->temp.submit_batches[batch_index].commandBufferCount; vk_foreach_struct_const(src, src_batch->pNext) { void *pnext = NULL; switch (src->sType) { case VK_STRUCTURE_TYPE_DEVICE_GROUP_SUBMIT_INFO: { uint32_t orig_cmd_count = 0; memcpy(&pnext_fix->group, src, sizeof(pnext_fix->group)); VkDeviceGroupSubmitInfo *src_device_group = (VkDeviceGroupSubmitInfo *)src; if (src_device_group->commandBufferCount) { orig_cmd_count = src_device_group->commandBufferCount; memcpy(submit->temp.dev_masks, src_device_group->pCommandBufferDeviceMasks, sizeof(uint32_t) * orig_cmd_count); } /* Set the group device mask. Unlike sync2, zero means skip. */ for (uint32_t i = orig_cmd_count; i < new_cmd_count; i++) { submit->temp.dev_masks[i] = dev->device_mask; } pnext_fix->group.commandBufferCount = new_cmd_count; pnext_fix->group.pCommandBufferDeviceMasks = submit->temp.dev_masks; pnext = &pnext_fix->group; break; } case VK_STRUCTURE_TYPE_PROTECTED_SUBMIT_INFO: memcpy(&pnext_fix->protected, src, sizeof(pnext_fix->protected)); pnext = &pnext_fix->protected; break; case VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO: memcpy(&pnext_fix->timeline, src, sizeof(pnext_fix->timeline)); pnext = &pnext_fix->timeline; break; default: /* The following structs are not supported by venus so are not * handled here. VkAmigoProfilingSubmitInfoSEC, * VkD3D12FenceSubmitInfoKHR, VkFrameBoundaryEXT, * VkLatencySubmissionPresentIdNV, VkPerformanceQuerySubmitInfoKHR, * VkWin32KeyedMutexAcquireReleaseInfoKHR, * VkWin32KeyedMutexAcquireReleaseInfoNV */ break; } if (pnext) { dst->pNext = pnext; dst = pnext; } } submit->temp.pnexts++; submit->temp.dev_masks += new_cmd_count; } static bool vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem); static VkResult vn_queue_submission_fix_batch_semaphores(struct vn_queue_submission *submit, uint32_t batch_index) { struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle); VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device); struct vn_device *dev = vn_device_from_handle(dev_handle); const uint32_t wait_count = vn_get_wait_semaphore_count(submit, batch_index); for (uint32_t i = 0; i < wait_count; i++) { VkSemaphore sem_handle = vn_get_wait_semaphore(submit, batch_index, i); struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle); const struct vn_sync_payload *payload = sem->payload; if (payload->type != VN_SYNC_TYPE_IMPORTED_SYNC_FD) continue; if (!vn_semaphore_wait_external(dev, sem)) return VK_ERROR_DEVICE_LOST; assert(dev->physical_device->renderer_sync_fd.semaphore_importable); const VkImportSemaphoreResourceInfoMESA res_info = { .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA, .semaphore = sem_handle, .resourceId = 0, }; vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, dev_handle, &res_info); } return VK_SUCCESS; } static void vn_queue_submission_count_batch_feedback(struct vn_queue_submission *submit, uint32_t batch_index) { const uint32_t signal_count = vn_get_signal_semaphore_count(submit, batch_index); uint32_t extra_cmd_count = 0; uint32_t feedback_types = 0; for (uint32_t i = 0; i < signal_count; i++) { struct vn_semaphore *sem = vn_semaphore_from_handle( vn_get_signal_semaphore(submit, batch_index, i)); if (sem->feedback.slot) { feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE; extra_cmd_count++; } } if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) { const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index); for (uint32_t i = 0; i < cmd_count; i++) { struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i); if (!list_is_empty(&cmd->builder.query_records)) feedback_types |= VN_FEEDBACK_TYPE_QUERY; /* If a cmd that was submitted previously and already has a feedback * cmd linked, as long as * VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT was not set we can * assume it has completed execution and is no longer in the pending * state so its safe to recycle the old feedback command. */ if (cmd->linked_qfb_cmd) { assert(!cmd->builder.is_simultaneous); vn_query_feedback_cmd_free(cmd->linked_qfb_cmd); cmd->linked_qfb_cmd = NULL; } } if (feedback_types & VN_FEEDBACK_TYPE_QUERY) extra_cmd_count++; if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE && batch_index == submit->batch_count - 1) { feedback_types |= VN_FEEDBACK_TYPE_FENCE; extra_cmd_count++; } /* Space to copy the original cmds to append feedback to it. * If the last batch is a zink sync batch which is an empty batch with * sem feedback, feedback will be appended to the second to last batch * so also need to copy the second to last batch's original cmds even * if it doesn't have feedback itself. */ if (feedback_types || (batch_index == submit->batch_count - 2 && submit->has_zink_sync_batch)) { extra_cmd_count += cmd_count; } } if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO && extra_cmd_count) { const VkDeviceGroupSubmitInfo *device_group = vk_find_struct_const( submit->submit_batches[batch_index].pNext, DEVICE_GROUP_SUBMIT_INFO); if (device_group) { submit->pnext_count++; submit->dev_mask_count += extra_cmd_count; } } submit->feedback_types |= feedback_types; submit->cmd_count += extra_cmd_count; } static VkResult vn_queue_submission_prepare(struct vn_queue_submission *submit) { struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle); struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle); assert(!fence || !fence->is_external || !fence->feedback.slot); if (fence && fence->feedback.slot) submit->feedback_types |= VN_FEEDBACK_TYPE_FENCE; if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) submit->has_zink_sync_batch = vn_has_zink_sync_batch(submit); submit->external_payload.ring_idx = queue->ring_idx; submit->wsi_mem = NULL; if (submit->batch_count == 1 && submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) { const struct wsi_memory_signal_submit_info *info = vk_find_struct_const( submit->submit_batches[0].pNext, WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA); if (info) { submit->wsi_mem = vn_device_memory_from_handle(info->memory); assert(submit->wsi_mem->base_bo); } } for (uint32_t i = 0; i < submit->batch_count; i++) { VkResult result = vn_queue_submission_fix_batch_semaphores(submit, i); if (result != VK_SUCCESS) return result; vn_queue_submission_count_batch_feedback(submit, i); } return VK_SUCCESS; } static VkResult vn_queue_submission_alloc_storage(struct vn_queue_submission *submit) { struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle); if (!submit->feedback_types) return VK_SUCCESS; /* for original batches or a new batch to hold feedback fence cmd */ const size_t total_batch_size = vn_get_batch_size(submit) * MAX2(submit->batch_count, 1); /* for fence, timeline semaphore and query feedback cmds */ const size_t total_cmd_size = vn_get_cmd_size(submit) * MAX2(submit->cmd_count, 1); /* for fixing command buffer counts in device group info, if it exists */ const size_t total_pnext_size = submit->pnext_count * sizeof(struct vn_submit_info_pnext_fix); const size_t total_dev_mask_size = submit->dev_mask_count * sizeof(uint32_t); submit->temp.storage = vn_cached_storage_get( &queue->storage, total_batch_size + total_cmd_size + total_pnext_size + total_dev_mask_size); if (!submit->temp.storage) return VK_ERROR_OUT_OF_HOST_MEMORY; submit->temp.batches = submit->temp.storage; submit->temp.cmds = submit->temp.storage + total_batch_size; submit->temp.pnexts = submit->temp.storage + total_batch_size + total_cmd_size; submit->temp.dev_masks = submit->temp.storage + total_batch_size + total_cmd_size + total_pnext_size; return VK_SUCCESS; } static VkResult vn_queue_submission_get_resolved_query_records( struct vn_queue_submission *submit, uint32_t batch_index, struct vn_feedback_cmd_pool *fb_cmd_pool, struct list_head *resolved_records) { struct vn_command_pool *cmd_pool = vn_command_pool_from_handle(fb_cmd_pool->pool_handle); struct list_head dropped_records; VkResult result = VK_SUCCESS; list_inithead(resolved_records); list_inithead(&dropped_records); const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index); for (uint32_t i = 0; i < cmd_count; i++) { struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i); list_for_each_entry(struct vn_cmd_query_record, record, &cmd->builder.query_records, head) { if (!record->copy) { list_for_each_entry_safe(struct vn_cmd_query_record, prev, resolved_records, head) { /* If we previously added a query feedback that is now getting * reset, remove it since it is now a no-op and the deferred * feedback copy will cause a hang waiting for the reset query * to become available. */ if (prev->copy && prev->query_pool == record->query_pool && prev->query >= record->query && prev->query < record->query + record->query_count) list_move_to(&prev->head, &dropped_records); } } simple_mtx_lock(&fb_cmd_pool->mutex); struct vn_cmd_query_record *curr = vn_cmd_pool_alloc_query_record( cmd_pool, record->query_pool, record->query, record->query_count, record->copy); simple_mtx_unlock(&fb_cmd_pool->mutex); if (!curr) { list_splicetail(resolved_records, &dropped_records); result = VK_ERROR_OUT_OF_HOST_MEMORY; goto out_free_dropped_records; } list_addtail(&curr->head, resolved_records); } } /* further resolve to batch sequential queries */ struct vn_cmd_query_record *curr = list_first_entry(resolved_records, struct vn_cmd_query_record, head); list_for_each_entry_safe(struct vn_cmd_query_record, next, resolved_records, head) { if (curr->query_pool == next->query_pool && curr->copy == next->copy) { if (curr->query + curr->query_count == next->query) { curr->query_count += next->query_count; list_move_to(&next->head, &dropped_records); } else if (curr->query == next->query + next->query_count) { curr->query = next->query; curr->query_count += next->query_count; list_move_to(&next->head, &dropped_records); } else { curr = next; } } else { curr = next; } } out_free_dropped_records: simple_mtx_lock(&fb_cmd_pool->mutex); vn_cmd_pool_free_query_records(cmd_pool, &dropped_records); simple_mtx_unlock(&fb_cmd_pool->mutex); return result; } static VkResult vn_queue_submission_add_query_feedback(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t *new_cmd_count) { struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle); struct vn_device *dev = (void *)queue_vk->base.device; VkResult result; struct vn_feedback_cmd_pool *fb_cmd_pool = NULL; for (uint32_t i = 0; i < dev->queue_family_count; i++) { if (dev->queue_families[i] == queue_vk->queue_family_index) { fb_cmd_pool = &dev->fb_cmd_pools[i]; break; } } assert(fb_cmd_pool); struct list_head resolved_records; result = vn_queue_submission_get_resolved_query_records( submit, batch_index, fb_cmd_pool, &resolved_records); if (result != VK_SUCCESS) return result; /* currently the reset query is always recorded */ assert(!list_is_empty(&resolved_records)); struct vn_query_feedback_cmd *qfb_cmd; result = vn_query_feedback_cmd_alloc(vn_device_to_handle(dev), fb_cmd_pool, &resolved_records, &qfb_cmd); if (result == VK_SUCCESS) { /* link query feedback cmd lifecycle with a cmd in the original batch so * that the feedback cmd can be reset and recycled when that cmd gets * reset/freed. * * Avoid cmd buffers with VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT * since we don't know if all its instances have completed execution. * Should be rare enough to just log and leak the feedback cmd. */ bool found_companion_cmd = false; const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index); for (uint32_t i = 0; i < cmd_count; i++) { struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i); if (!cmd->builder.is_simultaneous) { cmd->linked_qfb_cmd = qfb_cmd; found_companion_cmd = true; break; } } if (!found_companion_cmd) vn_log(dev->instance, "WARN: qfb cmd has leaked!"); vn_set_temp_cmd(submit, (*new_cmd_count)++, qfb_cmd->cmd_handle); } simple_mtx_lock(&fb_cmd_pool->mutex); vn_cmd_pool_free_query_records( vn_command_pool_from_handle(fb_cmd_pool->pool_handle), &resolved_records); simple_mtx_unlock(&fb_cmd_pool->mutex); return result; } struct vn_semaphore_feedback_cmd * vn_semaphore_get_feedback_cmd(struct vn_device *dev, struct vn_semaphore *sem); static VkResult vn_queue_submission_add_semaphore_feedback(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t signal_index, uint32_t *new_cmd_count) { struct vn_semaphore *sem = vn_semaphore_from_handle( vn_get_signal_semaphore(submit, batch_index, signal_index)); if (!sem->feedback.slot) return VK_SUCCESS; VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle); struct vn_device *dev = (void *)queue_vk->base.device; struct vn_semaphore_feedback_cmd *sfb_cmd = vn_semaphore_get_feedback_cmd(dev, sem); if (!sfb_cmd) return VK_ERROR_OUT_OF_HOST_MEMORY; const uint64_t counter = vn_get_signal_semaphore_counter(submit, batch_index, signal_index); vn_feedback_set_counter(sfb_cmd->src_slot, counter); for (uint32_t i = 0; i < dev->queue_family_count; i++) { if (dev->queue_families[i] == queue_vk->queue_family_index) { vn_set_temp_cmd(submit, (*new_cmd_count)++, sfb_cmd->cmd_handles[i]); return VK_SUCCESS; } } unreachable("bad feedback sem"); } static void vn_queue_submission_add_fence_feedback(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t *new_cmd_count) { VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle); struct vn_device *dev = (void *)queue_vk->base.device; struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle); VkCommandBuffer ffb_cmd_handle = VK_NULL_HANDLE; for (uint32_t i = 0; i < dev->queue_family_count; i++) { if (dev->queue_families[i] == queue_vk->queue_family_index) { ffb_cmd_handle = fence->feedback.commands[i]; } } assert(ffb_cmd_handle != VK_NULL_HANDLE); vn_set_temp_cmd(submit, (*new_cmd_count)++, ffb_cmd_handle); } static VkResult vn_queue_submission_add_feedback_cmds(struct vn_queue_submission *submit, uint32_t batch_index, uint32_t feedback_types) { VkResult result; uint32_t new_cmd_count = vn_get_cmd_count(submit, batch_index); if (feedback_types & VN_FEEDBACK_TYPE_QUERY) { result = vn_queue_submission_add_query_feedback(submit, batch_index, &new_cmd_count); if (result != VK_SUCCESS) return result; } if (feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE) { const uint32_t signal_count = vn_get_signal_semaphore_count(submit, batch_index); for (uint32_t i = 0; i < signal_count; i++) { result = vn_queue_submission_add_semaphore_feedback( submit, batch_index, i, &new_cmd_count); if (result != VK_SUCCESS) return result; } if (vn_fix_batch_cmd_count_for_zink_sync(submit, batch_index, new_cmd_count)) return VK_SUCCESS; } if (feedback_types & VN_FEEDBACK_TYPE_FENCE) { vn_queue_submission_add_fence_feedback(submit, batch_index, &new_cmd_count); } if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) { VkSubmitInfo2 *batch = &submit->temp.submit2_batches[batch_index]; batch->pCommandBufferInfos = submit->temp.cmd_infos; batch->commandBufferInfoCount = new_cmd_count; } else { VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index]; batch->pCommandBuffers = submit->temp.cmd_handles; batch->commandBufferCount = new_cmd_count; const VkDeviceGroupSubmitInfo *device_group = vk_find_struct_const( submit->submit_batches[batch_index].pNext, DEVICE_GROUP_SUBMIT_INFO); if (device_group) vn_fix_device_group_cmd_count(submit, batch_index); } return VK_SUCCESS; } static VkResult vn_queue_submission_setup_batch(struct vn_queue_submission *submit, uint32_t batch_index) { uint32_t feedback_types = 0; uint32_t extra_cmd_count = 0; const uint32_t signal_count = vn_get_signal_semaphore_count(submit, batch_index); for (uint32_t i = 0; i < signal_count; i++) { struct vn_semaphore *sem = vn_semaphore_from_handle( vn_get_signal_semaphore(submit, batch_index, i)); if (sem->feedback.slot) { feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE; extra_cmd_count++; } } const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index); for (uint32_t i = 0; i < cmd_count; i++) { struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i); if (!list_is_empty(&cmd->builder.query_records)) { feedback_types |= VN_FEEDBACK_TYPE_QUERY; extra_cmd_count++; break; } } if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE && batch_index == submit->batch_count - 1) { feedback_types |= VN_FEEDBACK_TYPE_FENCE; extra_cmd_count++; } /* If the batch has qfb, sfb or ffb, copy the original commands and append * feedback cmds. * If this is the second to last batch and the last batch a zink sync batch * which is empty but has feedback, also copy the original commands for * this batch so that the last batch's feedback can be appended to it. */ if (feedback_types || (batch_index == submit->batch_count - 2 && submit->has_zink_sync_batch)) { const size_t cmd_size = vn_get_cmd_size(submit); const size_t total_cmd_size = cmd_count * cmd_size; /* copy only needed for non-empty batches */ if (total_cmd_size) { memcpy(submit->temp.cmds, vn_get_cmds(submit, batch_index), total_cmd_size); } VkResult result = vn_queue_submission_add_feedback_cmds( submit, batch_index, feedback_types); if (result != VK_SUCCESS) return result; /* advance the temp cmds for working on next batch cmds */ submit->temp.cmds += total_cmd_size + (extra_cmd_count * cmd_size); } return VK_SUCCESS; } static VkResult vn_queue_submission_setup_batches(struct vn_queue_submission *submit) { assert(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2 || submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO); if (!submit->feedback_types) return VK_SUCCESS; /* For a submission that is: * - non-empty: copy batches for adding feedbacks * - empty: initialize a batch for fence feedback */ if (submit->batch_count) { memcpy(submit->temp.batches, submit->batches, vn_get_batch_size(submit) * submit->batch_count); } else { assert(submit->feedback_types & VN_FEEDBACK_TYPE_FENCE); if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) { submit->temp.submit2_batches[0] = (VkSubmitInfo2){ .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, }; } else { submit->temp.submit_batches[0] = (VkSubmitInfo){ .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, }; } submit->batch_count = 1; submit->batches = submit->temp.batches; } for (uint32_t i = 0; i < submit->batch_count; i++) { VkResult result = vn_queue_submission_setup_batch(submit, i); if (result != VK_SUCCESS) return result; } submit->batches = submit->temp.batches; return VK_SUCCESS; } static void vn_queue_submission_cleanup_semaphore_feedback( struct vn_queue_submission *submit) { struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle); VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device); for (uint32_t i = 0; i < submit->batch_count; i++) { const uint32_t wait_count = vn_get_wait_semaphore_count(submit, i); for (uint32_t j = 0; j < wait_count; j++) { VkSemaphore sem_handle = vn_get_wait_semaphore(submit, i, j); struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle); if (!sem->feedback.slot) continue; /* sfb pending cmds are recycled when signaled counter is updated */ uint64_t counter = 0; vn_GetSemaphoreCounterValue(dev_handle, sem_handle, &counter); } const uint32_t signal_count = vn_get_signal_semaphore_count(submit, i); for (uint32_t j = 0; j < signal_count; j++) { VkSemaphore sem_handle = vn_get_signal_semaphore(submit, i, j); struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle); if (!sem->feedback.slot) continue; /* sfb pending cmds are recycled when signaled counter is updated */ uint64_t counter = 0; vn_GetSemaphoreCounterValue(dev_handle, sem_handle, &counter); } } } static void vn_queue_submission_cleanup(struct vn_queue_submission *submit) { /* TODO clean up pending src feedbacks on failure? */ if (submit->feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE) vn_queue_submission_cleanup_semaphore_feedback(submit); } static VkResult vn_queue_submission_prepare_submit(struct vn_queue_submission *submit) { VkResult result = vn_queue_submission_prepare(submit); if (result != VK_SUCCESS) return result; result = vn_queue_submission_alloc_storage(submit); if (result != VK_SUCCESS) return result; result = vn_queue_submission_setup_batches(submit); if (result != VK_SUCCESS) { vn_queue_submission_cleanup(submit); return result; } return VK_SUCCESS; } static void vn_queue_wsi_present(struct vn_queue_submission *submit) { struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle); struct vn_device *dev = (void *)queue_vk->base.device; if (!submit->wsi_mem) return; if (dev->renderer->info.has_implicit_fencing) { struct vn_renderer_submit_batch batch = { .ring_idx = submit->external_payload.ring_idx, }; uint32_t local_data[8]; struct vn_cs_encoder local_enc = VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data)); if (submit->external_payload.ring_seqno_valid) { const uint64_t ring_id = vn_ring_get_id(dev->primary_ring); vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id, submit->external_payload.ring_seqno); batch.cs_data = local_data; batch.cs_size = vn_cs_encoder_get_len(&local_enc); } const struct vn_renderer_submit renderer_submit = { .bos = &submit->wsi_mem->base_bo, .bo_count = 1, .batches = &batch, .batch_count = 1, }; vn_renderer_submit(dev->renderer, &renderer_submit); } else { if (VN_DEBUG(WSI)) { static uint32_t num_rate_limit_warning = 0; if (num_rate_limit_warning++ < 10) vn_log(dev->instance, "forcing vkQueueWaitIdle before presenting"); } vn_QueueWaitIdle(submit->queue_handle); } } static VkResult vn_queue_submit(struct vn_queue_submission *submit) { struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle); struct vn_device *dev = (void *)queue->base.base.base.device; struct vn_instance *instance = dev->instance; VkResult result; /* To ensure external components waiting on the correct fence payload, * below sync primitives must be installed after the submission: * - explicit fencing: sync file export * - implicit fencing: dma-fence attached to the wsi bo * * We enforce above via an asynchronous vkQueueSubmit(2) via ring followed * by an asynchronous renderer submission to wait for the ring submission: * - struct wsi_memory_signal_submit_info * - fence is an external fence * - has an external signal semaphore */ result = vn_queue_submission_prepare_submit(submit); if (result != VK_SUCCESS) return vn_error(instance, result); /* skip no-op submit */ if (!submit->batch_count && submit->fence_handle == VK_NULL_HANDLE) return VK_SUCCESS; if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) { if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) { result = vn_call_vkQueueSubmit2( dev->primary_ring, submit->queue_handle, submit->batch_count, submit->submit2_batches, submit->fence_handle); } else { result = vn_call_vkQueueSubmit( dev->primary_ring, submit->queue_handle, submit->batch_count, submit->submit_batches, submit->fence_handle); } if (result != VK_SUCCESS) { vn_queue_submission_cleanup(submit); return vn_error(instance, result); } } else { struct vn_ring_submit_command ring_submit; if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) { vn_submit_vkQueueSubmit2( dev->primary_ring, 0, submit->queue_handle, submit->batch_count, submit->submit2_batches, submit->fence_handle, &ring_submit); } else { vn_submit_vkQueueSubmit(dev->primary_ring, 0, submit->queue_handle, submit->batch_count, submit->submit_batches, submit->fence_handle, &ring_submit); } if (!ring_submit.ring_seqno_valid) { vn_queue_submission_cleanup(submit); return vn_error(instance, VK_ERROR_DEVICE_LOST); } submit->external_payload.ring_seqno_valid = true; submit->external_payload.ring_seqno = ring_submit.ring_seqno; } /* If external fence, track the submission's ring_idx to facilitate * sync_file export. * * Imported syncs don't need a proxy renderer sync on subsequent export, * because an fd is already available. */ struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle); if (fence && fence->is_external) { assert(fence->payload->type == VN_SYNC_TYPE_DEVICE_ONLY); fence->external_payload = submit->external_payload; } for (uint32_t i = 0; i < submit->batch_count; i++) { const uint32_t signal_count = vn_get_signal_semaphore_count(submit, i); for (uint32_t j = 0; j < signal_count; j++) { struct vn_semaphore *sem = vn_semaphore_from_handle(vn_get_signal_semaphore(submit, i, j)); if (sem->is_external) { assert(sem->payload->type == VN_SYNC_TYPE_DEVICE_ONLY); sem->external_payload = submit->external_payload; } } } vn_queue_wsi_present(submit); vn_queue_submission_cleanup(submit); return VK_SUCCESS; } VkResult vn_QueueSubmit(VkQueue queue, uint32_t submitCount, const VkSubmitInfo *pSubmits, VkFence fence) { VN_TRACE_FUNC(); struct vn_queue_submission submit = { .batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO, .queue_handle = queue, .batch_count = submitCount, .submit_batches = pSubmits, .fence_handle = fence, }; return vn_queue_submit(&submit); } VkResult vn_QueueSubmit2(VkQueue queue, uint32_t submitCount, const VkSubmitInfo2 *pSubmits, VkFence fence) { VN_TRACE_FUNC(); struct vn_queue_submission submit = { .batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .queue_handle = queue, .batch_count = submitCount, .submit2_batches = pSubmits, .fence_handle = fence, }; return vn_queue_submit(&submit); } static VkResult vn_queue_bind_sparse_submit(struct vn_queue_submission *submit) { struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle); struct vn_device *dev = (void *)queue->base.base.base.device; struct vn_instance *instance = dev->instance; VkResult result; if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) { result = vn_call_vkQueueBindSparse( dev->primary_ring, submit->queue_handle, submit->batch_count, submit->sparse_batches, submit->fence_handle); if (result != VK_SUCCESS) return vn_error(instance, result); } else { struct vn_ring_submit_command ring_submit; vn_submit_vkQueueBindSparse(dev->primary_ring, 0, submit->queue_handle, submit->batch_count, submit->sparse_batches, submit->fence_handle, &ring_submit); if (!ring_submit.ring_seqno_valid) return vn_error(instance, VK_ERROR_DEVICE_LOST); } return VK_SUCCESS; } static VkResult vn_queue_bind_sparse_submit_batch(struct vn_queue_submission *submit, uint32_t batch_index) { struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle); VkDevice dev_handle = vk_device_to_handle(queue->base.base.base.device); const VkBindSparseInfo *sparse_info = &submit->sparse_batches[batch_index]; const VkSemaphore *signal_sem = sparse_info->pSignalSemaphores; uint32_t signal_sem_count = sparse_info->signalSemaphoreCount; VkResult result; struct vn_queue_submission sparse_batch = { .batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, .queue_handle = submit->queue_handle, .batch_count = 1, .fence_handle = VK_NULL_HANDLE, }; /* lazily create sparse semaphore */ if (queue->sparse_semaphore == VK_NULL_HANDLE) { queue->sparse_semaphore_counter = 1; const VkSemaphoreTypeCreateInfo sem_type_create_info = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO, .pNext = NULL, /* This must be timeline type to adhere to mesa's requirement * not to mix binary semaphores with wait-before-signal. */ .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = 1, }; const VkSemaphoreCreateInfo create_info = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &sem_type_create_info, .flags = 0, }; result = vn_CreateSemaphore(dev_handle, &create_info, NULL, &queue->sparse_semaphore); if (result != VK_SUCCESS) return result; } /* Setup VkTimelineSemaphoreSubmitInfo's for our queue sparse semaphore * so that the vkQueueSubmit waits on the vkQueueBindSparse signal. */ queue->sparse_semaphore_counter++; struct VkTimelineSemaphoreSubmitInfo wait_timeline_sem_info = { 0 }; wait_timeline_sem_info.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO; wait_timeline_sem_info.signalSemaphoreValueCount = 1; wait_timeline_sem_info.pSignalSemaphoreValues = &queue->sparse_semaphore_counter; struct VkTimelineSemaphoreSubmitInfo signal_timeline_sem_info = { 0 }; signal_timeline_sem_info.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO; signal_timeline_sem_info.waitSemaphoreValueCount = 1; signal_timeline_sem_info.pWaitSemaphoreValues = &queue->sparse_semaphore_counter; /* Split up the original wait and signal semaphores into its respective * vkTimelineSemaphoreSubmitInfo */ const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info = vk_find_struct_const(sparse_info->pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO); if (timeline_sem_info) { if (timeline_sem_info->waitSemaphoreValueCount) { wait_timeline_sem_info.waitSemaphoreValueCount = timeline_sem_info->waitSemaphoreValueCount; wait_timeline_sem_info.pWaitSemaphoreValues = timeline_sem_info->pWaitSemaphoreValues; } if (timeline_sem_info->signalSemaphoreValueCount) { signal_timeline_sem_info.signalSemaphoreValueCount = timeline_sem_info->signalSemaphoreValueCount; signal_timeline_sem_info.pSignalSemaphoreValues = timeline_sem_info->pSignalSemaphoreValues; } } /* Attach the original VkDeviceGroupBindSparseInfo if it exists */ struct VkDeviceGroupBindSparseInfo batch_device_group_info; const struct VkDeviceGroupBindSparseInfo *device_group_info = vk_find_struct_const(sparse_info->pNext, DEVICE_GROUP_BIND_SPARSE_INFO); if (device_group_info) { memcpy(&batch_device_group_info, device_group_info, sizeof(*device_group_info)); batch_device_group_info.pNext = NULL; wait_timeline_sem_info.pNext = &batch_device_group_info; } /* Copy the original batch VkBindSparseInfo modified to signal * our sparse semaphore. */ VkBindSparseInfo batch_sparse_info; memcpy(&batch_sparse_info, sparse_info, sizeof(*sparse_info)); batch_sparse_info.pNext = &wait_timeline_sem_info; batch_sparse_info.signalSemaphoreCount = 1; batch_sparse_info.pSignalSemaphores = &queue->sparse_semaphore; /* Set up the SubmitInfo to wait on our sparse semaphore before sending * feedback and signaling the original semaphores/fence * * Even if this VkBindSparse batch does not have feedback semaphores, * we still glue all the batches together to ensure the feedback * fence occurs after. */ VkPipelineStageFlags stage_masks = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; VkSubmitInfo batch_submit_info = { .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .pNext = &signal_timeline_sem_info, .waitSemaphoreCount = 1, .pWaitSemaphores = &queue->sparse_semaphore, .pWaitDstStageMask = &stage_masks, .signalSemaphoreCount = signal_sem_count, .pSignalSemaphores = signal_sem, }; /* Set the possible fence if on the last batch */ VkFence fence_handle = VK_NULL_HANDLE; if ((submit->feedback_types & VN_FEEDBACK_TYPE_FENCE) && batch_index == (submit->batch_count - 1)) { fence_handle = submit->fence_handle; } sparse_batch.sparse_batches = &batch_sparse_info; result = vn_queue_bind_sparse_submit(&sparse_batch); if (result != VK_SUCCESS) return result; result = vn_QueueSubmit(submit->queue_handle, 1, &batch_submit_info, fence_handle); if (result != VK_SUCCESS) return result; return VK_SUCCESS; } VkResult vn_QueueBindSparse(VkQueue queue, uint32_t bindInfoCount, const VkBindSparseInfo *pBindInfo, VkFence fence) { VN_TRACE_FUNC(); VkResult result; struct vn_queue_submission submit = { .batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO, .queue_handle = queue, .batch_count = bindInfoCount, .sparse_batches = pBindInfo, .fence_handle = fence, }; result = vn_queue_submission_prepare(&submit); if (result != VK_SUCCESS) return result; if (!submit.batch_count) { /* skip no-op submit */ if (submit.fence_handle == VK_NULL_HANDLE) return VK_SUCCESS; /* if empty batch, just send a vkQueueSubmit with the fence */ result = vn_QueueSubmit(submit.queue_handle, 0, NULL, submit.fence_handle); if (result != VK_SUCCESS) return result; } /* if feedback isn't used in the batch, can directly submit */ if (!submit.feedback_types) return vn_queue_bind_sparse_submit(&submit); for (uint32_t i = 0; i < submit.batch_count; i++) { result = vn_queue_bind_sparse_submit_batch(&submit, i); if (result != VK_SUCCESS) return result; } return VK_SUCCESS; } VkResult vn_QueueWaitIdle(VkQueue _queue) { VN_TRACE_FUNC(); struct vn_queue *queue = vn_queue_from_handle(_queue); VkDevice dev_handle = vk_device_to_handle(queue->base.base.base.device); struct vn_device *dev = vn_device_from_handle(dev_handle); VkResult result; /* lazily create queue wait fence for queue idle waiting */ if (queue->wait_fence == VK_NULL_HANDLE) { const VkFenceCreateInfo create_info = { .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, .flags = 0, }; result = vn_CreateFence(dev_handle, &create_info, NULL, &queue->wait_fence); if (result != VK_SUCCESS) return result; } result = vn_QueueSubmit(_queue, 0, NULL, queue->wait_fence); if (result != VK_SUCCESS) return result; result = vn_WaitForFences(dev_handle, 1, &queue->wait_fence, true, UINT64_MAX); vn_ResetFences(dev_handle, 1, &queue->wait_fence); return vn_result(dev->instance, result); } /* fence commands */ static void vn_sync_payload_release(UNUSED struct vn_device *dev, struct vn_sync_payload *payload) { if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD && payload->fd >= 0) close(payload->fd); payload->type = VN_SYNC_TYPE_INVALID; } static VkResult vn_fence_init_payloads(struct vn_device *dev, struct vn_fence *fence, bool signaled, const VkAllocationCallbacks *alloc) { fence->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY; fence->temporary.type = VN_SYNC_TYPE_INVALID; fence->payload = &fence->permanent; return VK_SUCCESS; } void vn_fence_signal_wsi(struct vn_device *dev, struct vn_fence *fence) { struct vn_sync_payload *temp = &fence->temporary; vn_sync_payload_release(dev, temp); temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD; temp->fd = -1; fence->payload = temp; } static VkResult vn_fence_feedback_init(struct vn_device *dev, struct vn_fence *fence, bool signaled, const VkAllocationCallbacks *alloc) { VkDevice dev_handle = vn_device_to_handle(dev); struct vn_feedback_slot *slot; VkCommandBuffer *cmd_handles; VkResult result; if (fence->is_external) return VK_SUCCESS; if (VN_PERF(NO_FENCE_FEEDBACK)) return VK_SUCCESS; slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_FENCE); if (!slot) return VK_ERROR_OUT_OF_HOST_MEMORY; vn_feedback_set_status(slot, signaled ? VK_SUCCESS : VK_NOT_READY); cmd_handles = vk_zalloc(alloc, sizeof(*cmd_handles) * dev->queue_family_count, VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!cmd_handles) { vn_feedback_pool_free(&dev->feedback_pool, slot); return VK_ERROR_OUT_OF_HOST_MEMORY; } for (uint32_t i = 0; i < dev->queue_family_count; i++) { result = vn_feedback_cmd_alloc(dev_handle, &dev->fb_cmd_pools[i], slot, NULL, &cmd_handles[i]); if (result != VK_SUCCESS) { for (uint32_t j = 0; j < i; j++) { vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[j], cmd_handles[j]); } break; } } if (result != VK_SUCCESS) { vk_free(alloc, cmd_handles); vn_feedback_pool_free(&dev->feedback_pool, slot); return result; } fence->feedback.slot = slot; fence->feedback.commands = cmd_handles; return VK_SUCCESS; } static void vn_fence_feedback_fini(struct vn_device *dev, struct vn_fence *fence, const VkAllocationCallbacks *alloc) { VkDevice dev_handle = vn_device_to_handle(dev); if (!fence->feedback.slot) return; for (uint32_t i = 0; i < dev->queue_family_count; i++) { vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[i], fence->feedback.commands[i]); } vn_feedback_pool_free(&dev->feedback_pool, fence->feedback.slot); vk_free(alloc, fence->feedback.commands); } VkResult vn_CreateFence(VkDevice device, const VkFenceCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkFence *pFence) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : &dev->base.base.alloc; const bool signaled = pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT; VkResult result; struct vn_fence *fence = vk_zalloc(alloc, sizeof(*fence), VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!fence) return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY); vn_object_base_init(&fence->base, VK_OBJECT_TYPE_FENCE, &dev->base); const struct VkExportFenceCreateInfo *export_info = vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO); fence->is_external = export_info && export_info->handleTypes; result = vn_fence_init_payloads(dev, fence, signaled, alloc); if (result != VK_SUCCESS) goto out_object_base_fini; result = vn_fence_feedback_init(dev, fence, signaled, alloc); if (result != VK_SUCCESS) goto out_payloads_fini; *pFence = vn_fence_to_handle(fence); vn_async_vkCreateFence(dev->primary_ring, device, pCreateInfo, NULL, pFence); return VK_SUCCESS; out_payloads_fini: vn_sync_payload_release(dev, &fence->permanent); vn_sync_payload_release(dev, &fence->temporary); out_object_base_fini: vn_object_base_fini(&fence->base); vk_free(alloc, fence); return vn_error(dev->instance, result); } void vn_DestroyFence(VkDevice device, VkFence _fence, const VkAllocationCallbacks *pAllocator) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_fence *fence = vn_fence_from_handle(_fence); const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : &dev->base.base.alloc; if (!fence) return; vn_async_vkDestroyFence(dev->primary_ring, device, _fence, NULL); vn_fence_feedback_fini(dev, fence, alloc); vn_sync_payload_release(dev, &fence->permanent); vn_sync_payload_release(dev, &fence->temporary); vn_object_base_fini(&fence->base); vk_free(alloc, fence); } VkResult vn_ResetFences(VkDevice device, uint32_t fenceCount, const VkFence *pFences) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); vn_async_vkResetFences(dev->primary_ring, device, fenceCount, pFences); for (uint32_t i = 0; i < fenceCount; i++) { struct vn_fence *fence = vn_fence_from_handle(pFences[i]); struct vn_sync_payload *perm = &fence->permanent; vn_sync_payload_release(dev, &fence->temporary); assert(perm->type == VN_SYNC_TYPE_DEVICE_ONLY); fence->payload = perm; if (fence->feedback.slot) vn_feedback_reset_status(fence->feedback.slot); } return VK_SUCCESS; } VkResult vn_GetFenceStatus(VkDevice device, VkFence _fence) { struct vn_device *dev = vn_device_from_handle(device); struct vn_fence *fence = vn_fence_from_handle(_fence); struct vn_sync_payload *payload = fence->payload; VkResult result; switch (payload->type) { case VN_SYNC_TYPE_DEVICE_ONLY: if (fence->feedback.slot) { result = vn_feedback_get_status(fence->feedback.slot); if (result == VK_SUCCESS) { /* When fence feedback slot gets signaled, the real fence * signal operation follows after but the signaling isr can be * deferred or preempted. To avoid racing, we let the * renderer wait for the fence. This also helps resolve * synchronization validation errors, because the layer no * longer sees any fence status checks and falsely believes the * caller does not sync. */ vn_async_vkWaitForFences(dev->primary_ring, device, 1, &_fence, VK_TRUE, UINT64_MAX); } } else { result = vn_call_vkGetFenceStatus(dev->primary_ring, device, _fence); } break; case VN_SYNC_TYPE_IMPORTED_SYNC_FD: if (payload->fd < 0 || sync_wait(payload->fd, 0) == 0) result = VK_SUCCESS; else result = errno == ETIME ? VK_NOT_READY : VK_ERROR_DEVICE_LOST; break; default: unreachable("unexpected fence payload type"); break; } return vn_result(dev->instance, result); } static VkResult vn_find_first_signaled_fence(VkDevice device, const VkFence *fences, uint32_t count) { for (uint32_t i = 0; i < count; i++) { VkResult result = vn_GetFenceStatus(device, fences[i]); if (result == VK_SUCCESS || result < 0) return result; } return VK_NOT_READY; } static VkResult vn_remove_signaled_fences(VkDevice device, VkFence *fences, uint32_t *count) { uint32_t cur = 0; for (uint32_t i = 0; i < *count; i++) { VkResult result = vn_GetFenceStatus(device, fences[i]); if (result != VK_SUCCESS) { if (result < 0) return result; fences[cur++] = fences[i]; } } *count = cur; return cur ? VK_NOT_READY : VK_SUCCESS; } static VkResult vn_update_sync_result(struct vn_device *dev, VkResult result, int64_t abs_timeout, struct vn_relax_state *relax_state) { switch (result) { case VK_NOT_READY: if (abs_timeout != OS_TIMEOUT_INFINITE && os_time_get_nano() >= abs_timeout) result = VK_TIMEOUT; else vn_relax(relax_state); break; default: assert(result == VK_SUCCESS || result < 0); break; } return result; } VkResult vn_WaitForFences(VkDevice device, uint32_t fenceCount, const VkFence *pFences, VkBool32 waitAll, uint64_t timeout) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); const int64_t abs_timeout = os_time_get_absolute_timeout(timeout); VkResult result = VK_NOT_READY; if (fenceCount > 1 && waitAll) { STACK_ARRAY(VkFence, fences, fenceCount); typed_memcpy(fences, pFences, fenceCount); struct vn_relax_state relax_state = vn_relax_init(dev->instance, VN_RELAX_REASON_FENCE); while (result == VK_NOT_READY) { result = vn_remove_signaled_fences(device, fences, &fenceCount); result = vn_update_sync_result(dev, result, abs_timeout, &relax_state); } vn_relax_fini(&relax_state); STACK_ARRAY_FINISH(fences); } else { struct vn_relax_state relax_state = vn_relax_init(dev->instance, VN_RELAX_REASON_FENCE); while (result == VK_NOT_READY) { result = vn_find_first_signaled_fence(device, pFences, fenceCount); result = vn_update_sync_result(dev, result, abs_timeout, &relax_state); } vn_relax_fini(&relax_state); } return vn_result(dev->instance, result); } static VkResult vn_create_sync_file(struct vn_device *dev, struct vn_sync_payload_external *external_payload, int *out_fd) { struct vn_renderer_sync *sync; VkResult result = vn_renderer_sync_create(dev->renderer, 0, VN_RENDERER_SYNC_BINARY, &sync); if (result != VK_SUCCESS) return vn_error(dev->instance, result); struct vn_renderer_submit_batch batch = { .syncs = &sync, .sync_values = &(const uint64_t){ 1 }, .sync_count = 1, .ring_idx = external_payload->ring_idx, }; uint32_t local_data[8]; struct vn_cs_encoder local_enc = VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data)); if (external_payload->ring_seqno_valid) { const uint64_t ring_id = vn_ring_get_id(dev->primary_ring); vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id, external_payload->ring_seqno); batch.cs_data = local_data; batch.cs_size = vn_cs_encoder_get_len(&local_enc); } const struct vn_renderer_submit submit = { .batches = &batch, .batch_count = 1, }; result = vn_renderer_submit(dev->renderer, &submit); if (result != VK_SUCCESS) { vn_renderer_sync_destroy(dev->renderer, sync); return vn_error(dev->instance, result); } *out_fd = vn_renderer_sync_export_syncobj(dev->renderer, sync, true); vn_renderer_sync_destroy(dev->renderer, sync); return *out_fd >= 0 ? VK_SUCCESS : VK_ERROR_TOO_MANY_OBJECTS; } static inline bool vn_sync_valid_fd(int fd) { /* the special value -1 for fd is treated like a valid sync file descriptor * referring to an object that has already signaled */ return (fd >= 0 && sync_valid_fd(fd)) || fd == -1; } VkResult vn_ImportFenceFdKHR(VkDevice device, const VkImportFenceFdInfoKHR *pImportFenceFdInfo) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_fence *fence = vn_fence_from_handle(pImportFenceFdInfo->fence); ASSERTED const bool sync_file = pImportFenceFdInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; const int fd = pImportFenceFdInfo->fd; assert(sync_file); if (!vn_sync_valid_fd(fd)) return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); struct vn_sync_payload *temp = &fence->temporary; vn_sync_payload_release(dev, temp); temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD; temp->fd = fd; fence->payload = temp; return VK_SUCCESS; } VkResult vn_GetFenceFdKHR(VkDevice device, const VkFenceGetFdInfoKHR *pGetFdInfo, int *pFd) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_fence *fence = vn_fence_from_handle(pGetFdInfo->fence); const bool sync_file = pGetFdInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; struct vn_sync_payload *payload = fence->payload; VkResult result; assert(sync_file); assert(dev->physical_device->renderer_sync_fd.fence_exportable); int fd = -1; if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) { result = vn_create_sync_file(dev, &fence->external_payload, &fd); if (result != VK_SUCCESS) return vn_error(dev->instance, result); vn_async_vkResetFenceResourceMESA(dev->primary_ring, device, pGetFdInfo->fence); vn_sync_payload_release(dev, &fence->temporary); fence->payload = &fence->permanent; #ifdef VN_USE_WSI_PLATFORM if (!dev->renderer->info.has_implicit_fencing) sync_wait(fd, -1); #endif } else { assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD); /* transfer ownership of imported sync fd to save a dup */ fd = payload->fd; payload->fd = -1; /* reset host fence in case in signaled state before import */ result = vn_ResetFences(device, 1, &pGetFdInfo->fence); if (result != VK_SUCCESS) { /* transfer sync fd ownership back on error */ payload->fd = fd; return result; } } *pFd = fd; return VK_SUCCESS; } /* semaphore commands */ static VkResult vn_semaphore_init_payloads(struct vn_device *dev, struct vn_semaphore *sem, uint64_t initial_val, const VkAllocationCallbacks *alloc) { sem->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY; sem->temporary.type = VN_SYNC_TYPE_INVALID; sem->payload = &sem->permanent; return VK_SUCCESS; } static bool vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem) { struct vn_sync_payload *temp = &sem->temporary; assert(temp->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD); if (temp->fd >= 0) { if (sync_wait(temp->fd, -1)) return false; } vn_sync_payload_release(dev, &sem->temporary); sem->payload = &sem->permanent; return true; } void vn_semaphore_signal_wsi(struct vn_device *dev, struct vn_semaphore *sem) { struct vn_sync_payload *temp = &sem->temporary; vn_sync_payload_release(dev, temp); temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD; temp->fd = -1; sem->payload = temp; } struct vn_semaphore_feedback_cmd * vn_semaphore_get_feedback_cmd(struct vn_device *dev, struct vn_semaphore *sem) { struct vn_semaphore_feedback_cmd *sfb_cmd = NULL; simple_mtx_lock(&sem->feedback.cmd_mtx); if (!list_is_empty(&sem->feedback.free_cmds)) { sfb_cmd = list_first_entry(&sem->feedback.free_cmds, struct vn_semaphore_feedback_cmd, head); list_move_to(&sfb_cmd->head, &sem->feedback.pending_cmds); sem->feedback.free_cmd_count--; } simple_mtx_unlock(&sem->feedback.cmd_mtx); if (!sfb_cmd) { sfb_cmd = vn_semaphore_feedback_cmd_alloc(dev, sem->feedback.slot); simple_mtx_lock(&sem->feedback.cmd_mtx); list_add(&sfb_cmd->head, &sem->feedback.pending_cmds); simple_mtx_unlock(&sem->feedback.cmd_mtx); } return sfb_cmd; } static VkResult vn_semaphore_feedback_init(struct vn_device *dev, struct vn_semaphore *sem, uint64_t initial_value, const VkAllocationCallbacks *alloc) { struct vn_feedback_slot *slot; assert(sem->type == VK_SEMAPHORE_TYPE_TIMELINE); if (sem->is_external) return VK_SUCCESS; if (VN_PERF(NO_SEMAPHORE_FEEDBACK)) return VK_SUCCESS; slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_SEMAPHORE); if (!slot) return VK_ERROR_OUT_OF_HOST_MEMORY; list_inithead(&sem->feedback.pending_cmds); list_inithead(&sem->feedback.free_cmds); vn_feedback_set_counter(slot, initial_value); simple_mtx_init(&sem->feedback.cmd_mtx, mtx_plain); simple_mtx_init(&sem->feedback.async_wait_mtx, mtx_plain); sem->feedback.signaled_counter = initial_value; sem->feedback.slot = slot; return VK_SUCCESS; } static void vn_semaphore_feedback_fini(struct vn_device *dev, struct vn_semaphore *sem) { if (!sem->feedback.slot) return; list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd, &sem->feedback.free_cmds, head) vn_semaphore_feedback_cmd_free(dev, sfb_cmd); list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd, &sem->feedback.pending_cmds, head) vn_semaphore_feedback_cmd_free(dev, sfb_cmd); simple_mtx_destroy(&sem->feedback.cmd_mtx); simple_mtx_destroy(&sem->feedback.async_wait_mtx); vn_feedback_pool_free(&dev->feedback_pool, sem->feedback.slot); } VkResult vn_CreateSemaphore(VkDevice device, const VkSemaphoreCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkSemaphore *pSemaphore) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : &dev->base.base.alloc; struct vn_semaphore *sem = vk_zalloc(alloc, sizeof(*sem), VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!sem) return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY); vn_object_base_init(&sem->base, VK_OBJECT_TYPE_SEMAPHORE, &dev->base); const VkSemaphoreTypeCreateInfo *type_info = vk_find_struct_const(pCreateInfo->pNext, SEMAPHORE_TYPE_CREATE_INFO); uint64_t initial_val = 0; if (type_info && type_info->semaphoreType == VK_SEMAPHORE_TYPE_TIMELINE) { sem->type = VK_SEMAPHORE_TYPE_TIMELINE; initial_val = type_info->initialValue; } else { sem->type = VK_SEMAPHORE_TYPE_BINARY; } const struct VkExportSemaphoreCreateInfo *export_info = vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO); sem->is_external = export_info && export_info->handleTypes; VkResult result = vn_semaphore_init_payloads(dev, sem, initial_val, alloc); if (result != VK_SUCCESS) goto out_object_base_fini; if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE) { result = vn_semaphore_feedback_init(dev, sem, initial_val, alloc); if (result != VK_SUCCESS) goto out_payloads_fini; } VkSemaphore sem_handle = vn_semaphore_to_handle(sem); vn_async_vkCreateSemaphore(dev->primary_ring, device, pCreateInfo, NULL, &sem_handle); *pSemaphore = sem_handle; return VK_SUCCESS; out_payloads_fini: vn_sync_payload_release(dev, &sem->permanent); vn_sync_payload_release(dev, &sem->temporary); out_object_base_fini: vn_object_base_fini(&sem->base); vk_free(alloc, sem); return vn_error(dev->instance, result); } void vn_DestroySemaphore(VkDevice device, VkSemaphore semaphore, const VkAllocationCallbacks *pAllocator) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore); const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : &dev->base.base.alloc; if (!sem) return; vn_async_vkDestroySemaphore(dev->primary_ring, device, semaphore, NULL); if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE) vn_semaphore_feedback_fini(dev, sem); vn_sync_payload_release(dev, &sem->permanent); vn_sync_payload_release(dev, &sem->temporary); vn_object_base_fini(&sem->base); vk_free(alloc, sem); } VkResult vn_GetSemaphoreCounterValue(VkDevice device, VkSemaphore semaphore, uint64_t *pValue) { struct vn_device *dev = vn_device_from_handle(device); struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore); ASSERTED struct vn_sync_payload *payload = sem->payload; assert(payload->type == VN_SYNC_TYPE_DEVICE_ONLY); if (sem->feedback.slot) { simple_mtx_lock(&sem->feedback.async_wait_mtx); const uint64_t counter = vn_feedback_get_counter(sem->feedback.slot); if (sem->feedback.signaled_counter < counter) { /* When the timeline semaphore feedback slot gets signaled, the real * semaphore signal operation follows after but the signaling isr can * be deferred or preempted. To avoid racing, we let the renderer * wait for the semaphore by sending an asynchronous wait call for * the feedback value. * We also cache the counter value to only send the async call once * per counter value to prevent spamming redundant async wait calls. * The cached counter value requires a lock to ensure multiple * threads querying for the same value are guaranteed to encode after * the async wait call. * * This also helps resolve synchronization validation errors, because * the layer no longer sees any semaphore status checks and falsely * believes the caller does not sync. */ VkSemaphoreWaitInfo wait_info = { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .pNext = NULL, .flags = 0, .semaphoreCount = 1, .pSemaphores = &semaphore, .pValues = &counter, }; vn_async_vkWaitSemaphores(dev->primary_ring, device, &wait_info, UINT64_MAX); /* search pending cmds for already signaled values */ simple_mtx_lock(&sem->feedback.cmd_mtx); list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd, &sem->feedback.pending_cmds, head) { if (counter >= vn_feedback_get_counter(sfb_cmd->src_slot)) { /* avoid over-caching more than normal runtime usage */ if (sem->feedback.free_cmd_count > 5) { list_del(&sfb_cmd->head); vn_semaphore_feedback_cmd_free(dev, sfb_cmd); } else { list_move_to(&sfb_cmd->head, &sem->feedback.free_cmds); sem->feedback.free_cmd_count++; } } } simple_mtx_unlock(&sem->feedback.cmd_mtx); sem->feedback.signaled_counter = counter; } simple_mtx_unlock(&sem->feedback.async_wait_mtx); *pValue = counter; return VK_SUCCESS; } else { return vn_call_vkGetSemaphoreCounterValue(dev->primary_ring, device, semaphore, pValue); } } VkResult vn_SignalSemaphore(VkDevice device, const VkSemaphoreSignalInfo *pSignalInfo) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_semaphore *sem = vn_semaphore_from_handle(pSignalInfo->semaphore); vn_async_vkSignalSemaphore(dev->primary_ring, device, pSignalInfo); if (sem->feedback.slot) { simple_mtx_lock(&sem->feedback.async_wait_mtx); vn_feedback_set_counter(sem->feedback.slot, pSignalInfo->value); /* Update async counters. Since we're signaling, we're aligned with * the renderer. */ sem->feedback.signaled_counter = pSignalInfo->value; simple_mtx_unlock(&sem->feedback.async_wait_mtx); } return VK_SUCCESS; } static VkResult vn_find_first_signaled_semaphore(VkDevice device, const VkSemaphore *semaphores, const uint64_t *values, uint32_t count) { for (uint32_t i = 0; i < count; i++) { uint64_t val = 0; VkResult result = vn_GetSemaphoreCounterValue(device, semaphores[i], &val); if (result != VK_SUCCESS || val >= values[i]) return result; } return VK_NOT_READY; } static VkResult vn_remove_signaled_semaphores(VkDevice device, VkSemaphore *semaphores, uint64_t *values, uint32_t *count) { uint32_t cur = 0; for (uint32_t i = 0; i < *count; i++) { uint64_t val = 0; VkResult result = vn_GetSemaphoreCounterValue(device, semaphores[i], &val); if (result != VK_SUCCESS) return result; if (val < values[i]) semaphores[cur++] = semaphores[i]; } *count = cur; return cur ? VK_NOT_READY : VK_SUCCESS; } VkResult vn_WaitSemaphores(VkDevice device, const VkSemaphoreWaitInfo *pWaitInfo, uint64_t timeout) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); const int64_t abs_timeout = os_time_get_absolute_timeout(timeout); VkResult result = VK_NOT_READY; if (pWaitInfo->semaphoreCount > 1 && !(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT)) { uint32_t semaphore_count = pWaitInfo->semaphoreCount; STACK_ARRAY(VkSemaphore, semaphores, semaphore_count); STACK_ARRAY(uint64_t, values, semaphore_count); typed_memcpy(semaphores, pWaitInfo->pSemaphores, semaphore_count); typed_memcpy(values, pWaitInfo->pValues, semaphore_count); struct vn_relax_state relax_state = vn_relax_init(dev->instance, VN_RELAX_REASON_SEMAPHORE); while (result == VK_NOT_READY) { result = vn_remove_signaled_semaphores(device, semaphores, values, &semaphore_count); result = vn_update_sync_result(dev, result, abs_timeout, &relax_state); } vn_relax_fini(&relax_state); STACK_ARRAY_FINISH(semaphores); STACK_ARRAY_FINISH(values); } else { struct vn_relax_state relax_state = vn_relax_init(dev->instance, VN_RELAX_REASON_SEMAPHORE); while (result == VK_NOT_READY) { result = vn_find_first_signaled_semaphore( device, pWaitInfo->pSemaphores, pWaitInfo->pValues, pWaitInfo->semaphoreCount); result = vn_update_sync_result(dev, result, abs_timeout, &relax_state); } vn_relax_fini(&relax_state); } return vn_result(dev->instance, result); } VkResult vn_ImportSemaphoreFdKHR( VkDevice device, const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_semaphore *sem = vn_semaphore_from_handle(pImportSemaphoreFdInfo->semaphore); ASSERTED const bool sync_file = pImportSemaphoreFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; const int fd = pImportSemaphoreFdInfo->fd; assert(sync_file); if (!vn_sync_valid_fd(fd)) return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE); struct vn_sync_payload *temp = &sem->temporary; vn_sync_payload_release(dev, temp); temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD; temp->fd = fd; sem->payload = temp; return VK_SUCCESS; } VkResult vn_GetSemaphoreFdKHR(VkDevice device, const VkSemaphoreGetFdInfoKHR *pGetFdInfo, int *pFd) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_semaphore *sem = vn_semaphore_from_handle(pGetFdInfo->semaphore); const bool sync_file = pGetFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; struct vn_sync_payload *payload = sem->payload; assert(sync_file); assert(dev->physical_device->renderer_sync_fd.semaphore_exportable); assert(dev->physical_device->renderer_sync_fd.semaphore_importable); int fd = -1; if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) { VkResult result = vn_create_sync_file(dev, &sem->external_payload, &fd); if (result != VK_SUCCESS) return vn_error(dev->instance, result); #ifdef VN_USE_WSI_PLATFORM if (!dev->renderer->info.has_implicit_fencing) sync_wait(fd, -1); #endif } else { assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD); /* transfer ownership of imported sync fd to save a dup */ fd = payload->fd; payload->fd = -1; } /* When payload->type is VN_SYNC_TYPE_IMPORTED_SYNC_FD, the current * payload is from a prior temporary sync_fd import. The permanent * payload of the sempahore might be in signaled state. So we do an * import here to ensure later wait operation is legit. With resourceId * 0, renderer does a signaled sync_fd -1 payload import on the host * semaphore. */ if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD) { const VkImportSemaphoreResourceInfoMESA res_info = { .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA, .semaphore = pGetFdInfo->semaphore, .resourceId = 0, }; vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, device, &res_info); } /* perform wait operation on the host semaphore */ vn_async_vkWaitSemaphoreResourceMESA(dev->primary_ring, device, pGetFdInfo->semaphore); vn_sync_payload_release(dev, &sem->temporary); sem->payload = &sem->permanent; *pFd = fd; return VK_SUCCESS; } /* event commands */ static VkResult vn_event_feedback_init(struct vn_device *dev, struct vn_event *ev) { struct vn_feedback_slot *slot; if (VN_PERF(NO_EVENT_FEEDBACK)) return VK_SUCCESS; slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_EVENT); if (!slot) return VK_ERROR_OUT_OF_HOST_MEMORY; /* newly created event object is in the unsignaled state */ vn_feedback_set_status(slot, VK_EVENT_RESET); ev->feedback_slot = slot; return VK_SUCCESS; } static inline void vn_event_feedback_fini(struct vn_device *dev, struct vn_event *ev) { if (ev->feedback_slot) vn_feedback_pool_free(&dev->feedback_pool, ev->feedback_slot); } VkResult vn_CreateEvent(VkDevice device, const VkEventCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkEvent *pEvent) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : &dev->base.base.alloc; struct vn_event *ev = vk_zalloc(alloc, sizeof(*ev), VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!ev) return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY); vn_object_base_init(&ev->base, VK_OBJECT_TYPE_EVENT, &dev->base); /* feedback is only needed to speed up host operations */ if (!(pCreateInfo->flags & VK_EVENT_CREATE_DEVICE_ONLY_BIT)) { VkResult result = vn_event_feedback_init(dev, ev); if (result != VK_SUCCESS) return vn_error(dev->instance, result); } VkEvent ev_handle = vn_event_to_handle(ev); vn_async_vkCreateEvent(dev->primary_ring, device, pCreateInfo, NULL, &ev_handle); *pEvent = ev_handle; return VK_SUCCESS; } void vn_DestroyEvent(VkDevice device, VkEvent event, const VkAllocationCallbacks *pAllocator) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_event *ev = vn_event_from_handle(event); const VkAllocationCallbacks *alloc = pAllocator ? pAllocator : &dev->base.base.alloc; if (!ev) return; vn_async_vkDestroyEvent(dev->primary_ring, device, event, NULL); vn_event_feedback_fini(dev, ev); vn_object_base_fini(&ev->base); vk_free(alloc, ev); } VkResult vn_GetEventStatus(VkDevice device, VkEvent event) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_event *ev = vn_event_from_handle(event); VkResult result; if (ev->feedback_slot) result = vn_feedback_get_status(ev->feedback_slot); else result = vn_call_vkGetEventStatus(dev->primary_ring, device, event); return vn_result(dev->instance, result); } VkResult vn_SetEvent(VkDevice device, VkEvent event) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_event *ev = vn_event_from_handle(event); if (ev->feedback_slot) { vn_feedback_set_status(ev->feedback_slot, VK_EVENT_SET); vn_async_vkSetEvent(dev->primary_ring, device, event); } else { VkResult result = vn_call_vkSetEvent(dev->primary_ring, device, event); if (result != VK_SUCCESS) return vn_error(dev->instance, result); } return VK_SUCCESS; } VkResult vn_ResetEvent(VkDevice device, VkEvent event) { VN_TRACE_FUNC(); struct vn_device *dev = vn_device_from_handle(device); struct vn_event *ev = vn_event_from_handle(event); if (ev->feedback_slot) { vn_feedback_reset_status(ev->feedback_slot); vn_async_vkResetEvent(dev->primary_ring, device, event); } else { VkResult result = vn_call_vkResetEvent(dev->primary_ring, device, event); if (result != VK_SUCCESS) return vn_error(dev->instance, result); } return VK_SUCCESS; }