/* * Copyright © 2020 Google, Inc. * SPDX-License-Identifier: MIT */ #include "tu_knl.h" #include #include #include #include #include #include #include #include "msm_kgsl.h" #include "ion/ion.h" #include "ion/ion_4.19.h" #include "vk_util.h" #include "util/os_file.h" #include "util/u_debug.h" #include "util/u_vector.h" #include "util/libsync.h" #include "util/timespec.h" #include "tu_cmd_buffer.h" #include "tu_cs.h" #include "tu_device.h" #include "tu_dynamic_rendering.h" #include "tu_rmv.h" /* ION_HEAP(ION_SYSTEM_HEAP_ID) */ #define KGSL_ION_SYSTEM_HEAP_MASK (1u << 25) static int safe_ioctl(int fd, unsigned long request, void *arg) { int ret; do { ret = ioctl(fd, request, arg); } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); return ret; } static int kgsl_submitqueue_new(struct tu_device *dev, int priority, uint32_t *queue_id) { struct kgsl_drawctxt_create req = { .flags = KGSL_CONTEXT_SAVE_GMEM | KGSL_CONTEXT_NO_GMEM_ALLOC | KGSL_CONTEXT_PREAMBLE, }; int ret = safe_ioctl(dev->physical_device->local_fd, IOCTL_KGSL_DRAWCTXT_CREATE, &req); if (ret) return ret; *queue_id = req.drawctxt_id; return 0; } static void kgsl_submitqueue_close(struct tu_device *dev, uint32_t queue_id) { struct kgsl_drawctxt_destroy req = { .drawctxt_id = queue_id, }; safe_ioctl(dev->physical_device->local_fd, IOCTL_KGSL_DRAWCTXT_DESTROY, &req); } static void kgsl_bo_finish(struct tu_device *dev, struct tu_bo *bo); static VkResult bo_init_new_dmaheap(struct tu_device *dev, struct tu_bo **out_bo, uint64_t size, enum tu_bo_alloc_flags flags) { struct dma_heap_allocation_data alloc = { .len = size, .fd_flags = O_RDWR | O_CLOEXEC, }; int ret; ret = safe_ioctl(dev->physical_device->kgsl_dma_fd, DMA_HEAP_IOCTL_ALLOC, &alloc); if (ret) { return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "DMA_HEAP_IOCTL_ALLOC failed (%s)", strerror(errno)); } return tu_bo_init_dmabuf(dev, out_bo, -1, alloc.fd); } static VkResult bo_init_new_ion(struct tu_device *dev, struct tu_bo **out_bo, uint64_t size, enum tu_bo_alloc_flags flags) { struct ion_new_allocation_data alloc = { .len = size, .heap_id_mask = KGSL_ION_SYSTEM_HEAP_MASK, .flags = 0, .fd = -1, }; int ret; ret = safe_ioctl(dev->physical_device->kgsl_dma_fd, ION_IOC_NEW_ALLOC, &alloc); if (ret) { return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "ION_IOC_NEW_ALLOC failed (%s)", strerror(errno)); } return tu_bo_init_dmabuf(dev, out_bo, -1, alloc.fd); } static VkResult bo_init_new_ion_legacy(struct tu_device *dev, struct tu_bo **out_bo, uint64_t size, enum tu_bo_alloc_flags flags) { struct ion_allocation_data alloc = { .len = size, .align = 4096, .heap_id_mask = KGSL_ION_SYSTEM_HEAP_MASK, .flags = 0, .handle = -1, }; int ret; ret = safe_ioctl(dev->physical_device->kgsl_dma_fd, ION_IOC_ALLOC, &alloc); if (ret) { return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "ION_IOC_ALLOC failed (%s)", strerror(errno)); } struct ion_fd_data share = { .handle = alloc.handle, .fd = -1, }; ret = safe_ioctl(dev->physical_device->kgsl_dma_fd, ION_IOC_SHARE, &share); if (ret) { return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "ION_IOC_SHARE failed (%s)", strerror(errno)); } struct ion_handle_data free = { .handle = alloc.handle, }; ret = safe_ioctl(dev->physical_device->kgsl_dma_fd, ION_IOC_FREE, &free); if (ret) { return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "ION_IOC_FREE failed (%s)", strerror(errno)); } return tu_bo_init_dmabuf(dev, out_bo, -1, share.fd); } static VkResult kgsl_bo_init(struct tu_device *dev, struct vk_object_base *base, struct tu_bo **out_bo, uint64_t size, uint64_t client_iova, VkMemoryPropertyFlags mem_property, enum tu_bo_alloc_flags flags, const char *name) { if (flags & TU_BO_ALLOC_SHAREABLE) { /* The Vulkan spec doesn't forbid allocating exportable memory with a * fixed address, only imported memory, but on kgsl we can't sensibly * implement it so just always reject it. */ if (client_iova) { return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS, "cannot allocate an exportable BO with a fixed address"); } switch(dev->physical_device->kgsl_dma_type) { case TU_KGSL_DMA_TYPE_DMAHEAP: return bo_init_new_dmaheap(dev, out_bo, size, flags); case TU_KGSL_DMA_TYPE_ION: return bo_init_new_ion(dev, out_bo, size, flags); case TU_KGSL_DMA_TYPE_ION_LEGACY: return bo_init_new_ion_legacy(dev, out_bo, size, flags); } } struct kgsl_gpumem_alloc_id req = { .size = size, }; if (mem_property & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) { if (mem_property & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) { req.flags |= KGSL_MEMFLAGS_IOCOHERENT; } req.flags |= KGSL_CACHEMODE_WRITEBACK << KGSL_CACHEMODE_SHIFT; } else { req.flags |= KGSL_CACHEMODE_WRITECOMBINE << KGSL_CACHEMODE_SHIFT; } if (flags & TU_BO_ALLOC_GPU_READ_ONLY) req.flags |= KGSL_MEMFLAGS_GPUREADONLY; if (flags & TU_BO_ALLOC_REPLAYABLE) req.flags |= KGSL_MEMFLAGS_USE_CPU_MAP; int ret; ret = safe_ioctl(dev->physical_device->local_fd, IOCTL_KGSL_GPUMEM_ALLOC_ID, &req); if (ret) { return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "GPUMEM_ALLOC_ID failed (%s)", strerror(errno)); } struct tu_bo* bo = tu_device_lookup_bo(dev, req.id); assert(bo && bo->gem_handle == 0); *bo = (struct tu_bo) { .gem_handle = req.id, .size = req.mmapsize, .iova = req.gpuaddr, .name = tu_debug_bos_add(dev, req.mmapsize, name), .refcnt = 1, .shared_fd = -1, .base = base, }; if (flags & TU_BO_ALLOC_REPLAYABLE) { uint64_t offset = req.id << 12; void *map = mmap((void *)client_iova, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED, dev->physical_device->local_fd, offset); if (map == MAP_FAILED) { kgsl_bo_finish(dev, bo); return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "mmap failed (%s)", strerror(errno)); } if (client_iova && (uint64_t)map != client_iova) { kgsl_bo_finish(dev, bo); return vk_errorf(dev, VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS, "mmap could not map the given address"); } bo->map = map; bo->iova = (uint64_t)map; /* Because we're using SVM, the CPU mapping and GPU mapping are the same * and the CPU mapping must stay fixed for the lifetime of the BO. */ bo->never_unmap = true; } *out_bo = bo; TU_RMV(bo_allocate, dev, bo); if (flags & TU_BO_ALLOC_INTERNAL_RESOURCE) { TU_RMV(internal_resource_create, dev, bo); TU_RMV(resource_name, dev, bo, name); } return VK_SUCCESS; } static VkResult kgsl_bo_init_dmabuf(struct tu_device *dev, struct tu_bo **out_bo, uint64_t size, int fd) { struct kgsl_gpuobj_import_dma_buf import_dmabuf = { .fd = fd, }; struct kgsl_gpuobj_import req = { .priv = (uintptr_t)&import_dmabuf, .priv_len = sizeof(import_dmabuf), .flags = 0, .type = KGSL_USER_MEM_TYPE_DMABUF, }; int ret; ret = safe_ioctl(dev->physical_device->local_fd, IOCTL_KGSL_GPUOBJ_IMPORT, &req); if (ret) return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "Failed to import dma-buf (%s)\n", strerror(errno)); struct kgsl_gpuobj_info info_req = { .id = req.id, }; ret = safe_ioctl(dev->physical_device->local_fd, IOCTL_KGSL_GPUOBJ_INFO, &info_req); if (ret) return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "Failed to get dma-buf info (%s)\n", strerror(errno)); struct tu_bo* bo = tu_device_lookup_bo(dev, req.id); assert(bo && bo->gem_handle == 0); *bo = (struct tu_bo) { .gem_handle = req.id, .size = info_req.size, .iova = info_req.gpuaddr, .name = tu_debug_bos_add(dev, info_req.size, "dmabuf"), .refcnt = 1, .shared_fd = os_dupfd_cloexec(fd), }; *out_bo = bo; return VK_SUCCESS; } static int kgsl_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo) { assert(bo->shared_fd != -1); return os_dupfd_cloexec(bo->shared_fd); } static VkResult kgsl_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr) { void *map = MAP_FAILED; if (bo->shared_fd == -1) { uint64_t offset = bo->gem_handle << 12; map = mmap(placed_addr, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED | (placed_addr != NULL ? MAP_FIXED : 0), dev->physical_device->local_fd, offset); } else { map = mmap(placed_addr, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED | (placed_addr != NULL ? MAP_FIXED : 0), bo->shared_fd, 0); } if (map == MAP_FAILED) return vk_error(dev, VK_ERROR_MEMORY_MAP_FAILED); bo->map = map; TU_RMV(bo_map, dev, bo); return VK_SUCCESS; } static void kgsl_bo_allow_dump(struct tu_device *dev, struct tu_bo *bo) { } static void kgsl_bo_finish(struct tu_device *dev, struct tu_bo *bo) { assert(bo->gem_handle); if (!p_atomic_dec_zero(&bo->refcnt)) return; if (bo->map) { TU_RMV(bo_unmap, dev, bo); munmap(bo->map, bo->size); } if (bo->shared_fd != -1) close(bo->shared_fd); TU_RMV(bo_destroy, dev, bo); struct kgsl_gpumem_free_id req = { .id = bo->gem_handle }; /* Tell sparse array that entry is free */ memset(bo, 0, sizeof(*bo)); safe_ioctl(dev->physical_device->local_fd, IOCTL_KGSL_GPUMEM_FREE_ID, &req); } static VkResult get_kgsl_prop(int fd, unsigned int type, void *value, size_t size) { struct kgsl_device_getproperty getprop = { .type = type, .value = value, .sizebytes = size, }; return safe_ioctl(fd, IOCTL_KGSL_DEVICE_GETPROPERTY, &getprop) ? VK_ERROR_UNKNOWN : VK_SUCCESS; } static bool kgsl_is_memory_type_supported(int fd, uint32_t flags) { struct kgsl_gpumem_alloc_id req_alloc = { .flags = flags, .size = 0x1000, }; int ret = safe_ioctl(fd, IOCTL_KGSL_GPUMEM_ALLOC_ID, &req_alloc); if (ret) { return false; } struct kgsl_gpumem_free_id req_free = { .id = req_alloc.id }; safe_ioctl(fd, IOCTL_KGSL_GPUMEM_FREE_ID, &req_free); return true; } enum kgsl_syncobj_state { KGSL_SYNCOBJ_STATE_UNSIGNALED, KGSL_SYNCOBJ_STATE_SIGNALED, KGSL_SYNCOBJ_STATE_TS, KGSL_SYNCOBJ_STATE_FD, }; struct kgsl_syncobj { struct vk_object_base base; enum kgsl_syncobj_state state; struct tu_queue *queue; uint32_t timestamp; int fd; }; struct tu_u_trace_syncobj { uint32_t msm_queue_id; uint32_t timestamp; }; static void kgsl_syncobj_init(struct kgsl_syncobj *s, bool signaled) { s->state = signaled ? KGSL_SYNCOBJ_STATE_SIGNALED : KGSL_SYNCOBJ_STATE_UNSIGNALED; s->timestamp = UINT32_MAX; s->fd = -1; } static void kgsl_syncobj_reset(struct kgsl_syncobj *s) { if (s->state == KGSL_SYNCOBJ_STATE_FD && s->fd >= 0) { ASSERTED int ret = close(s->fd); assert(ret == 0); s->fd = -1; } else if (s->state == KGSL_SYNCOBJ_STATE_TS) { s->timestamp = UINT32_MAX; } s->state = KGSL_SYNCOBJ_STATE_UNSIGNALED; } static void kgsl_syncobj_destroy(struct kgsl_syncobj *s) { kgsl_syncobj_reset(s); } static int timestamp_to_fd(struct tu_queue *queue, uint32_t timestamp) { int fd; struct kgsl_timestamp_event event = { .type = KGSL_TIMESTAMP_EVENT_FENCE, .timestamp = timestamp, .context_id = queue->msm_queue_id, .priv = &fd, .len = sizeof(fd), }; int ret = safe_ioctl(queue->device->fd, IOCTL_KGSL_TIMESTAMP_EVENT, &event); if (ret) return -1; return fd; } static int kgsl_syncobj_ts_to_fd(const struct kgsl_syncobj *syncobj) { assert(syncobj->state == KGSL_SYNCOBJ_STATE_TS); return timestamp_to_fd(syncobj->queue, syncobj->timestamp); } /* return true if timestamp a is greater (more recent) then b * this relies on timestamps never having a difference > (1<<31) */ static inline bool timestamp_cmp(uint32_t a, uint32_t b) { return (int32_t) (a - b) >= 0; } static uint32_t max_ts(uint32_t a, uint32_t b) { return timestamp_cmp(a, b) ? a : b; } static uint32_t min_ts(uint32_t a, uint32_t b) { return timestamp_cmp(a, b) ? b : a; } static int get_relative_ms(uint64_t abs_timeout_ns) { if (abs_timeout_ns >= INT64_MAX) /* We can assume that a wait with a value this high is a forever wait * and return -1 here as it's the infinite timeout for ppoll() while * being the highest unsigned integer value for the wait KGSL IOCTL */ return -1; uint64_t cur_time_ms = os_time_get_nano() / 1000000; uint64_t abs_timeout_ms = abs_timeout_ns / 1000000; if (abs_timeout_ms <= cur_time_ms) return 0; return abs_timeout_ms - cur_time_ms; } /* safe_ioctl is not enough as restarted waits would not adjust the timeout * which could lead to waiting substantially longer than requested */ static int wait_timestamp_safe(int fd, unsigned int context_id, unsigned int timestamp, uint64_t abs_timeout_ns) { struct kgsl_device_waittimestamp_ctxtid wait = { .context_id = context_id, .timestamp = timestamp, .timeout = get_relative_ms(abs_timeout_ns), }; while (true) { int ret = ioctl(fd, IOCTL_KGSL_DEVICE_WAITTIMESTAMP_CTXTID, &wait); if (ret == -1 && (errno == EINTR || errno == EAGAIN)) { int timeout_ms = get_relative_ms(abs_timeout_ns); /* update timeout to consider time that has passed since the start */ if (timeout_ms == 0) { errno = ETIME; return -1; } wait.timeout = timeout_ms; } else if (ret == -1 && errno == ETIMEDOUT) { /* The kernel returns ETIMEDOUT if the timeout is reached, but * we want to return ETIME instead. */ errno = ETIME; return -1; } else { return ret; } } } static VkResult kgsl_syncobj_wait(struct tu_device *device, struct kgsl_syncobj *s, uint64_t abs_timeout_ns) { if (s->state == KGSL_SYNCOBJ_STATE_UNSIGNALED) { /* If this syncobj is unsignaled we need to wait for it to resolve to a * valid syncobj prior to letting the rest of the wait continue, this * avoids needing kernel support for wait-before-signal semantics. */ if (abs_timeout_ns == 0) return VK_TIMEOUT; // If this is a simple poll then we can return early pthread_mutex_lock(&device->submit_mutex); struct timespec abstime; timespec_from_nsec(&abstime, abs_timeout_ns); while (s->state == KGSL_SYNCOBJ_STATE_UNSIGNALED) { int ret; if (abs_timeout_ns == UINT64_MAX) { ret = pthread_cond_wait(&device->timeline_cond, &device->submit_mutex); } else { ret = pthread_cond_timedwait(&device->timeline_cond, &device->submit_mutex, &abstime); } if (ret != 0) { assert(ret == ETIMEDOUT); pthread_mutex_unlock(&device->submit_mutex); return VK_TIMEOUT; } } pthread_mutex_unlock(&device->submit_mutex); } switch (s->state) { case KGSL_SYNCOBJ_STATE_SIGNALED: return VK_SUCCESS; case KGSL_SYNCOBJ_STATE_UNSIGNALED: return VK_TIMEOUT; case KGSL_SYNCOBJ_STATE_TS: { int ret = wait_timestamp_safe(device->fd, s->queue->msm_queue_id, s->timestamp, abs_timeout_ns); if (ret) { assert(errno == ETIME); return VK_TIMEOUT; } else { return VK_SUCCESS; } } case KGSL_SYNCOBJ_STATE_FD: { int ret = sync_wait(s->fd, get_relative_ms(abs_timeout_ns)); if (ret) { assert(errno == ETIME); return VK_TIMEOUT; } else { return VK_SUCCESS; } } default: unreachable("invalid syncobj state"); } } #define kgsl_syncobj_foreach_state(syncobjs, filter) \ for (uint32_t i = 0; sync = syncobjs[i], i < count; i++) \ if (sync->state == filter) static VkResult kgsl_syncobj_wait_any(struct tu_device* device, struct kgsl_syncobj **syncobjs, uint32_t count, uint64_t abs_timeout_ns) { if (count == 0) return VK_TIMEOUT; else if (count == 1) return kgsl_syncobj_wait(device, syncobjs[0], abs_timeout_ns); uint32_t num_fds = 0; struct tu_queue *queue = NULL; struct kgsl_syncobj *sync = NULL; /* Simple case, we already have a signal one */ kgsl_syncobj_foreach_state(syncobjs, KGSL_SYNCOBJ_STATE_SIGNALED) return VK_SUCCESS; kgsl_syncobj_foreach_state(syncobjs, KGSL_SYNCOBJ_STATE_FD) num_fds++; /* If we have TS from different queues we cannot compare them and would * have to convert them into FDs */ bool convert_ts_to_fd = false; kgsl_syncobj_foreach_state(syncobjs, KGSL_SYNCOBJ_STATE_TS) { if (queue != NULL && sync->queue != queue) { convert_ts_to_fd = true; break; } queue = sync->queue; } /* If we have no FD nor TS syncobjs then we can return immediately */ if (num_fds == 0 && queue == NULL) return VK_TIMEOUT; VkResult result = VK_TIMEOUT; struct u_vector poll_fds = { 0 }; uint32_t lowest_timestamp = 0; if (convert_ts_to_fd || num_fds > 0) u_vector_init(&poll_fds, 4, sizeof(struct pollfd)); if (convert_ts_to_fd) { kgsl_syncobj_foreach_state(syncobjs, KGSL_SYNCOBJ_STATE_TS) { struct pollfd *poll_fd = (struct pollfd *) u_vector_add(&poll_fds); poll_fd->fd = timestamp_to_fd(sync->queue, sync->timestamp); poll_fd->events = POLLIN; } } else { /* TSs could be merged by finding the one with the lowest timestamp */ bool first_ts = true; kgsl_syncobj_foreach_state(syncobjs, KGSL_SYNCOBJ_STATE_TS) { if (first_ts || timestamp_cmp(sync->timestamp, lowest_timestamp)) { first_ts = false; lowest_timestamp = sync->timestamp; } } if (num_fds) { struct pollfd *poll_fd = (struct pollfd *) u_vector_add(&poll_fds); poll_fd->fd = timestamp_to_fd(queue, lowest_timestamp); poll_fd->events = POLLIN; } } if (num_fds) { kgsl_syncobj_foreach_state(syncobjs, KGSL_SYNCOBJ_STATE_FD) { struct pollfd *poll_fd = (struct pollfd *) u_vector_add(&poll_fds); poll_fd->fd = sync->fd; poll_fd->events = POLLIN; } } if (u_vector_length(&poll_fds) == 0) { int ret = wait_timestamp_safe(device->fd, queue->msm_queue_id, lowest_timestamp, MIN2(abs_timeout_ns, INT64_MAX)); if (ret) { assert(errno == ETIME); result = VK_TIMEOUT; } else { result = VK_SUCCESS; } } else { int ret, i; struct pollfd *fds = (struct pollfd *) poll_fds.data; uint32_t fds_count = u_vector_length(&poll_fds); do { ret = poll(fds, fds_count, get_relative_ms(abs_timeout_ns)); if (ret > 0) { for (i = 0; i < fds_count; i++) { if (fds[i].revents & (POLLERR | POLLNVAL)) { errno = EINVAL; ret = -1; break; } } break; } else if (ret == 0) { errno = ETIME; break; } } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); for (uint32_t i = 0; i < fds_count - num_fds; i++) close(fds[i].fd); if (ret != 0) { assert(errno == ETIME); result = VK_TIMEOUT; } else { result = VK_SUCCESS; } } u_vector_finish(&poll_fds); return result; } static VkResult kgsl_syncobj_export(struct kgsl_syncobj *s, int *pFd) { if (!pFd) return VK_SUCCESS; switch (s->state) { case KGSL_SYNCOBJ_STATE_SIGNALED: case KGSL_SYNCOBJ_STATE_UNSIGNALED: /* Getting a sync FD from an unsignaled syncobj is UB in Vulkan */ *pFd = -1; return VK_SUCCESS; case KGSL_SYNCOBJ_STATE_FD: if (s->fd < 0) *pFd = -1; else *pFd = dup(s->fd); return VK_SUCCESS; case KGSL_SYNCOBJ_STATE_TS: *pFd = kgsl_syncobj_ts_to_fd(s); return VK_SUCCESS; default: unreachable("Invalid syncobj state"); } } static VkResult kgsl_syncobj_import(struct kgsl_syncobj *s, int fd) { kgsl_syncobj_reset(s); if (fd >= 0) { s->state = KGSL_SYNCOBJ_STATE_FD; s->fd = fd; } else { s->state = KGSL_SYNCOBJ_STATE_SIGNALED; } return VK_SUCCESS; } static int sync_merge_close(const char *name, int fd1, int fd2, bool close_fd2) { int fd = sync_merge(name, fd1, fd2); if (fd < 0) return -1; close(fd1); if (close_fd2) close(fd2); return fd; } /* Merges multiple kgsl_syncobjs into a single one which is only signalled * after all submitted syncobjs are signalled */ static struct kgsl_syncobj kgsl_syncobj_merge(const struct kgsl_syncobj **syncobjs, uint32_t count) { struct kgsl_syncobj ret; kgsl_syncobj_init(&ret, true); if (count == 0) return ret; for (uint32_t i = 0; i < count; ++i) { const struct kgsl_syncobj *sync = syncobjs[i]; switch (sync->state) { case KGSL_SYNCOBJ_STATE_SIGNALED: break; case KGSL_SYNCOBJ_STATE_UNSIGNALED: kgsl_syncobj_reset(&ret); return ret; case KGSL_SYNCOBJ_STATE_TS: if (ret.state == KGSL_SYNCOBJ_STATE_TS) { if (ret.queue == sync->queue) { ret.timestamp = max_ts(ret.timestamp, sync->timestamp); } else { ret.state = KGSL_SYNCOBJ_STATE_FD; int sync_fd = kgsl_syncobj_ts_to_fd(sync); ret.fd = sync_merge_close("tu_sync", ret.fd, sync_fd, true); assert(ret.fd >= 0); } } else if (ret.state == KGSL_SYNCOBJ_STATE_FD) { int sync_fd = kgsl_syncobj_ts_to_fd(sync); ret.fd = sync_merge_close("tu_sync", ret.fd, sync_fd, true); assert(ret.fd >= 0); } else { ret = *sync; } break; case KGSL_SYNCOBJ_STATE_FD: if (ret.state == KGSL_SYNCOBJ_STATE_FD) { ret.fd = sync_merge_close("tu_sync", ret.fd, sync->fd, false); assert(ret.fd >= 0); } else if (ret.state == KGSL_SYNCOBJ_STATE_TS) { ret.state = KGSL_SYNCOBJ_STATE_FD; int sync_fd = kgsl_syncobj_ts_to_fd(sync); ret.fd = sync_merge_close("tu_sync", ret.fd, sync_fd, true); assert(ret.fd >= 0); } else { ret = *sync; ret.fd = dup(ret.fd); assert(ret.fd >= 0); } break; default: unreachable("invalid syncobj state"); } } return ret; } struct vk_kgsl_syncobj { struct vk_sync vk; struct kgsl_syncobj syncobj; }; static VkResult vk_kgsl_sync_init(struct vk_device *device, struct vk_sync *sync, uint64_t initial_value) { struct vk_kgsl_syncobj *s = container_of(sync, struct vk_kgsl_syncobj, vk); kgsl_syncobj_init(&s->syncobj, initial_value != 0); return VK_SUCCESS; } static void vk_kgsl_sync_finish(struct vk_device *device, struct vk_sync *sync) { struct vk_kgsl_syncobj *s = container_of(sync, struct vk_kgsl_syncobj, vk); kgsl_syncobj_destroy(&s->syncobj); } static VkResult vk_kgsl_sync_reset(struct vk_device *device, struct vk_sync *sync) { struct vk_kgsl_syncobj *s = container_of(sync, struct vk_kgsl_syncobj, vk); kgsl_syncobj_reset(&s->syncobj); return VK_SUCCESS; } static VkResult vk_kgsl_sync_move(struct vk_device *device, struct vk_sync *dst, struct vk_sync *src) { struct vk_kgsl_syncobj *d = container_of(dst, struct vk_kgsl_syncobj, vk); struct vk_kgsl_syncobj *s = container_of(src, struct vk_kgsl_syncobj, vk); kgsl_syncobj_reset(&d->syncobj); d->syncobj = s->syncobj; kgsl_syncobj_init(&s->syncobj, false); return VK_SUCCESS; } static VkResult vk_kgsl_sync_wait(struct vk_device *_device, struct vk_sync *sync, uint64_t wait_value, enum vk_sync_wait_flags wait_flags, uint64_t abs_timeout_ns) { struct tu_device *device = container_of(_device, struct tu_device, vk); struct vk_kgsl_syncobj *s = container_of(sync, struct vk_kgsl_syncobj, vk); if (wait_flags & VK_SYNC_WAIT_PENDING) return VK_SUCCESS; return kgsl_syncobj_wait(device, &s->syncobj, abs_timeout_ns); } static VkResult vk_kgsl_sync_wait_many(struct vk_device *_device, uint32_t wait_count, const struct vk_sync_wait *waits, enum vk_sync_wait_flags wait_flags, uint64_t abs_timeout_ns) { struct tu_device *device = container_of(_device, struct tu_device, vk); if (wait_flags & VK_SYNC_WAIT_PENDING) return VK_SUCCESS; if (wait_flags & VK_SYNC_WAIT_ANY) { struct kgsl_syncobj *syncobjs[wait_count]; for (uint32_t i = 0; i < wait_count; i++) { syncobjs[i] = &container_of(waits[i].sync, struct vk_kgsl_syncobj, vk)->syncobj; } return kgsl_syncobj_wait_any(device, syncobjs, wait_count, abs_timeout_ns); } else { for (uint32_t i = 0; i < wait_count; i++) { struct vk_kgsl_syncobj *s = container_of(waits[i].sync, struct vk_kgsl_syncobj, vk); VkResult result = kgsl_syncobj_wait(device, &s->syncobj, abs_timeout_ns); if (result != VK_SUCCESS) return result; } return VK_SUCCESS; } } static VkResult vk_kgsl_sync_import_sync_file(struct vk_device *device, struct vk_sync *sync, int fd) { struct vk_kgsl_syncobj *s = container_of(sync, struct vk_kgsl_syncobj, vk); if (fd >= 0) { fd = dup(fd); if (fd < 0) { mesa_loge("vk_kgsl_sync_import_sync_file: dup failed: %s", strerror(errno)); return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); } } return kgsl_syncobj_import(&s->syncobj, fd); } static VkResult vk_kgsl_sync_export_sync_file(struct vk_device *device, struct vk_sync *sync, int *pFd) { struct vk_kgsl_syncobj *s = container_of(sync, struct vk_kgsl_syncobj, vk); return kgsl_syncobj_export(&s->syncobj, pFd); } const struct vk_sync_type vk_kgsl_sync_type = { .size = sizeof(struct vk_kgsl_syncobj), .features = (enum vk_sync_features) (VK_SYNC_FEATURE_BINARY | VK_SYNC_FEATURE_GPU_WAIT | VK_SYNC_FEATURE_GPU_MULTI_WAIT | VK_SYNC_FEATURE_CPU_WAIT | VK_SYNC_FEATURE_CPU_RESET | VK_SYNC_FEATURE_WAIT_ANY | VK_SYNC_FEATURE_WAIT_PENDING), .init = vk_kgsl_sync_init, .finish = vk_kgsl_sync_finish, .reset = vk_kgsl_sync_reset, .move = vk_kgsl_sync_move, .wait = vk_kgsl_sync_wait, .wait_many = vk_kgsl_sync_wait_many, .import_sync_file = vk_kgsl_sync_import_sync_file, .export_sync_file = vk_kgsl_sync_export_sync_file, }; static VkResult kgsl_queue_submit(struct tu_queue *queue, struct vk_queue_submit *vk_submit) { MESA_TRACE_FUNC(); bool u_trace_enabled = u_trace_should_process(&queue->device->trace_context); bool has_trace_points = false; if (vk_submit->command_buffer_count == 0) { pthread_mutex_lock(&queue->device->submit_mutex); const struct kgsl_syncobj *wait_semaphores[vk_submit->wait_count + 1]; for (uint32_t i = 0; i < vk_submit->wait_count; i++) { wait_semaphores[i] = &container_of(vk_submit->waits[i].sync, struct vk_kgsl_syncobj, vk) ->syncobj; } struct kgsl_syncobj last_submit_sync; if (queue->fence >= 0) last_submit_sync = (struct kgsl_syncobj) { .state = KGSL_SYNCOBJ_STATE_TS, .queue = queue, .timestamp = queue->fence, }; else last_submit_sync = (struct kgsl_syncobj) { .state = KGSL_SYNCOBJ_STATE_SIGNALED, }; wait_semaphores[vk_submit->wait_count] = &last_submit_sync; struct kgsl_syncobj wait_sync = kgsl_syncobj_merge(wait_semaphores, vk_submit->wait_count + 1); assert(wait_sync.state != KGSL_SYNCOBJ_STATE_UNSIGNALED); // Would wait forever for (uint32_t i = 0; i < vk_submit->signal_count; i++) { struct kgsl_syncobj *signal_sync = &container_of(vk_submit->signals[i].sync, struct vk_kgsl_syncobj, vk) ->syncobj; kgsl_syncobj_reset(signal_sync); *signal_sync = wait_sync; } pthread_mutex_unlock(&queue->device->submit_mutex); pthread_cond_broadcast(&queue->device->timeline_cond); return VK_SUCCESS; } uint32_t perf_pass_index = queue->device->perfcntrs_pass_cs ? vk_submit->perf_pass_index : ~0; if (TU_DEBUG(LOG_SKIP_GMEM_OPS)) tu_dbg_log_gmem_load_store_skips(queue->device); VkResult result = VK_SUCCESS; pthread_mutex_lock(&queue->device->submit_mutex); struct tu_cmd_buffer **cmd_buffers = (struct tu_cmd_buffer **) vk_submit->command_buffers; static_assert(offsetof(struct tu_cmd_buffer, vk) == 0, "vk must be first member of tu_cmd_buffer"); uint32_t cmdbuf_count = vk_submit->command_buffer_count; result = tu_insert_dynamic_cmdbufs(queue->device, &cmd_buffers, &cmdbuf_count); if (result != VK_SUCCESS) { pthread_mutex_unlock(&queue->device->submit_mutex); return result; } uint32_t entry_count = 0; for (uint32_t i = 0; i < cmdbuf_count; ++i) { struct tu_cmd_buffer *cmd_buffer = cmd_buffers[i]; if (perf_pass_index != ~0) entry_count++; entry_count += cmd_buffer->cs.entry_count; if (u_trace_enabled && u_trace_has_points(&cmd_buffers[i]->trace)) { if (!(cmd_buffers[i]->usage_flags & VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT)) entry_count++; has_trace_points = true; } } if (tu_autotune_submit_requires_fence(cmd_buffers, cmdbuf_count)) entry_count++; struct kgsl_command_object *cmds = (struct kgsl_command_object *) vk_alloc(&queue->device->vk.alloc, sizeof(*cmds) * entry_count, alignof(*cmds), VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); if (cmds == NULL) { pthread_mutex_unlock(&queue->device->submit_mutex); return vk_error(queue, VK_ERROR_OUT_OF_HOST_MEMORY); } uint32_t obj_count = 0; if (has_trace_points) obj_count++; struct kgsl_command_object *objs = (struct kgsl_command_object *) vk_alloc(&queue->device->vk.alloc, sizeof(*objs) * obj_count, alignof(*objs), VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); struct tu_u_trace_submission_data *u_trace_submission_data = NULL; if (has_trace_points) { tu_u_trace_submission_data_create( queue->device, cmd_buffers, cmdbuf_count, &u_trace_submission_data); mtx_lock(&queue->device->kgsl_profiling_mutex); tu_suballoc_bo_alloc(&u_trace_submission_data->kgsl_timestamp_bo, &queue->device->kgsl_profiling_suballoc, sizeof(struct kgsl_cmdbatch_profiling_buffer), 4); mtx_unlock(&queue->device->kgsl_profiling_mutex); } uint32_t entry_idx = 0; for (uint32_t i = 0; i < cmdbuf_count; i++) { struct tu_cmd_buffer *cmd_buffer = cmd_buffers[i]; struct tu_cs *cs = &cmd_buffer->cs; if (perf_pass_index != ~0) { struct tu_cs_entry *perf_cs_entry = &cmd_buffer->device->perfcntrs_pass_cs_entries[perf_pass_index]; cmds[entry_idx++] = (struct kgsl_command_object) { .gpuaddr = perf_cs_entry->bo->iova + perf_cs_entry->offset, .size = perf_cs_entry->size, .flags = KGSL_CMDLIST_IB, .id = perf_cs_entry->bo->gem_handle, }; } for (uint32_t j = 0; j < cs->entry_count; j++) { cmds[entry_idx++] = (struct kgsl_command_object) { .gpuaddr = cs->entries[j].bo->iova + cs->entries[j].offset, .size = cs->entries[j].size, .flags = KGSL_CMDLIST_IB, .id = cs->entries[j].bo->gem_handle, }; } if (u_trace_submission_data && u_trace_submission_data->cmd_trace_data[i].timestamp_copy_cs) { struct tu_cs_entry *trace_cs_entry = &u_trace_submission_data->cmd_trace_data[i] .timestamp_copy_cs->entries[0]; cmds[entry_idx++] = (struct kgsl_command_object) { .offset = trace_cs_entry->offset, .gpuaddr = trace_cs_entry->bo->iova, .size = trace_cs_entry->size, .flags = KGSL_CMDLIST_IB, .id = trace_cs_entry->bo->gem_handle, }; } } struct kgsl_cmdbatch_profiling_buffer *profiling_buffer = NULL; uint32_t obj_idx = 0; if (u_trace_submission_data) { struct tu_suballoc_bo *bo = &u_trace_submission_data->kgsl_timestamp_bo; objs[obj_idx++] = (struct kgsl_command_object) { .offset = bo->iova - bo->bo->iova, .gpuaddr = bo->bo->iova, .size = sizeof(struct kgsl_cmdbatch_profiling_buffer), .flags = KGSL_OBJLIST_MEMOBJ | KGSL_OBJLIST_PROFILE, .id = bo->bo->gem_handle, }; profiling_buffer = (struct kgsl_cmdbatch_profiling_buffer *) tu_suballoc_bo_map(bo); memset(profiling_buffer, 0, sizeof(*profiling_buffer)); } if (tu_autotune_submit_requires_fence(cmd_buffers, cmdbuf_count)) { struct tu_cs *autotune_cs = tu_autotune_on_submit( queue->device, &queue->device->autotune, cmd_buffers, cmdbuf_count); cmds[entry_idx++] = (struct kgsl_command_object) { .gpuaddr = autotune_cs->entries[0].bo->iova + autotune_cs->entries[0].offset, .size = autotune_cs->entries[0].size, .flags = KGSL_CMDLIST_IB, .id = autotune_cs->entries[0].bo->gem_handle, }; } const struct kgsl_syncobj *wait_semaphores[vk_submit->wait_count]; for (uint32_t i = 0; i < vk_submit->wait_count; i++) { wait_semaphores[i] = &container_of(vk_submit->waits[i].sync, struct vk_kgsl_syncobj, vk) ->syncobj; } struct kgsl_syncobj wait_sync = kgsl_syncobj_merge(wait_semaphores, vk_submit->wait_count); assert(wait_sync.state != KGSL_SYNCOBJ_STATE_UNSIGNALED); // Would wait forever struct kgsl_cmd_syncpoint_timestamp ts; struct kgsl_cmd_syncpoint_fence fn; struct kgsl_command_syncpoint sync = { 0 }; bool has_sync = false; switch (wait_sync.state) { case KGSL_SYNCOBJ_STATE_SIGNALED: break; case KGSL_SYNCOBJ_STATE_TS: ts.context_id = wait_sync.queue->msm_queue_id; ts.timestamp = wait_sync.timestamp; has_sync = true; sync.type = KGSL_CMD_SYNCPOINT_TYPE_TIMESTAMP; sync.priv = (uintptr_t) &ts; sync.size = sizeof(ts); break; case KGSL_SYNCOBJ_STATE_FD: fn.fd = wait_sync.fd; has_sync = true; sync.type = KGSL_CMD_SYNCPOINT_TYPE_FENCE; sync.priv = (uintptr_t) &fn; sync.size = sizeof(fn); break; default: unreachable("invalid syncobj state"); } struct kgsl_gpu_command req = { .flags = KGSL_CMDBATCH_SUBMIT_IB_LIST, .cmdlist = (uintptr_t) cmds, .cmdsize = sizeof(struct kgsl_command_object), .numcmds = entry_idx, .synclist = (uintptr_t) &sync, .syncsize = sizeof(sync), .numsyncs = has_sync != 0 ? 1 : 0, .context_id = queue->msm_queue_id, }; if (obj_idx) { req.flags |= KGSL_CMDBATCH_PROFILING; req.objlist = (uintptr_t) objs; req.objsize = sizeof(struct kgsl_command_object); req.numobjs = obj_idx; } int ret = safe_ioctl(queue->device->physical_device->local_fd, IOCTL_KGSL_GPU_COMMAND, &req); uint64_t gpu_offset = 0; #if HAVE_PERFETTO if (profiling_buffer) { /* We need to wait for KGSL to queue the GPU command before we can read * the timestamp. Since this is just for profiling and doesn't take too * long, we can just busy-wait for it. */ while (p_atomic_read(&profiling_buffer->gpu_ticks_queued) == 0); struct kgsl_perfcounter_read_group perf = { .groupid = KGSL_PERFCOUNTER_GROUP_ALWAYSON, .countable = 0, .value = 0 }; struct kgsl_perfcounter_read req = { .reads = &perf, .count = 1, }; ret = safe_ioctl(queue->device->fd, IOCTL_KGSL_PERFCOUNTER_READ, &req); /* Older KGSL has some kind of garbage in upper 32 bits */ uint64_t offseted_gpu_ts = perf.value & 0xffffffff; gpu_offset = tu_device_ticks_to_ns( queue->device, offseted_gpu_ts - profiling_buffer->gpu_ticks_queued); struct tu_perfetto_clocks clocks = { .cpu = profiling_buffer->wall_clock_ns, .gpu_ts = tu_device_ticks_to_ns(queue->device, profiling_buffer->gpu_ticks_queued), .gpu_ts_offset = gpu_offset, }; clocks = tu_perfetto_submit(queue->device, queue->device->submit_count, &clocks); gpu_offset = clocks.gpu_ts_offset; } #endif kgsl_syncobj_destroy(&wait_sync); if (ret) { result = vk_device_set_lost(&queue->device->vk, "submit failed: %s\n", strerror(errno)); goto fail_submit; } p_atomic_set(&queue->fence, req.timestamp); for (uint32_t i = 0; i < vk_submit->signal_count; i++) { struct kgsl_syncobj *signal_sync = &container_of(vk_submit->signals[i].sync, struct vk_kgsl_syncobj, vk) ->syncobj; kgsl_syncobj_reset(signal_sync); signal_sync->state = KGSL_SYNCOBJ_STATE_TS; signal_sync->queue = queue; signal_sync->timestamp = req.timestamp; } if (u_trace_submission_data) { struct tu_u_trace_submission_data *submission_data = u_trace_submission_data; submission_data->submission_id = queue->device->submit_count; submission_data->gpu_ts_offset = gpu_offset; /* We have to allocate it here since it is different between drm/kgsl */ submission_data->syncobj = (struct tu_u_trace_syncobj *) vk_alloc(&queue->device->vk.alloc, sizeof(struct tu_u_trace_syncobj), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); submission_data->syncobj->timestamp = req.timestamp; submission_data->syncobj->msm_queue_id = queue->msm_queue_id; u_trace_submission_data = NULL; for (uint32_t i = 0; i < submission_data->cmd_buffer_count; i++) { bool free_data = i == submission_data->last_buffer_with_tracepoints; if (submission_data->cmd_trace_data[i].trace) u_trace_flush(submission_data->cmd_trace_data[i].trace, submission_data, queue->device->vk.current_frame, free_data); if (!submission_data->cmd_trace_data[i].timestamp_copy_cs) { /* u_trace is owned by cmd_buffer */ submission_data->cmd_trace_data[i].trace = NULL; } } } queue->device->submit_count++; pthread_mutex_unlock(&queue->device->submit_mutex); pthread_cond_broadcast(&queue->device->timeline_cond); u_trace_context_process(&queue->device->trace_context, false); if (cmd_buffers != (struct tu_cmd_buffer **) vk_submit->command_buffers) vk_free(&queue->device->vk.alloc, cmd_buffers); vk_free(&queue->device->vk.alloc, cmds); return VK_SUCCESS; fail_submit: pthread_mutex_unlock(&queue->device->submit_mutex); if (result != VK_SUCCESS) { mtx_lock(&queue->device->kgsl_profiling_mutex); tu_suballoc_bo_free(&queue->device->kgsl_profiling_suballoc, &u_trace_submission_data->kgsl_timestamp_bo); mtx_unlock(&queue->device->kgsl_profiling_mutex); } if (cmd_buffers != (struct tu_cmd_buffer **) vk_submit->command_buffers) vk_free(&queue->device->vk.alloc, cmd_buffers); vk_free(&queue->device->vk.alloc, cmds); return result; } static VkResult kgsl_device_wait_u_trace(struct tu_device *dev, struct tu_u_trace_syncobj *syncobj) { struct kgsl_device_waittimestamp_ctxtid req = { .context_id = syncobj->msm_queue_id, .timestamp = syncobj->timestamp, .timeout = 5000, // 5s }; int ret = safe_ioctl(dev->fd, IOCTL_KGSL_DEVICE_WAITTIMESTAMP_CTXTID, &req); if (ret) { assert(errno == ETIME); return VK_TIMEOUT; } return VK_SUCCESS; } static VkResult kgsl_device_init(struct tu_device *dev) { dev->fd = dev->physical_device->local_fd; return VK_SUCCESS; } static void kgsl_device_finish(struct tu_device *dev) { /* No-op */ } static int kgsl_device_get_gpu_timestamp(struct tu_device *dev, uint64_t *ts) { unreachable(""); return 0; } static int kgsl_device_get_suspend_count(struct tu_device *dev, uint64_t *suspend_count) { /* kgsl doesn't have a way to get it */ *suspend_count = 0; return 0; } static VkResult kgsl_device_check_status(struct tu_device *device) { for (unsigned i = 0; i < TU_MAX_QUEUE_FAMILIES; i++) { for (unsigned q = 0; q < device->queue_count[i]; q++) { /* KGSL's KGSL_PROP_GPU_RESET_STAT takes the u32 msm_queue_id and returns a * KGSL_CTX_STAT_* for the worst reset that happened since the last time it * was queried on that queue. */ uint32_t value = device->queues[i][q].msm_queue_id; VkResult status = get_kgsl_prop(device->fd, KGSL_PROP_GPU_RESET_STAT, &value, sizeof(value)); if (status != VK_SUCCESS) return vk_device_set_lost(&device->vk, "Failed to get GPU reset status"); if (value != KGSL_CTX_STAT_NO_ERROR && value != KGSL_CTX_STAT_INNOCENT_CONTEXT_RESET_EXT) { return vk_device_set_lost(&device->vk, "GPU faulted or hung"); } } } return VK_SUCCESS; } static const struct tu_knl kgsl_knl_funcs = { .name = "kgsl", .device_init = kgsl_device_init, .device_finish = kgsl_device_finish, .device_get_gpu_timestamp = kgsl_device_get_gpu_timestamp, .device_get_suspend_count = kgsl_device_get_suspend_count, .device_check_status = kgsl_device_check_status, .submitqueue_new = kgsl_submitqueue_new, .submitqueue_close = kgsl_submitqueue_close, .bo_init = kgsl_bo_init, .bo_init_dmabuf = kgsl_bo_init_dmabuf, .bo_export_dmabuf = kgsl_bo_export_dmabuf, .bo_map = kgsl_bo_map, .bo_allow_dump = kgsl_bo_allow_dump, .bo_finish = kgsl_bo_finish, .device_wait_u_trace = kgsl_device_wait_u_trace, .queue_submit = kgsl_queue_submit, }; VkResult tu_knl_kgsl_load(struct tu_instance *instance, int fd) { if (instance->vk.enabled_extensions.KHR_display) { return vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED, "I can't KHR_display"); } struct tu_physical_device *device = (struct tu_physical_device *) vk_zalloc(&instance->vk.alloc, sizeof(*device), 8, VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); if (!device) { close(fd); return vk_error(instance, VK_ERROR_OUT_OF_HOST_MEMORY); } static const char dma_heap_path[] = "/dev/dma_heap/system"; static const char ion_path[] = "/dev/ion"; int dma_fd; dma_fd = open(dma_heap_path, O_RDONLY); if (dma_fd >= 0) { device->kgsl_dma_type = TU_KGSL_DMA_TYPE_DMAHEAP; } else { dma_fd = open(ion_path, O_RDONLY); if (dma_fd >= 0) { /* ION_IOC_FREE available only for legacy ION */ struct ion_handle_data free = { .handle = 0 }; if (safe_ioctl(dma_fd, ION_IOC_FREE, &free) >= 0 || errno != ENOTTY) device->kgsl_dma_type = TU_KGSL_DMA_TYPE_ION_LEGACY; else device->kgsl_dma_type = TU_KGSL_DMA_TYPE_ION; } else { mesa_logw( "Unable to open neither %s nor %s, VK_KHR_external_memory_fd would be " "unavailable: %s", dma_heap_path, ion_path, strerror(errno)); } } VkResult result = VK_ERROR_INITIALIZATION_FAILED; struct kgsl_devinfo info; if (get_kgsl_prop(fd, KGSL_PROP_DEVICE_INFO, &info, sizeof(info))) goto fail; uint64_t gmem_iova; if (get_kgsl_prop(fd, KGSL_PROP_UCHE_GMEM_VADDR, &gmem_iova, sizeof(gmem_iova))) goto fail; /* kgsl version check? */ device->instance = instance; device->master_fd = -1; device->local_fd = fd; device->kgsl_dma_fd = dma_fd; device->dev_id.gpu_id = ((info.chip_id >> 24) & 0xff) * 100 + ((info.chip_id >> 16) & 0xff) * 10 + ((info.chip_id >> 8) & 0xff); device->dev_id.chip_id = info.chip_id; device->gmem_size = debug_get_num_option("TU_GMEM", info.gmem_sizebytes); device->gmem_base = gmem_iova; device->submitqueue_priority_count = 1; device->timeline_type = vk_sync_timeline_get_type(&vk_kgsl_sync_type); device->sync_types[0] = &vk_kgsl_sync_type; device->sync_types[1] = &device->timeline_type.sync; device->sync_types[2] = NULL; device->heap.size = tu_get_system_heap_size(device); device->heap.used = 0u; device->heap.flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; device->has_set_iova = kgsl_is_memory_type_supported( fd, KGSL_MEMFLAGS_USE_CPU_MAP); /* Even if kernel is new enough, the GPU itself may not support it. */ device->has_cached_coherent_memory = kgsl_is_memory_type_supported( fd, KGSL_MEMFLAGS_IOCOHERENT | (KGSL_CACHEMODE_WRITEBACK << KGSL_CACHEMODE_SHIFT)); instance->knl = &kgsl_knl_funcs; result = tu_physical_device_init(device, instance); if (result != VK_SUCCESS) goto fail; list_addtail(&device->vk.link, &instance->vk.physical_devices.list); return VK_SUCCESS; fail: vk_free(&instance->vk.alloc, device); close(fd); if (dma_fd >= 0) close(dma_fd); return result; }