#include #include "graphics-hook.h" #define VK_USE_PLATFORM_WIN32_KHR #include #include #include #include #define COBJMACROS #include #include #include "dxgi-helpers.hpp" #include "vulkan-capture.h" /* ======================================================================== */ /* defs/statics */ /* use the loader's dispatch table pointer as a key for internal data maps */ #define GET_LDT(x) (*(void **)x) static bool vulkan_seen = false; /* ======================================================================== */ /* hook data */ struct vk_obj_node { uint64_t obj; struct vk_obj_node *next; }; struct vk_obj_list { struct vk_obj_node *root; SRWLOCK mutex; }; struct vk_swap_data { struct vk_obj_node node; VkExtent2D image_extent; VkFormat format; HWND hwnd; VkImage export_image; bool layout_initialized; VkDeviceMemory export_mem; VkImage *swap_images; uint32_t image_count; HANDLE handle; struct shtex_data *shtex_info; ID3D11Texture2D *d3d11_tex; bool captured; }; struct vk_queue_data { struct vk_obj_node node; uint32_t fam_idx; bool supports_transfer; struct vk_frame_data *frames; uint32_t frame_index; uint32_t frame_count; }; struct vk_swap_view_data { struct vk_obj_node node; }; #define OBS_COLOR_ATTACHMENT_LIMIT 8 struct vk_framebuffer_data { struct vk_obj_node node; VkFramebuffer alternates[1 << OBS_COLOR_ATTACHMENT_LIMIT]; uint32_t color_attachment_mask; }; struct vk_frame_data { VkCommandPool cmd_pool; VkCommandBuffer cmd_buffer; VkFence fence; bool cmd_buffer_busy; }; struct vk_surf_data { struct vk_obj_node node; HWND hwnd; }; struct vk_inst_data { struct vk_obj_node node; VkInstance instance; bool valid; struct vk_inst_funcs funcs; struct vk_obj_list surfaces; }; struct vk_data { struct vk_obj_node node; VkDevice device; bool valid; struct vk_device_funcs funcs; VkPhysicalDevice phy_device; struct vk_obj_list swaps; struct vk_swap_data *cur_swap; struct vk_obj_list queues; struct vk_obj_list swap_views; struct vk_obj_list framebuffers; VkExternalMemoryProperties external_mem_props; struct vk_inst_data *inst_data; VkAllocationCallbacks ac_storage; const VkAllocationCallbacks *ac; ID3D11Device *d3d11_device; ID3D11DeviceContext *d3d11_context; }; __declspec(thread) int vk_presenting = 0; /* ------------------------------------------------------------------------- */ static void *vk_alloc(const VkAllocationCallbacks *ac, size_t size, size_t alignment, enum VkSystemAllocationScope scope) { return ac ? ac->pfnAllocation(ac->pUserData, size, alignment, scope) : _aligned_malloc(size, alignment); } static void vk_free(const VkAllocationCallbacks *ac, void *memory) { if (ac) ac->pfnFree(ac->pUserData, memory); else _aligned_free(memory); } static void add_obj_data(struct vk_obj_list *list, uint64_t obj, void *data) { AcquireSRWLockExclusive(&list->mutex); struct vk_obj_node *const node = data; node->obj = obj; node->next = list->root; list->root = node; ReleaseSRWLockExclusive(&list->mutex); } static struct vk_obj_node *get_obj_data(struct vk_obj_list *list, uint64_t obj) { struct vk_obj_node *data = NULL; AcquireSRWLockExclusive(&list->mutex); struct vk_obj_node *node = list->root; while (node) { if (node->obj == obj) { data = node; break; } node = node->next; } ReleaseSRWLockExclusive(&list->mutex); return data; } static struct vk_obj_node *remove_obj_data(struct vk_obj_list *list, uint64_t obj) { struct vk_obj_node *data = NULL; AcquireSRWLockExclusive(&list->mutex); struct vk_obj_node *prev = NULL; struct vk_obj_node *node = list->root; while (node) { if (node->obj == obj) { data = node; if (prev) prev->next = node->next; else list->root = node->next; break; } prev = node; node = node->next; } ReleaseSRWLockExclusive(&list->mutex); return data; } static void init_obj_list(struct vk_obj_list *list) { list->root = NULL; InitializeSRWLock(&list->mutex); } static struct vk_obj_node *obj_walk_begin(struct vk_obj_list *list) { AcquireSRWLockExclusive(&list->mutex); return list->root; } static struct vk_obj_node *obj_walk_next(struct vk_obj_node *node) { return node->next; } static void obj_walk_end(struct vk_obj_list *list) { ReleaseSRWLockExclusive(&list->mutex); } /* ------------------------------------------------------------------------- */ static struct vk_obj_list devices; static struct vk_data *alloc_device_data(const VkAllocationCallbacks *ac) { struct vk_data *data = vk_alloc(ac, sizeof(struct vk_data), _Alignof(struct vk_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); return data; } static void init_device_data(struct vk_data *data, VkDevice device) { add_obj_data(&devices, (uint64_t)GET_LDT(device), data); data->device = device; } static struct vk_data *get_device_data(VkDevice device) { return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(device)); } static struct vk_data *get_device_data_by_queue(VkQueue queue) { return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(queue)); } static struct vk_data * get_device_data_by_command_buffer(VkCommandBuffer commandBuffer) { return (struct vk_data *)get_obj_data(&devices, (uint64_t)GET_LDT(commandBuffer)); } static struct vk_data *remove_device_data(VkDevice device) { return (struct vk_data *)remove_obj_data(&devices, (uint64_t)GET_LDT(device)); } static void free_device_data(struct vk_data *data, const VkAllocationCallbacks *ac) { vk_free(ac, data); } /* ------------------------------------------------------------------------- */ static struct vk_queue_data *add_queue_data(struct vk_data *data, VkQueue queue, uint32_t fam_idx, bool supports_transfer, const VkAllocationCallbacks *ac) { struct vk_queue_data *const queue_data = vk_alloc(ac, sizeof(struct vk_queue_data), _Alignof(struct vk_queue_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); add_obj_data(&data->queues, (uint64_t)queue, queue_data); queue_data->fam_idx = fam_idx; queue_data->supports_transfer = supports_transfer; queue_data->frames = NULL; queue_data->frame_index = 0; queue_data->frame_count = 0; return queue_data; } static struct vk_queue_data *get_queue_data(struct vk_data *data, VkQueue queue) { return (struct vk_queue_data *)get_obj_data(&data->queues, (uint64_t)queue); } static void remove_free_queue_all(struct vk_data *data, const VkAllocationCallbacks *ac) { struct vk_queue_data *queue_data = (struct vk_queue_data *)data->queues.root; while (data->queues.root) { remove_obj_data(&data->queues, queue_data->node.obj); vk_free(ac, queue_data); queue_data = (struct vk_queue_data *)data->queues.root; } } static struct vk_queue_data *queue_walk_begin(struct vk_data *data) { return (struct vk_queue_data *)obj_walk_begin(&data->queues); } static struct vk_queue_data *queue_walk_next(struct vk_queue_data *queue_data) { return (struct vk_queue_data *)obj_walk_next( (struct vk_obj_node *)queue_data); } static void queue_walk_end(struct vk_data *data) { obj_walk_end(&data->queues); } /* ------------------------------------------------------------------------- */ static struct vk_swap_view_data * add_swap_view_data(struct vk_data *data, VkImageView imageView, const VkAllocationCallbacks *ac) { struct vk_swap_view_data *const swap_view_data = vk_alloc(ac, sizeof(struct vk_swap_view_data), _Alignof(struct vk_swap_view_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); add_obj_data(&data->swap_views, (uint64_t)imageView, swap_view_data); return swap_view_data; } static struct vk_swap_view_data *get_swap_view_data(struct vk_data *data, VkImageView imageView) { return (struct vk_swap_view_data *)get_obj_data(&data->swap_views, (uint64_t)imageView); } static void remove_free_swap_view_data(struct vk_data *data, VkImageView imageView, const VkAllocationCallbacks *ac) { struct vk_swap_data *const swap_view_data = (struct vk_swap_data *)remove_obj_data(&data->swap_views, (uint64_t)imageView); vk_free(ac, swap_view_data); } /* ------------------------------------------------------------------------- */ static struct vk_framebuffer_data * add_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer, const VkAllocationCallbacks *ac) { struct vk_framebuffer_data *const framebuffer_data = vk_alloc(ac, sizeof(struct vk_framebuffer_data), _Alignof(struct vk_framebuffer_data), VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); add_obj_data(&data->framebuffers, (uint64_t)framebuffer, framebuffer_data); for (size_t i = 0; i < _countof(framebuffer_data->alternates); ++i) { framebuffer_data->alternates[i] = VK_NULL_HANDLE; } framebuffer_data->color_attachment_mask = 0; return framebuffer_data; } static struct vk_framebuffer_data * get_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer) { return (struct vk_framebuffer_data *)get_obj_data( &data->framebuffers, (uint64_t)framebuffer); } static void remove_free_framebuffer_data(struct vk_data *data, VkFramebuffer framebuffer, const VkAllocationCallbacks *ac) { struct vk_swap_data *const framebuffer_data = (struct vk_swap_data *)remove_obj_data(&data->framebuffers, (uint64_t)framebuffer); vk_free(ac, framebuffer_data); } /* ------------------------------------------------------------------------- */ static struct vk_swap_data *alloc_swap_data(const VkAllocationCallbacks *ac) { struct vk_swap_data *const swap_data = vk_alloc( ac, sizeof(struct vk_swap_data), _Alignof(struct vk_swap_data), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); return swap_data; } static void init_swap_data(struct vk_swap_data *swap_data, struct vk_data *data, VkSwapchainKHR sc) { add_obj_data(&data->swaps, (uint64_t)sc, swap_data); } static struct vk_swap_data *get_swap_data(struct vk_data *data, VkSwapchainKHR sc) { return (struct vk_swap_data *)get_obj_data(&data->swaps, (uint64_t)sc); } static void remove_free_swap_data(struct vk_data *data, VkSwapchainKHR sc, const VkAllocationCallbacks *ac) { struct vk_swap_data *const swap_data = (struct vk_swap_data *)remove_obj_data(&data->swaps, (uint64_t)sc); vk_free(ac, swap_data); } static struct vk_swap_data *swap_walk_begin(struct vk_data *data) { return (struct vk_swap_data *)obj_walk_begin(&data->swaps); } static struct vk_swap_data *swap_walk_next(struct vk_swap_data *swap_data) { return (struct vk_swap_data *)obj_walk_next( (struct vk_obj_node *)swap_data); } static void swap_walk_end(struct vk_data *data) { obj_walk_end(&data->swaps); } /* ------------------------------------------------------------------------- */ static void vk_shtex_clear_fence(const struct vk_data *data, struct vk_frame_data *frame_data) { const VkFence fence = frame_data->fence; if (frame_data->cmd_buffer_busy) { VkDevice device = data->device; const struct vk_device_funcs *funcs = &data->funcs; funcs->WaitForFences(device, 1, &fence, VK_TRUE, ~0ull); funcs->ResetFences(device, 1, &fence); frame_data->cmd_buffer_busy = false; } } static void vk_shtex_wait_until_pool_idle(struct vk_data *data, struct vk_queue_data *queue_data) { for (uint32_t frame_idx = 0; frame_idx < queue_data->frame_count; frame_idx++) { struct vk_frame_data *frame_data = &queue_data->frames[frame_idx]; if (frame_data->cmd_pool != VK_NULL_HANDLE) vk_shtex_clear_fence(data, frame_data); } } static void vk_shtex_wait_until_idle(struct vk_data *data) { struct vk_queue_data *queue_data = queue_walk_begin(data); while (queue_data) { vk_shtex_wait_until_pool_idle(data, queue_data); queue_data = queue_walk_next(queue_data); } queue_walk_end(data); } static void vk_shtex_free(struct vk_data *data) { capture_free(); vk_shtex_wait_until_idle(data); struct vk_swap_data *swap = swap_walk_begin(data); while (swap) { VkDevice device = data->device; if (swap->export_image) data->funcs.DestroyImage(device, swap->export_image, data->ac); if (swap->export_mem) data->funcs.FreeMemory(device, swap->export_mem, NULL); if (swap->d3d11_tex) { ID3D11Texture2D_Release(swap->d3d11_tex); } swap->handle = INVALID_HANDLE_VALUE; swap->d3d11_tex = NULL; swap->export_mem = VK_NULL_HANDLE; swap->export_image = VK_NULL_HANDLE; swap->captured = false; swap = swap_walk_next(swap); } swap_walk_end(data); if (data->d3d11_context) { ID3D11DeviceContext_Release(data->d3d11_context); data->d3d11_context = NULL; } if (data->d3d11_device) { ID3D11Device_Release(data->d3d11_device); data->d3d11_device = NULL; } data->cur_swap = NULL; hlog("------------------ vulkan capture freed ------------------"); } /* ------------------------------------------------------------------------- */ static void add_surf_data(struct vk_inst_data *idata, VkSurfaceKHR surf, HWND hwnd, const VkAllocationCallbacks *ac) { struct vk_surf_data *surf_data = vk_alloc( ac, sizeof(struct vk_surf_data), _Alignof(struct vk_surf_data), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (surf_data) { surf_data->hwnd = hwnd; add_obj_data(&idata->surfaces, (uint64_t)surf, surf_data); } } static HWND find_surf_hwnd(struct vk_inst_data *idata, VkSurfaceKHR surf) { struct vk_surf_data *surf_data = (struct vk_surf_data *)get_obj_data( &idata->surfaces, (uint64_t)surf); return surf_data->hwnd; } static void remove_free_surf_data(struct vk_inst_data *idata, VkSurfaceKHR surf, const VkAllocationCallbacks *ac) { struct vk_surf_data *surf_data = (struct vk_surf_data *)remove_obj_data( &idata->surfaces, (uint64_t)surf); vk_free(ac, surf_data); } /* ------------------------------------------------------------------------- */ static struct vk_obj_list instances; static struct vk_inst_data *alloc_inst_data(const VkAllocationCallbacks *ac) { struct vk_inst_data *idata = vk_alloc( ac, sizeof(struct vk_inst_data), _Alignof(struct vk_inst_data), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); return idata; } static void init_inst_data(struct vk_inst_data *idata, VkInstance instance) { add_obj_data(&instances, (uint64_t)GET_LDT(instance), idata); idata->instance = instance; } static struct vk_inst_data *get_inst_data(VkInstance instance) { return (struct vk_inst_data *)get_obj_data(&instances, (uint64_t)GET_LDT(instance)); } static struct vk_inst_funcs *get_inst_funcs(VkInstance instance) { struct vk_inst_data *idata = (struct vk_inst_data *)get_inst_data(instance); return &idata->funcs; } static struct vk_inst_data * get_inst_data_by_physical_device(VkPhysicalDevice physicalDevice) { return (struct vk_inst_data *)get_obj_data( &instances, (uint64_t)GET_LDT(physicalDevice)); } static struct vk_inst_funcs * get_inst_funcs_by_physical_device(VkPhysicalDevice physicalDevice) { struct vk_inst_data *idata = (struct vk_inst_data *)get_inst_data_by_physical_device( physicalDevice); return &idata->funcs; } static void remove_free_inst_data(VkInstance inst, const VkAllocationCallbacks *ac) { struct vk_inst_data *idata = (struct vk_inst_data *)remove_obj_data( &instances, (uint64_t)GET_LDT(inst)); vk_free(ac, idata); } /* ======================================================================== */ /* capture */ static inline bool vk_shtex_init_d3d11(struct vk_data *data) { D3D_FEATURE_LEVEL level_used; IDXGIFactory1 *factory; IDXGIAdapter1 *adapter; HRESULT hr; HMODULE d3d11 = load_system_library("d3d11.dll"); if (!d3d11) { flog("failed to load d3d11: %d", GetLastError()); return false; } HMODULE dxgi = load_system_library("dxgi.dll"); if (!dxgi) { flog("failed to load dxgi: %d", GetLastError()); return false; } HRESULT(WINAPI * create_factory) (REFIID, void **) = (void *)GetProcAddress(dxgi, "CreateDXGIFactory1"); if (!create_factory) { flog("failed to get CreateDXGIFactory1 address: %d", GetLastError()); return false; } PFN_D3D11_CREATE_DEVICE create = (void *)GetProcAddress(d3d11, "D3D11CreateDevice"); if (!create) { flog("failed to get D3D11CreateDevice address: %d", GetLastError()); return false; } hr = create_factory(&IID_IDXGIFactory1, &factory); if (FAILED(hr)) { flog_hr("failed to create factory", hr); return false; } hr = IDXGIFactory1_EnumAdapters1(factory, 0, &adapter); IDXGIFactory1_Release(factory); if (FAILED(hr)) { flog_hr("failed to create adapter", hr); return false; } static const D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, D3D_FEATURE_LEVEL_10_0, D3D_FEATURE_LEVEL_9_3, }; hr = create((IDXGIAdapter *)adapter, D3D_DRIVER_TYPE_UNKNOWN, NULL, 0, feature_levels, sizeof(feature_levels) / sizeof(D3D_FEATURE_LEVEL), D3D11_SDK_VERSION, &data->d3d11_device, &level_used, &data->d3d11_context); IDXGIAdapter1_Release(adapter); if (FAILED(hr)) { flog_hr("failed to create device", hr); return false; } return true; } static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data, struct vk_swap_data *swap) { IDXGIResource *dxgi_res; HRESULT hr; const UINT width = swap->image_extent.width; const UINT height = swap->image_extent.height; flog("OBS requesting %s texture format. capture dimensions: %ux%u", vk_format_to_str(swap->format), width, height); const DXGI_FORMAT format = vk_format_to_dxgi(swap->format); if (format == DXGI_FORMAT_UNKNOWN) { flog("cannot convert to DXGI format"); return false; } D3D11_TEXTURE2D_DESC desc = {0}; desc.Width = width; desc.Height = height; desc.MipLevels = 1; desc.ArraySize = 1; desc.Format = apply_dxgi_format_typeless( format, global_hook_info->allow_srgb_alias); desc.SampleDesc.Count = 1; desc.SampleDesc.Quality = 0; desc.Usage = D3D11_USAGE_DEFAULT; desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; hr = ID3D11Device_CreateTexture2D(data->d3d11_device, &desc, NULL, &swap->d3d11_tex); if (FAILED(hr)) { flog_hr("failed to create texture", hr); return false; } hr = ID3D11Texture2D_QueryInterface(swap->d3d11_tex, &IID_IDXGIResource, &dxgi_res); if (FAILED(hr)) { flog_hr("failed to get IDXGIResource", hr); return false; } hr = IDXGIResource_GetSharedHandle(dxgi_res, &swap->handle); IDXGIResource_Release(dxgi_res); if (FAILED(hr)) { flog_hr("failed to get shared handle", hr); return false; } return true; } static inline bool vk_shtex_init_vulkan_tex(struct vk_data *data, struct vk_swap_data *swap) { struct vk_device_funcs *funcs = &data->funcs; VkExternalMemoryFeatureFlags f = data->external_mem_props.externalMemoryFeatures; /* -------------------------------------------------------- */ /* create texture */ VkExternalMemoryImageCreateInfo emici; emici.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO; emici.pNext = NULL; emici.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT; VkImageCreateInfo ici; ici.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; ici.pNext = &emici; ici.flags = 0; ici.imageType = VK_IMAGE_TYPE_2D; ici.format = swap->format; ici.extent.width = swap->image_extent.width; ici.extent.height = swap->image_extent.height; ici.extent.depth = 1; ici.mipLevels = 1; ici.arrayLayers = 1; ici.samples = VK_SAMPLE_COUNT_1_BIT; ici.tiling = VK_IMAGE_TILING_OPTIMAL; ici.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; ici.sharingMode = VK_SHARING_MODE_EXCLUSIVE; ici.queueFamilyIndexCount = 0; ici.pQueueFamilyIndices = 0; ici.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; VkDevice device = data->device; VkResult res; res = funcs->CreateImage(device, &ici, data->ac, &swap->export_image); if (VK_SUCCESS != res) { flog("failed to CreateImage: %s", result_to_str(res)); swap->export_image = VK_NULL_HANDLE; return false; } swap->layout_initialized = false; /* -------------------------------------------------------- */ /* get image memory requirements */ VkMemoryRequirements mr; bool use_gimr2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT; if (use_gimr2) { VkMemoryDedicatedRequirements mdr = {0}; mdr.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS; mdr.pNext = NULL; VkMemoryRequirements2 mr2 = {0}; mr2.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2; mr2.pNext = &mdr; VkImageMemoryRequirementsInfo2 imri2 = {0}; imri2.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2; imri2.pNext = NULL; imri2.image = swap->export_image; funcs->GetImageMemoryRequirements2(device, &imri2, &mr2); mr = mr2.memoryRequirements; } else { funcs->GetImageMemoryRequirements(device, swap->export_image, &mr); } /* -------------------------------------------------------- */ /* get memory type index */ struct vk_inst_funcs *ifuncs = get_inst_funcs_by_physical_device(data->phy_device); VkPhysicalDeviceMemoryProperties pdmp; ifuncs->GetPhysicalDeviceMemoryProperties(data->phy_device, &pdmp); /* -------------------------------------------------------- */ /* allocate memory */ VkImportMemoryWin32HandleInfoKHR imw32hi; imw32hi.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_WIN32_HANDLE_INFO_KHR; imw32hi.pNext = NULL; imw32hi.name = NULL; imw32hi.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT; imw32hi.handle = swap->handle; VkMemoryAllocateInfo mai; mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; mai.pNext = &imw32hi; mai.allocationSize = mr.size; VkMemoryDedicatedAllocateInfo mdai; mdai.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; mdai.pNext = NULL; mdai.buffer = VK_NULL_HANDLE; if (data->external_mem_props.externalMemoryFeatures & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT) { mdai.image = swap->export_image; imw32hi.pNext = &mdai; } bool allocated = false; for (uint32_t i = 0; i < pdmp.memoryTypeCount; ++i) { if ((mr.memoryTypeBits & (1 << i)) && (pdmp.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) == VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) { mai.memoryTypeIndex = i; res = funcs->AllocateMemory(device, &mai, NULL, &swap->export_mem); allocated = res == VK_SUCCESS; if (allocated) break; flog("failed to AllocateMemory (DEVICE_LOCAL): %s (%d)", result_to_str(res), (int)res); } } if (!allocated) { /* Try again without DEVICE_LOCAL */ for (uint32_t i = 0; i < pdmp.memoryTypeCount; ++i) { if ((mr.memoryTypeBits & (1 << i)) && (pdmp.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) { mai.memoryTypeIndex = i; res = funcs->AllocateMemory(device, &mai, NULL, &swap->export_mem); allocated = res == VK_SUCCESS; if (allocated) break; flog("failed to AllocateMemory (not DEVICE_LOCAL): %s (%d)", result_to_str(res), (int)res); } } } if (!allocated) { flog("failed to allocate memory of any type"); funcs->DestroyImage(device, swap->export_image, data->ac); swap->export_image = VK_NULL_HANDLE; return false; } /* -------------------------------------------------------- */ /* bind image memory */ bool use_bi2 = f & VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT; if (use_bi2) { VkBindImageMemoryInfo bimi = {0}; bimi.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; bimi.image = swap->export_image; bimi.memory = swap->export_mem; bimi.memoryOffset = 0; res = funcs->BindImageMemory2(device, 1, &bimi); } else { res = funcs->BindImageMemory(device, swap->export_image, swap->export_mem, 0); } if (VK_SUCCESS != res) { flog("%s failed: %s", use_bi2 ? "BindImageMemory2" : "BindImageMemory", result_to_str(res)); funcs->DestroyImage(device, swap->export_image, data->ac); swap->export_image = VK_NULL_HANDLE; return false; } return true; } static bool vk_shtex_init(struct vk_data *data, HWND window, struct vk_swap_data *swap) { if (!vk_shtex_init_d3d11(data)) { return false; } if (!vk_shtex_init_d3d11_tex(data, swap)) { return false; } if (!vk_shtex_init_vulkan_tex(data, swap)) { return false; } data->cur_swap = swap; swap->captured = capture_init_shtex(&swap->shtex_info, window, swap->image_extent.width, swap->image_extent.height, (uint32_t)swap->format, false, (uintptr_t)swap->handle); if (!swap->captured) return false; if (global_hook_info->force_shmem) { flog("shared memory capture currently " "unsupported; ignoring"); } hlog("vulkan shared texture capture successful"); return true; } static void vk_shtex_create_frame_objects(struct vk_data *data, struct vk_queue_data *queue_data, uint32_t image_count) { queue_data->frames = vk_alloc(data->ac, image_count * sizeof(struct vk_frame_data), _Alignof(struct vk_frame_data), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); memset(queue_data->frames, 0, image_count * sizeof(struct vk_frame_data)); queue_data->frame_index = 0; queue_data->frame_count = image_count; VkDevice device = data->device; for (uint32_t image_index = 0; image_index < image_count; image_index++) { struct vk_frame_data *frame_data = &queue_data->frames[image_index]; VkCommandPoolCreateInfo cpci; cpci.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; cpci.pNext = NULL; cpci.flags = 0; cpci.queueFamilyIndex = queue_data->fam_idx; VkResult res = data->funcs.CreateCommandPool( device, &cpci, data->ac, &frame_data->cmd_pool); debug_res("CreateCommandPool", res); VkCommandBufferAllocateInfo cbai; cbai.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; cbai.pNext = NULL; cbai.commandPool = frame_data->cmd_pool; cbai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; cbai.commandBufferCount = 1; res = data->funcs.AllocateCommandBuffers( device, &cbai, &frame_data->cmd_buffer); debug_res("AllocateCommandBuffers", res); GET_LDT(frame_data->cmd_buffer) = GET_LDT(device); VkFenceCreateInfo fci = {0}; fci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; fci.pNext = NULL; fci.flags = 0; res = data->funcs.CreateFence(device, &fci, data->ac, &frame_data->fence); debug_res("CreateFence", res); } } static void vk_shtex_destroy_fence(struct vk_data *data, bool *cmd_buffer_busy, VkFence *fence) { VkDevice device = data->device; if (*cmd_buffer_busy) { data->funcs.WaitForFences(device, 1, fence, VK_TRUE, ~0ull); *cmd_buffer_busy = false; } data->funcs.DestroyFence(device, *fence, data->ac); *fence = VK_NULL_HANDLE; } static void vk_shtex_destroy_frame_objects(struct vk_data *data, struct vk_queue_data *queue_data) { VkDevice device = data->device; for (uint32_t frame_idx = 0; frame_idx < queue_data->frame_count; frame_idx++) { struct vk_frame_data *frame_data = &queue_data->frames[frame_idx]; bool *cmd_buffer_busy = &frame_data->cmd_buffer_busy; VkFence *fence = &frame_data->fence; vk_shtex_destroy_fence(data, cmd_buffer_busy, fence); data->funcs.DestroyCommandPool(device, frame_data->cmd_pool, data->ac); frame_data->cmd_pool = VK_NULL_HANDLE; } vk_free(data->ac, queue_data->frames); queue_data->frames = NULL; queue_data->frame_count = 0; } static void vk_shtex_capture(struct vk_data *data, struct vk_device_funcs *funcs, struct vk_swap_data *swap, uint32_t idx, VkQueue queue, const VkPresentInfoKHR *info) { VkResult res = VK_SUCCESS; VkCommandBufferBeginInfo begin_info; begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; begin_info.pNext = NULL; begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; begin_info.pInheritanceInfo = NULL; VkImageMemoryBarrier mb[2]; VkImageMemoryBarrier *src_mb = &mb[0]; VkImageMemoryBarrier *dst_mb = &mb[1]; /* ------------------------------------------------------ */ /* do image copy */ const uint32_t image_index = info->pImageIndices[idx]; VkImage cur_backbuffer = swap->swap_images[image_index]; struct vk_queue_data *queue_data = get_queue_data(data, queue); uint32_t fam_idx = queue_data->fam_idx; const uint32_t image_count = swap->image_count; if (queue_data->frame_count < image_count) { if (queue_data->frame_count > 0) vk_shtex_destroy_frame_objects(data, queue_data); vk_shtex_create_frame_objects(data, queue_data, image_count); } const uint32_t frame_index = queue_data->frame_index; struct vk_frame_data *frame_data = &queue_data->frames[frame_index]; queue_data->frame_index = (frame_index + 1) % queue_data->frame_count; vk_shtex_clear_fence(data, frame_data); VkDevice device = data->device; res = funcs->ResetCommandPool(device, frame_data->cmd_pool, 0); #ifdef MORE_DEBUGGING debug_res("ResetCommandPool", res); #endif const VkCommandBuffer cmd_buffer = frame_data->cmd_buffer; res = funcs->BeginCommandBuffer(cmd_buffer, &begin_info); #ifdef MORE_DEBUGGING debug_res("BeginCommandBuffer", res); #endif /* ------------------------------------------------------ */ /* transition shared texture if necessary */ if (!swap->layout_initialized) { VkImageMemoryBarrier imb; imb.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; imb.pNext = NULL; imb.srcAccessMask = 0; imb.dstAccessMask = 0; imb.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; imb.newLayout = VK_IMAGE_LAYOUT_GENERAL; imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; imb.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; imb.image = swap->export_image; imb.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imb.subresourceRange.baseMipLevel = 0; imb.subresourceRange.levelCount = 1; imb.subresourceRange.baseArrayLayer = 0; imb.subresourceRange.layerCount = 1; funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 0, NULL, 0, NULL, 1, &imb); swap->layout_initialized = true; } /* ------------------------------------------------------ */ /* transition cur_backbuffer to transfer source state */ src_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; src_mb->pNext = NULL; src_mb->srcAccessMask = VK_ACCESS_MEMORY_READ_BIT; src_mb->dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; src_mb->oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; src_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; src_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; src_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; src_mb->image = cur_backbuffer; src_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; src_mb->subresourceRange.baseMipLevel = 0; src_mb->subresourceRange.levelCount = 1; src_mb->subresourceRange.baseArrayLayer = 0; src_mb->subresourceRange.layerCount = 1; /* ------------------------------------------------------ */ /* transition exportedTexture to transfer dest state */ dst_mb->sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; dst_mb->pNext = NULL; dst_mb->srcAccessMask = 0; dst_mb->dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; dst_mb->oldLayout = VK_IMAGE_LAYOUT_GENERAL; dst_mb->newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; dst_mb->srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; dst_mb->dstQueueFamilyIndex = fam_idx; dst_mb->image = swap->export_image; dst_mb->subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; dst_mb->subresourceRange.baseMipLevel = 0; dst_mb->subresourceRange.levelCount = 1; dst_mb->subresourceRange.baseArrayLayer = 0; dst_mb->subresourceRange.layerCount = 1; funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 0, NULL, 2, mb); /* ------------------------------------------------------ */ /* copy cur_backbuffer's content to our interop image */ VkImageCopy cpy; cpy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; cpy.srcSubresource.mipLevel = 0; cpy.srcSubresource.baseArrayLayer = 0; cpy.srcSubresource.layerCount = 1; cpy.srcOffset.x = 0; cpy.srcOffset.y = 0; cpy.srcOffset.z = 0; cpy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; cpy.dstSubresource.mipLevel = 0; cpy.dstSubresource.baseArrayLayer = 0; cpy.dstSubresource.layerCount = 1; cpy.dstOffset.x = 0; cpy.dstOffset.y = 0; cpy.dstOffset.z = 0; cpy.extent.width = swap->image_extent.width; cpy.extent.height = swap->image_extent.height; cpy.extent.depth = 1; funcs->CmdCopyImage(cmd_buffer, cur_backbuffer, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, swap->export_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &cpy); /* ------------------------------------------------------ */ /* Restore the swap chain image layout to what it was * before. This may not be strictly needed, but it is * generally good to restore things to their original * state. */ src_mb->srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; src_mb->dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; src_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; src_mb->newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; dst_mb->srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; dst_mb->dstAccessMask = 0; dst_mb->oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; dst_mb->newLayout = VK_IMAGE_LAYOUT_GENERAL; dst_mb->srcQueueFamilyIndex = fam_idx; dst_mb->dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; funcs->CmdPipelineBarrier(cmd_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT | VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, NULL, 0, NULL, 2, mb); funcs->EndCommandBuffer(cmd_buffer); /* ------------------------------------------------------ */ VkSubmitInfo submit_info; submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submit_info.pNext = NULL; submit_info.waitSemaphoreCount = 0; submit_info.pWaitSemaphores = NULL; submit_info.pWaitDstStageMask = NULL; submit_info.commandBufferCount = 1; submit_info.pCommandBuffers = &cmd_buffer; submit_info.signalSemaphoreCount = 0; submit_info.pSignalSemaphores = NULL; const VkFence fence = frame_data->fence; res = funcs->QueueSubmit(queue, 1, &submit_info, fence); #ifdef MORE_DEBUGGING debug_res("QueueSubmit", res); #endif if (res == VK_SUCCESS) frame_data->cmd_buffer_busy = true; } static inline bool valid_rect(struct vk_swap_data *swap) { return !!swap->image_extent.width && !!swap->image_extent.height; } static void vk_capture(struct vk_data *data, VkQueue queue, const VkPresentInfoKHR *info) { struct vk_swap_data *swap = NULL; HWND window = NULL; uint32_t idx = 0; #ifdef MORE_DEBUGGING debug("QueuePresentKHR called on " "devicekey %p, swapchain count %d", &data->funcs, info->swapchainCount); #endif /* use first swap chain associated with a window */ for (; idx < info->swapchainCount; idx++) { struct vk_swap_data *cur_swap = get_swap_data(data, info->pSwapchains[idx]); if (cur_swap) { window = cur_swap->hwnd; if (window != NULL) { swap = cur_swap; break; } } } if (!window) { return; } if (capture_should_stop()) { vk_shtex_free(data); } if (capture_should_init()) { if (valid_rect(swap) && !vk_shtex_init(data, window, swap)) { vk_shtex_free(data); data->valid = false; flog("vk_shtex_init failed"); } } if (capture_ready()) { if (swap != data->cur_swap) { vk_shtex_free(data); return; } vk_shtex_capture(data, &data->funcs, swap, idx, queue, info); } } static VkResult VKAPI_CALL OBS_QueuePresentKHR(VkQueue queue, const VkPresentInfoKHR *info) { struct vk_data *const data = get_device_data_by_queue(queue); struct vk_queue_data *const queue_data = get_queue_data(data, queue); struct vk_device_funcs *const funcs = &data->funcs; if (data->valid && queue_data->supports_transfer) { vk_capture(data, queue, info); } if (vk_presenting != 0) { flog("non-zero vk_presenting: %d", vk_presenting); } vk_presenting++; VkResult res = funcs->QueuePresentKHR(queue, info); vk_presenting--; return res; } /* ======================================================================== */ /* setup hooks */ static inline bool is_inst_link_info(VkLayerInstanceCreateInfo *lici) { return lici->sType == VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO && lici->function == VK_LAYER_LINK_INFO; } static VkResult VKAPI_CALL OBS_CreateInstance(const VkInstanceCreateInfo *cinfo, const VkAllocationCallbacks *ac, VkInstance *p_inst) { VkInstanceCreateInfo info = *cinfo; /* -------------------------------------------------------- */ /* step through chain until we get to the link info */ VkLayerInstanceCreateInfo *lici = (void *)info.pNext; while (lici && !is_inst_link_info(lici)) { lici = (VkLayerInstanceCreateInfo *)lici->pNext; } if (lici == NULL) { return VK_ERROR_INITIALIZATION_FAILED; } PFN_vkGetInstanceProcAddr gpa = lici->u.pLayerInfo->pfnNextGetInstanceProcAddr; /* -------------------------------------------------------- */ /* move chain on for next layer */ lici->u.pLayerInfo = lici->u.pLayerInfo->pNext; /* -------------------------------------------------------- */ /* (HACK) Set api version to 1.1 if set to 1.0 */ /* We do this to get our extensions working properly */ VkApplicationInfo ai; if (info.pApplicationInfo) { ai = *info.pApplicationInfo; if (ai.apiVersion < VK_API_VERSION_1_1) ai.apiVersion = VK_API_VERSION_1_1; } else { ai.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; ai.pNext = NULL; ai.pApplicationName = NULL; ai.applicationVersion = 0; ai.pEngineName = NULL; ai.engineVersion = 0; ai.apiVersion = VK_API_VERSION_1_1; } info.pApplicationInfo = &ai; /* -------------------------------------------------------- */ /* allocate data node */ struct vk_inst_data *idata = alloc_inst_data(ac); if (!idata) return VK_ERROR_OUT_OF_HOST_MEMORY; /* -------------------------------------------------------- */ /* create instance */ PFN_vkCreateInstance create = (void *)gpa(NULL, "vkCreateInstance"); VkResult res = create(&info, ac, p_inst); bool valid = res == VK_SUCCESS; if (!valid) { /* try again with original arguments */ res = create(cinfo, ac, p_inst); if (res != VK_SUCCESS) { vk_free(ac, idata); return res; } } VkInstance inst = *p_inst; init_inst_data(idata, inst); /* -------------------------------------------------------- */ /* fetch the functions we need */ struct vk_inst_funcs *ifuncs = &idata->funcs; #define GETADDR(x) \ do { \ ifuncs->x = (void *)gpa(inst, "vk" #x); \ if (!ifuncs->x) { \ flog("could not get instance " \ "address for vk" #x); \ funcs_found = false; \ } \ } while (false) bool funcs_found = true; GETADDR(GetInstanceProcAddr); GETADDR(DestroyInstance); GETADDR(CreateWin32SurfaceKHR); GETADDR(DestroySurfaceKHR); GETADDR(GetPhysicalDeviceQueueFamilyProperties); GETADDR(GetPhysicalDeviceMemoryProperties); GETADDR(GetPhysicalDeviceImageFormatProperties2); GETADDR(EnumerateDeviceExtensionProperties); #undef GETADDR valid = valid && funcs_found; idata->valid = valid; if (valid) init_obj_list(&idata->surfaces); return res; } static void VKAPI_CALL OBS_DestroyInstance(VkInstance instance, const VkAllocationCallbacks *ac) { struct vk_inst_funcs *ifuncs = get_inst_funcs(instance); PFN_vkDestroyInstance destroy_instance = ifuncs->DestroyInstance; remove_free_inst_data(instance, ac); destroy_instance(instance, ac); } static bool vk_shared_tex_supported(struct vk_inst_funcs *funcs, VkPhysicalDevice phy_device, VkFormat format, VkImageUsageFlags usage, VkExternalMemoryProperties *external_mem_props) { VkPhysicalDeviceImageFormatInfo2 info; VkPhysicalDeviceExternalImageFormatInfo external_info; external_info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO; external_info.pNext = NULL; external_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_TEXTURE_KMT_BIT; info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2; info.pNext = &external_info; info.format = format; info.type = VK_IMAGE_TYPE_2D; info.tiling = VK_IMAGE_TILING_OPTIMAL; info.flags = 0; info.usage = usage; VkExternalImageFormatProperties external_props = {0}; external_props.sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES; external_props.pNext = NULL; VkImageFormatProperties2 props = {0}; props.sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2; props.pNext = &external_props; VkResult result = funcs->GetPhysicalDeviceImageFormatProperties2( phy_device, &info, &props); *external_mem_props = external_props.externalMemoryProperties; const VkExternalMemoryFeatureFlags features = external_mem_props->externalMemoryFeatures; return ((VK_SUCCESS == result) && (features & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT)); } static inline bool is_device_link_info(VkLayerDeviceCreateInfo *lici) { return lici->sType == VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO && lici->function == VK_LAYER_LINK_INFO; } static VkResult VKAPI_CALL OBS_CreateDevice(VkPhysicalDevice phy_device, const VkDeviceCreateInfo *info, const VkAllocationCallbacks *ac, VkDevice *p_device) { struct vk_inst_data *idata = get_inst_data_by_physical_device(phy_device); struct vk_inst_funcs *ifuncs = &idata->funcs; struct vk_data *data = NULL; VkResult ret = VK_ERROR_INITIALIZATION_FAILED; VkLayerDeviceCreateInfo *ldci = (void *)info->pNext; /* -------------------------------------------------------- */ /* step through chain until we get to the link info */ while (ldci && !is_device_link_info(ldci)) { ldci = (VkLayerDeviceCreateInfo *)ldci->pNext; } if (!ldci) { goto fail; } PFN_vkGetInstanceProcAddr gipa; PFN_vkGetDeviceProcAddr gdpa; gipa = ldci->u.pLayerInfo->pfnNextGetInstanceProcAddr; gdpa = ldci->u.pLayerInfo->pfnNextGetDeviceProcAddr; /* -------------------------------------------------------- */ /* move chain on for next layer */ ldci->u.pLayerInfo = ldci->u.pLayerInfo->pNext; /* -------------------------------------------------------- */ /* allocate data node */ data = alloc_device_data(ac); if (!data) return VK_ERROR_OUT_OF_HOST_MEMORY; init_obj_list(&data->queues); init_obj_list(&data->swap_views); init_obj_list(&data->framebuffers); /* -------------------------------------------------------- */ /* create device and initialize hook data */ PFN_vkCreateDevice createFunc = (PFN_vkCreateDevice)gipa(idata->instance, "vkCreateDevice"); ret = createFunc(phy_device, info, ac, p_device); if (ret != VK_SUCCESS) { vk_free(ac, data); return ret; } VkDevice device = *p_device; init_device_data(data, device); data->valid = false; /* set true below if it doesn't go to fail */ data->phy_device = phy_device; /* -------------------------------------------------------- */ /* fetch the functions we need */ struct vk_device_funcs *dfuncs = &data->funcs; bool funcs_found = true; #define GETADDR(x) \ do { \ dfuncs->x = (void *)gdpa(device, "vk" #x); \ if (!dfuncs->x) { \ flog("could not get device " \ "address for vk" #x); \ funcs_found = false; \ } \ } while (false) #define GETADDR_OPTIONAL(x) \ do { \ dfuncs->x = (void *)gdpa(device, "vk" #x); \ } while (false) GETADDR(GetDeviceProcAddr); GETADDR(DestroyDevice); GETADDR(CreateSwapchainKHR); GETADDR(DestroySwapchainKHR); GETADDR(QueuePresentKHR); GETADDR(AllocateMemory); GETADDR(FreeMemory); GETADDR(BindImageMemory); GETADDR(BindImageMemory2); GETADDR(GetSwapchainImagesKHR); GETADDR(CreateImage); GETADDR(DestroyImage); GETADDR(GetImageMemoryRequirements); GETADDR(GetImageMemoryRequirements2); GETADDR(ResetCommandPool); GETADDR(BeginCommandBuffer); GETADDR(EndCommandBuffer); GETADDR(CmdCopyImage); GETADDR(CmdPipelineBarrier); GETADDR(GetDeviceQueue); GETADDR(QueueSubmit); GETADDR(CreateCommandPool); GETADDR(DestroyCommandPool); GETADDR(AllocateCommandBuffers); GETADDR(CreateFence); GETADDR(DestroyFence); GETADDR(WaitForFences); GETADDR(ResetFences); GETADDR(CreateImageView); GETADDR(DestroyImageView); GETADDR(CreateFramebuffer); GETADDR(DestroyFramebuffer); GETADDR(CmdBeginRenderPass); GETADDR_OPTIONAL(CmdBeginRenderPass2KHR); GETADDR_OPTIONAL(CmdBeginRenderPass2); #undef GETADDR_OPTIONAL #undef GETADDR if (!funcs_found) { goto fail; } if (!idata->valid) { flog("instance not valid"); goto fail; } const char *required_device_extensions[] = { VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME}; uint32_t device_extension_count = 0; ret = ifuncs->EnumerateDeviceExtensionProperties( phy_device, NULL, &device_extension_count, NULL); if (ret != VK_SUCCESS) goto fail; VkExtensionProperties *device_extensions = _malloca( sizeof(VkExtensionProperties) * device_extension_count); ret = ifuncs->EnumerateDeviceExtensionProperties( phy_device, NULL, &device_extension_count, device_extensions); if (ret != VK_SUCCESS) { _freea(device_extensions); goto fail; } bool extensions_found = true; for (uint32_t i = 0; i < _countof(required_device_extensions); i++) { const char *const required_extension = required_device_extensions[i]; bool found = false; for (uint32_t j = 0; j < device_extension_count; j++) { if (!strcmp(required_extension, device_extensions[j].extensionName)) { found = true; break; } } if (!found) { flog("missing device extension: %s", required_extension); extensions_found = false; } } _freea(device_extensions); if (!extensions_found) goto fail; VkFormat format = VK_FORMAT_R8G8B8A8_UNORM; VkImageUsageFlags usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; if (!vk_shared_tex_supported(ifuncs, phy_device, format, usage, &data->external_mem_props)) { flog("texture sharing is not supported"); goto fail; } data->inst_data = idata; data->ac = NULL; if (ac) { data->ac_storage = *ac; data->ac = &data->ac_storage; } uint32_t queue_family_property_count = 0; ifuncs->GetPhysicalDeviceQueueFamilyProperties( phy_device, &queue_family_property_count, NULL); VkQueueFamilyProperties *queue_family_properties = _malloca( sizeof(VkQueueFamilyProperties) * queue_family_property_count); ifuncs->GetPhysicalDeviceQueueFamilyProperties( phy_device, &queue_family_property_count, queue_family_properties); for (uint32_t info_index = 0, info_count = info->queueCreateInfoCount; info_index < info_count; ++info_index) { const VkDeviceQueueCreateInfo *queue_info = &info->pQueueCreateInfos[info_index]; for (uint32_t queue_index = 0, queue_count = queue_info->queueCount; queue_index < queue_count; ++queue_index) { const uint32_t family_index = queue_info->queueFamilyIndex; VkQueue queue; data->funcs.GetDeviceQueue(device, family_index, queue_index, &queue); const bool supports_transfer = (queue_family_properties[family_index] .queueFlags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT)) != 0; add_queue_data(data, queue, family_index, supports_transfer, ac); } } _freea(queue_family_properties); init_obj_list(&data->swaps); data->cur_swap = NULL; data->d3d11_device = NULL; data->d3d11_context = NULL; data->valid = true; fail: return ret; } static void VKAPI_CALL OBS_DestroyDevice(VkDevice device, const VkAllocationCallbacks *ac) { struct vk_data *data = remove_device_data(device); if (data->valid) { struct vk_queue_data *queue_data = queue_walk_begin(data); while (queue_data) { vk_shtex_destroy_frame_objects(data, queue_data); queue_data = queue_walk_next(queue_data); } queue_walk_end(data); remove_free_queue_all(data, ac); } PFN_vkDestroyDevice destroy_device = data->funcs.DestroyDevice; vk_free(ac, data); destroy_device(device, ac); } static VkResult VKAPI_CALL OBS_CreateSwapchainKHR(VkDevice device, const VkSwapchainCreateInfoKHR *cinfo, const VkAllocationCallbacks *ac, VkSwapchainKHR *p_sc) { struct vk_data *data = get_device_data(device); struct vk_device_funcs *funcs = &data->funcs; if (!data->valid) return funcs->CreateSwapchainKHR(device, cinfo, ac, p_sc); VkSwapchainCreateInfoKHR info = *cinfo; info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; VkResult res = funcs->CreateSwapchainKHR(device, &info, ac, p_sc); debug_res("CreateSwapchainKHR", res); if (res != VK_SUCCESS) { /* try again with original imageUsage flags */ return funcs->CreateSwapchainKHR(device, cinfo, ac, p_sc); } VkSwapchainKHR sc = *p_sc; uint32_t count = 0; res = funcs->GetSwapchainImagesKHR(device, sc, &count, NULL); debug_res("GetSwapchainImagesKHR", res); if ((res == VK_SUCCESS) && (count > 0)) { struct vk_swap_data *swap_data = alloc_swap_data(ac); if (swap_data) { swap_data->swap_images = vk_alloc( ac, count * sizeof(VkImage), _Alignof(VkImage), VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); res = funcs->GetSwapchainImagesKHR( device, sc, &count, swap_data->swap_images); debug_res("GetSwapchainImagesKHR", res); swap_data->image_extent = cinfo->imageExtent; swap_data->format = cinfo->imageFormat; swap_data->hwnd = find_surf_hwnd(data->inst_data, cinfo->surface); swap_data->export_image = VK_NULL_HANDLE; swap_data->layout_initialized = false; swap_data->export_mem = VK_NULL_HANDLE; swap_data->image_count = count; swap_data->handle = INVALID_HANDLE_VALUE; swap_data->shtex_info = NULL; swap_data->d3d11_tex = NULL; swap_data->captured = false; init_swap_data(swap_data, data, sc); } } return VK_SUCCESS; } static VkResult VKAPI_CALL OBS_CreateImageView(VkDevice device, const VkImageViewCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkImageView *pView) { bool from_swap_chain = false; struct vk_data *const data = get_device_data(device); if (data->valid) { struct vk_swap_data *swap = swap_walk_begin(data); while (swap) { for (uint32_t i = 0, count = swap->image_count; i < count; ++i) { from_swap_chain = swap->swap_images[i] == pCreateInfo->image; if (from_swap_chain) break; } if (from_swap_chain) break; swap = swap_walk_next(swap); } swap_walk_end(data); } if (from_swap_chain) { const void *pCurrent = pCreateInfo->pNext; while (pCurrent) { VkBaseInStructure baseIn; memcpy(&baseIn, pCurrent, sizeof(baseIn)); if (baseIn.sType == VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO) { ((VkImageViewUsageCreateInfo *)pCurrent) ->usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; break; } pCurrent = baseIn.pNext; } } const PFN_vkCreateImageView func = data->funcs.CreateImageView; VkResult result = func(device, pCreateInfo, pAllocator, pView); if ((result == VK_SUCCESS) && from_swap_chain) add_swap_view_data(data, *pView, pAllocator); return result; } static void VKAPI_CALL OBS_DestroyImageView(VkDevice device, VkImageView imageView, const VkAllocationCallbacks *pAllocator) { struct vk_data *const data = get_device_data(device); if (data->valid && (imageView != VK_NULL_HANDLE)) { struct vk_swap_view_data *swap_view_data = get_swap_view_data(data, imageView); if (swap_view_data) remove_free_swap_view_data(data, imageView, pAllocator); } data->funcs.DestroyImageView(device, imageView, pAllocator); } static void generate_framebuffer_variants( PFN_vkCreateFramebuffer func, VkDevice device, const VkFramebufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, struct vk_framebuffer_data *framebuffer_data, const VkFramebufferAttachmentsCreateInfo *pAttachmentsCreateInfo, uint32_t colorCount) { const uint32_t variantCount = 1 << colorCount; for (uint32_t colorMask = 0; colorMask < variantCount; ++colorMask) { VkImageUsageFlags pPreviousUsage[OBS_COLOR_ATTACHMENT_LIMIT]; uint32_t colorIndex = 0; for (uint32_t infoIndex = 0, count = pAttachmentsCreateInfo ->attachmentImageInfoCount; infoIndex < count; ++infoIndex) { const VkFramebufferAttachmentImageInfo *const pInfo = &pAttachmentsCreateInfo ->pAttachmentImageInfos[infoIndex]; const VkImageUsageFlags usage = pInfo->usage; if (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) { pPreviousUsage[colorIndex] = usage; if (colorMask & (1 << colorIndex)) { ((VkFramebufferAttachmentImageInfo *) pInfo) ->usage |= VK_IMAGE_USAGE_TRANSFER_SRC_BIT; } ++colorIndex; if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT) break; } } VkFramebuffer framebuffer; const VkResult result = func(device, pCreateInfo, pAllocator, &framebuffer); if (result == VK_SUCCESS) framebuffer_data->alternates[colorMask] = framebuffer; colorIndex = 0; for (uint32_t infoIndex = 0, count = pAttachmentsCreateInfo ->attachmentImageInfoCount; infoIndex < count; ++infoIndex) { const VkFramebufferAttachmentImageInfo *const pInfo = &pAttachmentsCreateInfo ->pAttachmentImageInfos[infoIndex]; if (pInfo->usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) { if (colorMask & (1 << colorIndex)) { ((VkFramebufferAttachmentImageInfo *) pInfo) ->usage = pPreviousUsage[colorIndex]; } ++colorIndex; if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT) break; } } } } static VkResult VKAPI_CALL OBS_CreateFramebuffer( VkDevice device, const VkFramebufferCreateInfo *pCreateInfo, const VkAllocationCallbacks *pAllocator, VkFramebuffer *pFramebuffer) { struct vk_data *const data = get_device_data(device); const PFN_vkCreateFramebuffer func = data->funcs.CreateFramebuffer; VkResult result = func(device, pCreateInfo, pAllocator, pFramebuffer); if (data->valid && (result == VK_SUCCESS) && (pCreateInfo->flags & VK_FRAMEBUFFER_CREATE_IMAGELESS_BIT)) { struct vk_framebuffer_data *const framebuffer_data = add_framebuffer_data(data, *pFramebuffer, pAllocator); const void *pCurrent = pCreateInfo->pNext; while (pCurrent) { VkBaseInStructure baseIn; memcpy(&baseIn, pCurrent, sizeof(baseIn)); if (baseIn.sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_ATTACHMENTS_CREATE_INFO) { uint32_t attachmentMask = 0; uint32_t colorCount = 0; const VkFramebufferAttachmentsCreateInfo *const pAttachmentsCreateInfo = pCurrent; for (uint32_t infoIndex = 0, count = min( sizeof(attachmentMask) * 8, pAttachmentsCreateInfo ->attachmentImageInfoCount); infoIndex < count; ++infoIndex) { if (pAttachmentsCreateInfo ->pAttachmentImageInfos [infoIndex] .usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) { attachmentMask |= 1 << infoIndex; ++colorCount; if (colorCount == OBS_COLOR_ATTACHMENT_LIMIT) break; } } generate_framebuffer_variants( func, device, pCreateInfo, pAllocator, framebuffer_data, pAttachmentsCreateInfo, colorCount); framebuffer_data->color_attachment_mask = attachmentMask; break; } pCurrent = baseIn.pNext; } } return result; } static void VKAPI_CALL OBS_DestroyFramebuffer(VkDevice device, VkFramebuffer framebuffer, const VkAllocationCallbacks *pAllocator) { struct vk_data *const data = get_device_data(device); struct vk_device_funcs *const funcs = &data->funcs; if (data->valid && (framebuffer != VK_NULL_HANDLE)) { struct vk_framebuffer_data *framebuffer_data = get_framebuffer_data(data, framebuffer); if (framebuffer_data) { for (size_t i = 0; i < _countof(framebuffer_data->alternates); ++i) { if (framebuffer_data->alternates[i] != VK_NULL_HANDLE) { funcs->DestroyFramebuffer( device, framebuffer_data->alternates[i], pAllocator); framebuffer_data->alternates[i] = VK_NULL_HANDLE; } } remove_free_framebuffer_data(data, framebuffer, pAllocator); } } funcs->DestroyFramebuffer(device, framebuffer, pAllocator); } static const VkRenderPassBeginInfo * process_render_pass_begin_info(const VkRenderPassBeginInfo *pRenderPassBegin, VkRenderPassBeginInfo *pAlternateBegin, struct vk_data *data) { const void *pCurrent = pRenderPassBegin->pNext; while (pCurrent) { VkBaseInStructure baseIn; memcpy(&baseIn, pCurrent, sizeof(baseIn)); if (baseIn.sType == VK_STRUCTURE_TYPE_RENDER_PASS_ATTACHMENT_BEGIN_INFO) { struct vk_framebuffer_data *const framebuffer_data = get_framebuffer_data( data, pRenderPassBegin->framebuffer); if (framebuffer_data) { const VkRenderPassAttachmentBeginInfo *const pAttachmentInfo = pCurrent; uint32_t swapMask = 0; uint32_t colorIndex = 0; for (uint32_t infoIndex = 0, count = pAttachmentInfo ->attachmentCount; infoIndex < count; ++infoIndex) { if (framebuffer_data ->color_attachment_mask & (1 << infoIndex)) { if (get_swap_view_data( data, pAttachmentInfo->pAttachments [infoIndex])) { swapMask |= 1 << colorIndex; } ++colorIndex; if (colorIndex == OBS_COLOR_ATTACHMENT_LIMIT) break; } } if (swapMask > 0) { VkFramebuffer alternate = framebuffer_data ->alternates[swapMask]; if (alternate != VK_NULL_HANDLE) { *pAlternateBegin = *pRenderPassBegin; pAlternateBegin->framebuffer = framebuffer_data->alternates [swapMask]; pRenderPassBegin = pAlternateBegin; } } } break; } pCurrent = baseIn.pNext; } return pRenderPassBegin; } static void VKAPI_CALL OBS_CmdBeginRenderPass(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo *pRenderPassBegin, VkSubpassContents contents) { struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer); VkRenderPassBeginInfo alternateBegin; if (data->valid) { pRenderPassBegin = process_render_pass_begin_info( pRenderPassBegin, &alternateBegin, data); } data->funcs.CmdBeginRenderPass(commandBuffer, pRenderPassBegin, contents); } static void VKAPI_CALL OBS_CmdBeginRenderPass2KHR(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo *pRenderPassBegin, const VkSubpassBeginInfo *pSubpassBeginInfo) { struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer); VkRenderPassBeginInfo alternateBegin; if (data->valid) { pRenderPassBegin = process_render_pass_begin_info( pRenderPassBegin, &alternateBegin, data); } data->funcs.CmdBeginRenderPass2KHR(commandBuffer, pRenderPassBegin, pSubpassBeginInfo); } static void VKAPI_CALL OBS_CmdBeginRenderPass2(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo *pRenderPassBegin, const VkSubpassBeginInfo *pSubpassBeginInfo) { struct vk_data *const data = get_device_data_by_command_buffer(commandBuffer); VkRenderPassBeginInfo alternateBegin; if (data->valid) { pRenderPassBegin = process_render_pass_begin_info( pRenderPassBegin, &alternateBegin, data); } data->funcs.CmdBeginRenderPass2(commandBuffer, pRenderPassBegin, pSubpassBeginInfo); } static void VKAPI_CALL OBS_DestroySwapchainKHR(VkDevice device, VkSwapchainKHR sc, const VkAllocationCallbacks *ac) { struct vk_data *data = get_device_data(device); struct vk_device_funcs *funcs = &data->funcs; PFN_vkDestroySwapchainKHR destroy_swapchain = funcs->DestroySwapchainKHR; if ((sc != VK_NULL_HANDLE) && data->valid) { struct vk_swap_data *swap = get_swap_data(data, sc); if (swap) { if (data->cur_swap == swap) { vk_shtex_free(data); } vk_free(ac, swap->swap_images); remove_free_swap_data(data, sc, ac); } } destroy_swapchain(device, sc, ac); } static VkResult VKAPI_CALL OBS_CreateWin32SurfaceKHR( VkInstance inst, const VkWin32SurfaceCreateInfoKHR *info, const VkAllocationCallbacks *ac, VkSurfaceKHR *surf) { struct vk_inst_data *idata = get_inst_data(inst); struct vk_inst_funcs *ifuncs = &idata->funcs; VkResult res = ifuncs->CreateWin32SurfaceKHR(inst, info, ac, surf); if ((res == VK_SUCCESS) && idata->valid) add_surf_data(idata, *surf, info->hwnd, ac); return res; } static void VKAPI_CALL OBS_DestroySurfaceKHR(VkInstance inst, VkSurfaceKHR surf, const VkAllocationCallbacks *ac) { struct vk_inst_data *idata = get_inst_data(inst); struct vk_inst_funcs *ifuncs = &idata->funcs; PFN_vkDestroySurfaceKHR destroy_surface = ifuncs->DestroySurfaceKHR; if ((surf != VK_NULL_HANDLE) && idata->valid) remove_free_surf_data(idata, surf, ac); destroy_surface(inst, surf, ac); } /* clang-format off */ #define GETPROCADDR(func) \ if (!strcmp(pName, "vk" #func)) \ return (PFN_vkVoidFunction)&OBS_##func; #define GETPROCADDR_IF_SUPPORTED(func) \ if (!strcmp(pName, "vk" #func)) \ return funcs->func ? (PFN_vkVoidFunction)&OBS_##func : NULL; /* clang-format on */ static PFN_vkVoidFunction VKAPI_CALL OBS_GetDeviceProcAddr(VkDevice device, const char *pName) { struct vk_data *data = get_device_data(device); struct vk_device_funcs *funcs = &data->funcs; debug_procaddr("vkGetDeviceProcAddr(%p, \"%s\")", device, pName); GETPROCADDR(GetDeviceProcAddr); GETPROCADDR(DestroyDevice); GETPROCADDR_IF_SUPPORTED(CreateSwapchainKHR); GETPROCADDR_IF_SUPPORTED(DestroySwapchainKHR); GETPROCADDR_IF_SUPPORTED(QueuePresentKHR); GETPROCADDR(CreateImageView); GETPROCADDR(DestroyImageView); GETPROCADDR(CreateFramebuffer); GETPROCADDR(DestroyFramebuffer); GETPROCADDR(CmdBeginRenderPass); GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2KHR); GETPROCADDR_IF_SUPPORTED(CmdBeginRenderPass2); if (funcs->GetDeviceProcAddr == NULL) return NULL; return funcs->GetDeviceProcAddr(device, pName); } /* bad layers require spec violation */ #define RETURN_FP_FOR_NULL_INSTANCE 1 static PFN_vkVoidFunction VKAPI_CALL OBS_GetInstanceProcAddr(VkInstance instance, const char *pName) { debug_procaddr("vkGetInstanceProcAddr(%p, \"%s\")", instance, pName); /* instance chain functions we intercept */ GETPROCADDR(GetInstanceProcAddr); GETPROCADDR(CreateInstance); #if RETURN_FP_FOR_NULL_INSTANCE /* other instance chain functions we intercept */ GETPROCADDR(DestroyInstance); GETPROCADDR(CreateWin32SurfaceKHR); GETPROCADDR(DestroySurfaceKHR); /* device chain functions we intercept */ GETPROCADDR(GetDeviceProcAddr); GETPROCADDR(CreateDevice); GETPROCADDR(DestroyDevice); if (instance == NULL) return NULL; struct vk_inst_funcs *const funcs = get_inst_funcs(instance); #else if (instance == NULL) return NULL; struct vk_inst_funcs *const funcs = get_inst_funcs(instance); /* other instance chain functions we intercept */ GETPROCADDR(DestroyInstance); GETPROCADDR_IF_SUPPORTED(CreateWin32SurfaceKHR); GETPROCADDR_IF_SUPPORTED(DestroySurfaceKHR); /* device chain functions we intercept */ GETPROCADDR(GetDeviceProcAddr); GETPROCADDR(CreateDevice); GETPROCADDR(DestroyDevice); #endif const PFN_vkGetInstanceProcAddr gipa = funcs->GetInstanceProcAddr; return gipa ? gipa(instance, pName) : NULL; } #undef GETPROCADDR #ifndef _WIN64 #pragma comment(linker, "/EXPORT:OBS_Negotiate=_OBS_Negotiate@4") #endif __declspec(dllexport) VkResult VKAPI_CALL OBS_Negotiate(VkNegotiateLayerInterface *nli) { if (nli->loaderLayerInterfaceVersion >= 2) { nli->sType = LAYER_NEGOTIATE_INTERFACE_STRUCT; nli->pNext = NULL; nli->pfnGetInstanceProcAddr = OBS_GetInstanceProcAddr; nli->pfnGetDeviceProcAddr = OBS_GetDeviceProcAddr; nli->pfnGetPhysicalDeviceProcAddr = NULL; } const uint32_t cur_ver = CURRENT_LOADER_LAYER_INTERFACE_VERSION; if (nli->loaderLayerInterfaceVersion > cur_ver) { nli->loaderLayerInterfaceVersion = cur_ver; } if (!vulkan_seen) { init_obj_list(&instances); init_obj_list(&devices); vulkan_seen = true; } return VK_SUCCESS; } bool hook_vulkan(void) { static bool hooked = false; if (!hooked && vulkan_seen) { hlog("Hooked Vulkan"); hooked = true; } return hooked; }