From 30f6870b2304db87b159f5abfd28e1b96eb1eb75 Mon Sep 17 00:00:00 2001 From: jpark37 Date: Tue, 12 May 2020 12:06:03 -0700 Subject: [PATCH] win-capture: Fail on unsupported Vulkan formats Don't allow unsupported Vulkan formats to fall back to B8G8R8A8. Probably better to fail completely than do an illegal copy. Also remove bad conversion for VK_FORMAT_A2R10G10B10_UNORM_PACK32. Red and blue channels were reversed, and there's no DXGI equivalent. Addresses #2796. We can do more later if justified. --- .../graphics-hook/vulkan-capture.c | 22 +++++++++++++------ .../graphics-hook/vulkan-capture.h | 6 ----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/plugins/win-capture/graphics-hook/vulkan-capture.c b/plugins/win-capture/graphics-hook/vulkan-capture.c index 5576b60e8..dc1f98d46 100644 --- a/plugins/win-capture/graphics-hook/vulkan-capture.c +++ b/plugins/win-capture/graphics-hook/vulkan-capture.c @@ -457,16 +457,24 @@ static inline bool vk_shtex_init_d3d11_tex(struct vk_data *data, IDXGIResource *dxgi_res; HRESULT hr; + const UINT width = swap->image_extent.width; + const UINT height = swap->image_extent.height; + + flog("OBS requesting %s texture format. capture dimensions: %ux%u", + vk_format_to_str(swap->format), width, height); + + const DXGI_FORMAT format = vk_format_to_dxgi(swap->format); + if (format == DXGI_FORMAT_UNKNOWN) { + flog("cannot convert to DXGI format"); + return false; + } + D3D11_TEXTURE2D_DESC desc = {0}; - desc.Width = swap->image_extent.width; - desc.Height = swap->image_extent.height; + desc.Width = width; + desc.Height = height; desc.MipLevels = 1; desc.ArraySize = 1; - - flog("OBS requesting %s texture format. capture dimensions: %dx%d", - vk_format_to_str(swap->format), (int)desc.Width, (int)desc.Height); - - desc.Format = vk_format_to_dxgi(swap->format); + desc.Format = format; desc.SampleDesc.Count = 1; desc.SampleDesc.Quality = 0; desc.Usage = D3D11_USAGE_DEFAULT; diff --git a/plugins/win-capture/graphics-hook/vulkan-capture.h b/plugins/win-capture/graphics-hook/vulkan-capture.h index 15651153f..3687b84bf 100644 --- a/plugins/win-capture/graphics-hook/vulkan-capture.h +++ b/plugins/win-capture/graphics-hook/vulkan-capture.h @@ -473,7 +473,6 @@ DXGI_FORMAT vk_format_to_dxgi(VkFormat format) case VK_FORMAT_A8B8G8R8_SRGB_PACK32: break; case VK_FORMAT_A2R10G10B10_UNORM_PACK32: - dxgi_format = DXGI_FORMAT_R10G10B10A2_UNORM; break; case VK_FORMAT_A2R10G10B10_SNORM_PACK32: break; @@ -841,11 +840,6 @@ DXGI_FORMAT vk_format_to_dxgi(VkFormat format) case VK_FORMAT_PVRTC2_4BPP_SRGB_BLOCK_IMG: break; } - if (dxgi_format == DXGI_FORMAT_UNKNOWN) { - flog("unknown swapchain format, " - "defaulting to B8G8R8A8_UNORM"); - dxgi_format = DXGI_FORMAT_B8G8R8A8_UNORM; - } return dxgi_format; }