Skip to content

layer: Force bypass when requested format is unsupported on fallback #2183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
matte-schwartz wants to merge 1 commit into
base: master
Choose a base branch
from
+51 −35
Open

layer: Force bypass when requested format is unsupported on fallback #2183

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 51 additions & 35 deletions layer/VkLayer_FROG_gamescope_wsi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,6 +500,7 @@ namespace GamescopeWSILayer {
VkSurfaceKHR surface; // Always the Gamescope Surface surface -- so the Wayland one.
bool isWayland;
bool isBypassingXWayland;
bool forcedBypass;
bool forceFifo;
VkPresentModeKHR presentMode;
VkExtent2D extent;
Expand Down Expand Up @@ -1111,20 +1112,10 @@ namespace GamescopeWSILayer {
return pDispatch->CreateSwapchainKHR(device, pCreateInfo, pAllocator, pSwapchain);
}

const bool canBypass = gamescopeSurface->canBypassXWayland();
bool canBypass = gamescopeSurface->canBypassXWayland();

VkSwapchainCreateInfoKHR swapchainInfo = *pCreateInfo;

if (pCreateInfo->oldSwapchain) {
if (auto gamescopeSwapchain = GamescopeSwapchain::get(pCreateInfo->oldSwapchain)) {
gamescopeSwapchain->retired = true;
// If we are going to/from being able to bypass XWayland, make sure
// we NULL out oldSwapchain, as they'll be for different surfaces and swapchain types.
if (gamescopeSwapchain->isBypassingXWayland != canBypass)
swapchainInfo.oldSwapchain = VK_NULL_HANDLE;
}
}

if (gamescopeSurface->flags & GamescopeLayerClient::Flag::ForceSwapchainExtent) {
if (!gamescopeSurface->isWayland()) {
auto rect = xcb::getWindowRect(gamescopeSurface->connection, gamescopeSurface->window);
Expand All @@ -1135,6 +1126,42 @@ namespace GamescopeWSILayer {
}
}

auto isFormatSupportedOnSurface = [&](VkSurfaceKHR vkSurface, VkFormat format) {
std::vector<VkSurfaceFormatKHR> formats;
vkroots::helpers::enumerate(
pDispatch->pPhysicalDeviceDispatch->pInstanceDispatch->GetPhysicalDeviceSurfaceFormatsKHR,
formats,
pDispatch->PhysicalDevice,
vkSurface);
return std::ranges::any_of(
formats,
std::bind_front(std::equal_to{}, format),
&VkSurfaceFormatKHR::format);
};

// If bypass supports the format but the XCB fallback doesn't, force bypass.
// Refusing here would leave the app with no recovery path.
bool forcedBypass = false;
if (!canBypass
&& !isFormatSupportedOnSurface(gamescopeSurface->fallbackSurface, pCreateInfo->imageFormat)
&& isFormatSupportedOnSurface(pCreateInfo->surface, pCreateInfo->imageFormat)) {
fprintf(stderr, "[Gamescope WSI] Forcing bypass (format unsupported on fallback) for xid: 0x%0x - format: %s\n",
gamescopeSurface->window,
vkroots::helpers::enumString(pCreateInfo->imageFormat));
canBypass = true;
forcedBypass = true;
}

if (pCreateInfo->oldSwapchain) {
if (auto gamescopeSwapchain = GamescopeSwapchain::get(pCreateInfo->oldSwapchain)) {
gamescopeSwapchain->retired = true;
// If we are going to/from being able to bypass XWayland, make sure
// we NULL out oldSwapchain, as they'll be for different surfaces and swapchain types.
if (gamescopeSwapchain->isBypassingXWayland != canBypass)
swapchainInfo.oldSwapchain = VK_NULL_HANDLE;
}
}

// If we can't flip, fallback to the regular XCB surface on the XCB window.
if (!canBypass)
swapchainInfo.surface = gamescopeSurface->fallbackSurface;
Expand Down Expand Up @@ -1175,28 +1202,14 @@ namespace GamescopeWSILayer {

// Check for VkFormat support and return VK_ERROR_INITIALIZATION_FAILED
// if that VkFormat is unsupported for the underlying surface.
{
std::vector<VkSurfaceFormatKHR> supportedSurfaceFormats;
vkroots::helpers::enumerate(
pDispatch->pPhysicalDeviceDispatch->pInstanceDispatch->GetPhysicalDeviceSurfaceFormatsKHR,
supportedSurfaceFormats,
pDispatch->PhysicalDevice,
swapchainInfo.surface);

bool supportedSwapchainFormat = std::ranges::any_of(
supportedSurfaceFormats,
std::bind_front(std::equal_to{}, swapchainInfo.imageFormat),
&VkSurfaceFormatKHR::format) ;

if (!supportedSwapchainFormat) {
fprintf(stderr, "[Gamescope WSI] Refusing to make swapchain (unsupported VkFormat) for xid: 0x%0x - format: %s - colorspace: %s - flip: %s\n",
gamescopeSurface->window,
vkroots::helpers::enumString(pCreateInfo->imageFormat),
vkroots::helpers::enumString(pCreateInfo->imageColorSpace),
canBypass ? "true" : "false");

return VK_ERROR_INITIALIZATION_FAILED;
}
if (!isFormatSupportedOnSurface(swapchainInfo.surface, swapchainInfo.imageFormat)) {
fprintf(stderr, "[Gamescope WSI] Refusing to make swapchain (unsupported VkFormat) for xid: 0x%0x - format: %s - colorspace: %s - flip: %s\n",
gamescopeSurface->window,
vkroots::helpers::enumString(pCreateInfo->imageFormat),
vkroots::helpers::enumString(pCreateInfo->imageColorSpace),
canBypass ? "true" : "false");

return VK_ERROR_INITIALIZATION_FAILED;
}

uint32_t serverId = ~0u;
Expand Down Expand Up @@ -1232,6 +1245,7 @@ namespace GamescopeWSILayer {
.surface = pCreateInfo->surface, // Always the Wayland side surface.
.isWayland = gamescopeSurface->isWayland(),
.isBypassingXWayland = canBypass,
.forcedBypass = forcedBypass,
.forceFifo = gamescopeIsForcingFifo(), // Were we forcing fifo when this swapchain was made?
.presentMode = pCreateInfo->presentMode, // The new present mode.
.extent = pCreateInfo->imageExtent,
Expand Down Expand Up @@ -1431,8 +1445,10 @@ namespace GamescopeWSILayer {
if (canBypass) {
if (!(gamescopeSurface->flags & GamescopeLayerClient::Flag::NoSuboptimal))
UpdateSwapchainResult(VK_SUBOPTIMAL_KHR);
} else {
UpdateSwapchainResult(VK_ERROR_OUT_OF_DATE_KHR);
} else if (!gamescopeSwapchain->forcedBypass) {
// A forced-bypass swapchain has no fallback path; recreating
// would re-force bypass and loop.
UpdateSwapchainResult(VK_ERROR_OUT_OF_DATE_KHR);
}
}

Expand Down