[Vulkan] Fix swapchain creation for Wayland

VkSurfaceCapabilitiesKHR.currentExtent is 0xFFFFFFFF when
using Wayland. This commit fixes swapchain creation to
properly handle this case.

Author: hyazinthh

RELEASE_NOTES.md

@@ -1,3 +1,5 @@
+- [Vulkan] Fixed swapchain creation for Wayland
+
 ### 5.6.0
 - https://github.com/aardvark-platform/aardvark.rendering/wiki/Aardvark-Rendering-5.6-changelog
 

src/Aardvark.Rendering.Vulkan/Swapchain/Surface.fs

@@ -97,7 +97,6 @@ type Surface(device : Device, handle : VkSurfaceKHR) =
 
     let physical = device.PhysicalDevice
     let family = device.GraphicsFamily
-        
 
     let supported =
         native {
@@ -107,25 +106,34 @@ type Surface(device : Device, handle : VkSurfaceKHR) =
             return !!ptr
         }
 
-    let surfaceCaps = 
-        if supported <> 0u then
-            native {
-                let! pSurfaceCaps = VkSurfaceCapabilitiesKHR()
-                VkRaw.vkGetPhysicalDeviceSurfaceCapabilitiesKHR(physical.Handle, handle, pSurfaceCaps) 
-                    |> check "could not get Surface capabilities"
-                return !!pSurfaceCaps
-            }
-        else 
-            VkSurfaceCapabilitiesKHR()
-        
+    let getSurfaceCaps() =
+        let mutable result = VkSurfaceCapabilitiesKHR.Empty
+
+        if supported = VkTrue then
+            use ptr = fixed &result
+            VkRaw.vkGetPhysicalDeviceSurfaceCapabilitiesKHR(physical.Handle, handle, ptr)
+                |> check "could not get Surface capabilities"
+        else
+            result.minImageCount <- 1u
+            result.maxImageCount <- UInt32.MaxValue
+            result.maxImageArrayLayers <- UInt32.MaxValue
+            result.maxImageExtent <- VkExtent2D(UInt32.MaxValue, UInt32.MaxValue)
+
+        result
+
+    let toInt =
+        min <| (uint32 Int32.MaxValue) >> int
+
+    let surfaceCaps = getSurfaceCaps()
+
     let supportedTransforms = unbox<VkSurfaceTransformFlagsKHR> (int surfaceCaps.supportedTransforms) |> VkSurfaceTransformFlagsKHR.toImageTrafos
     let supportedCompositeAlpha = unbox<VkCompositeAlphaFlagsKHR> (int surfaceCaps.supportedCompositeAlpha)
     let supportedUsage = surfaceCaps.supportedUsageFlags
-    let minSize = V2i(int surfaceCaps.minImageExtent.width, int surfaceCaps.minImageExtent.height)
-    let maxSize = V2i(int surfaceCaps.maxImageExtent.width, int surfaceCaps.maxImageExtent.height)
-    let maxSlices = int surfaceCaps.maxImageArrayLayers
-    let minImageCount = int surfaceCaps.minImageCount
-    let maxImageCount = int surfaceCaps.maxImageCount
+    let minSize = V2i(toInt surfaceCaps.minImageExtent.width, toInt surfaceCaps.minImageExtent.height)
+    let maxSize = V2i(toInt surfaceCaps.maxImageExtent.width, toInt surfaceCaps.maxImageExtent.height)
+    let maxSlices = toInt surfaceCaps.maxImageArrayLayers
+    let minImageCount = toInt surfaceCaps.minImageCount
+    let maxImageCount = toInt surfaceCaps.maxImageCount
 
 
     let presentModes =
@@ -184,20 +192,18 @@ type Surface(device : Device, handle : VkSurfaceKHR) =
     member x.MaxArraySlices = maxSlices
     member x.MinImageCount = minImageCount
     member x.MaxImageCount = maxImageCount
-       
+
     member x.Size =
         if supported <> 0u && handle.IsValid then
-            let surfaceCaps =
-                native {
-                    let! ptr = VkSurfaceCapabilitiesKHR()
-                    VkRaw.vkGetPhysicalDeviceSurfaceCapabilitiesKHR(physical.Handle, handle, ptr) 
-                        |> check "could not get Surface capabilities"
-                    return !!ptr
-                }
-            V2i(int surfaceCaps.currentExtent.width, int surfaceCaps.currentExtent.height)
+            let surfaceCaps = getSurfaceCaps()
+            let size = V2ui(surfaceCaps.currentExtent.width, surfaceCaps.currentExtent.height)
+            if size <> V2ui(0xFFFFFFFFu) then
+                V2i(toInt size.X, toInt size.Y)
+            else
+                V2i.Zero // See doc for VkSurfaceCapabilitiesKHR.currentExtent
         else
             V2i.Zero
-        
+
     member x.HasCompositeAlpha (t : VkCompositeAlphaFlagsKHR) = (t &&& supportedCompositeAlpha) = t
     member x.HasUsage (t : VkImageUsageFlags) = (t &&& supportedUsage) = t
 

src/Aardvark.Rendering.Vulkan/Swapchain/Swapchain.fs

@@ -3,34 +3,32 @@
 open System
 open Aardvark.Base
 open Aardvark.Rendering
-open FSharp.Data.Adaptive
 open Aardvark.Rendering.Vulkan
 open System.Runtime.CompilerServices
-open Microsoft.FSharp.NativeInterop
 open KHRSwapchain
 open KHRSurface
 
 #nowarn "9"
 #nowarn "51"
 
-[<AutoOpen>]          
+[<AutoOpen>]
 module ImageTrafoExtensions =
     type Box3i with
         member x.Transformed(t : ImageTrafo) =
             match t with
-                | ImageTrafo.Identity -> 
-                    x
+            | ImageTrafo.Identity ->
+                x
 
-                | ImageTrafo.MirrorX -> 
-                    Box3i(V3i(x.Max.X, x.Min.Y, x.Min.Z), V3i(x.Min.X, x.Max.Y, x.Max.Z))
-                    
-                | ImageTrafo.MirrorY -> 
-                    Box3i(V3i(x.Min.X, x.Max.Y, x.Min.Z), V3i(x.Max.X, x.Min.Y, x.Max.Z))
+            | ImageTrafo.MirrorX ->
+                Box3i(V3i(x.Max.X, x.Min.Y, x.Min.Z), V3i(x.Min.X, x.Max.Y, x.Max.Z))
 
-                | _ ->
-                    failwithf "box cannot be transformed using %A" t
+            | ImageTrafo.MirrorY ->
+                Box3i(V3i(x.Min.X, x.Max.Y, x.Min.Z), V3i(x.Max.X, x.Min.Y, x.Max.Z))
+
+            | _ ->
+                failwithf "box cannot be transformed using %A" t
 
-type Swapchain(device : Device, description : SwapchainDescription) =
+type Swapchain(device : Device, initialSize : V2i, description : SwapchainDescription) =
     let fence = device.CreateFence()
     let surface = description.surface
     let renderPass = description.renderPass
@@ -179,7 +177,11 @@ type Swapchain(device : Device, description : SwapchainDescription) =
 
     let update() =
         if surface.Handle.IsValid && disposed = 0 then
-            let newSize = surface.Size
+            let newSize =
+                let surfaceSize = surface.Size
+                let currentSize = if surfaceSize <> V2i.Zero then surfaceSize else initialSize
+                currentSize |> clamp surface.MinSize surface.MaxSize
+
             if newSize <> size || handle.IsNull then
                 // delete old things
 
@@ -193,7 +195,7 @@ type Swapchain(device : Device, description : SwapchainDescription) =
                 buffers |> Array.iter Disposable.dispose
 
                 //handle, buffers, framebuffer, resolvedView
-                let (newHandle, newBuffers, newFramebuffer, newResolvedImage) = recreate handle newSize
+                let newHandle, newBuffers, newFramebuffer, newResolvedImage = recreate handle newSize
                 if handle.IsValid then VkRaw.vkDestroySwapchainKHR(device.Handle, handle, NativePtr.zero)
 
                 handle <- newHandle
@@ -203,6 +205,10 @@ type Swapchain(device : Device, description : SwapchainDescription) =
                 resolvedImage <- newResolvedImage
                 currentBuffer <- 0u
 
+    [<Obsolete>]
+    new (device : Device, description : SwapchainDescription) =
+        new Swapchain(device, V2i(1024, 768), description)
+
     member x.Size = update(); size
     member x.Description = description
     member x.Samples = description.samples
@@ -384,12 +390,21 @@ type Swapchain(device : Device, description : SwapchainDescription) =
 
 [<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
 module Swapchain =
+
+    let create2 (size : V2i) (description : SwapchainDescription) (device : Device) =
+        new Swapchain(device, size, description)
+
+    [<Obsolete("Use Swapchain.create2 instead.")>]
     let create (desc : SwapchainDescription) (device : Device) =
-        new Swapchain(device, desc)
+        create2 (V2i(1024, 768)) desc device
 
 [<AbstractClass; Sealed; Extension>]
 type DeviceSwapchainExtensions private() =
 
+    [<Extension>]
+    static member CreateSwapchain(this : Device, size : V2i, description : SwapchainDescription) =
+        this |> Swapchain.create2 size description
+
     [<Extension>]
     static member CreateSwapchain(this : Device, description : SwapchainDescription) =
-        this |> Swapchain.create description
+        this.CreateSwapchain(V2i(1024, 768), description)

src/Application/Aardvark.Application.Slim.Vulkan/Application.fs

@@ -62,10 +62,10 @@ module private Vulkan =
             device.CreateSwapchainDescription(surf, graphicsMode)
 
         { new IWindowSurface with
-            override __.Signature = 
+            override _.Signature =
                 description.renderPass :> IFramebufferSignature
             override this.CreateSwapchain(size: V2i) = 
-                let swap = device.CreateSwapchain(description)
+                let swap = device.CreateSwapchain(size, description)
                 { new ISwapchain with
                     override this.Dispose() = 
                         swap.Dispose()
@@ -87,16 +87,16 @@ module private Vulkan =
 
     let interop =
         { new IWindowInterop with
-            override __.Boot(_) =
+            override _.Boot(_) =
                 ()
 
-            override __.CreateSurface(runtime : IRuntime, cfg: WindowConfig, glfw: Glfw, win: nativeptr<WindowHandle>) = 
+            override _.CreateSurface(runtime : IRuntime, cfg: WindowConfig, glfw: Glfw, win: nativeptr<WindowHandle>) =
                 createSurface (runtime :?> _) cfg glfw win
 
-            override __.WindowHints(cfg: WindowConfig, glfw: Glfw) = 
+            override _.WindowHints(cfg: WindowConfig, glfw: Glfw) =
                 glfw.WindowHint(WindowHintClientApi.ClientApi, ClientApi.NoApi)
 
-            override __.Dispose() =
+            override _.Dispose() =
                 ()
         }