Vulkan: Support macOS via MoltenVK

The path to the MoltenVK library can be specified by the
LIBMOLTENVK_PATH environment variable, otherwise it assumes it is
located in the application bundle's Contents/MacOS directory.

Source/Core/VideoBackends/Vulkan/SwapChain.cpp

@@ -118,6 +118,19 @@ VkSurfaceKHR SwapChain::CreateVulkanSurface(VkInstance instance, void* display_h
 
   return surface;
 
+#elif defined(VK_USE_PLATFORM_MACOS_MVK)
+  VkMacOSSurfaceCreateInfoMVK surface_create_info = {
+      VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK, nullptr, 0, hwnd};
+
+  VkSurfaceKHR surface;
+  VkResult res = vkCreateMacOSSurfaceMVK(instance, &surface_create_info, nullptr, &surface);
+  if (res != VK_SUCCESS)
+  {
+    LOG_VULKAN_ERROR(res, "vkCreateMacOSSurfaceMVK failed: ");
+    return VK_NULL_HANDLE;
+  }
+
+  return surface;
 #else
   return VK_NULL_HANDLE;
 #endif

Source/Core/VideoBackends/Vulkan/VideoBackend.h

@@ -18,5 +18,6 @@ public:
   std::string GetName() const override { return "Vulkan"; }
   std::string GetDisplayName() const override { return _trans("Vulkan"); }
   void InitBackendInfo() override;
+  void PrepareWindow(const WindowSystemInfo& wsi) override;
 };
 }  // namespace Vulkan

Source/Core/VideoBackends/Vulkan/VulkanContext.cpp

@@ -186,6 +186,9 @@ bool VulkanContext::SelectInstanceExtensions(ExtensionList* extension_list, bool
 #elif defined(VK_USE_PLATFORM_ANDROID_KHR)
   if (enable_surface && !SupportsExtension(VK_KHR_ANDROID_SURFACE_EXTENSION_NAME, true))
     return false;
+#elif defined(VK_USE_PLATFORM_MACOS_MVK)
+  if (enable_surface && !SupportsExtension(VK_MVK_MACOS_SURFACE_EXTENSION_NAME, true))
+    return false;
 #endif
 
   // VK_EXT_debug_report
@@ -833,6 +836,13 @@ void VulkanContext::InitDriverDetails()
     driver = DriverDetails::DRIVER_UNKNOWN;
   }
 
+#ifdef __APPLE__
+  // Vulkan on macOS goes through Metal, and is not susceptible to the same bugs
+  // as the vendor's native Vulkan drivers. We use a different driver fields to
+  // differentiate MoltenVK.
+  driver = DriverDetails::DRIVER_PORTABILITY;
+#endif
+
   DriverDetails::Init(DriverDetails::API_VULKAN, vendor, driver,
                       static_cast<double>(m_device_properties.driverVersion),
                       DriverDetails::Family::UNKNOWN);

Source/Core/VideoBackends/Vulkan/VulkanEntryPoints.inl

@@ -178,6 +178,10 @@ VULKAN_INSTANCE_ENTRY_POINT(vkGetPhysicalDeviceXcbPresentationSupportKHR, false)
 
 VULKAN_INSTANCE_ENTRY_POINT(vkCreateAndroidSurfaceKHR, false)
 
+#elif defined(VK_USE_PLATFORM_MACOS_MVK)
+
+VULKAN_INSTANCE_ENTRY_POINT(vkCreateMacOSSurfaceMVK, false)
+
 #endif
 
 VULKAN_INSTANCE_ENTRY_POINT(vkCreateDebugReportCallbackEXT, false)

Source/Core/VideoBackends/Vulkan/VulkanLoader.cpp

@@ -4,8 +4,10 @@
 
 #include <atomic>
 #include <cstdarg>
+#include <cstdlib>
 
 #include "Common/CommonFuncs.h"
+#include "Common/FileUtil.h"
 #include "Common/Logging/Log.h"
 #include "Common/StringUtil.h"
 
@@ -14,7 +16,8 @@
 #if defined(VK_USE_PLATFORM_WIN32_KHR)
 #include <Windows.h>
 #elif defined(VK_USE_PLATFORM_XLIB_KHR) || defined(VK_USE_PLATFORM_XCB_KHR) ||                     \
-    defined(VK_USE_PLATFORM_ANDROID_KHR) || defined(USE_HEADLESS)
+    defined(VK_USE_PLATFORM_ANDROID_KHR) || defined(VK_USE_PLATFORM_MACOS_MVK) ||                  \
+    defined(USE_HEADLESS)
 #include <dlfcn.h>
 #endif
 
@@ -98,7 +101,8 @@ void UnloadVulkanLibrary()
 }
 
 #elif defined(VK_USE_PLATFORM_XLIB_KHR) || defined(VK_USE_PLATFORM_XCB_KHR) ||                     \
-    defined(VK_USE_PLATFORM_ANDROID_KHR) || defined(USE_HEADLESS)
+    defined(VK_USE_PLATFORM_ANDROID_KHR) || defined(VK_USE_PLATFORM_MACOS_MVK) ||                  \
+    defined(USE_HEADLESS)
 
 static void* vulkan_module;
 static std::atomic_int vulkan_module_ref_count = {0};
@@ -112,15 +116,27 @@ bool LoadVulkanLibrary()
     return true;
   }
 
+#if defined(__APPLE__)
+  // Check if a path to a specific Vulkan library has been specified.
+  char* libvulkan_env = getenv("LIBVULKAN_PATH");
+  if (libvulkan_env)
+    vulkan_module = dlopen(libvulkan_env, RTLD_NOW);
+  if (!vulkan_module)
+  {
+    // Use the libvulkan.dylib from the application bundle.
+    std::string path = File::GetBundleDirectory() + "/Contents/Frameworks/libvulkan.dylib";
+    vulkan_module = dlopen(path.c_str(), RTLD_NOW);
+  }
+#else
   // Names of libraries to search. Desktop should use libvulkan.so.1 or libvulkan.so.
   static const char* search_lib_names[] = {"libvulkan.so.1", "libvulkan.so"};
-
   for (size_t i = 0; i < ArraySize(search_lib_names); i++)
   {
     vulkan_module = dlopen(search_lib_names[i], RTLD_NOW);
     if (vulkan_module)
       break;
   }
+#endif
 
   if (!vulkan_module)
   {
@@ -164,6 +180,7 @@ void UnloadVulkanLibrary()
   dlclose(vulkan_module);
   vulkan_module = nullptr;
 }
+
 #else
 
 //#warning Unknown platform, not compiling loader.

Source/Core/VideoBackends/Vulkan/VulkanLoader.h

@@ -15,6 +15,8 @@
 //#define VK_USE_PLATFORM_XCB_KHR
 #elif defined(ANDROID)
 #define VK_USE_PLATFORM_ANDROID_KHR
+#elif defined(__APPLE__)
+#define VK_USE_PLATFORM_MACOS_MVK
 #else
 //#warning Unknown platform
 #endif

Source/Core/VideoBackends/Vulkan/main.cpp

@@ -24,6 +24,10 @@
 #include "VideoCommon/VideoBackendBase.h"
 #include "VideoCommon/VideoConfig.h"
 
+#if defined(VK_USE_PLATFORM_MACOS_MVK)
+#include <objc/message.h>
+#endif
+
 namespace Vulkan
 {
 void VideoBackend::InitBackendInfo()
@@ -272,4 +276,33 @@ void VideoBackend::Shutdown()
   ShutdownShared();
   UnloadVulkanLibrary();
 }
+
+void VideoBackend::PrepareWindow(const WindowSystemInfo& wsi)
+{
+#if defined(VK_USE_PLATFORM_MACOS_MVK)
+  // This is kinda messy, but it avoids having to write Objective C++ just to create a metal layer.
+  id view = reinterpret_cast<id>(wsi.render_surface);
+  Class clsCAMetalLayer = objc_getClass("CAMetalLayer");
+  if (!clsCAMetalLayer)
+  {
+    ERROR_LOG(VIDEO, "Failed to get CAMetalLayer class.");
+    return;
+  }
+
+  // [CAMetalLayer layer]
+  id layer = reinterpret_cast<id (*)(Class, SEL)>(objc_msgSend)(objc_getClass("CAMetalLayer"),
+                                                                sel_getUid("layer"));
+  if (!layer)
+  {
+    ERROR_LOG(VIDEO, "Failed to create Metal layer.");
+    return;
+  }
+
+  // [view setWantsLayer:YES]
+  reinterpret_cast<void (*)(id, SEL, BOOL)>(objc_msgSend)(view, sel_getUid("setWantsLayer:"), YES);
+
+  // [view setLayer:layer]
+  reinterpret_cast<void (*)(id, SEL, id)>(objc_msgSend)(view, sel_getUid("setLayer:"), layer);
+#endif
+}
 }  // namespace Vulkan

Source/Core/VideoCommon/DriverDetails.h

@@ -61,6 +61,7 @@ enum Driver
   DRIVER_FREEDRENO,    // OSS Adreno driver
   DRIVER_IMGTEC,       // Official PowerVR driver
   DRIVER_VIVANTE,      // Official Vivante driver
+  DRIVER_PORTABILITY,  // Vulkan via Metal on macOS
   DRIVER_UNKNOWN       // Unknown driver, default to official hardware driver
 };
 
@@ -283,4 +284,4 @@ void Init(API api, Vendor vendor, Driver driver, const double version, const Fam
 // Once Vendor and driver version is set, this will return if it has the applicable bug passed to
 // it.
 bool HasBug(Bug bug);
-}
+}  // namespace DriverDetails

Source/Core/VideoCommon/VideoBackendBase.cpp

@@ -25,9 +25,7 @@
 #include "VideoBackends/Null/VideoBackend.h"
 #include "VideoBackends/OGL/VideoBackend.h"
 #include "VideoBackends/Software/VideoBackend.h"
-#ifndef __APPLE__
 #include "VideoBackends/Vulkan/VideoBackend.h"
-#endif
 
 #include "VideoCommon/AsyncRequests.h"
 #include "VideoCommon/BPStructs.h"
@@ -187,9 +185,7 @@ void VideoBackendBase::PopulateList()
 #ifdef _WIN32
   g_available_video_backends.push_back(std::make_unique<DX11::VideoBackend>());
 #endif
-#ifndef __APPLE__
   g_available_video_backends.push_back(std::make_unique<Vulkan::VideoBackend>());
-#endif
   g_available_video_backends.push_back(std::make_unique<SW::VideoSoftware>());
   g_available_video_backends.push_back(std::make_unique<Null::VideoBackend>());