diff --git a/Source/Core/InputCommon/GCAdapter.cpp b/Source/Core/InputCommon/GCAdapter.cpp
index c16b1ce635..1baaceee12 100644
--- a/Source/Core/InputCommon/GCAdapter.cpp
+++ b/Source/Core/InputCommon/GCAdapter.cpp
@@ -170,17 +170,8 @@ void Init()
 
   s_libusb_driver_not_supported = false;
 
-  s_libusb_context = LibusbContext::Get();
-  if (!s_libusb_context)
-  {
-    s_libusb_driver_not_supported = true;
-    Shutdown();
-  }
-  else
-  {
-    if (UseAdapter())
-      StartScanThread();
-  }
+  if (UseAdapter())
+    StartScanThread();
 }
 
 void StartScanThread()
@@ -188,6 +179,9 @@ void StartScanThread()
   if (s_adapter_detect_thread_running.IsSet())
     return;
 
+  s_libusb_context = LibusbContext::Get();
+  if (!s_libusb_context)
+    return;
   s_adapter_detect_thread_running.Set(true);
   s_adapter_detect_thread = std::thread(ScanThreadFunc);
 }
@@ -334,7 +328,7 @@ void Shutdown()
 {
   StopScanThread();
 #if defined(LIBUSB_API_VERSION) && LIBUSB_API_VERSION >= 0x01000102
-  if (s_libusb_hotplug_enabled)
+  if (s_libusb_context && s_libusb_hotplug_enabled)
     libusb_hotplug_deregister_callback(s_libusb_context.get(), s_hotplug_handle);
 #endif
   Reset();
diff --git a/Source/Core/UICommon/UICommon.cpp b/Source/Core/UICommon/UICommon.cpp
index 8de73bd151..0842d511bf 100644
--- a/Source/Core/UICommon/UICommon.cpp
+++ b/Source/Core/UICommon/UICommon.cpp
@@ -30,7 +30,6 @@ void Init()
   VideoBackendBase::PopulateList();
   WiimoteReal::LoadSettings();
   GCAdapter::Init();
-  USBUtils::Init();
   VideoBackendBase::ActivateBackend(SConfig::GetInstance().m_strVideoBackend);
 
   SetEnableAlert(SConfig::GetInstance().bUsePanicHandlers);
@@ -42,7 +41,6 @@ void Shutdown()
   WiimoteReal::Shutdown();
   VideoBackendBase::ClearList();
   SConfig::Shutdown();
-  USBUtils::Shutdown();
   LogManager::Shutdown();
 }
 
diff --git a/Source/Core/UICommon/USBUtils.cpp b/Source/Core/UICommon/USBUtils.cpp
index da7c603580..4fbabda856 100644
--- a/Source/Core/UICommon/USBUtils.cpp
+++ b/Source/Core/UICommon/USBUtils.cpp
@@ -3,6 +3,7 @@
 // Refer to the license.txt file included.
 
 #include <memory>
+#include <mutex>
 
 #ifdef __LIBUSB__
 #include <libusb.h>
@@ -14,6 +15,7 @@
 #include "UICommon/USBUtils.h"
 
 #ifdef __LIBUSB__
+static std::once_flag s_tried_libusb_init;
 static std::shared_ptr<libusb_context> s_libusb_context;
 #endif
 
@@ -36,25 +38,12 @@ static const std::map<std::pair<u16, u16>, std::string> s_wii_peripherals = {{
 
 namespace USBUtils
 {
-void Init()
-{
-#ifdef __LIBUSB__
-  s_libusb_context = LibusbContext::Get();
-#endif
-}
-
-void Shutdown()
-{
-#ifdef __LIBUSB__
-  s_libusb_context = nullptr;
-#endif
-}
-
 std::map<std::pair<u16, u16>, std::string> GetInsertedDevices()
 {
   std::map<std::pair<u16, u16>, std::string> devices;
 
 #ifdef __LIBUSB__
+  std::call_once(s_tried_libusb_init, []() { s_libusb_context = LibusbContext::Get(); });
   if (!s_libusb_context)
     return devices;
 
diff --git a/Source/Core/UICommon/USBUtils.h b/Source/Core/UICommon/USBUtils.h
index 6ae778e8a4..84d54a5381 100644
--- a/Source/Core/UICommon/USBUtils.h
+++ b/Source/Core/UICommon/USBUtils.h
@@ -12,9 +12,6 @@
 
 namespace USBUtils
 {
-void Init();
-void Shutdown();
-
 std::map<std::pair<u16, u16>, std::string> GetInsertedDevices();
 std::string GetDeviceName(std::pair<u16, u16> vid_pid);
 }  // namespace USBUtils