diff --git a/Source/Core/InputCommon/ControllerInterface/ControllerInterface.cpp b/Source/Core/InputCommon/ControllerInterface/ControllerInterface.cpp
index 8f7bed3af1..9c272fdec4 100644
--- a/Source/Core/InputCommon/ControllerInterface/ControllerInterface.cpp
+++ b/Source/Core/InputCommon/ControllerInterface/ControllerInterface.cpp
@@ -128,7 +128,7 @@ void ControllerInterface::RefreshDevices(RefreshReason reason)
   // or removing them as we are populating them (causing missing or duplicate devices).
   std::lock_guard lk_population(m_devices_population_mutex);
 
-#if defined(CIFACE_USE_WIN32) && !defined(CIFACE_USE_XLIB) && !defined(CIFACE_USE_OSX)
+#if defined(CIFACE_USE_WIN32) && !defined(CIFACE_USE_OSX)
   // If only the window changed, avoid removing and re-adding all devices.
   // Instead only refresh devices that require the window handle.
   if (reason == RefreshReason::WindowChangeOnly)
diff --git a/Source/Core/InputCommon/ControllerInterface/Xlib/XInput2.cpp b/Source/Core/InputCommon/ControllerInterface/Xlib/XInput2.cpp
index 5de0840354..e4b717e3af 100644
--- a/Source/Core/InputCommon/ControllerInterface/Xlib/XInput2.cpp
+++ b/Source/Core/InputCommon/ControllerInterface/Xlib/XInput2.cpp
@@ -66,11 +66,14 @@ constexpr int XINPUT_MAJOR = 2, XINPUT_MINOR = 1;
 
 namespace ciface::XInput2
 {
+constexpr std::string_view SOURCE_NAME = "XInput2";
+
 class InputBackend final : public ciface::InputBackend
 {
 public:
   using ciface::InputBackend::InputBackend;
   void PopulateDevices() override;
+  void HandleWindowChange() override;
 };
 
 std::unique_ptr<ciface::InputBackend> CreateInputBackend(ControllerInterface* controller_interface)
@@ -78,6 +81,14 @@ std::unique_ptr<ciface::InputBackend> CreateInputBackend(ControllerInterface* co
   return std::make_unique<InputBackend>(controller_interface);
 }
 
+void InputBackend::HandleWindowChange()
+{
+  GetControllerInterface().RemoveDevice(
+      [](const auto* dev) { return dev->GetSource() == SOURCE_NAME; }, true);
+
+  PopulateDevices();
+}
+
 // This function will add zero or more KeyboardMouse objects to devices.
 void InputBackend::PopulateDevices()
 {
@@ -400,7 +411,7 @@ std::string KeyboardMouse::GetName() const
 
 std::string KeyboardMouse::GetSource() const
 {
-  return "XInput2";
+  return std::string(SOURCE_NAME);
 }
 
 KeyboardMouse::Key::Key(Display* const display, KeyCode keycode, const char* keyboard)