diff --git a/app/src/main/java/emu/skyline/input/onscreen/OnScreenControllerView.kt b/app/src/main/java/emu/skyline/input/onscreen/OnScreenControllerView.kt
index f1e28c02..9d7901a9 100644
--- a/app/src/main/java/emu/skyline/input/onscreen/OnScreenControllerView.kt
+++ b/app/src/main/java/emu/skyline/input/onscreen/OnScreenControllerView.kt
@@ -68,7 +68,6 @@ class OnScreenControllerView @JvmOverloads constructor(context : Context, attrs
         }
 
     internal val editInfo = OnScreenEditInfo()
-    val isEditing get() = editInfo.isEditing
     val editButton get() = editInfo.editButton
     fun setOnEditButtonChangedListener(listener : OnEditButtonChangedListener?) {
         editInfo.onEditButtonChangedListener = listener
@@ -283,9 +282,12 @@ class OnScreenControllerView @JvmOverloads constructor(context : Context, attrs
         setOnTouchListener(playingTouchHandler)
     }
 
-    fun setEditMode(editMode : EditMode) {
-        editInfo.editMode = editMode
-        setOnTouchListener(if (isEditing) editingTouchHandler else playingTouchHandler)
+    fun setEditMode(isEdit : Boolean) {
+        // Select all buttons when entering edit if we weren't already editing
+        if (!editInfo.isEditing)
+            selectAllButtons()
+        editInfo.isEditing = isEdit
+        setOnTouchListener(if (isEdit) editingTouchHandler else playingTouchHandler)
     }
 
     fun selectAllButtons() {
diff --git a/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditActivity.kt b/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditActivity.kt
index 8df4ddc0..a0709d3f 100644
--- a/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditActivity.kt
+++ b/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditActivity.kt
@@ -133,8 +133,7 @@ class OnScreenEditActivity : AppCompatActivity() {
 
         binding.dragHandle.setOnTouchListener(dragPanelListener)
 
-        binding.onScreenControllerView.setEditMode(EditMode.Move)
-        binding.onScreenControllerView.selectAllButtons()
+        binding.onScreenControllerView.setEditMode(true)
     }
 
     override fun onResume() {
diff --git a/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditInfo.kt b/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditInfo.kt
index e9c53bd6..d73b8e30 100644
--- a/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditInfo.kt
+++ b/app/src/main/java/emu/skyline/input/onscreen/OnScreenEditInfo.kt
@@ -8,11 +8,6 @@ package emu.skyline.input.onscreen
 import android.content.res.Resources
 import android.util.TypedValue
 
-enum class EditMode {
-    None,
-    Move
-}
-
 typealias OnEditButtonChangedListener = ((ConfigurableButton) -> Unit)
 
 /**
@@ -21,9 +16,9 @@ typealias OnEditButtonChangedListener = ((ConfigurableButton) -> Unit)
  */
 class OnScreenEditInfo {
     /**
-     * The current edit mode
+     * Whether the buttons are currently in edit mode
      */
-    var editMode : EditMode = EditMode.None
+    var isEditing : Boolean = false
 
     /**
      * The button that is currently being edited
@@ -47,8 +42,6 @@ class OnScreenEditInfo {
 
     var arrowKeyMoveAmount : Int = ArrowKeyMoveAmount
 
-    val isEditing get() = editMode != EditMode.None
-
     companion object {
         /**
          * The size of the grid, calculated from the value of 8dp