diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
index 9028c5c0cf..c0cd19508a 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
@@ -298,10 +298,6 @@ class LibraryController(
         recycler.setHasFixedSize(true)
         recycler.adapter = adapter
 
-        fast_scroller.addOnScrollStateChangeListener {
-            swipe_refresh.isEnabled = !it
-        }
-
         adapter.fastScroller = fast_scroller
         recycler.addOnScrollListener(scrollListener)
 
diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/MaterialFastScroll.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/MaterialFastScroll.kt
index 786a6c9025..573fbbec41 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/MaterialFastScroll.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/MaterialFastScroll.kt
@@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.ui.library
 import android.content.Context
 import android.util.AttributeSet
 import android.view.MotionEvent
+import androidx.core.view.ViewCompat
 import androidx.recyclerview.widget.LinearLayoutManager
 import androidx.recyclerview.widget.RecyclerView
 import androidx.recyclerview.widget.StaggeredGridLayoutManager
@@ -15,18 +16,59 @@ import kotlin.math.abs
 class MaterialFastScroll @JvmOverloads constructor(context: Context, attrs: AttributeSet? = null) :
     FastScroller(context, attrs) {
 
+    var canScroll = false
+    var startY = 0f
     var scrollOffset = 0
     init {
         setViewsToUse(
             R.layout.material_fastscroll, R.id.fast_scroller_bubble, R.id.fast_scroller_handle
         )
         autoHideEnabled = true
-        ignoreTouchesOutsideHandle = true
+        ignoreTouchesOutsideHandle = false
         updateScrollListener()
     }
 
+    // Overriding to force a distance moved before scrolling
     override fun onTouchEvent(event: MotionEvent): Boolean {
-        if (isHidden) return false
+        if (recyclerView.computeVerticalScrollRange() <= recyclerView.computeVerticalScrollExtent()) {
+            return super.onTouchEvent(event)
+        }
+
+        when (event.action) {
+            MotionEvent.ACTION_DOWN -> {
+                if (event.x < handle.x - ViewCompat.getPaddingStart(handle)) return false
+                val y = event.y
+                startY = event.y
+                if (canScroll) {
+                    handle.isSelected = true
+                    notifyScrollStateChange(true)
+                    showBubble()
+                    showScrollbar()
+                    setBubbleAndHandlePosition(y)
+                    setRecyclerViewPosition(y)
+                }
+                return true
+            }
+            MotionEvent.ACTION_MOVE -> {
+                val y = event.y
+                if (!canScroll && abs(y - startY) > 10) {
+                    canScroll = true
+                    handle.isSelected = true
+                    notifyScrollStateChange(true)
+                    showBubble()
+                    showScrollbar()
+                }
+                if (canScroll) {
+                    setBubbleAndHandlePosition(y)
+                    setRecyclerViewPosition(y)
+                }
+                return true
+            }
+            MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
+                startY = 0f
+                canScroll = false
+            }
+        }
         return super.onTouchEvent(event)
     }
 
diff --git a/app/src/main/res/layout/library_list_controller.xml b/app/src/main/res/layout/library_list_controller.xml
index e926e7c88b..99e8a46a6e 100644
--- a/app/src/main/res/layout/library_list_controller.xml
+++ b/app/src/main/res/layout/library_list_controller.xml
@@ -51,12 +51,6 @@
 
             <include layout="@layout/library_grid_recycler" />
 
-            <eu.kanade.tachiyomi.ui.library.MaterialFastScroll
-                android:id="@+id/fast_scroller"
-                android:layout_width="match_parent"
-                android:layout_height="match_parent"
-                app:fastScrollerBubbleEnabled="true" />
-
             <View
                 android:id="@+id/recycler_cover"
                 android:layout_width="match_parent"
@@ -67,6 +61,12 @@
 
     </androidx.swiperefreshlayout.widget.SwipeRefreshLayout>
 
+    <eu.kanade.tachiyomi.ui.library.MaterialFastScroll
+        android:id="@+id/fast_scroller"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        app:fastScrollerBubbleEnabled="true" />
+
     <eu.kanade.tachiyomi.widget.EmptyView
         android:id="@+id/empty_view"
         android:layout_width="wrap_content"
diff --git a/app/src/main/res/layout/material_fastscroll.xml b/app/src/main/res/layout/material_fastscroll.xml
index 4b83ba20d1..971bacf2e9 100644
--- a/app/src/main/res/layout/material_fastscroll.xml
+++ b/app/src/main/res/layout/material_fastscroll.xml
@@ -32,7 +32,7 @@
             android:maxLines="5"
             android:visibility="gone"
             app:layout_constraintHorizontal_bias="1.0"
-            tools:text="sdfs fas d fsA"
+            tools:text="Tool tip text"
             tools:visibility="visible"/>
 
         <!-- Padding is here to have better grab -->
@@ -45,7 +45,7 @@
             app:layout_constraintTop_toTopOf="parent"
             app:layout_constraintBottom_toBottomOf="parent"
             app:layout_constraintEnd_toEndOf="parent"
-            android:paddingStart="6dp"
+            android:paddingStart="2dp"
             android:paddingEnd="4dp"
             android:src="@drawable/thumb_drawable"/>