Revert "Prevent view from being removed if a to is found (#5135) (#5152)"

This reverts commit 974275a429.

app/src/main/java/eu/kanade/tachiyomi/ui/base/controller/OneWayFadeChangeHandler.kt

@@ -31,12 +31,11 @@ class OneWayFadeChangeHandler : FadeChangeHandler {
         if (to != null) {
             val start: Float = if (toAddedToContainer) 0F else to.alpha
             animator.play(ObjectAnimator.ofFloat(to, View.ALPHA, start, 1f))
-        } else {
+        }
+
         if (from != null && (!isPush || removesFromViewOnPush())) {
             container.removeView(from)
         }
-        }
-
         return animator
     }