Even more optimizing to Recents

Ungrouped/Grouped are now a single db call using union

Also this only confirmed my hate of SQL and especially SQLite

Anyway this should close #726

app/src/main/java/eu/kanade/tachiyomi/data/database/queries/HistoryQueries.kt

@@ -8,7 +8,6 @@ import eu.kanade.tachiyomi.data.database.models.MangaChapterHistory
 import eu.kanade.tachiyomi.data.database.resolvers.HistoryLastReadPutResolver
 import eu.kanade.tachiyomi.data.database.resolvers.MangaChapterHistoryGetResolver
 import eu.kanade.tachiyomi.data.database.tables.HistoryTable
-import eu.kanade.tachiyomi.data.database.tables.MangaTable
 import eu.kanade.tachiyomi.util.lang.sqLite
 
 interface HistoryQueries : DbProvider {
@@ -36,23 +35,6 @@ interface HistoryQueries : DbProvider {
 //        .withGetResolver(MangaChapterHistoryGetResolver.INSTANCE)
 //        .prepare()
 
-    /**
-     * Returns history of recent manga containing last read chapter in 25s
-     * @param date recent date range
-     * @offset offset the db by
-     */
-    fun getRecentlyAdded(search: String = "", endless: Boolean, offset: Int, isResuming: Boolean) = db.get()
-        .listOfObjects(MangaChapterHistory::class.java)
-        .withQuery(
-            RawQuery.builder()
-                .query(getRecentAdditionsQuery(search.sqLite, endless, offset, isResuming))
-//                .args(date.time, startDate.time)
-                .observesTables(MangaTable.TABLE)
-                .build()
-        )
-        .withGetResolver(MangaChapterHistoryGetResolver.INSTANCE)
-        .prepare()
-
     /**
      * Returns history of recent manga containing last read chapter in 25s
      * @param date recent date range
@@ -75,12 +57,12 @@ interface HistoryQueries : DbProvider {
      * @param date recent date range
      * @offset offset the db by
      */
-    fun getAllRecents(search: String = "", includeRead: Boolean, endless: Boolean, offset: Int, isResuming: Boolean) = db.get()
+    fun getAllRecentsTypes(search: String = "", includeRead: Boolean, endless: Boolean, offset: Int, isResuming: Boolean) = db.get()
         .listOfObjects(MangaChapterHistory::class.java)
         .withQuery(
             RawQuery.builder()
                 .query(
-                    getRecentRead(
+                    getAllRecentsType(
                         search.sqLite,
                         includeRead,
                         endless,

app/src/main/java/eu/kanade/tachiyomi/data/database/queries/RawQueries.kt

@@ -67,9 +67,9 @@ fun getRecentAdditionsQuery(search: String, endless: Boolean, offset: Int, isRes
 
 fun limitAndOffset(endless: Boolean, isResuming: Boolean, offset: Int): String {
     return when {
-        isResuming && endless -> "LIMIT $offset"
+        isResuming && endless && offset > 0 -> "LIMIT $offset"
         endless -> "LIMIT ${RecentsPresenter.ENDLESS_LIMIT}\nOFFSET $offset"
-        else -> "LIMIT 8"
+        else -> "LIMIT 12"
     }
 }
 
@@ -134,42 +134,87 @@ fun getRecentMangasLimitQuery(
  * The max_last_read table contains the most recent chapters grouped by manga
  * The select statement returns all information of chapters that have the same id as the chapter in max_last_read
  * and are read after the given time period
+ * The Second Union/Select gets recents chapters
+ * Final Union gets newly added manga
  */
-fun getRecentRead(
+fun getAllRecentsType(
     search: String = "",
     includeRead: Boolean,
     endless: Boolean,
     offset: Int = 0,
     isResuming: Boolean
-) =
+) = """
-    """
+	SELECT * FROM
-    SELECT ${Manga.TABLE}.${Manga.COL_URL} as mangaUrl, ${Manga.TABLE}.*, ${Chapter.TABLE}.*, ${History.TABLE}.*
+	(SELECT mangas.url as mangaUrl, mangas.*, chapters.*, history.*
     FROM (
-        SELECT ${Manga.TABLE}.*
+        SELECT mangas.*
-        FROM ${Manga.TABLE}
+        FROM mangas
         LEFT JOIN (
-            SELECT ${Chapter.COL_MANGA_ID}, COUNT(*) AS unread
+            SELECT manga_id, COUNT(*) AS unread
-            FROM ${Chapter.TABLE}
+            FROM chapters
-            GROUP BY ${Chapter.COL_MANGA_ID}
+            WHERE read = 0
+            GROUP BY manga_id
         ) AS C
-        ON ${Manga.COL_ID} = C.${Chapter.COL_MANGA_ID}
+        ON _id = C.manga_id
         ${if (includeRead) "" else "WHERE C.unread > 0"}
-        GROUP BY ${Manga.COL_ID}
+        GROUP BY _id
-        ORDER BY ${Manga.COL_TITLE}
+        ORDER BY title
-    ) AS ${Manga.TABLE}
+    ) AS mangas
+    JOIN chapters
+    ON mangas._id = chapters.manga_id
+    JOIN history
+    ON chapters._id = history.history_chapter_id
+    JOIN (
+        SELECT chapters.manga_id,chapters._id as history_chapter_id, MAX(history.history_last_read) as history_last_read
+        FROM chapters JOIN history
+        ON chapters._id = history.history_chapter_id
+        GROUP BY chapters.manga_id) AS max_last_read
+    ON chapters.manga_id = max_last_read.manga_id
+    AND max_last_read.history_chapter_id = history.history_chapter_id
+    AND lower(mangas.title) LIKE '%$search%')
+	UNION
+	SELECT * FROM
+	(SELECT ${Manga.TABLE}.${Manga.COL_URL} as mangaUrl, ${Manga.TABLE}.*, ${Chapter.TABLE}.*, 
+        Null as history_id, 
+        Null as history_chapter_id, 
+        chapters.date_fetch as history_last_read, 
+        Null as history_time_read
+    FROM ${Manga.TABLE}
     JOIN ${Chapter.TABLE}
     ON ${Manga.TABLE}.${Manga.COL_ID} = ${Chapter.TABLE}.${Chapter.COL_MANGA_ID}
-    JOIN ${History.TABLE}
-    ON ${Chapter.TABLE}.${Chapter.COL_ID} = ${History.TABLE}.${History.COL_CHAPTER_ID}
     JOIN (
-        SELECT ${Chapter.TABLE}.${Chapter.COL_MANGA_ID},${Chapter.TABLE}.${Chapter.COL_ID} as ${History.COL_CHAPTER_ID}, MAX(${History.TABLE}.${History.COL_LAST_READ}) as ${History.COL_LAST_READ}
+        SELECT ${Chapter.TABLE}.${Chapter.COL_MANGA_ID},${Chapter.TABLE}.${Chapter.COL_ID} as ${History.COL_CHAPTER_ID},MAX(${Chapter.TABLE}.${Chapter.COL_DATE_UPLOAD}) 
-        FROM ${Chapter.TABLE} JOIN ${History.TABLE}
+        FROM ${Chapter.TABLE} JOIN ${Manga.TABLE}
-        ON ${Chapter.TABLE}.${Chapter.COL_ID} = ${History.TABLE}.${History.COL_CHAPTER_ID}
+        ON ${Manga.TABLE}.${Manga.COL_ID} = ${Chapter.TABLE}.${Chapter.COL_MANGA_ID}
-        GROUP BY ${Chapter.TABLE}.${Chapter.COL_MANGA_ID}) AS max_last_read
+        WHERE ${Chapter.COL_READ} = 0
-    ON ${Chapter.TABLE}.${Chapter.COL_MANGA_ID} = max_last_read.${Chapter.COL_MANGA_ID}
+        GROUP BY ${Chapter.TABLE}.${Chapter.COL_MANGA_ID}) AS newest_chapter
-    AND max_last_read.${History.COL_CHAPTER_ID} = ${History.TABLE}.${History.COL_CHAPTER_ID}
+    ON ${Chapter.TABLE}.${Chapter.COL_MANGA_ID} = newest_chapter.${Chapter.COL_MANGA_ID}
-    AND lower(${Manga.TABLE}.${Manga.COL_TITLE}) LIKE '%$search%'
+    WHERE ${Manga.COL_FAVORITE} = 1
-    ORDER BY max_last_read.${History.COL_LAST_READ} DESC
+    AND newest_chapter.${History.COL_CHAPTER_ID} = ${Chapter.TABLE}.${Chapter.COL_ID}
+    AND ${Chapter.COL_DATE_FETCH} > ${Manga.COL_DATE_ADDED}
+    AND lower(${Manga.COL_TITLE}) LIKE '%$search%')
+    UNION
+    SELECT * FROM
+    (SELECT mangas.url as mangaUrl, 
+        mangas.*,
+		Null as _id,
+		Null as manga_id,
+		Null as url,
+		Null as name,
+		Null as read,
+		Null as scanlator,
+		Null as bookmark,
+		Null as date_fetch,
+		Null as date_upload,
+		Null as last_page_read,
+		Null as pages_left,
+		Null as chapter_number,
+		Null as source_order,
+		Null as history_id, Null as history_chapter_id, mangas.date_added as history_last_read, Null as history_time_read
+		FROM mangas
+    WHERE ${Manga.COL_FAVORITE} = 1
+    AND lower(${Manga.COL_TITLE}) LIKE '%$search%')
+    ORDER BY history_last_read DESC
     ${limitAndOffset(endless, isResuming, offset)}
 """
 

app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaChapterHistoryGetResolver.kt

@@ -49,7 +49,13 @@ class MangaChapterHistoryGetResolver : DefaultGetResolver<MangaChapterHistory>()
         val history =
             if (!cursor.isNull(cursor.getColumnIndex(HistoryTable.COL_ID))) historyGetResolver.mapFromCursor(
                 cursor
-            ) else HistoryImpl()
+            ) else HistoryImpl().apply {
+                last_read = try {
+                    cursor.getLong(cursor.getColumnIndex(HistoryTable.COL_LAST_READ))
+                } catch (e: Exception) {
+                    0L
+                }
+            }
 
         // Make certain column conflicts are dealt with
         if (chapter.id != null) {

app/src/main/java/eu/kanade/tachiyomi/ui/recents/RecentsPresenter.kt

@@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.data.library.LibraryServiceListener
 import eu.kanade.tachiyomi.data.library.LibraryUpdateService
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.source.SourceManager
+import eu.kanade.tachiyomi.util.system.executeOnIO
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
@@ -57,27 +58,19 @@ class RecentsPresenter(
     var heldItems: HashMap<Int, List<RecentMangaItem>> = hashMapOf()
     private var shouldMoveToTop = false
     var viewType: Int = preferences.recentsViewType().get()
-        set(value) {
-            field = value
-            ENDLESS_LIMIT = if (value == VIEW_TYPE_UNGROUP_ALL) 25 else 50
-        }
 
     private fun resetOffsets() {
         finished = false
         shouldMoveToTop = true
-        updatesOffset = 0
+        pageOffset = 0
-        historyOffset = 0
-        additionsOffset = 0
     }
 
-    private var updatesOffset = 0
+    private var pageOffset = 0
-    private var historyOffset = 0
-    private var additionsOffset = 0
     var isLoading = false
         private set
 
     private val isOnFirstPage: Boolean
-        get() = additionsOffset + historyOffset + updatesOffset == 0
+        get() = pageOffset == 0
 
     init {
         preferences.showReadInAllRecents()
@@ -131,82 +124,55 @@ class RecentsPresenter(
         }
         val viewType = customViewType ?: viewType
 
-//        Timber.d("starting up items new page: $updatePageCount")
         val showRead = preferences.showReadInAllRecents().get() && !limit
         val isUngrouped = viewType > VIEW_TYPE_GROUP_ALL && query.isEmpty()
 
         val isCustom = customViewType != null
         val isEndless = isUngrouped && !limit
-//        Timber.d("set up cal items")
+        val cReading = when {
-        val (cReading, rUpdates, nAdditions) = db.inTransactionReturn {
+            query.isNotEmpty() || viewType <= VIEW_TYPE_UNGROUP_ALL -> {
-            val cReading = if (viewType != VIEW_TYPE_ONLY_UPDATES) {
+                db.getAllRecentsTypes(
-                if (query.isEmpty() && viewType != VIEW_TYPE_ONLY_HISTORY) {
-                    db.getAllRecents(
-                        query,
-                        showRead,
-                        isEndless,
-                        if (isCustom) ENDLESS_LIMIT else historyOffset,
-                        !updatePageCount && !isOnFirstPage
-                    )
-                } else {
-                    db.getRecentMangaLimit(
-                        query,
-                        isEndless,
-                        if (isCustom) ENDLESS_LIMIT else historyOffset,
-                        !updatePageCount && !isOnFirstPage
-                    )
-                }.executeAsBlocking()
-            } else emptyList()
-//            Timber.d("set up cReader items: ${cReading.size}")
-            val rUpdates = when {
-                viewType == VIEW_TYPE_ONLY_UPDATES -> db.getRecentChapters(
                     query,
-                    if (isCustom) ENDLESS_LIMIT else updatesOffset,
+                    showRead,
+                    isEndless,
+                    if (isCustom) ENDLESS_LIMIT else pageOffset,
                     !updatePageCount && !isOnFirstPage
-                ).executeAsBlocking().map {
+                ).executeOnIO()
-                    MangaChapterHistory(it.manga, it.chapter, HistoryImpl())
+            }
+            viewType == VIEW_TYPE_ONLY_HISTORY -> {
+                db.getRecentMangaLimit(
+                    query,
+                    isEndless,
+                    if (isCustom) ENDLESS_LIMIT else pageOffset,
+                    !updatePageCount && !isOnFirstPage
+                ).executeOnIO()
+            }
+            viewType == VIEW_TYPE_ONLY_UPDATES -> {
+                db.getRecentChapters(
+                    query,
+                    if (isCustom) ENDLESS_LIMIT else pageOffset,
+                    !updatePageCount && !isOnFirstPage
+                ).executeOnIO().map {
+                    MangaChapterHistory(
+                        it.manga,
+                        it.chapter,
+                        HistoryImpl().apply {
+                            last_read = it.chapter.date_fetch
+                        }
+                    )
                 }
-                viewType != VIEW_TYPE_ONLY_HISTORY -> db.getUpdatedManga(
-                    query,
-                    isEndless,
-                    if (isCustom) ENDLESS_LIMIT else updatesOffset,
-                    !updatePageCount && !isOnFirstPage
-                ).executeAsBlocking()
-                else -> emptyList()
             }
-            rUpdates.forEach {
+            else -> emptyList()
-                it.history.last_read = it.chapter.date_fetch
-            }
-//            Timber.d("set up rUpdates items: ${rUpdates.size}")
-            val nAdditions = if (viewType < VIEW_TYPE_ONLY_HISTORY) {
-                db.getRecentlyAdded(
-                    query,
-                    isEndless,
-                    if (isCustom) ENDLESS_LIMIT else additionsOffset,
-                    !updatePageCount && !isOnFirstPage
-                ).executeAsBlocking()
-            } else emptyList()
-            nAdditions.forEach {
-                it.history.last_read = it.manga.date_added
-            }
-//            Timber.d("set up nAdditons items: ${nAdditions.size}")
-            Triple(cReading, rUpdates, nAdditions)
         }
 
         if (!isCustom &&
-            (historyOffset + updatesOffset + additionsOffset == 0 || updatePageCount)
+            (pageOffset == 0 || updatePageCount)
         ) {
-            additionsOffset += nAdditions.size
+            pageOffset += cReading.size
-            historyOffset += cReading.size
-            updatesOffset += rUpdates.size
         }
 
-//        Timber.d("loaded items: ")
         if (query != oldQuery) return
-//        Timber.d("query matches items: ")
+        val mangaList = cReading.distinctBy {
-        val mangaList = (cReading + rUpdates + nAdditions).sortedByDescending {
-            it.history.last_read
-        }.distinctBy {
             if (query.isEmpty() && viewType != VIEW_TYPE_ONLY_HISTORY && viewType != VIEW_TYPE_ONLY_UPDATES) it.manga.id else it.chapter.id
         }.filter { mch ->
             if (updatePageCount && !isOnFirstPage && query.isEmpty()) {
@@ -232,7 +198,6 @@ class RecentsPresenter(
             else null
             else Pair(it, chapter)
         }
-//        Timber.d("setting new items")
         val newItems = if (query.isEmpty() && !isUngrouped) {
             val nChaptersItems =
                 pairs.asSequence()
@@ -289,7 +254,6 @@ class RecentsPresenter(
                 }
             } else pairs.map { RecentMangaItem(it.first, it.second, null) }
         }
-//        Timber.d("setting some items")
         if (customViewType == null) {
             recentItems = if (isOnFirstPage || !updatePageCount) {
                 newItems
@@ -302,7 +266,6 @@ class RecentsPresenter(
         val newCount = itemCount + newItems.size
         val hasNewItems = newItems.isNotEmpty()
         if (updatePageCount && newCount < 25 && viewType != VIEW_TYPE_GROUP_ALL && query.isEmpty() && !limit) {
-            // Timber.d("needs to retry. has New items: $hasNewItems, offset: $subUpdatesOffset")
             runRecents(oldQuery, true, retryCount + (if (hasNewItems) 0 else 1), newCount)
             return
         }
@@ -314,15 +277,6 @@ class RecentsPresenter(
                     isLoading = false
                     shouldMoveToTop = false
                 }
-//                scope.launchIO {
-//                    (0..3).map {
-//                        async {
-//                            if (this@RecentsPresenter.viewType != it) {
-//                                runRecents(oldQuery, customViewType = it)
-//                            }
-//                        }
-//                    }.awaitAll()
-//                }
             }
         }
     }
@@ -507,7 +461,7 @@ class RecentsPresenter(
         const val VIEW_TYPE_UNGROUP_ALL = 1
         const val VIEW_TYPE_ONLY_HISTORY = 2
         const val VIEW_TYPE_ONLY_UPDATES = 3
-        var ENDLESS_LIMIT = 50
+        const val ENDLESS_LIMIT = 50
 
         suspend fun getRecentManga(): List<Pair<Manga, Long>> {
             val presenter = RecentsPresenter(null)