From 120943a8b37eaf847ca1073676a8293288c28e12 Mon Sep 17 00:00:00 2001 From: AntsyLich <59261191+AntsyLich@users.noreply.github.com> Date: Sat, 11 Jun 2022 21:38:39 +0600 Subject: [PATCH] Make `syncChaptersWithSource` use sqldelight (#7263) * Make `syncChaptersWithSource` use sqldelight Will break chapter list live update on current ui Co-Authored-By: Ivan Iskandar <12537387+ivaniskandar@users.noreply.github.com> * Review Changes Co-authored-by: Ivan Iskandar <12537387+ivaniskandar@users.noreply.github.com> --- .../data/chapter/ChapterRepositoryImpl.kt | 70 +++++++ .../kanade/data/manga/MangaRepositoryImpl.kt | 8 + .../java/eu/kanade/domain/DomainModule.kt | 6 + .../interactor/ShouldUpdateDbChapter.kt | 13 ++ .../interactor/SyncChaptersWithSource.kt | 179 ++++++++++++++++++ .../eu/kanade/domain/chapter/model/Chapter.kt | 62 +++++- .../domain/chapter/model/ChapterUpdate.kt | 4 + .../chapter/repository/ChapterRepository.kt | 9 + .../manga/interactor/UpdateMangaLastUpdate.kt | 12 ++ .../eu/kanade/domain/manga/model/Manga.kt | 28 +++ .../manga/repository/MangaRepository.kt | 2 + .../data/backup/AbstractBackupManager.kt | 2 +- .../tachiyomi/data/database/models/Manga.kt | 24 +++ .../data/library/LibraryUpdateService.kt | 2 +- .../tachiyomi/source/online/HttpSource.kt | 3 +- .../migration/search/SearchPresenter.kt | 2 +- .../tachiyomi/ui/manga/MangaPresenter.kt | 2 +- .../util/chapter/ChapterSourceSync.kt | 168 ++-------------- app/src/main/sqldelight/data/chapters.sq | 37 +++- app/src/main/sqldelight/data/mangas.sq | 7 +- 20 files changed, 478 insertions(+), 162 deletions(-) create mode 100644 app/src/main/java/eu/kanade/domain/chapter/interactor/ShouldUpdateDbChapter.kt create mode 100644 app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt create mode 100644 app/src/main/java/eu/kanade/domain/manga/interactor/UpdateMangaLastUpdate.kt diff --git a/app/src/main/java/eu/kanade/data/chapter/ChapterRepositoryImpl.kt b/app/src/main/java/eu/kanade/data/chapter/ChapterRepositoryImpl.kt index 1fa5d9fd23..58fb236cc5 100644 --- a/app/src/main/java/eu/kanade/data/chapter/ChapterRepositoryImpl.kt +++ b/app/src/main/java/eu/kanade/data/chapter/ChapterRepositoryImpl.kt @@ -2,6 +2,7 @@ package eu.kanade.data.chapter import eu.kanade.data.DatabaseHandler import eu.kanade.data.toLong +import eu.kanade.domain.chapter.model.Chapter import eu.kanade.domain.chapter.model.ChapterUpdate import eu.kanade.domain.chapter.repository.ChapterRepository import eu.kanade.tachiyomi.util.system.logcat @@ -11,6 +12,33 @@ class ChapterRepositoryImpl( private val handler: DatabaseHandler, ) : ChapterRepository { + override suspend fun addAll(chapters: List): List { + return try { + handler.await(inTransaction = true) { + chapters.map { chapter -> + chaptersQueries.insert( + chapter.mangaId, + chapter.url, + chapter.name, + chapter.scanlator, + chapter.read, + chapter.bookmark, + chapter.lastPageRead, + chapter.chapterNumber, + chapter.sourceOrder, + chapter.dateFetch, + chapter.dateUpload, + ) + val lastInsertId = chaptersQueries.selectLastInsertedRowId().executeAsOne() + chapter.copy(id = lastInsertId) + } + } + } catch (e: Exception) { + logcat(LogPriority.ERROR, e) + emptyList() + } + } + override suspend fun update(chapterUpdate: ChapterUpdate) { try { handler.await { @@ -33,4 +61,46 @@ class ChapterRepositoryImpl( logcat(LogPriority.ERROR, e) } } + + override suspend fun updateAll(chapterUpdates: List) { + try { + handler.await(inTransaction = true) { + chapterUpdates.forEach { chapterUpdate -> + chaptersQueries.update( + chapterUpdate.mangaId, + chapterUpdate.url, + chapterUpdate.name, + chapterUpdate.scanlator, + chapterUpdate.read?.toLong(), + chapterUpdate.bookmark?.toLong(), + chapterUpdate.lastPageRead, + chapterUpdate.chapterNumber?.toDouble(), + chapterUpdate.sourceOrder, + chapterUpdate.dateFetch, + chapterUpdate.dateUpload, + chapterId = chapterUpdate.id, + ) + } + } + } catch (e: Exception) { + logcat(LogPriority.ERROR, e) + } + } + + override suspend fun removeChaptersWithIds(chapterIds: List) { + try { + handler.await { chaptersQueries.removeChaptersWithIds(chapterIds) } + } catch (e: Exception) { + logcat(LogPriority.ERROR, e) + } + } + + override suspend fun getChapterByMangaId(mangaId: Long): List { + return try { + handler.awaitList { chaptersQueries.getChapterByMangaId(mangaId, chapterMapper) } + } catch (e: Exception) { + logcat(LogPriority.ERROR, e) + emptyList() + } + } } diff --git a/app/src/main/java/eu/kanade/data/manga/MangaRepositoryImpl.kt b/app/src/main/java/eu/kanade/data/manga/MangaRepositoryImpl.kt index f5d0788f65..addc238640 100644 --- a/app/src/main/java/eu/kanade/data/manga/MangaRepositoryImpl.kt +++ b/app/src/main/java/eu/kanade/data/manga/MangaRepositoryImpl.kt @@ -24,4 +24,12 @@ class MangaRepositoryImpl( false } } + + override suspend fun updateLastUpdate(mangaId: Long, lastUpdate: Long) { + try { + handler.await { mangasQueries.updateLastUpdate(lastUpdate, mangaId) } + } catch (e: Exception) { + logcat(LogPriority.ERROR, e) + } + } } diff --git a/app/src/main/java/eu/kanade/domain/DomainModule.kt b/app/src/main/java/eu/kanade/domain/DomainModule.kt index 3e5d483a3c..9d7023a06b 100644 --- a/app/src/main/java/eu/kanade/domain/DomainModule.kt +++ b/app/src/main/java/eu/kanade/domain/DomainModule.kt @@ -4,6 +4,8 @@ import eu.kanade.data.chapter.ChapterRepositoryImpl import eu.kanade.data.history.HistoryRepositoryImpl import eu.kanade.data.manga.MangaRepositoryImpl import eu.kanade.data.source.SourceRepositoryImpl +import eu.kanade.domain.chapter.interactor.ShouldUpdateDbChapter +import eu.kanade.domain.chapter.interactor.SyncChaptersWithSource import eu.kanade.domain.chapter.interactor.UpdateChapter import eu.kanade.domain.chapter.repository.ChapterRepository import eu.kanade.domain.extension.interactor.GetExtensionLanguages @@ -19,6 +21,7 @@ import eu.kanade.domain.history.interactor.UpsertHistory import eu.kanade.domain.history.repository.HistoryRepository import eu.kanade.domain.manga.interactor.GetFavoritesBySourceId import eu.kanade.domain.manga.interactor.ResetViewerFlags +import eu.kanade.domain.manga.interactor.UpdateMangaLastUpdate import eu.kanade.domain.manga.repository.MangaRepository import eu.kanade.domain.source.interactor.GetEnabledSources import eu.kanade.domain.source.interactor.GetLanguagesWithSources @@ -42,9 +45,12 @@ class DomainModule : InjektModule { addFactory { GetFavoritesBySourceId(get()) } addFactory { GetNextChapter(get()) } addFactory { ResetViewerFlags(get()) } + addFactory { UpdateMangaLastUpdate(get()) } addSingletonFactory { ChapterRepositoryImpl(get()) } addFactory { UpdateChapter(get()) } + addFactory { ShouldUpdateDbChapter() } + addFactory { SyncChaptersWithSource(get(), get(), get(), get()) } addSingletonFactory { HistoryRepositoryImpl(get()) } addFactory { DeleteHistoryTable(get()) } diff --git a/app/src/main/java/eu/kanade/domain/chapter/interactor/ShouldUpdateDbChapter.kt b/app/src/main/java/eu/kanade/domain/chapter/interactor/ShouldUpdateDbChapter.kt new file mode 100644 index 0000000000..e0d455feb5 --- /dev/null +++ b/app/src/main/java/eu/kanade/domain/chapter/interactor/ShouldUpdateDbChapter.kt @@ -0,0 +1,13 @@ +package eu.kanade.domain.chapter.interactor + +import eu.kanade.domain.chapter.model.Chapter + +class ShouldUpdateDbChapter { + + fun await(dbChapter: Chapter, sourceChapter: Chapter): Boolean { + return dbChapter.scanlator != sourceChapter.scanlator || dbChapter.name != sourceChapter.name || + dbChapter.dateUpload != sourceChapter.dateUpload || + dbChapter.chapterNumber != sourceChapter.chapterNumber || + dbChapter.sourceOrder != sourceChapter.sourceOrder + } +} diff --git a/app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt b/app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt new file mode 100644 index 0000000000..a9358c20dd --- /dev/null +++ b/app/src/main/java/eu/kanade/domain/chapter/interactor/SyncChaptersWithSource.kt @@ -0,0 +1,179 @@ +package eu.kanade.domain.chapter.interactor + +import eu.kanade.data.chapter.NoChaptersException +import eu.kanade.domain.chapter.model.Chapter +import eu.kanade.domain.chapter.model.toChapterUpdate +import eu.kanade.domain.chapter.model.toDbChapter +import eu.kanade.domain.chapter.repository.ChapterRepository +import eu.kanade.domain.manga.interactor.UpdateMangaLastUpdate +import eu.kanade.domain.manga.model.Manga +import eu.kanade.domain.manga.model.toDbManga +import eu.kanade.tachiyomi.data.download.DownloadManager +import eu.kanade.tachiyomi.source.LocalSource +import eu.kanade.tachiyomi.source.Source +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.online.HttpSource +import eu.kanade.tachiyomi.util.chapter.ChapterRecognition +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import java.lang.Long.max +import java.util.Date +import java.util.TreeSet + +class SyncChaptersWithSource( + private val downloadManager: DownloadManager = Injekt.get(), + private val chapterRepository: ChapterRepository = Injekt.get(), + private val shouldUpdateDbChapter: ShouldUpdateDbChapter = Injekt.get(), + private val updateMangaLastUpdate: UpdateMangaLastUpdate = Injekt.get(), +) { + + suspend fun await( + rawSourceChapters: List, + manga: Manga, + source: Source, + ): Pair, List> { + if (rawSourceChapters.isEmpty() && source.id != LocalSource.ID) { + throw NoChaptersException() + } + + val sourceChapters = rawSourceChapters + .distinctBy { it.url } + .mapIndexed { i, sChapter -> + Chapter.create() + .copyFromSChapter(sChapter) + .copy(mangaId = manga.id, sourceOrder = i.toLong()) + } + + // Chapters from db. + val dbChapters = chapterRepository.getChapterByMangaId(manga.id) + + // Chapters from the source not in db. + val toAdd = mutableListOf() + + // Chapters whose metadata have changed. + val toChange = mutableListOf() + + // Chapters from the db not in source. + val toDelete = dbChapters.filterNot { dbChapter -> + sourceChapters.any { sourceChapter -> + dbChapter.url == sourceChapter.url + } + } + + val rightNow = Date().time + + // Used to not set upload date of older chapters + // to a higher value than newer chapters + var maxSeenUploadDate = 0L + + val sManga = manga.toSManga() + for (sourceChapter in sourceChapters) { + var chapter = sourceChapter + + // Update metadata from source if necessary. + if (source is HttpSource) { + val sChapter = chapter.toSChapter() + source.prepareNewChapter(sChapter, sManga) + chapter = chapter.copyFromSChapter(sChapter) + } + + // Recognize chapter number for the chapter. + val chapterNumber = ChapterRecognition.parseChapterNumber(manga.title, chapter.name, chapter.chapterNumber) + chapter = chapter.copy(chapterNumber = chapterNumber) + + val dbChapter = dbChapters.find { it.url == chapter.url } + + if (dbChapter == null) { + if (chapter.dateUpload == 0L) { + val altDateUpload = if (maxSeenUploadDate == 0L) rightNow else maxSeenUploadDate + chapter = chapter.copy(dateUpload = altDateUpload) + } else { + maxSeenUploadDate = max(maxSeenUploadDate, sourceChapter.dateUpload) + } + toAdd.add(chapter) + } else { + if (shouldUpdateDbChapter.await(dbChapter, chapter)) { + if (dbChapter.name != chapter.name && downloadManager.isChapterDownloaded(dbChapter.toDbChapter(), manga.toDbManga())) { + downloadManager.renameChapter(source, manga.toDbManga(), dbChapter.toDbChapter(), chapter.toDbChapter()) + } + chapter = dbChapter.copy( + name = sourceChapter.name, + chapterNumber = sourceChapter.chapterNumber, + scanlator = sourceChapter.scanlator, + sourceOrder = sourceChapter.sourceOrder, + ) + if (sourceChapter.dateUpload != 0L) { + chapter = chapter.copy(dateUpload = sourceChapter.dateUpload) + } + toChange.add(chapter) + } + } + } + + // Return if there's nothing to add, delete or change, avoiding unnecessary db transactions. + if (toAdd.isEmpty() && toDelete.isEmpty() && toChange.isEmpty()) { + return Pair(emptyList(), emptyList()) + } + + val reAdded = mutableListOf() + + val deletedChapterNumbers = TreeSet() + val deletedReadChapterNumbers = TreeSet() + + toDelete.forEach { chapter -> + if (chapter.read) { + deletedReadChapterNumbers.add(chapter.chapterNumber) + } + deletedChapterNumbers.add(chapter.chapterNumber) + } + + val deletedChapterNumberDateFetchMap = toDelete.sortedByDescending { it.dateFetch } + .associate { it.chapterNumber to it.dateFetch } + + // Date fetch is set in such a way that the upper ones will have bigger value than the lower ones + // Sources MUST return the chapters from most to less recent, which is common. + val now = Date().time + + var itemCount = toAdd.size + var updatedToAdd = toAdd.map { toAddItem -> + var chapter = toAddItem.copy(dateFetch = now + itemCount--) + + if (chapter.isRecognizedNumber.not() && chapter.chapterNumber !in deletedChapterNumbers) return@map chapter + + if (chapter.chapterNumber in deletedReadChapterNumbers) { + chapter = chapter.copy(read = true) + } + + // Try to to use the fetch date of the original entry to not pollute 'Updates' tab + val oldDateFetch = deletedChapterNumberDateFetchMap[chapter.chapterNumber] + oldDateFetch?.let { + chapter = chapter.copy(dateFetch = it) + } + + reAdded.add(chapter) + + chapter + } + + if (toDelete.isNotEmpty()) { + val toDeleteIds = toDelete.map { it.id } + chapterRepository.removeChaptersWithIds(toDeleteIds) + } + + if (updatedToAdd.isNotEmpty()) { + updatedToAdd = chapterRepository.addAll(updatedToAdd) + } + + if (toChange.isNotEmpty()) { + val chapterUpdates = toChange.map { it.toChapterUpdate() } + chapterRepository.updateAll(chapterUpdates) + } + + // Set this manga as updated since chapters were changed + // Note that last_update actually represents last time the chapter list changed at all + updateMangaLastUpdate.await(manga.id, Date().time) + + @Suppress("ConvertArgumentToSet") // See tachiyomiorg/tachiyomi#6372. + return Pair(updatedToAdd.subtract(reAdded).toList(), toDelete.subtract(reAdded).toList()) + } +} diff --git a/app/src/main/java/eu/kanade/domain/chapter/model/Chapter.kt b/app/src/main/java/eu/kanade/domain/chapter/model/Chapter.kt index 27ed9a46f9..ce9005786e 100644 --- a/app/src/main/java/eu/kanade/domain/chapter/model/Chapter.kt +++ b/app/src/main/java/eu/kanade/domain/chapter/model/Chapter.kt @@ -1,5 +1,8 @@ package eu.kanade.domain.chapter.model +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.data.database.models.Chapter as DbChapter + data class Chapter( val id: Long, val mangaId: Long, @@ -13,4 +16,61 @@ data class Chapter( val dateUpload: Long, val chapterNumber: Float, val scanlator: String?, -) +) { + val isRecognizedNumber: Boolean + get() = chapterNumber >= 0f + + fun toSChapter(): SChapter { + return SChapter.create().also { + it.url = url + it.name = name + it.date_upload = dateUpload + it.chapter_number = chapterNumber + it.scanlator = scanlator + } + } + + fun copyFromSChapter(sChapter: SChapter): Chapter { + return this.copy( + name = sChapter.name, + url = sChapter.url, + dateUpload = sChapter.date_upload, + chapterNumber = sChapter.chapter_number, + scanlator = sChapter.scanlator, + ) + } + + companion object { + fun create(): Chapter { + return Chapter( + id = -1, + mangaId = -1, + read = false, + bookmark = false, + lastPageRead = 0, + dateFetch = 0, + sourceOrder = 0, + url = "", + name = "", + dateUpload = -1, + chapterNumber = -1f, + scanlator = null, + ) + } + } +} + +// TODO: Remove when all deps are migrated +fun Chapter.toDbChapter(): DbChapter = DbChapter.create().also { + it.id = id + it.manga_id = mangaId + it.url = url + it.name = name + it.scanlator = scanlator + it.read = read + it.bookmark = bookmark + it.last_page_read = lastPageRead.toInt() + it.date_fetch = dateFetch + it.chapter_number = chapterNumber + it.source_order = sourceOrder.toInt() +} diff --git a/app/src/main/java/eu/kanade/domain/chapter/model/ChapterUpdate.kt b/app/src/main/java/eu/kanade/domain/chapter/model/ChapterUpdate.kt index 2c9042c477..8e073f358f 100644 --- a/app/src/main/java/eu/kanade/domain/chapter/model/ChapterUpdate.kt +++ b/app/src/main/java/eu/kanade/domain/chapter/model/ChapterUpdate.kt @@ -14,3 +14,7 @@ data class ChapterUpdate( val chapterNumber: Float? = null, val scanlator: String? = null, ) + +fun Chapter.toChapterUpdate(): ChapterUpdate { + return ChapterUpdate(id, mangaId, read, bookmark, lastPageRead, dateFetch, sourceOrder, url, name, dateUpload, chapterNumber, scanlator) +} diff --git a/app/src/main/java/eu/kanade/domain/chapter/repository/ChapterRepository.kt b/app/src/main/java/eu/kanade/domain/chapter/repository/ChapterRepository.kt index 9f3bb73d74..9ffa372601 100644 --- a/app/src/main/java/eu/kanade/domain/chapter/repository/ChapterRepository.kt +++ b/app/src/main/java/eu/kanade/domain/chapter/repository/ChapterRepository.kt @@ -1,8 +1,17 @@ package eu.kanade.domain.chapter.repository +import eu.kanade.domain.chapter.model.Chapter import eu.kanade.domain.chapter.model.ChapterUpdate interface ChapterRepository { + suspend fun addAll(chapters: List): List + suspend fun update(chapterUpdate: ChapterUpdate) + + suspend fun updateAll(chapterUpdates: List) + + suspend fun removeChaptersWithIds(chapterIds: List) + + suspend fun getChapterByMangaId(mangaId: Long): List } diff --git a/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateMangaLastUpdate.kt b/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateMangaLastUpdate.kt new file mode 100644 index 0000000000..6411927448 --- /dev/null +++ b/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateMangaLastUpdate.kt @@ -0,0 +1,12 @@ +package eu.kanade.domain.manga.interactor + +import eu.kanade.domain.manga.repository.MangaRepository + +class UpdateMangaLastUpdate( + private val mangaRepository: MangaRepository, +) { + + suspend fun await(mangaId: Long, lastUpdate: Long) { + mangaRepository.updateLastUpdate(mangaId, lastUpdate) + } +} diff --git a/app/src/main/java/eu/kanade/domain/manga/model/Manga.kt b/app/src/main/java/eu/kanade/domain/manga/model/Manga.kt index bed5f1d61b..312bff395a 100644 --- a/app/src/main/java/eu/kanade/domain/manga/model/Manga.kt +++ b/app/src/main/java/eu/kanade/domain/manga/model/Manga.kt @@ -1,5 +1,8 @@ package eu.kanade.domain.manga.model +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.data.database.models.Manga as DbManga + data class Manga( val id: Long, val source: Long, @@ -23,6 +26,20 @@ data class Manga( val sorting: Long get() = chapterFlags and CHAPTER_SORTING_MASK + fun toSManga(): SManga { + return SManga.create().also { + it.url = url + it.title = title + it.artist = artist + it.author = author + it.description = description + it.genre = genre.orEmpty().joinToString() + it.status = status.toInt() + it.thumbnail_url = thumbnailUrl + it.initialized = initialized + } + } + companion object { // Generic filter that does not filter anything @@ -34,3 +51,14 @@ data class Manga( const val CHAPTER_SORTING_MASK = 0x00000300L } } + +// TODO: Remove when all deps are migrated +fun Manga.toDbManga(): DbManga = DbManga.create(url, title, source).also { + it.id = id + it.favorite = favorite + it.last_update = lastUpdate + it.date_added = dateAdded + it.viewer_flags = viewerFlags.toInt() + it.chapter_flags = chapterFlags.toInt() + it.cover_last_modified = coverLastModified +} diff --git a/app/src/main/java/eu/kanade/domain/manga/repository/MangaRepository.kt b/app/src/main/java/eu/kanade/domain/manga/repository/MangaRepository.kt index 0dcf680822..83367d103e 100644 --- a/app/src/main/java/eu/kanade/domain/manga/repository/MangaRepository.kt +++ b/app/src/main/java/eu/kanade/domain/manga/repository/MangaRepository.kt @@ -8,4 +8,6 @@ interface MangaRepository { fun getFavoritesBySourceId(sourceId: Long): Flow> suspend fun resetViewerFlags(): Boolean + + suspend fun updateLastUpdate(mangaId: Long, lastUpdate: Long) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/AbstractBackupManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/AbstractBackupManager.kt index cf1b4b8e6e..21b0ae55f2 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/AbstractBackupManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/AbstractBackupManager.kt @@ -43,7 +43,7 @@ abstract class AbstractBackupManager(protected val context: Context) { internal suspend fun restoreChapters(source: Source, manga: Manga, chapters: List): Pair, List> { val fetchedChapters = source.getChapterList(manga.toMangaInfo()) .map { it.toSChapter() } - val syncedChapters = syncChaptersWithSource(db, fetchedChapters, manga, source) + val syncedChapters = syncChaptersWithSource(fetchedChapters, manga, source) if (syncedChapters.first.isNotEmpty()) { chapters.forEach { it.manga_id = manga.id } updateChapters(chapters) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt index 35efbe1ffe..716f195bec 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt @@ -4,6 +4,7 @@ import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.ui.reader.setting.OrientationType import eu.kanade.tachiyomi.ui.reader.setting.ReadingModeType import tachiyomi.source.model.MangaInfo +import eu.kanade.domain.manga.model.Manga as DomainManga interface Manga : SManga { @@ -128,3 +129,26 @@ fun Manga.toMangaInfo(): MangaInfo { title = this.title, ) } + +fun Manga.toDomainManga(): DomainManga? { + val mangaId = id ?: return null + return DomainManga( + id = mangaId, + source = source, + favorite = favorite, + lastUpdate = last_update, + dateAdded = date_added, + viewerFlags = viewer_flags.toLong(), + chapterFlags = chapter_flags.toLong(), + coverLastModified = cover_last_modified, + url = url, + title = title, + artist = artist, + author = author, + description = description, + genre = getGenres(), + status = status.toLong(), + thumbnailUrl = thumbnail_url, + initialized = initialized, + ) +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt index 0c0143c07e..a702ba5483 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt @@ -426,7 +426,7 @@ class LibraryUpdateService( // [dbmanga] was used so that manga data doesn't get overwritten // in case manga gets new chapter - return syncChaptersWithSource(db, chapters, dbManga, source) + return syncChaptersWithSource(chapters, dbManga, source) } private suspend fun updateCovers() { diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/HttpSource.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/HttpSource.kt index ba4f782c9c..0d46164643 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/HttpSource.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/HttpSource.kt @@ -362,8 +362,7 @@ abstract class HttpSource : CatalogueSource { * @param chapter the chapter to be added. * @param manga the manga of the chapter. */ - open fun prepareNewChapter(chapter: SChapter, manga: SManga) { - } + open fun prepareNewChapter(chapter: SChapter, manga: SManga) {} /** * Returns the list of filters for the source. diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/browse/migration/search/SearchPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/browse/migration/search/SearchPresenter.kt index 45ff3e341b..36e54f580d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/browse/migration/search/SearchPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/browse/migration/search/SearchPresenter.kt @@ -115,7 +115,7 @@ class SearchPresenter( // Update chapters read if (migrateChapters) { try { - syncChaptersWithSource(db, sourceChapters, manga, source) + syncChaptersWithSource(sourceChapters, manga, source) } catch (e: Exception) { // Worst case, chapters won't be synced } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaPresenter.kt index 8f458beb42..55563488e7 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaPresenter.kt @@ -417,7 +417,7 @@ class MangaPresenter( val chapters = source.getChapterList(manga.toMangaInfo()) .map { it.toSChapter() } - val (newChapters, _) = syncChaptersWithSource(db, chapters, manga, source) + val (newChapters, _) = syncChaptersWithSource(chapters, manga, source) if (manualFetch) { downloadNewChapters(newChapters) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt b/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt index 7439c7f75c..cec3615c68 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt @@ -1,175 +1,37 @@ package eu.kanade.tachiyomi.util.chapter -import eu.kanade.data.chapter.NoChaptersException -import eu.kanade.tachiyomi.data.database.DatabaseHelper -import eu.kanade.tachiyomi.data.database.models.Chapter -import eu.kanade.tachiyomi.data.database.models.Manga -import eu.kanade.tachiyomi.data.download.DownloadManager -import eu.kanade.tachiyomi.source.LocalSource +import eu.kanade.domain.chapter.interactor.SyncChaptersWithSource +import eu.kanade.domain.chapter.model.toDbChapter +import eu.kanade.tachiyomi.data.database.models.toDomainManga import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.online.HttpSource +import kotlinx.coroutines.runBlocking import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get -import java.util.Date -import java.util.TreeSet -import kotlin.math.max +import eu.kanade.tachiyomi.data.database.models.Chapter as DbChapter +import eu.kanade.tachiyomi.data.database.models.Manga as DbManga /** * Helper method for syncing the list of chapters from the source with the ones from the database. * - * @param db the database. * @param rawSourceChapters a list of chapters from the source. * @param manga the manga of the chapters. * @param source the source of the chapters. * @return a pair of new insertions and deletions. */ fun syncChaptersWithSource( - db: DatabaseHelper, rawSourceChapters: List, - manga: Manga, + manga: DbManga, source: Source, -): Pair, List> { - if (rawSourceChapters.isEmpty() && source !is LocalSource) { - throw NoChaptersException() + syncChaptersWithSource: SyncChaptersWithSource = Injekt.get(), +): Pair, List> { + val domainManga = manga.toDomainManga() ?: return Pair(emptyList(), emptyList()) + val (added, deleted) = runBlocking { + syncChaptersWithSource.await(rawSourceChapters, domainManga, source) } - val downloadManager: DownloadManager = Injekt.get() + val addedDbChapters = added.map { it.toDbChapter() } + val deletedDbChapters = deleted.map { it.toDbChapter() } - // Chapters from db. - val dbChapters = db.getChapters(manga).executeAsBlocking() - - val sourceChapters = rawSourceChapters - .distinctBy { it.url } - .mapIndexed { i, sChapter -> - Chapter.create().apply { - copyFrom(sChapter) - manga_id = manga.id - source_order = i - } - } - - // Chapters from the source not in db. - val toAdd = mutableListOf() - - // Chapters whose metadata have changed. - val toChange = mutableListOf() - - // Chapters from the db not in source. - val toDelete = dbChapters.filterNot { dbChapter -> - sourceChapters.any { sourceChapter -> - dbChapter.url == sourceChapter.url - } - } - - var maxTimestamp = 0L // in previous chapters to add - val rightNow = Date().time - - for (sourceChapter in sourceChapters) { - // This forces metadata update for the main viewable things in the chapter list. - if (source is HttpSource) { - source.prepareNewChapter(sourceChapter, manga) - } - // Recognize chapter number for the chapter. - sourceChapter.chapter_number = ChapterRecognition.parseChapterNumber(manga.title, sourceChapter.name, sourceChapter.chapter_number) - - val dbChapter = dbChapters.find { it.url == sourceChapter.url } - - // Add the chapter if not in db already, or update if the metadata changed. - if (dbChapter == null) { - if (sourceChapter.date_upload == 0L) { - sourceChapter.date_upload = if (maxTimestamp == 0L) rightNow else maxTimestamp - } else { - maxTimestamp = max(maxTimestamp, sourceChapter.date_upload) - } - toAdd.add(sourceChapter) - } else { - if (shouldUpdateDbChapter(dbChapter, sourceChapter)) { - if (dbChapter.name != sourceChapter.name && downloadManager.isChapterDownloaded(dbChapter, manga)) { - downloadManager.renameChapter(source, manga, dbChapter, sourceChapter) - } - dbChapter.scanlator = sourceChapter.scanlator - dbChapter.name = sourceChapter.name - dbChapter.chapter_number = sourceChapter.chapter_number - dbChapter.source_order = sourceChapter.source_order - if (sourceChapter.date_upload != 0L) { - dbChapter.date_upload = sourceChapter.date_upload - } - toChange.add(dbChapter) - } - } - } - - // Return if there's nothing to add, delete or change, avoiding unnecessary db transactions. - if (toAdd.isEmpty() && toDelete.isEmpty() && toChange.isEmpty()) { - return Pair(emptyList(), emptyList()) - } - - // Keep it a List instead of a Set. See #6372. - val readded = mutableListOf() - - db.inTransaction { - val deletedChapterNumbers = TreeSet() - val deletedReadChapterNumbers = TreeSet() - - if (toDelete.isNotEmpty()) { - for (chapter in toDelete) { - if (chapter.read) { - deletedReadChapterNumbers.add(chapter.chapter_number) - } - deletedChapterNumbers.add(chapter.chapter_number) - } - db.deleteChapters(toDelete).executeAsBlocking() - } - - if (toAdd.isNotEmpty()) { - // Set the date fetch for new items in reverse order to allow another sorting method. - // Sources MUST return the chapters from most to less recent, which is common. - var now = Date().time - - for (i in toAdd.indices.reversed()) { - val chapter = toAdd[i] - chapter.date_fetch = now++ - - if (chapter.isRecognizedNumber && chapter.chapter_number in deletedChapterNumbers) { - // Try to mark already read chapters as read when the source deletes them - if (chapter.chapter_number in deletedReadChapterNumbers) { - chapter.read = true - } - // Try to to use the fetch date it originally had to not pollute 'Updates' tab - toDelete.filter { it.chapter_number == chapter.chapter_number } - .minByOrNull { it.date_fetch }!!.let { - chapter.date_fetch = it.date_fetch - } - readded.add(chapter) - } - } - val chapters = db.insertChapters(toAdd).executeAsBlocking() - toAdd.forEach { chapter -> - chapter.id = chapters.results().getValue(chapter).insertedId() - } - } - - if (toChange.isNotEmpty()) { - db.insertChapters(toChange).executeAsBlocking() - } - - // Fix order in source. - db.fixChaptersSourceOrder(sourceChapters).executeAsBlocking() - - // Set this manga as updated since chapters were changed - // Note that last_update actually represents last time the chapter list changed at all - manga.last_update = Date().time - db.updateLastUpdated(manga).executeAsBlocking() - } - - @Suppress("ConvertArgumentToSet") - return Pair(toAdd.subtract(readded).toList(), toDelete.subtract(readded).toList()) -} - -private fun shouldUpdateDbChapter(dbChapter: Chapter, sourceChapter: Chapter): Boolean { - return dbChapter.scanlator != sourceChapter.scanlator || dbChapter.name != sourceChapter.name || - dbChapter.date_upload != sourceChapter.date_upload || - dbChapter.chapter_number != sourceChapter.chapter_number || - dbChapter.source_order != sourceChapter.source_order + return Pair(addedDbChapters, deletedDbChapters) } diff --git a/app/src/main/sqldelight/data/chapters.sq b/app/src/main/sqldelight/data/chapters.sq index 226d9b6bdf..c7d8634a51 100644 --- a/app/src/main/sqldelight/data/chapters.sq +++ b/app/src/main/sqldelight/data/chapters.sq @@ -28,6 +28,38 @@ SELECT * FROM chapters WHERE manga_id = :mangaId; +removeChaptersWithIds: +DELETE FROM chapters +WHERE _id IN :chapterIds; + +insert: +INSERT INTO chapters( + manga_id, + url, + name, + scanlator, + read, + bookmark, + last_page_read, + chapter_number, + source_order, + date_fetch, + date_upload +) +VALUES ( + :mangaId, + :url, + :name, + :scanlator, + :read, + :bookmark, + :lastPageRead, + :chapterNumber, + :sourceOrder, + :dateFetch, + :dateUpload +); + update: UPDATE chapters SET manga_id = coalesce(:mangaId, manga_id), @@ -41,4 +73,7 @@ SET manga_id = coalesce(:mangaId, manga_id), source_order = coalesce(:sourceOrder, source_order), date_fetch = coalesce(:dateFetch, date_fetch), date_upload = coalesce(:dateUpload, date_upload) -WHERE _id = :chapterId; \ No newline at end of file +WHERE _id = :chapterId; + +selectLastInsertedRowId: +SELECT last_insert_rowid(); \ No newline at end of file diff --git a/app/src/main/sqldelight/data/mangas.sq b/app/src/main/sqldelight/data/mangas.sq index ccb351da2e..154a2c0800 100644 --- a/app/src/main/sqldelight/data/mangas.sq +++ b/app/src/main/sqldelight/data/mangas.sq @@ -56,4 +56,9 @@ GROUP BY source; deleteMangasNotInLibraryBySourceIds: DELETE FROM mangas -WHERE favorite = 0 AND source IN :sourceIds; \ No newline at end of file +WHERE favorite = 0 AND source IN :sourceIds; + +updateLastUpdate: +UPDATE mangas +SET last_update = :lastUpdate +WHERE _id = :mangaId;