From 4d3e13b0d13a902f0755714e0f56d284acad5a5f Mon Sep 17 00:00:00 2001 From: Ivan Iskandar <12537387+ivaniskandar@users.noreply.github.com> Date: Fri, 17 Mar 2023 09:18:11 +0700 Subject: [PATCH] Initialize download index disk cache (#9179) --- .../tachiyomi/data/download/DownloadCache.kt | 241 ++++++++++++------ .../data/download/DownloadManager.kt | 2 +- .../tachiyomi/data/download/Downloader.kt | 2 +- 3 files changed, 163 insertions(+), 82 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt index cf91e5810c..03caeb551c 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt @@ -1,6 +1,8 @@ package eu.kanade.tachiyomi.data.download +import android.app.Application import android.content.Context +import android.net.Uri import androidx.core.net.toUri import com.hippo.unifile.UniFile import eu.kanade.core.util.mapNotNullKeys @@ -14,6 +16,7 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.debounce @@ -23,7 +26,20 @@ import kotlinx.coroutines.flow.onStart import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.flow.shareIn import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.serialization.KSerializer +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromByteArray +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encodeToByteArray +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.protobuf.ProtoBuf import logcat.LogPriority import tachiyomi.core.util.lang.launchIO import tachiyomi.core.util.lang.launchNonCancellable @@ -34,7 +50,7 @@ import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.source.service.SourceManager import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get -import java.util.concurrent.ConcurrentHashMap +import java.io.File import kotlin.time.Duration.Companion.hours import kotlin.time.Duration.Companion.seconds @@ -76,7 +92,11 @@ class DownloadCache( .debounce(1000L) // Don't notify if it finishes quickly enough .stateIn(scope, SharingStarted.WhileSubscribed(), false) - private var rootDownloadsDir = RootDirectory(getDirectoryFromPreference()) + private val diskCacheFile: File + get() = File(context.cacheDir, "dl_index_cache") + + private val rootDownloadsDirLock = Mutex() + private var rootDownloadsDir: RootDirectory init { downloadPreferences.downloadsDirectory().changes() @@ -85,6 +105,21 @@ class DownloadCache( invalidateCache() } .launchIn(scope) + + rootDownloadsDir = runBlocking(Dispatchers.IO) { + try { + val diskCache = diskCacheFile.inputStream().use { + ProtoBuf.decodeFromByteArray(it.readBytes()) + } + lastRenew = 1 // Just so that the banner won't show up + diskCache + } catch (e: Throwable) { + diskCacheFile.delete() + null + } + } ?: RootDirectory(getDirectoryFromPreference()) + + notifyChanges() } /** @@ -158,27 +193,28 @@ class DownloadCache( * @param mangaUniFile the directory of the manga. * @param manga the manga of the chapter. */ - @Synchronized - fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) { - // Retrieve the cached source directory or cache a new one - var sourceDir = rootDownloadsDir.sourceDirs[manga.source] - if (sourceDir == null) { - val source = sourceManager.get(manga.source) ?: return - val sourceUniFile = provider.findSourceDir(source) ?: return - sourceDir = SourceDirectory(sourceUniFile) - rootDownloadsDir.sourceDirs += manga.source to sourceDir - } + suspend fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) { + rootDownloadsDirLock.withLock { + // Retrieve the cached source directory or cache a new one + var sourceDir = rootDownloadsDir.sourceDirs[manga.source] + if (sourceDir == null) { + val source = sourceManager.get(manga.source) ?: return + val sourceUniFile = provider.findSourceDir(source) ?: return + sourceDir = SourceDirectory(sourceUniFile) + rootDownloadsDir.sourceDirs += manga.source to sourceDir + } - // Retrieve the cached manga directory or cache a new one - val mangaDirName = provider.getMangaDirName(manga.title) - var mangaDir = sourceDir.mangaDirs[mangaDirName] - if (mangaDir == null) { - mangaDir = MangaDirectory(mangaUniFile) - sourceDir.mangaDirs += mangaDirName to mangaDir - } + // Retrieve the cached manga directory or cache a new one + val mangaDirName = provider.getMangaDirName(manga.title) + var mangaDir = sourceDir.mangaDirs[mangaDirName] + if (mangaDir == null) { + mangaDir = MangaDirectory(mangaUniFile) + sourceDir.mangaDirs += mangaDirName to mangaDir + } - // Save the chapter directory - mangaDir.chapterDirs += chapterDirName + // Save the chapter directory + mangaDir.chapterDirs += chapterDirName + } notifyChanges() } @@ -189,13 +225,14 @@ class DownloadCache( * @param chapter the chapter to remove. * @param manga the manga of the chapter. */ - @Synchronized - fun removeChapter(chapter: Chapter, manga: Manga) { - val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return - val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return - provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach { - if (it in mangaDir.chapterDirs) { - mangaDir.chapterDirs -= it + suspend fun removeChapter(chapter: Chapter, manga: Manga) { + rootDownloadsDirLock.withLock { + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return + val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return + provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach { + if (it in mangaDir.chapterDirs) { + mangaDir.chapterDirs -= it + } } } @@ -208,14 +245,15 @@ class DownloadCache( * @param chapters the list of chapter to remove. * @param manga the manga of the chapter. */ - @Synchronized - fun removeChapters(chapters: List, manga: Manga) { - val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return - val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return - chapters.forEach { chapter -> - provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach { - if (it in mangaDir.chapterDirs) { - mangaDir.chapterDirs -= it + suspend fun removeChapters(chapters: List, manga: Manga) { + rootDownloadsDirLock.withLock { + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return + val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return + chapters.forEach { chapter -> + provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach { + if (it in mangaDir.chapterDirs) { + mangaDir.chapterDirs -= it + } } } } @@ -228,20 +266,22 @@ class DownloadCache( * * @param manga the manga to remove. */ - @Synchronized - fun removeManga(manga: Manga) { - val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return - val mangaDirName = provider.getMangaDirName(manga.title) - if (sourceDir.mangaDirs.containsKey(mangaDirName)) { - sourceDir.mangaDirs -= mangaDirName + suspend fun removeManga(manga: Manga) { + rootDownloadsDirLock.withLock { + val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return + val mangaDirName = provider.getMangaDirName(manga.title) + if (sourceDir.mangaDirs.containsKey(mangaDirName)) { + sourceDir.mangaDirs -= mangaDirName + } } notifyChanges() } - @Synchronized - fun removeSource(source: Source) { - rootDownloadsDir.sourceDirs -= source.id + suspend fun removeSource(source: Source) { + rootDownloadsDirLock.withLock { + rootDownloadsDir.sourceDirs -= source.id + } notifyChanges() } @@ -287,46 +327,48 @@ class DownloadCache( } } - val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty() - .associate { it.name to SourceDirectory(it) } - .mapNotNullKeys { entry -> - sources.find { - provider.getSourceDirName(it).equals(entry.key, ignoreCase = true) - }?.id - } + rootDownloadsDirLock.withLock { + val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty() + .associate { it.name to SourceDirectory(it) } + .mapNotNullKeys { entry -> + sources.find { + provider.getSourceDirName(it).equals(entry.key, ignoreCase = true) + }?.id + } - rootDownloadsDir.sourceDirs = sourceDirs + rootDownloadsDir.sourceDirs = sourceDirs - sourceDirs.values - .map { sourceDir -> - async { - val mangaDirs = sourceDir.dir.listFiles().orEmpty() - .filterNot { it.name.isNullOrBlank() } - .associate { it.name!! to MangaDirectory(it) } + sourceDirs.values + .map { sourceDir -> + async { + sourceDir.mangaDirs = sourceDir.dir.listFiles().orEmpty() + .filterNot { it.name.isNullOrBlank() } + .associate { it.name!! to MangaDirectory(it) } - sourceDir.mangaDirs = ConcurrentHashMap(mangaDirs) - - mangaDirs.values.forEach { mangaDir -> - val chapterDirs = mangaDir.dir.listFiles().orEmpty() - .mapNotNull { - when { - // Ignore incomplete downloads - it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null - // Folder of images - it.isDirectory -> it.name - // CBZ files - it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast(".cbz") - // Anything else is irrelevant - else -> null + sourceDir.mangaDirs.values.forEach { mangaDir -> + val chapterDirs = mangaDir.dir.listFiles().orEmpty() + .mapNotNull { + when { + // Ignore incomplete downloads + it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null + // Folder of images + it.isDirectory -> it.name + // CBZ files + it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast( + ".cbz", + ) + // Anything else is irrelevant + else -> null + } } - } - .toMutableSet() + .toMutableSet() - mangaDir.chapterDirs = chapterDirs + mangaDir.chapterDirs = chapterDirs + } } } - } - .awaitAll() + .awaitAll() + } _isInitializing.emit(false) }.also { @@ -335,6 +377,7 @@ class DownloadCache( logcat(LogPriority.ERROR, exception) { "Failed to create download cache" } } lastRenew = System.currentTimeMillis() + notifyChanges() } } @@ -351,29 +394,67 @@ class DownloadCache( scope.launchNonCancellable { _changes.send(Unit) } + updateDiskCache() + } + + private var updateDiskCacheJob: Job? = null + private fun updateDiskCache() { + updateDiskCacheJob?.cancel() + updateDiskCacheJob = scope.launchIO { + delay(1000) + ensureActive() + val bytes = ProtoBuf.encodeToByteArray(rootDownloadsDir) + ensureActive() + try { + diskCacheFile.writeBytes(bytes) + } catch (e: Throwable) { + logcat( + priority = LogPriority.ERROR, + throwable = e, + message = { "Failed to write disk cache file" }, + ) + } + } } } /** * Class to store the files under the root downloads directory. */ +@Serializable private class RootDirectory( + @Serializable(with = UniFileAsStringSerializer::class) val dir: UniFile, - var sourceDirs: ConcurrentHashMap = ConcurrentHashMap(), + var sourceDirs: Map = mapOf(), ) /** * Class to store the files under a source directory. */ +@Serializable private class SourceDirectory( + @Serializable(with = UniFileAsStringSerializer::class) val dir: UniFile, - var mangaDirs: ConcurrentHashMap = ConcurrentHashMap(), + var mangaDirs: Map = mapOf(), ) /** * Class to store the files under a manga directory. */ +@Serializable private class MangaDirectory( + @Serializable(with = UniFileAsStringSerializer::class) val dir: UniFile, var chapterDirs: MutableSet = mutableSetOf(), ) + +private object UniFileAsStringSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("UniFile", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: UniFile) { + return encoder.encodeString(value.uri.toString()) + } + override fun deserialize(decoder: Decoder): UniFile { + return UniFile.fromUri(Injekt.get(), Uri.parse(decoder.decodeString())) + } +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt index 3bb7e97482..fc9b80dedf 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt @@ -325,7 +325,7 @@ class DownloadManager( * @param oldChapter the existing chapter with the old name. * @param newChapter the target chapter with the new name. */ - fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) { + suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) { val oldNames = provider.getValidChapterDirNames(oldChapter.name, oldChapter.scanlator) val mangaDir = provider.getMangaDir(manga.title, source) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt index 4e61a120a0..9f4682ad67 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt @@ -527,7 +527,7 @@ class Downloader( * @param tmpDir the directory where the download is currently stored. * @param dirname the real (non temporary) directory name of the download. */ - private fun ensureSuccessfulDownload( + private suspend fun ensureSuccessfulDownload( download: Download, mangaDir: UniFile, tmpDir: UniFile,