From 813a3c6e68ff1abccc7188d28ad0b27b10fb9498 Mon Sep 17 00:00:00 2001 From: Carlos <2092019+CarlosEsco@users.noreply.github.com> Date: Sun, 9 Aug 2020 14:55:53 -0400 Subject: [PATCH] Download updates (#546) * Add rename download attempt from preview * clean up renew cache some to help with readability * Download on WiFi regardless of metered status (From Tachiyomi) * no reason to show md2 beta message for new debug installs * update clean up downloads to be a little faster and hit some edge cases not thought of before * fix bug where manga name does not match the file name and was deleting manga downloads of manga with special characters Co-authored-by: Chris Allan Co-authored-by: Ken Swenson <2048861+flat@users.noreply.github.com> --- .../tachiyomi/data/download/DownloadCache.kt | 38 ++++---- .../data/download/DownloadManager.kt | 63 ++++++++++-- .../data/download/DownloadProvider.kt | 38 ++++++-- .../data/download/DownloadService.kt | 3 +- .../kanade/tachiyomi/ui/main/MainActivity.kt | 8 +- .../ui/setting/SettingsAdvancedController.kt | 95 ++++++++++++++----- .../util/chapter/ChapterSourceSync.kt | 8 ++ app/src/main/res/values/arrays.xml | 7 ++ app/src/main/res/values/strings.xml | 3 + 9 files changed, 197 insertions(+), 66 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt index 0d4387ff80..33ccbb0fd7 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt @@ -134,30 +134,19 @@ class DownloadCache( val mangas = db.getMangas().executeAsBlocking().groupBy { it.source } sourceDirs.forEach { sourceValue -> - val sourceMangasRaw = mangas[sourceValue.key]?.toMutableSet() ?: return@forEach - val sourceMangas = arrayOf(sourceMangasRaw.filter { it.favorite }, - sourceMangasRaw.filterNot { it.favorite }) + val sourceMangaRaw = mangas[sourceValue.key]?.toMutableSet() ?: return@forEach + val sourceMangaPair = sourceMangaRaw.partition { it.favorite } + val sourceDir = sourceValue.value - val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull { - val name = it.name ?: return@mapNotNull null - name to MangaDirectory(it) + + val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull { mangaDir -> + val name = mangaDir.name ?: return@mapNotNull null + val chapterDirs = mangaDir.listFiles().orEmpty().mapNotNull { chapterFile -> chapterFile.name }.toHashSet() + name to MangaDirectory(mangaDir, chapterDirs) }.toMap() - mangaDirs.values.forEach { mangaDir -> - val chapterDirs = - mangaDir.dir.listFiles().orEmpty().mapNotNull { it.name }.toHashSet() - mangaDir.files = chapterDirs - } val trueMangaDirs = mangaDirs.mapNotNull { mangaDir -> - val manga = sourceMangas.firstOrNull()?.find { - DiskUtil.buildValidFilename( - it.originalTitle - ).toLowerCase() == mangaDir.key.toLowerCase() && it.source == sourceValue.key - } ?: sourceMangas.lastOrNull()?.find { - DiskUtil.buildValidFilename( - it.originalTitle - ).toLowerCase() == mangaDir.key.toLowerCase() && it.source == sourceValue.key - } + val manga = findManga(sourceMangaPair.first, mangaDir.key, sourceValue.key) ?: findManga(sourceMangaPair.second, mangaDir.key, sourceValue.key) val id = manga?.id ?: return@mapNotNull null id to mangaDir.value.files }.toMap() @@ -166,6 +155,15 @@ class DownloadCache( } } + /** + * Searches a manga list and matches the given mangakey and source key + */ + private fun findManga(mangaList: List, mangaKey: String, sourceKey: Long): Manga? { + return mangaList.find { + DiskUtil.buildValidFilename(it.originalTitle).toLowerCase() == mangaKey.toLowerCase() && it.source == sourceKey + } + } + /** * Adds a chapter that has just been download to this cache. * diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt index 3f02669c31..ebae08cc5f 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt @@ -11,6 +11,7 @@ import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.model.Page import rx.Observable +import timber.log.Timber import uy.kohesive.injekt.injectLazy /** @@ -252,6 +253,13 @@ class DownloadManager(val context: Context) { } } + /** + * return the list of all manga folders + */ + fun getMangaFolders(source: Source): List { + return provider.findSourceDir(source)?.listFiles()?.toList() ?: emptyList() + } + /** * Deletes the directories of chapters that were read or have no match * @@ -259,19 +267,39 @@ class DownloadManager(val context: Context) { * @param manga the manga of the chapters. * @param source the source of the chapters. */ - fun cleanupChapters(allChapters: List, manga: Manga, source: Source): Int { + fun cleanupChapters(allChapters: List, manga: Manga, source: Source, removeRead: Boolean, removeNonFavorite: Boolean): Int { var cleaned = 0 + + if (removeNonFavorite && !manga.favorite) { + val mangaFolder = provider.getMangaDir(manga, source) + cleaned += 1 + (mangaFolder.listFiles()?.size ?: 0) + mangaFolder.delete() + cache.removeManga(manga) + return cleaned + } + val filesWithNoChapter = provider.findUnmatchedChapterDirs(allChapters, manga, source) cleaned += filesWithNoChapter.size cache.removeFolders(filesWithNoChapter.mapNotNull { it.name }, manga) filesWithNoChapter.forEach { it.delete() } - val readChapters = allChapters.filter { it.read } - val readChapterDirs = provider.findChapterDirs(readChapters, manga, source) - readChapterDirs.forEach { it.delete() } - cleaned += readChapterDirs.size - cache.removeChapters(readChapters, manga) + + if (removeRead) { + val readChapters = allChapters.filter { it.read } + val readChapterDirs = provider.findChapterDirs(readChapters, manga, source) + readChapterDirs.forEach { it.delete() } + cleaned += readChapterDirs.size + cache.removeChapters(readChapters, manga) + } + if (cache.getDownloadCount(manga) == 0) { - provider.findChapterDirs(allChapters, manga, source).firstOrNull()?.parentFile?.delete() // Delete manga directory if empty + val mangaFolder = provider.getMangaDir(manga, source) + val size = mangaFolder.listFiles()?.size ?: 0 + if (size == 0) { + mangaFolder.delete() + cache.removeManga(manga) + } else { + Timber.e("Cache and download folder doesn't match for %s", manga.title) + } } return cleaned } @@ -310,6 +338,27 @@ class DownloadManager(val context: Context) { } } + /** + * Renames an already downloaded chapter + * + * @param manga the manga of the chapter. + * @param oldChapter the existing chapter with the old name. + * @param newChapter the target chapter with the new name. + */ + fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) { + val oldName = provider.getChapterDirName(oldChapter) + val newName = provider.getChapterDirName(newChapter) + val mangaDir = provider.getMangaDir(manga, source) + + val oldFolder = mangaDir.findFile(oldName) + if (oldFolder?.renameTo(newName) == true) { + cache.removeChapters(listOf(oldChapter), manga) + cache.addChapter(newName, mangaDir, manga) + } else { + Timber.e("Could not rename downloaded chapter: %s.", oldName) + } + } + fun addListener(listener: DownloadQueue.DownloadListener) = queue.addListener(listener) fun removeListener(listener: DownloadQueue.DownloadListener) = queue.removeListener(listener) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt index 1c2cf6d42d..440fdce817 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt @@ -98,8 +98,19 @@ class DownloadProvider(private val context: Context) { */ fun findChapterDirs(chapters: List, manga: Manga, source: Source): List { val mangaDir = findMangaDir(manga, source) ?: return emptyList() - return chapters.mapNotNull { chp -> - getValidChapterDirNames(chp).mapNotNull { mangaDir.findFile(it) }.firstOrNull() + val chapterNameHashSet = chapters.map { it.name }.toHashSet() + val scanalatorNameHashSet = chapters.map { getChapterDirName(it) }.toHashSet() + + return mangaDir.listFiles()!!.asList().filter { file -> + file.name?.let { fileName -> + if (scanalatorNameHashSet.contains(fileName)) { + return@filter true + } + val afterScanlatorCheck = fileName.substringAfter("_") + return@filter chapterNameHashSet.contains(fileName) || chapterNameHashSet.contains(afterScanlatorCheck) + + } + return@filter false } } @@ -150,12 +161,25 @@ class DownloadProvider(private val context: Context) { source: Source ): List { val mangaDir = findMangaDir(manga, source) ?: return emptyList() - return mangaDir.listFiles()!!.asList().filter { - (chapters.find { chp -> - getValidChapterDirNames(chp).any { dir -> - mangaDir.findFile(dir) != null + val chapterNameHashSet = chapters.map { it.name }.toHashSet() + val scanalatorNameHashSet = chapters.map { getChapterDirName(it) }.toHashSet() + + + return mangaDir.listFiles()!!.asList().filter { file -> + file.name?.let { fileName -> + if (fileName.endsWith(Downloader.TMP_DIR_SUFFIX)) { + return@filter true } - } == null) || it.name?.endsWith("_tmp") == true + //check this first because this is the normal name format + if (scanalatorNameHashSet.contains(fileName)) { + return@filter false + } + val afterScanlatorCheck = fileName.substringAfter("_") + //check both these dont exist because who knows how a chapter name is and it might not trim scanlator correctly + return@filter !chapterNameHashSet.contains(fileName) && !chapterNameHashSet.contains(afterScanlatorCheck) + } + //everything else is considered true + return@filter true } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadService.kt index 6697f4eafe..23254f5b92 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadService.kt @@ -4,6 +4,7 @@ import android.app.Notification import android.app.Service import android.content.Context import android.content.Intent +import android.net.ConnectivityManager import android.net.NetworkInfo.State.CONNECTED import android.net.NetworkInfo.State.DISCONNECTED import android.os.Build @@ -176,7 +177,7 @@ class DownloadService : Service() { private fun onNetworkStateChanged(connectivity: Connectivity) { when (connectivity.state) { CONNECTED -> { - if (preferences.downloadOnlyOverWifi() && connectivityManager.isActiveNetworkMetered) { + if (preferences.downloadOnlyOverWifi() && connectivityManager.activeNetworkInfo?.type != ConnectivityManager.TYPE_WIFI) { downloadManager.stopDownloads(getString(R.string.no_wifi_connection)) } else { val started = downloadManager.startDownloads() diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt index dfc6587b71..e352d42e9d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt @@ -287,11 +287,9 @@ open class MainActivity : BaseActivity(), DownloadServiceListener { if (savedInstanceState == null) { // Show changelog if needed if (Migrations.upgrade(preferences)) { - if (BuildConfig.DEBUG) { - MaterialDialog(this).title(text = "Welcome to the J2K MD2 Beta").message( - text = "This beta is for testing the upcoming release. Requests for new additions for this beta will ignored (however suggestions on how to better implement a feature in this beta are welcome).\n\nFor any bugs you come across, there is a bug report button in settings.\n\nAs a reminder this is a *BETA* build; bugs may happen, features may be missing/not implemented yet, and screens can change.\n\nEnjoy and thanks for testing!" - ).positiveButton(android.R.string.ok).cancelOnTouchOutside(false).show() - } else ChangelogDialogController().showDialog(router) + if (!BuildConfig.DEBUG) { + ChangelogDialogController().showDialog(router) + } } } preferences.extensionUpdatesCount().asObservable().subscribe { diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt index 992328a33e..0b5f9b2eb8 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt @@ -13,11 +13,13 @@ import android.widget.Toast import androidx.core.net.toUri import androidx.preference.PreferenceScreen import com.afollestad.materialdialogs.MaterialDialog +import com.afollestad.materialdialogs.list.listItemsMultiChoice import eu.kanade.tachiyomi.R import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.cache.CoverCache import eu.kanade.tachiyomi.data.database.DatabaseHelper import eu.kanade.tachiyomi.data.download.DownloadManager +import eu.kanade.tachiyomi.data.download.DownloadProvider import eu.kanade.tachiyomi.data.library.LibraryUpdateService import eu.kanade.tachiyomi.data.library.LibraryUpdateService.Target import eu.kanade.tachiyomi.network.NetworkHelper @@ -109,7 +111,11 @@ class SettingsAdvancedController : SettingsController() { summaryRes = R.string.delete_unused_chapters - onClick { cleanupDownloads() } + onClick { + val ctrl = CleanupDownloadsDialogController() + ctrl.targetController = this@SettingsAdvancedController + ctrl.showDialog(router) + } } if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { @@ -134,18 +140,51 @@ class SettingsAdvancedController : SettingsController() { } } - private fun cleanupDownloads() { + class CleanupDownloadsDialogController() : DialogController() { + override fun onCreateDialog(savedViewState: Bundle?): Dialog { + + return MaterialDialog(activity!!).show { + + title(R.string.clean_up_downloaded_chapters) + .listItemsMultiChoice(R.array.clean_up_downloads, disabledIndices = intArrayOf(0), initialSelection = intArrayOf(0, 1, 2)) { dialog, selections, items -> + val deleteRead = selections.contains(1) + val deleteNonFavorite = selections.contains(2) + (targetController as? SettingsAdvancedController)?.cleanupDownloads(deleteRead, deleteNonFavorite) + } + positiveButton(android.R.string.ok) + negativeButton(android.R.string.cancel) + } + } + } + + private fun cleanupDownloads(removeRead: Boolean, removeNonFavorite: Boolean) { if (job?.isActive == true) return activity?.toast(R.string.starting_cleanup) job = GlobalScope.launch(Dispatchers.IO, CoroutineStart.DEFAULT) { val mangaList = db.getMangas().executeAsBlocking() val sourceManager: SourceManager = Injekt.get() val downloadManager: DownloadManager = Injekt.get() + val downloadProvider = DownloadProvider(activity!!) var foldersCleared = 0 - for (manga in mangaList) { - val chapterList = db.getChapters(manga).executeAsBlocking() - val source = sourceManager.getOrStub(manga.source) - foldersCleared += downloadManager.cleanupChapters(chapterList, manga, source) + val sources = sourceManager.getOnlineSources() + + for (source in sources) { + val mangaFolders = downloadManager.getMangaFolders(source) + val sourceManga = mangaList.filter { it.source == source.id } + + for (mangaFolder in mangaFolders) { + val manga = sourceManga.find { downloadProvider.getMangaDirName(it) == mangaFolder.name } + if (manga == null) { + //download is orphaned and not even in the db delete it if remove non favorited is enabled + if (removeNonFavorite) { + foldersCleared += 1 + (mangaFolder.listFiles()?.size ?: 0) + mangaFolder.delete() + } + continue + } + val chapterList = db.getChapters(manga).executeAsBlocking() + foldersCleared += downloadManager.cleanupChapters(chapterList, manga, source, removeRead, removeNonFavorite) + } } launchUI { val activity = activity ?: return@launchUI @@ -168,32 +207,36 @@ class SettingsAdvancedController : SettingsController() { var deletedFiles = 0 Observable.defer { Observable.from(files) } - .doOnNext { file -> - if (chapterCache.removeFileFromCache(file.name)) { - deletedFiles++ - } + .doOnNext { file -> + if (chapterCache.removeFileFromCache(file.name)) { + deletedFiles++ } - .subscribeOn(Schedulers.io()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe({ - }, { - activity?.toast(R.string.cache_delete_error) - }, { - activity?.toast(resources?.getQuantityString(R.plurals.cache_cleared, - deletedFiles, deletedFiles)) - findPreference(CLEAR_CACHE_KEY)?.summary = - resources?.getString(R.string.used_, chapterCache.readableSize) - }) + } + .subscribeOn(Schedulers.io()) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe({ + }, { + activity?.toast(R.string.cache_delete_error) + }, { + activity?.toast( + resources?.getQuantityString( + R.plurals.cache_cleared, + deletedFiles, deletedFiles + ) + ) + findPreference(CLEAR_CACHE_KEY)?.summary = + resources?.getString(R.string.used_, chapterCache.readableSize) + }) } class ClearDatabaseDialogController : DialogController() { override fun onCreateDialog(savedViewState: Bundle?): Dialog { return MaterialDialog(activity!!) - .message(R.string.clear_database_confirmation) - .positiveButton(android.R.string.ok) { - (targetController as? SettingsAdvancedController)?.clearDatabase() - } - .negativeButton(android.R.string.cancel) + .message(R.string.clear_database_confirmation) + .positiveButton(android.R.string.ok) { + (targetController as? SettingsAdvancedController)?.clearDatabase() + } + .negativeButton(android.R.string.cancel) } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt b/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt index 85394c5d15..79b13bb6dc 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt @@ -3,9 +3,12 @@ package eu.kanade.tachiyomi.util.chapter import eu.kanade.tachiyomi.data.database.DatabaseHelper import eu.kanade.tachiyomi.data.database.models.Chapter import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.download.DownloadManager import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.online.HttpSource +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get import java.util.Date import java.util.TreeSet @@ -29,6 +32,8 @@ fun syncChaptersWithSource( throw Exception("No chapters found") } + val downloadManager: DownloadManager = Injekt.get() + // Chapters from db. val dbChapters = db.getChapters(manga).executeAsBlocking() @@ -61,6 +66,9 @@ fun syncChaptersWithSource( ChapterRecognition.parseChapterNumber(sourceChapter, manga) if (shouldUpdateDbChapter(dbChapter, sourceChapter)) { + if (dbChapter.name != sourceChapter.name && downloadManager.isChapterDownloaded(dbChapter, manga)) { + downloadManager.renameChapter(source, manga, dbChapter, sourceChapter) + } dbChapter.scanlator = sourceChapter.scanlator dbChapter.name = sourceChapter.name dbChapter.date_upload = sourceChapter.date_upload diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml index 705ef2d436..b9baa0c991 100644 --- a/app/src/main/res/values/arrays.xml +++ b/app/src/main/res/values/arrays.xml @@ -61,4 +61,11 @@ @string/multiply @string/screen + + + @string/clean_orphaned_downloads + @string/clean_read_downloads + @string/clean_read_manga_not_in_library + + diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 90ea4db38f..0219da7732 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -545,6 +545,9 @@ Clean up downloaded chapters Delete non-existent, partially downloaded, and read chapter folders + Clean orphaned + Clean read + Clean manga not in library Starting cleanup No folders to cleanup Disable battery optimization