Download updates (#546)

* Add rename download attempt from preview

* clean up renew cache some to help with readability

* Download on WiFi regardless of metered status  (From Tachiyomi)

* no reason to show md2 beta message for new debug installs

* update clean up downloads to be a little faster and hit some edge cases not thought of before

* fix bug where manga name does not match the file name and was deleting manga downloads of manga with special characters

Co-authored-by: Chris Allan <chrisallan@pm.me>
Co-authored-by: Ken Swenson <2048861+flat@users.noreply.github.com>

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt

@@ -134,30 +134,19 @@ class DownloadCache(
         val mangas = db.getMangas().executeAsBlocking().groupBy { it.source }
 
         sourceDirs.forEach { sourceValue ->
-            val sourceMangasRaw = mangas[sourceValue.key]?.toMutableSet() ?: return@forEach
+            val sourceMangaRaw = mangas[sourceValue.key]?.toMutableSet() ?: return@forEach
-            val sourceMangas = arrayOf(sourceMangasRaw.filter { it.favorite },
+            val sourceMangaPair = sourceMangaRaw.partition { it.favorite }
-                sourceMangasRaw.filterNot { it.favorite })
+
             val sourceDir = sourceValue.value
-            val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull {
+
-                val name = it.name ?: return@mapNotNull null
+            val mangaDirs = sourceDir.dir.listFiles().orEmpty().mapNotNull { mangaDir ->
-                name to MangaDirectory(it)
+                val name = mangaDir.name ?: return@mapNotNull null
+                val chapterDirs = mangaDir.listFiles().orEmpty().mapNotNull { chapterFile -> chapterFile.name }.toHashSet()
+                name to MangaDirectory(mangaDir, chapterDirs)
             }.toMap()
 
-            mangaDirs.values.forEach { mangaDir ->
-                val chapterDirs =
-                    mangaDir.dir.listFiles().orEmpty().mapNotNull { it.name }.toHashSet()
-                mangaDir.files = chapterDirs
-            }
             val trueMangaDirs = mangaDirs.mapNotNull { mangaDir ->
-                val manga = sourceMangas.firstOrNull()?.find {
+                val manga = findManga(sourceMangaPair.first, mangaDir.key, sourceValue.key) ?: findManga(sourceMangaPair.second, mangaDir.key, sourceValue.key)
-                    DiskUtil.buildValidFilename(
-                        it.originalTitle
-                    ).toLowerCase() == mangaDir.key.toLowerCase() && it.source == sourceValue.key
-                } ?: sourceMangas.lastOrNull()?.find {
-                    DiskUtil.buildValidFilename(
-                        it.originalTitle
-                    ).toLowerCase() == mangaDir.key.toLowerCase() && it.source == sourceValue.key
-                }
                 val id = manga?.id ?: return@mapNotNull null
                 id to mangaDir.value.files
             }.toMap()
@@ -166,6 +155,15 @@ class DownloadCache(
         }
     }
 
+    /**
+     * Searches a manga list and matches the given mangakey and source key
+     */
+    private fun findManga(mangaList: List<Manga>, mangaKey: String, sourceKey: Long): Manga? {
+        return mangaList.find {
+            DiskUtil.buildValidFilename(it.originalTitle).toLowerCase() == mangaKey.toLowerCase() && it.source == sourceKey
+        }
+    }
+
     /**
      * Adds a chapter that has just been download to this cache.
      *

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt

@@ -11,6 +11,7 @@ import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.source.model.Page
 import rx.Observable
+import timber.log.Timber
 import uy.kohesive.injekt.injectLazy
 
 /**
@@ -252,6 +253,13 @@ class DownloadManager(val context: Context) {
         }
     }
 
+    /**
+     * return the list of all manga folders
+     */
+    fun getMangaFolders(source: Source): List<UniFile> {
+        return provider.findSourceDir(source)?.listFiles()?.toList() ?: emptyList()
+    }
+
     /**
      * Deletes the directories of chapters that were read or have no match
      *
@@ -259,19 +267,39 @@ class DownloadManager(val context: Context) {
      * @param manga the manga of the chapters.
      * @param source the source of the chapters.
      */
-    fun cleanupChapters(allChapters: List<Chapter>, manga: Manga, source: Source): Int {
+    fun cleanupChapters(allChapters: List<Chapter>, manga: Manga, source: Source, removeRead: Boolean, removeNonFavorite: Boolean): Int {
         var cleaned = 0
+
+        if (removeNonFavorite && !manga.favorite) {
+            val mangaFolder = provider.getMangaDir(manga, source)
+            cleaned += 1 + (mangaFolder.listFiles()?.size ?: 0)
+            mangaFolder.delete()
+            cache.removeManga(manga)
+            return cleaned
+        }
+
         val filesWithNoChapter = provider.findUnmatchedChapterDirs(allChapters, manga, source)
         cleaned += filesWithNoChapter.size
         cache.removeFolders(filesWithNoChapter.mapNotNull { it.name }, manga)
         filesWithNoChapter.forEach { it.delete() }
-        val readChapters = allChapters.filter { it.read }
+
-        val readChapterDirs = provider.findChapterDirs(readChapters, manga, source)
+        if (removeRead) {
-        readChapterDirs.forEach { it.delete() }
+            val readChapters = allChapters.filter { it.read }
-        cleaned += readChapterDirs.size
+            val readChapterDirs = provider.findChapterDirs(readChapters, manga, source)
-        cache.removeChapters(readChapters, manga)
+            readChapterDirs.forEach { it.delete() }
+            cleaned += readChapterDirs.size
+            cache.removeChapters(readChapters, manga)
+        }
+
         if (cache.getDownloadCount(manga) == 0) {
-            provider.findChapterDirs(allChapters, manga, source).firstOrNull()?.parentFile?.delete() // Delete manga directory if empty
+            val mangaFolder = provider.getMangaDir(manga, source)
+            val size = mangaFolder.listFiles()?.size ?: 0
+            if (size == 0) {
+                mangaFolder.delete()
+                cache.removeManga(manga)
+            } else {
+                Timber.e("Cache and download folder doesn't match for %s", manga.title)
+            }
         }
         return cleaned
     }
@@ -310,6 +338,27 @@ class DownloadManager(val context: Context) {
         }
     }
 
+    /**
+     * Renames an already downloaded chapter
+     *
+     * @param manga the manga of the chapter.
+     * @param oldChapter the existing chapter with the old name.
+     * @param newChapter the target chapter with the new name.
+     */
+    fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
+        val oldName = provider.getChapterDirName(oldChapter)
+        val newName = provider.getChapterDirName(newChapter)
+        val mangaDir = provider.getMangaDir(manga, source)
+
+        val oldFolder = mangaDir.findFile(oldName)
+        if (oldFolder?.renameTo(newName) == true) {
+            cache.removeChapters(listOf(oldChapter), manga)
+            cache.addChapter(newName, mangaDir, manga)
+        } else {
+            Timber.e("Could not rename downloaded chapter: %s.", oldName)
+        }
+    }
+
     fun addListener(listener: DownloadQueue.DownloadListener) = queue.addListener(listener)
     fun removeListener(listener: DownloadQueue.DownloadListener) = queue.removeListener(listener)
 }

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadProvider.kt

@@ -98,8 +98,19 @@ class DownloadProvider(private val context: Context) {
      */
     fun findChapterDirs(chapters: List<Chapter>, manga: Manga, source: Source): List<UniFile> {
         val mangaDir = findMangaDir(manga, source) ?: return emptyList()
-        return chapters.mapNotNull { chp ->
+        val chapterNameHashSet = chapters.map { it.name }.toHashSet()
-            getValidChapterDirNames(chp).mapNotNull { mangaDir.findFile(it) }.firstOrNull()
+        val scanalatorNameHashSet = chapters.map { getChapterDirName(it) }.toHashSet()
+
+        return mangaDir.listFiles()!!.asList().filter { file ->
+            file.name?.let { fileName ->
+                if (scanalatorNameHashSet.contains(fileName)) {
+                    return@filter true
+                }
+                val afterScanlatorCheck = fileName.substringAfter("_")
+                return@filter chapterNameHashSet.contains(fileName) || chapterNameHashSet.contains(afterScanlatorCheck)
+
+            }
+            return@filter false
         }
     }
 
@@ -150,12 +161,25 @@ class DownloadProvider(private val context: Context) {
         source: Source
     ): List<UniFile> {
         val mangaDir = findMangaDir(manga, source) ?: return emptyList()
-        return mangaDir.listFiles()!!.asList().filter {
+        val chapterNameHashSet = chapters.map { it.name }.toHashSet()
-            (chapters.find { chp ->
+        val scanalatorNameHashSet = chapters.map { getChapterDirName(it) }.toHashSet()
-                getValidChapterDirNames(chp).any { dir ->
+
-                    mangaDir.findFile(dir) != null
+
+        return mangaDir.listFiles()!!.asList().filter { file ->
+            file.name?.let { fileName ->
+                if (fileName.endsWith(Downloader.TMP_DIR_SUFFIX)) {
+                    return@filter true
                 }
-            } == null) || it.name?.endsWith("_tmp") == true
+                //check this first because this is the normal name format
+                if (scanalatorNameHashSet.contains(fileName)) {
+                    return@filter false
+                }
+                val afterScanlatorCheck = fileName.substringAfter("_")
+                //check both these dont exist because who knows how a chapter name is and it might not trim scanlator correctly
+                return@filter !chapterNameHashSet.contains(fileName) && !chapterNameHashSet.contains(afterScanlatorCheck)
+            }
+            //everything else is considered true
+            return@filter true
         }
     }
 

app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadService.kt

@@ -4,6 +4,7 @@ import android.app.Notification
 import android.app.Service
 import android.content.Context
 import android.content.Intent
+import android.net.ConnectivityManager
 import android.net.NetworkInfo.State.CONNECTED
 import android.net.NetworkInfo.State.DISCONNECTED
 import android.os.Build
@@ -176,7 +177,7 @@ class DownloadService : Service() {
     private fun onNetworkStateChanged(connectivity: Connectivity) {
         when (connectivity.state) {
             CONNECTED -> {
-                if (preferences.downloadOnlyOverWifi() && connectivityManager.isActiveNetworkMetered) {
+                if (preferences.downloadOnlyOverWifi() && connectivityManager.activeNetworkInfo?.type != ConnectivityManager.TYPE_WIFI) {
                     downloadManager.stopDownloads(getString(R.string.no_wifi_connection))
                 } else {
                     val started = downloadManager.startDownloads()

app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt

@@ -287,11 +287,9 @@ open class MainActivity : BaseActivity(), DownloadServiceListener {
         if (savedInstanceState == null) {
             // Show changelog if needed
             if (Migrations.upgrade(preferences)) {
-                if (BuildConfig.DEBUG) {
+                if (!BuildConfig.DEBUG) {
-                    MaterialDialog(this).title(text = "Welcome to the J2K MD2 Beta").message(
+                    ChangelogDialogController().showDialog(router)
-                        text = "This beta is for testing the upcoming release. Requests for new additions for this beta will ignored (however suggestions on how to better implement a feature in this beta are welcome).\n\nFor any bugs you come across, there is a bug report button in settings.\n\nAs a reminder this is a *BETA* build; bugs may happen, features may be missing/not implemented yet, and screens can change.\n\nEnjoy and thanks for testing!"
+                }
-                    ).positiveButton(android.R.string.ok).cancelOnTouchOutside(false).show()
-                } else ChangelogDialogController().showDialog(router)
             }
         }
         preferences.extensionUpdatesCount().asObservable().subscribe {

app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt

@@ -13,11 +13,13 @@ import android.widget.Toast
 import androidx.core.net.toUri
 import androidx.preference.PreferenceScreen
 import com.afollestad.materialdialogs.MaterialDialog
+import com.afollestad.materialdialogs.list.listItemsMultiChoice
 import eu.kanade.tachiyomi.R
 import eu.kanade.tachiyomi.data.cache.ChapterCache
 import eu.kanade.tachiyomi.data.cache.CoverCache
 import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.download.DownloadManager
+import eu.kanade.tachiyomi.data.download.DownloadProvider
 import eu.kanade.tachiyomi.data.library.LibraryUpdateService
 import eu.kanade.tachiyomi.data.library.LibraryUpdateService.Target
 import eu.kanade.tachiyomi.network.NetworkHelper
@@ -109,7 +111,11 @@ class SettingsAdvancedController : SettingsController() {
 
             summaryRes = R.string.delete_unused_chapters
 
-            onClick { cleanupDownloads() }
+            onClick {
+                val ctrl = CleanupDownloadsDialogController()
+                ctrl.targetController = this@SettingsAdvancedController
+                ctrl.showDialog(router)
+            }
         }
 
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
@@ -134,18 +140,51 @@ class SettingsAdvancedController : SettingsController() {
         }
     }
 
-    private fun cleanupDownloads() {
+    class CleanupDownloadsDialogController() : DialogController() {
+        override fun onCreateDialog(savedViewState: Bundle?): Dialog {
+
+            return MaterialDialog(activity!!).show {
+
+                title(R.string.clean_up_downloaded_chapters)
+                    .listItemsMultiChoice(R.array.clean_up_downloads, disabledIndices = intArrayOf(0), initialSelection = intArrayOf(0, 1, 2)) { dialog, selections, items ->
+                        val deleteRead = selections.contains(1)
+                        val deleteNonFavorite = selections.contains(2)
+                        (targetController as? SettingsAdvancedController)?.cleanupDownloads(deleteRead, deleteNonFavorite)
+                    }
+                positiveButton(android.R.string.ok)
+                negativeButton(android.R.string.cancel)
+            }
+        }
+    }
+
+    private fun cleanupDownloads(removeRead: Boolean, removeNonFavorite: Boolean) {
         if (job?.isActive == true) return
         activity?.toast(R.string.starting_cleanup)
         job = GlobalScope.launch(Dispatchers.IO, CoroutineStart.DEFAULT) {
             val mangaList = db.getMangas().executeAsBlocking()
             val sourceManager: SourceManager = Injekt.get()
             val downloadManager: DownloadManager = Injekt.get()
+            val downloadProvider = DownloadProvider(activity!!)
             var foldersCleared = 0
-            for (manga in mangaList) {
+            val sources = sourceManager.getOnlineSources()
-                val chapterList = db.getChapters(manga).executeAsBlocking()
+
-                val source = sourceManager.getOrStub(manga.source)
+            for (source in sources) {
-                foldersCleared += downloadManager.cleanupChapters(chapterList, manga, source)
+                val mangaFolders = downloadManager.getMangaFolders(source)
+                val sourceManga = mangaList.filter { it.source == source.id }
+
+                for (mangaFolder in mangaFolders) {
+                    val manga = sourceManga.find { downloadProvider.getMangaDirName(it) == mangaFolder.name }
+                    if (manga == null) {
+                    //download is orphaned and not even in the db delete it if remove non favorited is enabled
+                        if (removeNonFavorite) {
+                            foldersCleared += 1 + (mangaFolder.listFiles()?.size ?: 0)
+                            mangaFolder.delete()
+                        }
+                        continue
+                    }
+                    val chapterList = db.getChapters(manga).executeAsBlocking()
+                    foldersCleared += downloadManager.cleanupChapters(chapterList, manga, source, removeRead, removeNonFavorite)
+                }
             }
             launchUI {
                 val activity = activity ?: return@launchUI
@@ -168,32 +207,36 @@ class SettingsAdvancedController : SettingsController() {
         var deletedFiles = 0
 
         Observable.defer { Observable.from(files) }
-                .doOnNext { file ->
+            .doOnNext { file ->
-                    if (chapterCache.removeFileFromCache(file.name)) {
+                if (chapterCache.removeFileFromCache(file.name)) {
-                        deletedFiles++
+                    deletedFiles++
-                    }
                 }
-                .subscribeOn(Schedulers.io())
+            }
-                .observeOn(AndroidSchedulers.mainThread())
+            .subscribeOn(Schedulers.io())
-                .subscribe({
+            .observeOn(AndroidSchedulers.mainThread())
-                }, {
+            .subscribe({
-                    activity?.toast(R.string.cache_delete_error)
+            }, {
-                }, {
+                activity?.toast(R.string.cache_delete_error)
-                    activity?.toast(resources?.getQuantityString(R.plurals.cache_cleared,
+            }, {
-                        deletedFiles, deletedFiles))
+                activity?.toast(
-                    findPreference(CLEAR_CACHE_KEY)?.summary =
+                    resources?.getQuantityString(
-                            resources?.getString(R.string.used_, chapterCache.readableSize)
+                        R.plurals.cache_cleared,
-                })
+                        deletedFiles, deletedFiles
+                    )
+                )
+                findPreference(CLEAR_CACHE_KEY)?.summary =
+                    resources?.getString(R.string.used_, chapterCache.readableSize)
+            })
     }
 
     class ClearDatabaseDialogController : DialogController() {
         override fun onCreateDialog(savedViewState: Bundle?): Dialog {
             return MaterialDialog(activity!!)
-                    .message(R.string.clear_database_confirmation)
+                .message(R.string.clear_database_confirmation)
-                    .positiveButton(android.R.string.ok) {
+                .positiveButton(android.R.string.ok) {
-                        (targetController as? SettingsAdvancedController)?.clearDatabase()
+                    (targetController as? SettingsAdvancedController)?.clearDatabase()
-                    }
+                }
-                    .negativeButton(android.R.string.cancel)
+                .negativeButton(android.R.string.cancel)
         }
     }
 

app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt

@@ -3,9 +3,12 @@ package eu.kanade.tachiyomi.util.chapter
 import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.database.models.Chapter
 import eu.kanade.tachiyomi.data.database.models.Manga
+import eu.kanade.tachiyomi.data.download.DownloadManager
 import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.online.HttpSource
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
 import java.util.Date
 import java.util.TreeSet
 
@@ -29,6 +32,8 @@ fun syncChaptersWithSource(
         throw Exception("No chapters found")
     }
 
+    val downloadManager: DownloadManager = Injekt.get()
+
     // Chapters from db.
     val dbChapters = db.getChapters(manga).executeAsBlocking()
 
@@ -61,6 +66,9 @@ fun syncChaptersWithSource(
             ChapterRecognition.parseChapterNumber(sourceChapter, manga)
 
             if (shouldUpdateDbChapter(dbChapter, sourceChapter)) {
+                if (dbChapter.name != sourceChapter.name && downloadManager.isChapterDownloaded(dbChapter, manga)) {
+                    downloadManager.renameChapter(source, manga, dbChapter, sourceChapter)
+                }
                 dbChapter.scanlator = sourceChapter.scanlator
                 dbChapter.name = sourceChapter.name
                 dbChapter.date_upload = sourceChapter.date_upload

app/src/main/res/values/arrays.xml

@@ -61,4 +61,11 @@
         <item>@string/multiply</item>
         <item>@string/screen</item>
     </string-array>
+
+    <string-array name="clean_up_downloads">
+        <item>@string/clean_orphaned_downloads</item>
+        <item>@string/clean_read_downloads</item>
+        <item>@string/clean_read_manga_not_in_library</item>
+    </string-array>
+
 </resources>

app/src/main/res/values/strings.xml

@@ -545,6 +545,9 @@
     <string name="clean_up_downloaded_chapters">Clean up downloaded chapters</string>
     <string name="delete_unused_chapters">Delete non-existent, partially downloaded,
             and read chapter folders</string>
+    <string name="clean_orphaned_downloads">Clean orphaned</string>
+    <string name="clean_read_downloads">Clean read</string>
+    <string name="clean_read_manga_not_in_library">Clean manga not in library</string>
     <string name="starting_cleanup">Starting cleanup</string>
     <string name="no_folders_to_cleanup">No folders to cleanup</string>
     <string name="disable_battery_optimization">Disable battery optimization</string>