Minor updates to downloader from upstream

Such as available space check and using io thread instead of ui

Co-Authored-By: arkon <4098258+arkon@users.noreply.github.com>
This commit is contained in:
Jays2Kings 2021-04-19 00:14:24 -04:00
parent 4462e75ccd
commit 94a90c6cec
4 changed files with 86 additions and 48 deletions

View File

@ -5,6 +5,7 @@ import android.webkit.MimeTypeMap
import com.hippo.unifile.UniFile
import com.jakewharton.rxrelay.BehaviorRelay
import com.jakewharton.rxrelay.PublishRelay
import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.cache.ChapterCache
import eu.kanade.tachiyomi.data.database.models.Chapter
import eu.kanade.tachiyomi.data.database.models.Manga
@ -20,8 +21,8 @@ import eu.kanade.tachiyomi.util.lang.plusAssign
import eu.kanade.tachiyomi.util.storage.DiskUtil
import eu.kanade.tachiyomi.util.storage.saveTo
import eu.kanade.tachiyomi.util.system.ImageUtil
import eu.kanade.tachiyomi.util.system.launchIO
import eu.kanade.tachiyomi.util.system.launchNow
import eu.kanade.tachiyomi.util.system.launchUI
import kotlinx.coroutines.async
import okhttp3.Response
import rx.Observable
@ -29,6 +30,7 @@ import rx.android.schedulers.AndroidSchedulers
import rx.schedulers.Schedulers
import rx.subscriptions.CompositeSubscription
import timber.log.Timber
import uy.kohesive.injekt.injectLazy
import java.io.File
/**
@ -52,6 +54,8 @@ class Downloader(
private val sourceManager: SourceManager
) {
private val chapterCache: ChapterCache by injectLazy()
/**
* Store for persisting downloads across restarts.
*/
@ -119,7 +123,9 @@ class Downloader(
*/
fun stop(reason: String? = null) {
destroySubscriptions()
queue.filter { it.status == Download.DOWNLOADING }.forEach { it.status = Download.ERROR }
queue
.filter { it.status == Download.DOWNLOADING }
.forEach { it.status = Download.ERROR }
if (reason != null) {
notifier.onWarning(reason)
@ -142,7 +148,9 @@ class Downloader(
*/
fun pause() {
destroySubscriptions()
queue.filter { it.status == Download.DOWNLOADING }.forEach { it.status = Download.QUEUE }
queue
.filter { it.status == Download.DOWNLOADING }
.forEach { it.status = Download.QUEUE }
notifier.paused = true
}
@ -161,7 +169,8 @@ class Downloader(
// Needed to update the chapter view
if (isNotification) {
queue.filter { it.status == Download.QUEUE }
queue
.filter { it.status == Download.QUEUE }
.forEach { it.status = Download.NOT_DOWNLOADED }
}
queue.clear()
@ -207,7 +216,7 @@ class Downloader(
},
5
)
.onBackpressureBuffer()
.onBackpressureLatest()
.observeOn(AndroidSchedulers.mainThread())
.subscribe(
{
@ -239,8 +248,8 @@ class Downloader(
* @param chapters the list of chapters to download.
* @param autoStart whether to start the downloader after enqueing the chapters.
*/
fun queueChapters(manga: Manga, chapters: List<Chapter>, autoStart: Boolean) = launchUI {
val source = sourceManager.get(manga.source) as? HttpSource ?: return@launchUI
fun queueChapters(manga: Manga, chapters: List<Chapter>, autoStart: Boolean) = launchIO {
val source = sourceManager.get(manga.source) as? HttpSource ?: return@launchIO
val wasEmpty = queue.isEmpty()
// Called in background thread, the operation can be slow with SAF.
val chaptersWithoutDir = async {
@ -281,15 +290,23 @@ class Downloader(
* @param download the chapter to be downloaded.
*/
private fun downloadChapter(download: Download): Observable<Download> = Observable.defer {
val chapterDirname = provider.getChapterDirName(download.chapter)
val mangaDir = provider.getMangaDir(download.manga, download.source)
val availSpace = DiskUtil.getAvailableStorageSpace(mangaDir)
if (availSpace != -1L && availSpace < MIN_DISK_SPACE) {
download.status = Download.ERROR
notifier.onError(context.getString(R.string.couldnt_download_low_space), download.chapter.name)
return@defer Observable.just(download)
}
val chapterDirname = provider.getChapterDirName(download.chapter)
val tmpDir = mangaDir.createDirectory(chapterDirname + TMP_DIR_SUFFIX)
val pageListObservable = if (download.pages == null) {
// Pull page list from network and add them to download object
download.source.fetchPageList(download.chapter).doOnNext { pages ->
if (pages.isEmpty()) {
throw Exception("Page list is empty")
throw Exception(context.getString(R.string.no_pages_found))
}
download.pages = pages
}
@ -298,20 +315,25 @@ class Downloader(
Observable.just(download.pages!!)
}
pageListObservable.doOnNext { _ ->
// Delete all temporary (unfinished) files
tmpDir.listFiles()?.filter { it.name!!.endsWith(".tmp") }?.forEach { it.delete() }
pageListObservable
.doOnNext { _ ->
// Delete all temporary (unfinished) files
tmpDir.listFiles()
?.filter { it.name!!.endsWith(".tmp") }
?.forEach { it.delete() }
download.downloadedImages = 0
download.status = Download.DOWNLOADING
}
download.downloadedImages = 0
download.status = Download.DOWNLOADING
}
// Get all the URLs to the source images, fetch pages if necessary
.flatMap { download.source.fetchAllImageUrlsFromPageList(it) }
// Start downloading images, consider we can have downloaded images already
// Concurrently do 5 pages at a time
.flatMap({ page -> getOrDownloadImage(page, download, tmpDir) }, 5)
// Do when page is downloaded.
.doOnNext { notifier.onProgressChange(download) }.toList().map { _ -> download }
.doOnNext { notifier.onProgressChange(download) }
.toList()
.map { download }
// Do after download completes
.doOnNext { ensureSuccessfulDownload(download, mangaDir, tmpDir, chapterDirname) }
// If the page list threw, it will resume here
@ -336,7 +358,9 @@ class Downloader(
tmpDir: UniFile
): Observable<Page> {
// If the image URL is empty, do nothing
if (page.imageUrl == null) return Observable.just(page)
if (page.imageUrl == null) {
return Observable.just(page)
}
val filename = String.format("%03d", page.number)
val tmpFile = tmpDir.findFile("$filename.tmp")
@ -346,16 +370,11 @@ class Downloader(
// Try to find the image file.
val imageFile = tmpDir.listFiles()!!.find { it.name!!.startsWith("$filename.") }
val cache = ChapterCache(context)
// If the image is already downloaded, do nothing. Otherwise download from network
val pageObservable = when {
imageFile != null -> Observable.just(imageFile)
cache.isImageInCache(page.imageUrl!!) -> moveFromCache(
page,
cache.getImageFile(page.imageUrl!!),
tmpDir,
filename
)
chapterCache.isImageInCache(page.imageUrl!!) -> moveImageFromCache(chapterCache.getImageFile(page.imageUrl!!), tmpDir, filename)
else -> downloadImage(page, download.source, tmpDir, filename)
}
@ -366,7 +385,8 @@ class Downloader(
page.progress = 100
download.downloadedImages++
page.status = Page.READY
}.map { page }
}
.map { page }
// Mark this page as error and allow to download the remaining
.onErrorReturn {
page.progress = 0
@ -376,30 +396,27 @@ class Downloader(
}
/**
* Returns the observable which takes from the downloaded image from cache
* Return the observable which copies the image from cache.
*
* @param page the page to download.
* @param file the file from cache
* @param cacheFile the file from cache.
* @param tmpDir the temporary directory of the download.
* @param filename the filename of the image.
*/
private fun moveFromCache(
page: Page,
file: File,
private fun moveImageFromCache(
cacheFile: File,
tmpDir: UniFile,
filename: String
): Observable<UniFile> {
return Observable.just(file).map {
return Observable.just(cacheFile).map {
val tmpFile = tmpDir.createFile("$filename.tmp")
val inputStream = file.inputStream()
inputStream.use { input ->
cacheFile.inputStream().use { input ->
tmpFile.openOutputStream().use { output ->
input.copyTo(output)
}
}
val extension = ImageUtil.findImageType(file.inputStream()) ?: return@map tmpFile
val extension = ImageUtil.findImageType(cacheFile.inputStream()) ?: return@map tmpFile
tmpFile.renameTo("$filename.${extension.extension}")
file.delete()
cacheFile.delete()
tmpFile
}
}
@ -420,19 +437,20 @@ class Downloader(
): Observable<UniFile> {
page.status = Page.DOWNLOAD_IMAGE
page.progress = 0
return source.fetchImage(page).map { response ->
val file = tmpDir.createFile("$filename.tmp")
try {
response.body!!.source().saveTo(file.openOutputStream())
val extension = getImageExtension(response, file)
file.renameTo("$filename.$extension")
} catch (e: Exception) {
response.close()
file.delete()
throw e
return source.fetchImage(page)
.map { response ->
val file = tmpDir.createFile("$filename.tmp")
try {
response.body!!.source().saveTo(file.openOutputStream())
val extension = getImageExtension(response, file)
file.renameTo("$filename.$extension")
} catch (e: Exception) {
response.close()
file.delete()
throw e
}
file
}
file
}
// Retry 3 times, waiting 2, 4 and 8 seconds between attempts.
.retryWhen(RetryWithDelay(3, { (2 shl it - 1) * 1000 }, Schedulers.trampoline()))
}
@ -514,5 +532,8 @@ class Downloader(
companion object {
const val TMP_DIR_SUFFIX = "_tmp"
// Arbitrary minimum required space to start a download: 50 MB
const val MIN_DISK_SPACE = 50 * 1024 * 1024
}
}

View File

@ -4,6 +4,7 @@ import android.content.Context
import android.content.Intent
import android.net.Uri
import android.os.Environment
import android.os.StatFs
import androidx.core.content.ContextCompat
import androidx.core.os.EnvironmentCompat
import com.hippo.unifile.UniFile
@ -28,6 +29,18 @@ object DiskUtil {
return size
}
/**
* Gets the available space for the disk that a file path points to, in bytes.
*/
fun getAvailableStorageSpace(f: UniFile): Long {
return try {
val stat = StatFs(f.uri.path)
stat.availableBlocksLong * stat.blockSizeLong
} catch (_: Exception) {
-1L
}
}
/**
* Returns the root folders of all the available external storages.
*/

View File

@ -11,6 +11,9 @@ import kotlinx.coroutines.withContext
fun launchUI(block: suspend CoroutineScope.() -> Unit): Job =
GlobalScope.launch(Dispatchers.Main, CoroutineStart.DEFAULT, block)
fun launchIO(block: suspend CoroutineScope.() -> Unit): Job =
GlobalScope.launch(Dispatchers.IO, CoroutineStart.DEFAULT, block)
fun launchNow(block: suspend CoroutineScope.() -> Unit): Job =
GlobalScope.launch(Dispatchers.Main, CoroutineStart.UNDISPATCHED, block)

View File

@ -760,6 +760,7 @@
<string name="manage_whats_downloading">Manage what\'s downloading</string>
<string name="visit_recents_for_download_queue">Visit the recents tab to access the download
queue. You can also double tap or press and hold for quicker access</string>
<string name="couldnt_download_low_space">Couldn\'t download chapters due to low disk space</string>
<!-- Download Notification -->
<string name="could_not_download_unexpected_error">Could not download chapter due to unexpected error</string>