Fix delay between URL fetch and image download (#9452)

Fetch each source image URL immediately before downloading each image
instead of fetching all URLs and then downloading all images.

Source image URLs may change, so the downloader may fail if there is
too long a delay between fetching the image URL and downloading the
image.
This commit is contained in:
Two-Ai 2023-05-05 22:17:51 -04:00 committed by GitHub
parent cb2d43c0d1
commit bbe0ab1dd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -338,21 +338,21 @@ class Downloader(
download.status = Download.State.DOWNLOADING download.status = Download.State.DOWNLOADING
// Get all the URLs to the source images, fetch pages if necessary
pageList.filter { it.imageUrl.isNullOrEmpty() }.forEach { page ->
page.status = Page.State.LOAD_PAGE
try {
page.imageUrl = download.source.fetchImageUrl(page).awaitSingle()
} catch (e: Throwable) {
page.status = Page.State.ERROR
}
}
// Start downloading images, consider we can have downloaded images already // Start downloading images, consider we can have downloaded images already
// Concurrently do 2 pages at a time // Concurrently do 2 pages at a time
pageList.asFlow() pageList.asFlow()
.flatMapMerge(concurrency = 2) { page -> .flatMapMerge(concurrency = 2) { page ->
flow { flow {
// Fetch image URL if necessary
if (page.imageUrl.isNullOrEmpty()) {
page.status = Page.State.LOAD_PAGE
try {
page.imageUrl = download.source.fetchImageUrl(page).awaitSingle()
} catch (e: Throwable) {
page.status = Page.State.ERROR
}
}
withIOContext { getOrDownloadImage(page, download, tmpDir) } withIOContext { getOrDownloadImage(page, download, tmpDir) }
emit(page) emit(page)
}.flowOn(Dispatchers.IO) }.flowOn(Dispatchers.IO)