From b0a8740e8d42cc26e9519b31178550ea541a3ce0 Mon Sep 17 00:00:00 2001 From: inorichi Date: Wed, 4 Nov 2015 10:51:49 +0100 Subject: [PATCH] Improve download manager. Add an option to select the number of threads for downloads. --- .../data/helpers/DownloadManager.java | 183 +++++++++++++----- .../data/helpers/PreferencesHelper.java | 4 + .../mangafeed/data/models/Download.java | 20 ++ .../presenter/MangaChaptersPresenter.java | 26 ++- .../ui/fragment/MangaChaptersFragment.java | 22 +-- app/src/main/res/values/arrays.xml | 6 + app/src/main/res/values/keys.xml | 1 + app/src/main/res/values/strings.xml | 1 + app/src/main/res/xml/pref_downloads.xml | 7 + 9 files changed, 197 insertions(+), 73 deletions(-) create mode 100644 app/src/main/java/eu/kanade/mangafeed/data/models/Download.java diff --git a/app/src/main/java/eu/kanade/mangafeed/data/helpers/DownloadManager.java b/app/src/main/java/eu/kanade/mangafeed/data/helpers/DownloadManager.java index 3aa2ee4f58..63dc149494 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/helpers/DownloadManager.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/helpers/DownloadManager.java @@ -2,11 +2,21 @@ package eu.kanade.mangafeed.data.helpers; import android.content.Context; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; + import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; import java.io.IOException; +import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; import eu.kanade.mangafeed.data.models.Chapter; +import eu.kanade.mangafeed.data.models.Download; import eu.kanade.mangafeed.data.models.Manga; import eu.kanade.mangafeed.data.models.Page; import eu.kanade.mangafeed.events.DownloadChapterEvent; @@ -20,77 +30,107 @@ import rx.subjects.PublishSubject; public class DownloadManager { private PublishSubject downloadsSubject; - private Subscription downloadsSubscription; + private Subscription downloadSubscription; private Context context; private SourceManager sourceManager; private PreferencesHelper preferences; + private Gson gson; + + private List queue; public DownloadManager(Context context, SourceManager sourceManager, PreferencesHelper preferences) { this.context = context; this.sourceManager = sourceManager; this.preferences = preferences; + this.gson = new Gson(); + + queue = new ArrayList<>(); initializeDownloadSubscription(); } + public PublishSubject getDownloadsSubject() { + return downloadsSubject; + } + private void initializeDownloadSubscription() { - if (downloadsSubscription != null && !downloadsSubscription.isUnsubscribed()) { - downloadsSubscription.unsubscribe(); + if (downloadSubscription != null && !downloadSubscription.isUnsubscribed()) { + downloadSubscription.unsubscribe(); } downloadsSubject = PublishSubject.create(); - downloadsSubscription = downloadsSubject + // Listen for download events, add them to queue and download + downloadSubscription = downloadsSubject .subscribeOn(Schedulers.io()) - .concatMap(event -> downloadChapter(event.getManga(), event.getChapter())) + .filter(event -> !isChapterDownloaded(event)) + .flatMap(this::createDownload) + .window(preferences.getDownloadThreads()) + .concatMap(concurrentDownloads -> concurrentDownloads + .concatMap(this::downloadChapter)) .onBackpressureBuffer() .subscribe(); } - public Observable downloadChapter(Manga manga, Chapter chapter) { - final Source source = sourceManager.get(manga.source); - final File chapterDirectory = getAbsoluteChapterDirectory(source, manga, chapter); + // Check if a chapter is already downloaded + private boolean isChapterDownloaded(DownloadChapterEvent event) { + final Source source = sourceManager.get(event.getManga().source); - return source - .pullPageListFromNetwork(chapter.url) - // Ensure we don't download a chapter already downloaded - .filter(pages -> !isChapterDownloaded(chapterDirectory, pages)) + // If the chapter is already queued, don't add it again + for (Download download : queue) { + if (download.chapter.id == event.getChapter().id) + return true; + } + + // If the directory doesn't exist, the chapter isn't downloaded + File dir = getAbsoluteChapterDirectory(source, event.getManga(), event.getChapter()); + if (!dir.exists()) + return false; + + // If the page list doesn't exist, the chapter isn't download (or maybe it's, + // but we consider it's not) + List savedPages = getSavedPageList(source, event.getManga(), event.getChapter()); + if (savedPages == null) + return false; + + // If the number of files matches the number of pages, the chapter is downloaded. + // We have the index file, so we check one file less + return (dir.listFiles().length - 1) == savedPages.size(); + } + + // Create a download object and add it to the downloads queue + private Observable createDownload(DownloadChapterEvent event) { + Download download = new Download( + sourceManager.get(event.getManga().source), + event.getManga(), + event.getChapter()); + + download.directory = getAbsoluteChapterDirectory( + download.source, download.manga, download.chapter); + + queue.add(download); + return Observable.just(download); + } + + // Download the entire chapter + private Observable downloadChapter(Download download) { + return download.source + .pullPageListFromNetwork(download.chapter.url) + .subscribeOn(Schedulers.io()) + // Add resulting pages to download object + .doOnNext(pages -> download.pages = pages) // Get all the URLs to the source images, fetch pages if necessary .flatMap(pageList -> Observable.merge( Observable.from(pageList).filter(page -> page.getImageUrl() != null), - source.getRemainingImageUrlsFromPageList(pageList))) - // Start downloading images - .flatMap(page -> getDownloadedImage(page, source, chapterDirectory)); - } - - public File getAbsoluteChapterDirectory(Source source, Manga manga, Chapter chapter) { - return new File(preferences.getDownloadsDirectory(), - getChapterDirectory(source, manga, chapter)); - } - - public String getChapterDirectory(Source source, Manga manga, Chapter chapter) { - return source.getName() + - File.separator + - manga.title.replaceAll("[^a-zA-Z0-9.-]", "_") + - File.separator + - chapter.name.replaceAll("[^a-zA-Z0-9.-]", "_"); - } - - private String getImageFilename(Page page) { - return page.getImageUrl().substring( - page.getImageUrl().lastIndexOf("/") + 1, - page.getImageUrl().length()); - } - - private boolean isChapterDownloaded(File chapterDir, List pages) { - return chapterDir.exists() && chapterDir.listFiles().length == pages.size(); - } - - private boolean isImageDownloaded(File imagePath) { - return imagePath.exists() && !imagePath.isDirectory(); + download.source.getRemainingImageUrlsFromPageList(pageList))) + // Start downloading images, consider we can have downloaded images already + .concatMap(page -> getDownloadedImage(page, download.source, download.directory)) + // Remove from the queue + .doOnCompleted(() -> removeFromQueue(download)); } + // Get downloaded image if exists, otherwise download it with the method below public Observable getDownloadedImage(final Page page, Source source, File chapterDir) { Observable obs = Observable.just(page); if (page.getImageUrl() == null) @@ -114,6 +154,7 @@ public class DownloadManager { }); } + // Download the image private Observable downloadImage(final Page page, Source source, File chapterDir, String imageFilename) { return source.getImageProgressResponse(page) .flatMap(resp -> { @@ -127,8 +168,62 @@ public class DownloadManager { }); } - public PublishSubject getDownloadsSubject() { - return downloadsSubject; + // Get the filename for an image given the page + private String getImageFilename(Page page) { + return page.getImageUrl().substring( + page.getImageUrl().lastIndexOf("/") + 1, + page.getImageUrl().length()); + } + + private boolean isImageDownloaded(File imagePath) { + return imagePath.exists() && !imagePath.isDirectory(); + } + + private void removeFromQueue(final Download download) { + savePageList(download.source, download.manga, download.chapter, download.pages); + queue.remove(download); + } + + // Return the page list from the chapter's directory if it exists, null otherwise + public List getSavedPageList(Source source, Manga manga, Chapter chapter) { + File chapterDir = getAbsoluteChapterDirectory(source, manga, chapter); + File pagesFile = new File(chapterDir, "index.json"); + + try { + JsonReader reader = new JsonReader(new FileReader(pagesFile.getAbsolutePath())); + + Type collectionType = new TypeToken>() {}.getType(); + return gson.fromJson(reader, collectionType); + } catch (FileNotFoundException e) { + return null; + } + } + + // Save the page list to the chapter's directory + public void savePageList(Source source, Manga manga, Chapter chapter, List pages) { + File chapterDir = getAbsoluteChapterDirectory(source, manga, chapter); + File pagesFile = new File(chapterDir, "index.json"); + + FileOutputStream out; + try { + out = new FileOutputStream(pagesFile); + out.write(gson.toJson(pages).getBytes()); + out.flush(); + out.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + // Get the absolute path to the chapter directory + public File getAbsoluteChapterDirectory(Source source, Manga manga, Chapter chapter) { + String chapterRelativePath = source.getName() + + File.separator + + manga.title.replaceAll("[^a-zA-Z0-9.-]", "_") + + File.separator + + chapter.name.replaceAll("[^a-zA-Z0-9.-]", "_"); + + return new File(preferences.getDownloadsDirectory(), chapterRelativePath); } } diff --git a/app/src/main/java/eu/kanade/mangafeed/data/helpers/PreferencesHelper.java b/app/src/main/java/eu/kanade/mangafeed/data/helpers/PreferencesHelper.java index 51b400eec6..c2b3f7ca01 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/helpers/PreferencesHelper.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/helpers/PreferencesHelper.java @@ -59,4 +59,8 @@ public class PreferencesHelper { DiskUtils.getStorageDirectories(context)[0]); } + public int getDownloadThreads() { + return Integer.parseInt(mPref.getString(getKey(R.string.pref_download_threads_key), "1")); + } + } diff --git a/app/src/main/java/eu/kanade/mangafeed/data/models/Download.java b/app/src/main/java/eu/kanade/mangafeed/data/models/Download.java new file mode 100644 index 0000000000..f55d184dd7 --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/data/models/Download.java @@ -0,0 +1,20 @@ +package eu.kanade.mangafeed.data.models; + +import java.io.File; +import java.util.List; + +import eu.kanade.mangafeed.sources.base.Source; + +public class Download { + public Source source; + public Manga manga; + public Chapter chapter; + public List pages; + public File directory; + + public Download(Source source, Manga manga, Chapter chapter) { + this.source = source; + this.manga = manga; + this.chapter = chapter; + } +} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/mangafeed/presenter/MangaChaptersPresenter.java b/app/src/main/java/eu/kanade/mangafeed/presenter/MangaChaptersPresenter.java index fa41f545fe..90c59577e6 100644 --- a/app/src/main/java/eu/kanade/mangafeed/presenter/MangaChaptersPresenter.java +++ b/app/src/main/java/eu/kanade/mangafeed/presenter/MangaChaptersPresenter.java @@ -15,6 +15,7 @@ import eu.kanade.mangafeed.data.helpers.SourceManager; import eu.kanade.mangafeed.data.models.Chapter; import eu.kanade.mangafeed.data.models.Manga; import eu.kanade.mangafeed.events.ChapterCountEvent; +import eu.kanade.mangafeed.events.DownloadChapterEvent; import eu.kanade.mangafeed.events.SourceMangaChapterEvent; import eu.kanade.mangafeed.sources.base.Source; import eu.kanade.mangafeed.ui.fragment.MangaChaptersFragment; @@ -38,7 +39,8 @@ public class MangaChaptersPresenter extends BasePresenter private static final int DB_CHAPTERS = 1; private static final int ONLINE_CHAPTERS = 2; - private Subscription menuOperationSubscription; + private Subscription markReadSubscription; + private Subscription downloadSubscription; @Override protected void onCreate(Bundle savedState) { @@ -90,10 +92,6 @@ public class MangaChaptersPresenter extends BasePresenter } } - public Manga getManga() { - return manga; - } - public void refreshChapters() { if (getView() != null) getView().setSwipeRefreshing(); @@ -120,10 +118,10 @@ public class MangaChaptersPresenter extends BasePresenter } public void markChaptersRead(Observable selectedChapters, boolean read) { - if (menuOperationSubscription != null) - remove(menuOperationSubscription); + if (markReadSubscription != null) + remove(markReadSubscription); - add(menuOperationSubscription = selectedChapters + add(markReadSubscription = selectedChapters .subscribeOn(Schedulers.io()) .map(chapter -> { chapter.read = read; @@ -137,6 +135,18 @@ public class MangaChaptersPresenter extends BasePresenter })); } + public void downloadChapters(Observable selectedChapters) { + if (downloadSubscription != null) + remove(downloadSubscription); + + add(downloadSubscription = selectedChapters + .subscribeOn(Schedulers.io()) + .subscribe(chapter -> { + EventBus.getDefault().post( + new DownloadChapterEvent(manga, chapter)); + })); + } + public void checkIsChapterDownloaded(Chapter chapter) { File dir = downloadManager.getAbsoluteChapterDirectory(source, manga, chapter); diff --git a/app/src/main/java/eu/kanade/mangafeed/ui/fragment/MangaChaptersFragment.java b/app/src/main/java/eu/kanade/mangafeed/ui/fragment/MangaChaptersFragment.java index 0fb6d30e42..cdaae91e68 100644 --- a/app/src/main/java/eu/kanade/mangafeed/ui/fragment/MangaChaptersFragment.java +++ b/app/src/main/java/eu/kanade/mangafeed/ui/fragment/MangaChaptersFragment.java @@ -18,11 +18,9 @@ import java.util.List; import butterknife.Bind; import butterknife.ButterKnife; -import de.greenrobot.event.EventBus; import eu.kanade.mangafeed.R; import eu.kanade.mangafeed.data.models.Chapter; import eu.kanade.mangafeed.data.services.DownloadService; -import eu.kanade.mangafeed.events.DownloadChapterEvent; import eu.kanade.mangafeed.presenter.MangaChaptersPresenter; import eu.kanade.mangafeed.ui.activity.MangaDetailActivity; import eu.kanade.mangafeed.ui.activity.ReaderActivity; @@ -31,8 +29,6 @@ import eu.kanade.mangafeed.ui.adapter.ChaptersAdapter; import eu.kanade.mangafeed.ui.fragment.base.BaseRxFragment; import nucleus.factory.RequiresPresenter; import rx.Observable; -import rx.Subscription; -import rx.schedulers.Schedulers; @RequiresPresenter(MangaChaptersPresenter.class) public class MangaChaptersFragment extends BaseRxFragment implements @@ -44,7 +40,6 @@ public class MangaChaptersFragment extends BaseRxFragment { - EventBus.getDefault().post( - new DownloadChapterEvent(getPresenter().getManga(), chapter)); - downloadSubscription.unsubscribe(); - }); - } - } diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml index a89cd1024c..81aa3dc977 100644 --- a/app/src/main/res/values/arrays.xml +++ b/app/src/main/res/values/arrays.xml @@ -14,4 +14,10 @@ 4 + + 1 + 2 + 3 + + \ No newline at end of file diff --git a/app/src/main/res/values/keys.xml b/app/src/main/res/values/keys.xml index 3e7b28ee4b..04ea8b2c25 100644 --- a/app/src/main/res/values/keys.xml +++ b/app/src/main/res/values/keys.xml @@ -6,4 +6,5 @@ pref_fullscreen_key pref_default_viewer_key pref_download_directory_key + pref_download_threads_key \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 2656784956..a084b9b6f9 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -92,5 +92,6 @@ Update completed No new chapters found Found new chapters for: + Download threads diff --git a/app/src/main/res/xml/pref_downloads.xml b/app/src/main/res/xml/pref_downloads.xml index 70056be23d..7c79e587da 100644 --- a/app/src/main/res/xml/pref_downloads.xml +++ b/app/src/main/res/xml/pref_downloads.xml @@ -2,4 +2,11 @@ + + \ No newline at end of file