Improve download manager. Add an option to select the number of threads for downloads.

app/src/main/java/eu/kanade/mangafeed/data/helpers/DownloadManager.java

@@ -2,11 +2,21 @@ package eu.kanade.mangafeed.data.helpers;
 
 import android.content.Context;
 
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import com.google.gson.stream.JsonReader;
+
 import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
 import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
 import java.util.List;
 
 import eu.kanade.mangafeed.data.models.Chapter;
+import eu.kanade.mangafeed.data.models.Download;
 import eu.kanade.mangafeed.data.models.Manga;
 import eu.kanade.mangafeed.data.models.Page;
 import eu.kanade.mangafeed.events.DownloadChapterEvent;
@@ -20,77 +30,107 @@ import rx.subjects.PublishSubject;
 public class DownloadManager {
 
     private PublishSubject<DownloadChapterEvent> downloadsSubject;
-    private Subscription downloadsSubscription;
+    private Subscription downloadSubscription;
 
     private Context context;
     private SourceManager sourceManager;
     private PreferencesHelper preferences;
+    private Gson gson;
+
+    private List<Download> queue;
 
     public DownloadManager(Context context, SourceManager sourceManager, PreferencesHelper preferences) {
         this.context = context;
         this.sourceManager = sourceManager;
         this.preferences = preferences;
+        this.gson = new Gson();
+
+        queue = new ArrayList<>();
 
         initializeDownloadSubscription();
     }
 
+    public PublishSubject<DownloadChapterEvent> getDownloadsSubject() {
+        return downloadsSubject;
+    }
+
     private void initializeDownloadSubscription() {
-        if (downloadsSubscription != null && !downloadsSubscription.isUnsubscribed()) {
+        if (downloadSubscription != null && !downloadSubscription.isUnsubscribed()) {
-            downloadsSubscription.unsubscribe();
+            downloadSubscription.unsubscribe();
         }
 
         downloadsSubject = PublishSubject.create();
 
-        downloadsSubscription = downloadsSubject
+        // Listen for download events, add them to queue and download
+        downloadSubscription = downloadsSubject
                 .subscribeOn(Schedulers.io())
-                .concatMap(event -> downloadChapter(event.getManga(), event.getChapter()))
+                .filter(event -> !isChapterDownloaded(event))
+                .flatMap(this::createDownload)
+                .window(preferences.getDownloadThreads())
+                .concatMap(concurrentDownloads -> concurrentDownloads
+                        .concatMap(this::downloadChapter))
                 .onBackpressureBuffer()
                 .subscribe();
     }
 
-    public Observable<Page> downloadChapter(Manga manga, Chapter chapter) {
+    // Check if a chapter is already downloaded
-        final Source source = sourceManager.get(manga.source);
+    private boolean isChapterDownloaded(DownloadChapterEvent event) {
-        final File chapterDirectory = getAbsoluteChapterDirectory(source, manga, chapter);
+        final Source source = sourceManager.get(event.getManga().source);
 
-        return source
+        // If the chapter is already queued, don't add it again
-                .pullPageListFromNetwork(chapter.url)
+        for (Download download : queue) {
-                // Ensure we don't download a chapter already downloaded
+            if (download.chapter.id == event.getChapter().id)
-                .filter(pages -> !isChapterDownloaded(chapterDirectory, pages))
+                return true;
+        }
+
+        // If the directory doesn't exist, the chapter isn't downloaded
+        File dir = getAbsoluteChapterDirectory(source, event.getManga(), event.getChapter());
+        if (!dir.exists())
+            return false;
+
+        // If the page list doesn't exist, the chapter isn't download (or maybe it's,
+        // but we consider it's not)
+        List<Page> savedPages = getSavedPageList(source, event.getManga(), event.getChapter());
+        if (savedPages == null)
+            return false;
+
+        // If the number of files matches the number of pages, the chapter is downloaded.
+        // We have the index file, so we check one file less
+        return (dir.listFiles().length - 1) == savedPages.size();
+    }
+
+    // Create a download object and add it to the downloads queue
+    private Observable<Download> createDownload(DownloadChapterEvent event) {
+        Download download = new Download(
+                sourceManager.get(event.getManga().source),
+                event.getManga(),
+                event.getChapter());
+
+        download.directory = getAbsoluteChapterDirectory(
+                download.source, download.manga, download.chapter);
+
+        queue.add(download);
+        return Observable.just(download);
+    }
+
+    // Download the entire chapter
+    private Observable<Page> downloadChapter(Download download) {
+        return download.source
+                .pullPageListFromNetwork(download.chapter.url)
+                .subscribeOn(Schedulers.io())
+                // Add resulting pages to download object
+                .doOnNext(pages -> download.pages = pages)
                 // Get all the URLs to the source images, fetch pages if necessary
                 .flatMap(pageList -> Observable.merge(
                         Observable.from(pageList).filter(page -> page.getImageUrl() != null),
-                        source.getRemainingImageUrlsFromPageList(pageList)))
+                        download.source.getRemainingImageUrlsFromPageList(pageList)))
-                // Start downloading images
+                // Start downloading images, consider we can have downloaded images already
-                .flatMap(page -> getDownloadedImage(page, source, chapterDirectory));
+                .concatMap(page -> getDownloadedImage(page, download.source, download.directory))
-    }
+                // Remove from the queue
-    
+                .doOnCompleted(() -> removeFromQueue(download));
-    public File getAbsoluteChapterDirectory(Source source, Manga manga, Chapter chapter) {
-        return new File(preferences.getDownloadsDirectory(),
-                getChapterDirectory(source, manga, chapter));
-    }
-
-    public String getChapterDirectory(Source source, Manga manga, Chapter chapter) {
-        return source.getName() +
-                File.separator +
-                manga.title.replaceAll("[^a-zA-Z0-9.-]", "_") +
-                File.separator +
-                chapter.name.replaceAll("[^a-zA-Z0-9.-]", "_");
-    }
-
-    private String getImageFilename(Page page) {
-        return page.getImageUrl().substring(
-                page.getImageUrl().lastIndexOf("/") + 1,
-                page.getImageUrl().length());
-    }
-
-    private boolean isChapterDownloaded(File chapterDir, List<Page> pages) {
-        return chapterDir.exists() && chapterDir.listFiles().length == pages.size();
-    }
-
-    private boolean isImageDownloaded(File imagePath) {
-        return imagePath.exists() && !imagePath.isDirectory();
     }
 
+    // Get downloaded image if exists, otherwise download it with the method below
     public Observable<Page> getDownloadedImage(final Page page, Source source, File chapterDir) {
         Observable<Page> obs = Observable.just(page);
         if (page.getImageUrl() == null)
@@ -114,6 +154,7 @@ public class DownloadManager {
         });
     }
 
+    // Download the image
     private Observable<Page> downloadImage(final Page page, Source source, File chapterDir, String imageFilename) {
         return source.getImageProgressResponse(page)
                 .flatMap(resp -> {
@@ -127,8 +168,62 @@ public class DownloadManager {
                 });
     }
 
-    public PublishSubject<DownloadChapterEvent> getDownloadsSubject() {
+    // Get the filename for an image given the page
-        return downloadsSubject;
+    private String getImageFilename(Page page) {
+        return page.getImageUrl().substring(
+                page.getImageUrl().lastIndexOf("/") + 1,
+                page.getImageUrl().length());
+    }
+
+    private boolean isImageDownloaded(File imagePath) {
+        return imagePath.exists() && !imagePath.isDirectory();
+    }
+
+    private void removeFromQueue(final Download download) {
+        savePageList(download.source, download.manga, download.chapter, download.pages);
+        queue.remove(download);
+    }
+
+    // Return the page list from the chapter's directory if it exists, null otherwise
+    public List<Page> getSavedPageList(Source source, Manga manga, Chapter chapter) {
+        File chapterDir = getAbsoluteChapterDirectory(source, manga, chapter);
+        File pagesFile = new File(chapterDir, "index.json");
+
+        try {
+            JsonReader reader = new JsonReader(new FileReader(pagesFile.getAbsolutePath()));
+
+            Type collectionType = new TypeToken<List<Page>>() {}.getType();
+            return gson.fromJson(reader, collectionType);
+        } catch (FileNotFoundException e) {
+            return null;
+        }
+    }
+
+    // Save the page list to the chapter's directory
+    public void savePageList(Source source, Manga manga, Chapter chapter, List<Page> pages) {
+        File chapterDir = getAbsoluteChapterDirectory(source, manga, chapter);
+        File pagesFile = new File(chapterDir, "index.json");
+
+        FileOutputStream out;
+        try {
+            out = new FileOutputStream(pagesFile);
+            out.write(gson.toJson(pages).getBytes());
+            out.flush();
+            out.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    // Get the absolute path to the chapter directory
+    public File getAbsoluteChapterDirectory(Source source, Manga manga, Chapter chapter) {
+        String chapterRelativePath = source.getName() +
+                File.separator +
+                manga.title.replaceAll("[^a-zA-Z0-9.-]", "_") +
+                File.separator +
+                chapter.name.replaceAll("[^a-zA-Z0-9.-]", "_");
+
+        return new File(preferences.getDownloadsDirectory(), chapterRelativePath);
     }
 
 }

app/src/main/java/eu/kanade/mangafeed/data/helpers/PreferencesHelper.java

@@ -59,4 +59,8 @@ public class PreferencesHelper {
                 DiskUtils.getStorageDirectories(context)[0]);
     }
 
+    public int getDownloadThreads() {
+        return Integer.parseInt(mPref.getString(getKey(R.string.pref_download_threads_key), "1"));
+    }
+
 }

app/src/main/java/eu/kanade/mangafeed/data/models/Download.java

@@ -0,0 +1,20 @@
+package eu.kanade.mangafeed.data.models;
+
+import java.io.File;
+import java.util.List;
+
+import eu.kanade.mangafeed.sources.base.Source;
+
+public class Download {
+    public Source source;
+    public Manga manga;
+    public Chapter chapter;
+    public List<Page> pages;
+    public File directory;
+
+    public Download(Source source, Manga manga, Chapter chapter) {
+        this.source = source;
+        this.manga = manga;
+        this.chapter = chapter;
+    }
+}

app/src/main/java/eu/kanade/mangafeed/presenter/MangaChaptersPresenter.java

@@ -15,6 +15,7 @@ import eu.kanade.mangafeed.data.helpers.SourceManager;
 import eu.kanade.mangafeed.data.models.Chapter;
 import eu.kanade.mangafeed.data.models.Manga;
 import eu.kanade.mangafeed.events.ChapterCountEvent;
+import eu.kanade.mangafeed.events.DownloadChapterEvent;
 import eu.kanade.mangafeed.events.SourceMangaChapterEvent;
 import eu.kanade.mangafeed.sources.base.Source;
 import eu.kanade.mangafeed.ui.fragment.MangaChaptersFragment;
@@ -38,7 +39,8 @@ public class MangaChaptersPresenter extends BasePresenter<MangaChaptersFragment>
     private static final int DB_CHAPTERS = 1;
     private static final int ONLINE_CHAPTERS = 2;
 
-    private Subscription menuOperationSubscription;
+    private Subscription markReadSubscription;
+    private Subscription downloadSubscription;
 
     @Override
     protected void onCreate(Bundle savedState) {
@@ -90,10 +92,6 @@ public class MangaChaptersPresenter extends BasePresenter<MangaChaptersFragment>
         }
     }
 
-    public Manga getManga() {
-        return manga;
-    }
-
     public void refreshChapters() {
         if (getView() != null)
             getView().setSwipeRefreshing();
@@ -120,10 +118,10 @@ public class MangaChaptersPresenter extends BasePresenter<MangaChaptersFragment>
     }
 
     public void markChaptersRead(Observable<Chapter> selectedChapters, boolean read) {
-        if (menuOperationSubscription != null)
+        if (markReadSubscription != null)
-            remove(menuOperationSubscription);
+            remove(markReadSubscription);
 
-        add(menuOperationSubscription = selectedChapters
+        add(markReadSubscription = selectedChapters
                 .subscribeOn(Schedulers.io())
                 .map(chapter -> {
                     chapter.read = read;
@@ -137,6 +135,18 @@ public class MangaChaptersPresenter extends BasePresenter<MangaChaptersFragment>
                 }));
     }
 
+    public void downloadChapters(Observable<Chapter> selectedChapters) {
+        if (downloadSubscription != null)
+            remove(downloadSubscription);
+
+        add(downloadSubscription = selectedChapters
+                .subscribeOn(Schedulers.io())
+                .subscribe(chapter -> {
+                    EventBus.getDefault().post(
+                            new DownloadChapterEvent(manga, chapter));
+                }));
+    }
+
     public void checkIsChapterDownloaded(Chapter chapter) {
         File dir = downloadManager.getAbsoluteChapterDirectory(source, manga, chapter);
 

app/src/main/java/eu/kanade/mangafeed/ui/fragment/MangaChaptersFragment.java

@@ -18,11 +18,9 @@ import java.util.List;
 
 import butterknife.Bind;
 import butterknife.ButterKnife;
-import de.greenrobot.event.EventBus;
 import eu.kanade.mangafeed.R;
 import eu.kanade.mangafeed.data.models.Chapter;
 import eu.kanade.mangafeed.data.services.DownloadService;
-import eu.kanade.mangafeed.events.DownloadChapterEvent;
 import eu.kanade.mangafeed.presenter.MangaChaptersPresenter;
 import eu.kanade.mangafeed.ui.activity.MangaDetailActivity;
 import eu.kanade.mangafeed.ui.activity.ReaderActivity;
@@ -31,8 +29,6 @@ import eu.kanade.mangafeed.ui.adapter.ChaptersAdapter;
 import eu.kanade.mangafeed.ui.fragment.base.BaseRxFragment;
 import nucleus.factory.RequiresPresenter;
 import rx.Observable;
-import rx.Subscription;
-import rx.schedulers.Schedulers;
 
 @RequiresPresenter(MangaChaptersPresenter.class)
 public class MangaChaptersFragment extends BaseRxFragment<MangaChaptersPresenter> implements
@@ -44,7 +40,6 @@ public class MangaChaptersFragment extends BaseRxFragment<MangaChaptersPresenter
     private ChaptersAdapter adapter;
 
     private ActionMode actionMode;
-    private Subscription downloadSubscription;
 
     public static Fragment newInstance() {
         return new MangaChaptersFragment();
@@ -146,7 +141,7 @@ public class MangaChaptersFragment extends BaseRxFragment<MangaChaptersPresenter
                 getPresenter().markChaptersRead(getSelectedChapters(), false);
                 return true;
             case R.id.action_download:
-                onDownloadChapters();
+                getPresenter().downloadChapters(getSelectedChapters());
                 return true;
         }
         return false;
@@ -207,19 +202,4 @@ public class MangaChaptersFragment extends BaseRxFragment<MangaChaptersPresenter
         actionMode.setTitle(getString(R.string.selected_chapters_title, count));
     }
 
-    private void onDownloadChapters() {
-        if (downloadSubscription != null && !downloadSubscription.isUnsubscribed()) {
-            downloadSubscription.unsubscribe();
-            downloadSubscription = null;
-        }
-
-        downloadSubscription = getSelectedChapters()
-                .subscribeOn(Schedulers.io())
-                .subscribe(chapter -> {
-                    EventBus.getDefault().post(
-                            new DownloadChapterEvent(getPresenter().getManga(), chapter));
-                    downloadSubscription.unsubscribe();
-                });
-    }
-
 }

app/src/main/res/values/arrays.xml

@@ -14,4 +14,10 @@
         <item>4</item>
     </string-array>
 
+    <string-array name="download_threads">
+        <item>1</item>
+        <item>2</item>
+        <item>3</item>
+    </string-array>
+
 </resources>

app/src/main/res/values/keys.xml

@@ -6,4 +6,5 @@
     <string name="pref_fullscreen_key">pref_fullscreen_key</string>
     <string name="pref_default_viewer_key">pref_default_viewer_key</string>
     <string name="pref_download_directory_key">pref_download_directory_key</string>
+    <string name="pref_download_threads_key">pref_download_threads_key</string>
 </resources>

app/src/main/res/values/strings.xml

@@ -92,5 +92,6 @@
     <string name="notification_completed">Update completed</string>
     <string name="notification_no_new_chapters">No new chapters found</string>
     <string name="notification_new_chapters">Found new chapters for:</string>
+    <string name="pref_download_threads">Download threads</string>
 
 </resources>

app/src/main/res/xml/pref_downloads.xml

@@ -2,4 +2,11 @@
 <PreferenceScreen xmlns:android="http://schemas.android.com/apk/res/android"
     android:orderingFromXml="true">
 
+    <ListPreference android:title="@string/pref_download_threads"
+        android:key="@string/pref_download_threads_key"
+        android:entries="@array/download_threads"
+        android:entryValues="@array/download_threads"
+        android:defaultValue="1"
+        android:summary="%s"/>
+
 </PreferenceScreen>