From a055cc07d830c28decb7ce9f4d7e7dded025130f Mon Sep 17 00:00:00 2001 From: inorichi Date: Sun, 15 Nov 2015 17:51:14 +0100 Subject: [PATCH] Improve sources' paging --- .../mangafeed/data/source/SourceManager.java | 6 +- .../data/source/base/BaseSource.java | 25 +++++--- .../mangafeed/data/source/base/Source.java | 33 ++++++---- .../data/source/model/MangasPage.java | 18 ++++++ .../data/source/online/english/Batoto.java | 60 +++++++++++++------ .../data/source/online/english/Mangafox.java | 43 ++++++++----- .../data/source/online/english/Mangahere.java | 56 ++++++++++------- .../ui/catalogue/CatalogueFragment.java | 2 +- .../ui/catalogue/CataloguePresenter.java | 25 +++++--- 9 files changed, 186 insertions(+), 82 deletions(-) create mode 100644 app/src/main/java/eu/kanade/mangafeed/data/source/model/MangasPage.java diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/SourceManager.java b/app/src/main/java/eu/kanade/mangafeed/data/source/SourceManager.java index 05bd4aead9..35098ec667 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/SourceManager.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/SourceManager.java @@ -6,10 +6,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import eu.kanade.mangafeed.data.source.online.english.Batoto; -import eu.kanade.mangafeed.data.source.online.english.Mangahere; -import eu.kanade.mangafeed.data.source.online.english.Mangafox; import eu.kanade.mangafeed.data.source.base.Source; +import eu.kanade.mangafeed.data.source.online.english.Batoto; +import eu.kanade.mangafeed.data.source.online.english.Mangafox; +import eu.kanade.mangafeed.data.source.online.english.Mangahere; public class SourceManager { diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/base/BaseSource.java b/app/src/main/java/eu/kanade/mangafeed/data/source/base/BaseSource.java index 8526d92986..921e9c8997 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/base/BaseSource.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/base/BaseSource.java @@ -3,10 +3,13 @@ package eu.kanade.mangafeed.data.source.base; import com.squareup.okhttp.Headers; import com.squareup.okhttp.Response; +import org.jsoup.nodes.Document; + import java.util.List; import eu.kanade.mangafeed.data.database.models.Chapter; import eu.kanade.mangafeed.data.database.models.Manga; +import eu.kanade.mangafeed.data.source.model.MangasPage; import rx.Observable; public abstract class BaseSource { @@ -20,17 +23,23 @@ public abstract class BaseSource { // True if the source requires a login public abstract boolean isLoginRequired(); - // Given a page number, it should return the URL of the page where the manga list is found - protected abstract String getUrlFromPageNumber(int page); + // Return the initial popular mangas URL + protected abstract String getInitialPopularMangasUrl(); - // From the URL obtained before, this method must return a list of mangas - protected abstract List parsePopularMangasFromHtml(String unparsedHtml); + // Return the initial search url given a query + protected abstract String getInitialSearchUrl(String query); - // Given a query and a page number, return the URL of the results - protected abstract String getSearchUrl(String query, int page); + // Get the popular list of mangas from the source's parsed document + protected abstract List parsePopularMangasFromHtml(Document parsedHtml); - // From the URL obtained before, this method must return a list of mangas - protected abstract List parseSearchFromHtml(String unparsedHtml); + // Get the next popular page URL or null if it's the last + protected abstract String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page); + + // Get the searched list of mangas from the source's parsed document + protected abstract List parseSearchFromHtml(Document parsedHtml); + + // Get the next search page URL or null if it's the last + protected abstract String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query); // Given the URL of a manga and the result of the request, return the details of the manga protected abstract Manga parseHtmlToManga(String mangaUrl, String unparsedHtml); diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java b/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java index 2c3398e75a..0fe51cf6f0 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java @@ -1,11 +1,12 @@ package eu.kanade.mangafeed.data.source.base; - import android.content.Context; import com.squareup.okhttp.Headers; import com.squareup.okhttp.Response; +import org.jsoup.Jsoup; + import java.util.ArrayList; import java.util.List; @@ -13,10 +14,11 @@ import javax.inject.Inject; import eu.kanade.mangafeed.App; import eu.kanade.mangafeed.data.cache.CacheManager; -import eu.kanade.mangafeed.data.network.NetworkHelper; -import eu.kanade.mangafeed.data.preference.PreferencesHelper; import eu.kanade.mangafeed.data.database.models.Chapter; import eu.kanade.mangafeed.data.database.models.Manga; +import eu.kanade.mangafeed.data.network.NetworkHelper; +import eu.kanade.mangafeed.data.preference.PreferencesHelper; +import eu.kanade.mangafeed.data.source.model.MangasPage; import eu.kanade.mangafeed.data.source.model.Page; import rx.Observable; import rx.schedulers.Schedulers; @@ -34,18 +36,29 @@ public abstract class Source extends BaseSource { } // Get the most popular mangas from the source - public Observable> pullPopularMangasFromNetwork(int page) { - String url = getUrlFromPageNumber(page); + public Observable pullPopularMangasFromNetwork(MangasPage page) { + if (page.page == 1) + page.url = getInitialPopularMangasUrl(); + return mNetworkService - .getStringResponse(url, mRequestHeaders, null) - .flatMap(response -> Observable.just(parsePopularMangasFromHtml(response))); + .getStringResponse(page.url, mRequestHeaders, null) + .map(Jsoup::parse) + .doOnNext(doc -> page.mangas = parsePopularMangasFromHtml(doc)) + .doOnNext(doc -> page.nextPageUrl = parseNextPopularMangasUrl(doc, page)) + .map(response -> page); } // Get mangas from the source with a query - public Observable> searchMangasFromNetwork(String query, int page) { + public Observable searchMangasFromNetwork(MangasPage page, String query) { + if (page.page == 1) + page.url = getInitialSearchUrl(query); + return mNetworkService - .getStringResponse(getSearchUrl(query, page), mRequestHeaders, null) - .flatMap(response -> Observable.just(parseSearchFromHtml(response))); + .getStringResponse(page.url, mRequestHeaders, null) + .map(Jsoup::parse) + .doOnNext(doc -> page.mangas = parseSearchFromHtml(doc)) + .doOnNext(doc -> page.nextPageUrl = parseNextSearchUrl(doc, page, query)) + .map(response -> page); } // Get manga details from the source diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/model/MangasPage.java b/app/src/main/java/eu/kanade/mangafeed/data/source/model/MangasPage.java new file mode 100644 index 0000000000..b63dc719ec --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/model/MangasPage.java @@ -0,0 +1,18 @@ +package eu.kanade.mangafeed.data.source.model; + +import java.util.List; + +import eu.kanade.mangafeed.data.database.models.Manga; + +public class MangasPage { + + public List mangas; + public int page; + public String url; + public String nextPageUrl; + + public MangasPage(int page) { + this.page = page; + } + +} diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java index b9158dd20a..6477bfe694 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java @@ -25,13 +25,14 @@ import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.database.models.Chapter; import eu.kanade.mangafeed.data.database.models.Manga; import eu.kanade.mangafeed.data.source.base.Source; +import eu.kanade.mangafeed.data.source.model.MangasPage; import rx.Observable; public class Batoto extends Source { public static final String NAME = "Batoto (EN)"; public static final String BASE_URL = "http://bato.to"; - public static final String INITIAL_UPDATE_URL = + public static final String INITIAL_POPULAR_MANGAS_URL = "http://bato.to/search_ajax?order_cond=views&order=desc&p="; public static final String INITIAL_SEARCH_URL = "http://bato.to/search_ajax?"; public static final String INITIAL_PAGE_URL = "http://bato.to/areader?"; @@ -112,13 +113,13 @@ public class Batoto extends Source { } @Override - protected String getUrlFromPageNumber(int page) { - return INITIAL_UPDATE_URL + page; + public String getInitialPopularMangasUrl() { + return INITIAL_POPULAR_MANGAS_URL + "1"; } @Override - protected String getSearchUrl(String query, int page) { - return INITIAL_SEARCH_URL + "name=" + query + "&p=" + page; + public String getInitialSearchUrl(String query) { + return INITIAL_SEARCH_URL + "name=" + query + "&p=1"; } @Override @@ -141,33 +142,49 @@ public class Batoto extends Source { return INITIAL_PAGE_URL + "id=" + id + "&p=" + defaultPageUrl.substring(end+1); } - private List parseMangasFromHtml(String unparsedHtml) { - if (unparsedHtml.contains("No (more) comics found!")) { + @Override + protected List parsePopularMangasFromHtml(Document parsedHtml) { + if (parsedHtml.text().contains("No (more) comics found!")) { return new ArrayList<>(); } - Document parsedDocument = Jsoup.parse(unparsedHtml); + List mangaList = new ArrayList<>(); - List updatedMangaList = new ArrayList<>(); - - Elements updatedHtmlBlocks = parsedDocument.select("tr:not([id]):not([class])"); + Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])"); for (Element currentHtmlBlock : updatedHtmlBlocks) { Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock); - updatedMangaList.add(currentlyUpdatedManga); + mangaList.add(currentlyUpdatedManga); } - return updatedMangaList; + return mangaList; } @Override - public List parsePopularMangasFromHtml(String unparsedHtml) { - return parseMangasFromHtml(unparsedHtml); + protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { + Element next = parsedHtml.select("#show_more_row").first(); + if (next == null) + return null; + + return INITIAL_POPULAR_MANGAS_URL + (page.page + 1); } @Override - protected List parseSearchFromHtml(String unparsedHtml) { - return parseMangasFromHtml(unparsedHtml); + protected List parseSearchFromHtml(Document parsedHtml) { + if (parsedHtml.text().contains("No (more) comics found!")) { + return new ArrayList<>(); + } + + List mangaList = new ArrayList<>(); + + Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])"); + for (Element currentHtmlBlock : updatedHtmlBlocks) { + Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock); + + mangaList.add(currentlyUpdatedManga); + } + + return mangaList; } private Manga constructMangaFromHtmlBlock(Element htmlBlock) { @@ -195,6 +212,15 @@ public class Batoto extends Source { return mangaFromHtmlBlock; } + @Override + protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { + Element next = parsedHtml.select("#show_more_row").first(); + if (next == null) + return null; + + return INITIAL_SEARCH_URL + "name=" + query + "&p=" + (page.page + 1); + } + private long parseUpdateFromElement(Element updateElement) { String updatedDateAsString = updateElement.text(); diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java index 1d4fe2b470..ca36d79312 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java @@ -14,13 +14,14 @@ import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.database.models.Chapter; import eu.kanade.mangafeed.data.database.models.Manga; import eu.kanade.mangafeed.data.source.base.Source; +import eu.kanade.mangafeed.data.source.model.MangasPage; public class Mangafox extends Source { public static final String NAME = "Mangafox (EN)"; - - private static final String INITIAL_UPDATE_URL = "http://mangafox.me/directory/"; - private static final String INITIAL_SEARCH_URL = + public static final String BASE_URL = "http://mangafox.me"; + public static final String INITIAL_POPULAR_MANGAS_URL = "http://mangafox.me/directory/"; + public static final String INITIAL_SEARCH_URL = "http://mangafox.me/search.php?name_method=cw&advopts=1&order=az&sort=name"; public Mangafox(Context context) { @@ -43,17 +44,20 @@ public class Mangafox extends Source { } @Override - protected String getUrlFromPageNumber(int page) { - return INITIAL_UPDATE_URL + page + ".htm"; + protected String getInitialPopularMangasUrl() { + return INITIAL_POPULAR_MANGAS_URL; } @Override - public List parsePopularMangasFromHtml(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); + protected String getInitialSearchUrl(String query) { + return INITIAL_SEARCH_URL + "&name=" + query + "&page=1"; + } + @Override + protected List parsePopularMangasFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedDocument.select("div#mangalist > ul.list > li"); + Elements mangaHtmlBlocks = parsedHtml.select("div#mangalist > ul.list > li"); for (Element currentHtmlBlock : mangaHtmlBlocks) { Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); @@ -77,17 +81,19 @@ public class Mangafox extends Source { } @Override - protected String getSearchUrl(String query, int page) { - return INITIAL_SEARCH_URL + "&name=" + query + "&page=" + page; + protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { + Element next = parsedHtml.select("a:has(span.next)").first(); + if (next == null) + return null; + + return INITIAL_POPULAR_MANGAS_URL + next.attr("href"); } @Override - protected List parseSearchFromHtml(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - + protected List parseSearchFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedDocument.select("table#listing > tbody > tr:gt(0)"); + Elements mangaHtmlBlocks = parsedHtml.select("table#listing > tbody > tr:gt(0)"); for (Element currentHtmlBlock : mangaHtmlBlocks) { Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); @@ -110,6 +116,15 @@ public class Mangafox extends Source { return mangaFromHtmlBlock; } + @Override + protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { + Element next = parsedHtml.select("a:has(span.next)").first(); + if (next == null) + return null; + + return BASE_URL + next.attr("href"); + } + @Override protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); diff --git a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java index 2b2655c082..7088fbcf13 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java @@ -15,18 +15,19 @@ import java.util.Date; import java.util.List; import java.util.Locale; -import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.database.models.Chapter; import eu.kanade.mangafeed.data.database.models.Manga; +import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.source.base.Source; +import eu.kanade.mangafeed.data.source.model.MangasPage; import rx.Observable; public class Mangahere extends Source { public static final String NAME = "Mangahere (EN)"; - public static final String BASE_URL = "www.mangahere.co"; + public static final String BASE_URL = "http://www.mangahere.co"; - private static final String INITIAL_UPDATE_URL = "http://www.mangahere.co/directory/"; + private static final String INITIAL_POPULAR_MANGAS_URL = "http://www.mangahere.co/directory/"; private static final String INITIAL_SEARCH_URL = "http://www.mangahere.co/search.php?"; public Mangahere(Context context) { @@ -43,22 +44,21 @@ public class Mangahere extends Source { return SourceManager.MANGAHERE; } - @Override - protected String getUrlFromPageNumber(int page) { - return INITIAL_UPDATE_URL + page + ".htm?views.za"; - } - - @Override - protected String getSearchUrl(String query, int page) { - return INITIAL_SEARCH_URL + "name=" + query + "&page=" + page; - } - - @Override public boolean isLoginRequired() { return false; } + @Override + protected String getInitialPopularMangasUrl() { + return INITIAL_POPULAR_MANGAS_URL; + } + + @Override + protected String getInitialSearchUrl(String query) { + return INITIAL_SEARCH_URL + "name=" + query + "&page=1"; + } + public Observable> getGenres() { List genres = new ArrayList<>(30); @@ -98,12 +98,10 @@ public class Mangahere extends Source { } @Override - public List parsePopularMangasFromHtml(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - + public List parsePopularMangasFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedDocument.select("div.directory_list > ul > li"); + Elements mangaHtmlBlocks = parsedHtml.select("div.directory_list > ul > li"); for (Element currentHtmlBlock : mangaHtmlBlocks) { Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); @@ -127,12 +125,19 @@ public class Mangahere extends Source { } @Override - protected List parseSearchFromHtml(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); + protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { + Element next = parsedHtml.select("div.next-page > a.next").first(); + if (next == null) + return null; + return INITIAL_POPULAR_MANGAS_URL + next.attr("href"); + } + + @Override + protected List parseSearchFromHtml(Document parsedHtml) { List mangaList = new ArrayList<>(); - Elements mangaHtmlBlocks = parsedDocument.select("div.result_search > dl"); + Elements mangaHtmlBlocks = parsedHtml.select("div.result_search > dl"); for (Element currentHtmlBlock : mangaHtmlBlocks) { Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); mangaList.add(currentManga); @@ -155,6 +160,15 @@ public class Mangahere extends Source { return mangaFromHtmlBlock; } + @Override + protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { + Element next = parsedHtml.select("div.next-page > a.next").first(); + if (next == null) + return null; + + return BASE_URL + next.attr("href"); + } + private long parseUpdateFromElement(Element updateElement) { String updatedDateAsString = updateElement.text(); diff --git a/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CatalogueFragment.java b/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CatalogueFragment.java index 61fd11891f..79d43c91cd 100644 --- a/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CatalogueFragment.java +++ b/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CatalogueFragment.java @@ -22,8 +22,8 @@ import butterknife.ButterKnife; import butterknife.OnItemClick; import eu.kanade.mangafeed.R; import eu.kanade.mangafeed.data.database.models.Manga; -import eu.kanade.mangafeed.ui.manga.MangaActivity; import eu.kanade.mangafeed.ui.base.fragment.BaseRxFragment; +import eu.kanade.mangafeed.ui.manga.MangaActivity; import eu.kanade.mangafeed.util.PageBundle; import eu.kanade.mangafeed.widget.EndlessScrollListener; import nucleus.factory.RequiresPresenter; diff --git a/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CataloguePresenter.java b/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CataloguePresenter.java index 85dca61aa2..52b3940f06 100644 --- a/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CataloguePresenter.java +++ b/app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CataloguePresenter.java @@ -10,9 +10,10 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import eu.kanade.mangafeed.data.database.DatabaseHelper; -import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.database.models.Manga; +import eu.kanade.mangafeed.data.source.SourceManager; import eu.kanade.mangafeed.data.source.base.Source; +import eu.kanade.mangafeed.data.source.model.MangasPage; import eu.kanade.mangafeed.ui.base.presenter.BasePresenter; import eu.kanade.mangafeed.util.PageBundle; import eu.kanade.mangafeed.util.RxPager; @@ -36,6 +37,7 @@ public class CataloguePresenter extends BasePresenter { private int mCurrentPage; private RxPager pager; + private MangasPage lastMangasPage; private Subscription mQueryDebouncerSubscription; private Subscription mMangaDetailFetchSubscription; @@ -91,21 +93,28 @@ public class CataloguePresenter extends BasePresenter { } public void requestNext() { - if (getView() != null) - getView().showGridProgressBar(); - pager.requestNext(++mCurrentPage); } private Observable> getMangaObs(int page) { - Observable> obs; + MangasPage nextMangasPage = new MangasPage(page); + if (page != 1) { + if (lastMangasPage.nextPageUrl == null) + return Observable.empty(); + nextMangasPage.url = lastMangasPage.nextPageUrl; + } + if (getView() != null) + getView().showGridProgressBar(); + + Observable obs; if (mSearchMode) - obs = selectedSource.searchMangasFromNetwork(mSearchName, page); + obs = selectedSource.searchMangasFromNetwork(nextMangasPage, mSearchName); else - obs = selectedSource.pullPopularMangasFromNetwork(page); + obs = selectedSource.pullPopularMangasFromNetwork(nextMangasPage); return obs.subscribeOn(Schedulers.io()) - .flatMap(Observable::from) + .doOnNext(mangasPage -> lastMangasPage = mangasPage) + .flatMap(mangasPage -> Observable.from(mangasPage.mangas)) .map(this::networkToLocalManga) .toList(); }