From e74f81d41a818ad51e62d01a3050b46ea7ecb160 Mon Sep 17 00:00:00 2001 From: inorichi Date: Sat, 14 Nov 2015 15:33:07 +0100 Subject: [PATCH] New source (unfinished) --- .../mangafeed/data/helpers/SourceManager.java | 9 +- .../presenter/CataloguePresenter.java | 40 ++-- .../eu/kanade/mangafeed/sources/Mangafox.java | 179 ++++++++++++++++++ .../{MangaHere.java => Mangahere.java} | 65 ++++--- .../kanade/mangafeed/sources/base/Source.java | 4 +- .../mangafeed/ui/activity/MainActivity.java | 6 +- 6 files changed, 247 insertions(+), 56 deletions(-) create mode 100644 app/src/main/java/eu/kanade/mangafeed/sources/Mangafox.java rename app/src/main/java/eu/kanade/mangafeed/sources/{MangaHere.java => Mangahere.java} (87%) diff --git a/app/src/main/java/eu/kanade/mangafeed/data/helpers/SourceManager.java b/app/src/main/java/eu/kanade/mangafeed/data/helpers/SourceManager.java index 25dbc3a0a2..9087e12341 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/helpers/SourceManager.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/helpers/SourceManager.java @@ -7,13 +7,15 @@ import java.util.HashMap; import java.util.List; import eu.kanade.mangafeed.sources.Batoto; -import eu.kanade.mangafeed.sources.MangaHere; +import eu.kanade.mangafeed.sources.Mangahere; +import eu.kanade.mangafeed.sources.Mangafox; import eu.kanade.mangafeed.sources.base.Source; public class SourceManager { public static final int BATOTO = 1; public static final int MANGAHERE = 2; + public static final int MANGAFOX = 3; private HashMap mSourcesMap; private Context context; @@ -37,7 +39,9 @@ public class SourceManager { case BATOTO: return new Batoto(context); case MANGAHERE: - return new MangaHere(context); + return new Mangahere(context); + case MANGAFOX: + return new Mangafox(context); } return null; @@ -46,6 +50,7 @@ public class SourceManager { private void initializeSources() { mSourcesMap.put(BATOTO, createSource(BATOTO)); mSourcesMap.put(MANGAHERE, createSource(MANGAHERE)); + mSourcesMap.put(MANGAFOX, createSource(MANGAFOX)); } public List getSources() { diff --git a/app/src/main/java/eu/kanade/mangafeed/presenter/CataloguePresenter.java b/app/src/main/java/eu/kanade/mangafeed/presenter/CataloguePresenter.java index 6b28d581a2..2e5441ab2c 100644 --- a/app/src/main/java/eu/kanade/mangafeed/presenter/CataloguePresenter.java +++ b/app/src/main/java/eu/kanade/mangafeed/presenter/CataloguePresenter.java @@ -4,7 +4,6 @@ import android.os.Bundle; import com.pushtorefresh.storio.sqlite.operations.put.PutResult; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -40,8 +39,8 @@ public class CataloguePresenter extends BasePresenter { private Subscription mQueryDebouncerSubscription; private Subscription mMangaDetailFetchSubscription; - private PublishSubject> mQueryDebouncerSubject; - private PublishSubject>> mMangaDetailPublishSubject; + private PublishSubject mQueryDebouncerSubject; + private PublishSubject> mMangaDetailPublishSubject; private static final int GET_MANGA_LIST = 1; @@ -59,7 +58,7 @@ public class CataloguePresenter extends BasePresenter { view.hideProgressBar(); view.onAddPage(page); if (mMangaDetailPublishSubject != null) - mMangaDetailPublishSubject.onNext(Observable.just(page.data)); + mMangaDetailPublishSubject.onNext(page.data); }); initializeSearch(); @@ -130,7 +129,7 @@ public class CataloguePresenter extends BasePresenter { mSearchMode = false; mQueryDebouncerSubject = PublishSubject.create(); - add(mQueryDebouncerSubscription = Observable.switchOnNext(mQueryDebouncerSubject) + add(mQueryDebouncerSubscription = mQueryDebouncerSubject .debounce(SEARCH_TIMEOUT, TimeUnit.MILLISECONDS) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) @@ -143,25 +142,12 @@ public class CataloguePresenter extends BasePresenter { mMangaDetailPublishSubject = PublishSubject.create(); - add(mMangaDetailFetchSubscription = Observable.switchOnNext(mMangaDetailPublishSubject) + add(mMangaDetailFetchSubscription = mMangaDetailPublishSubject .subscribeOn(Schedulers.io()) .flatMap(Observable::from) .filter(manga -> !manga.initialized) - .buffer(3) - .concatMap(localMangas -> { - List> mangaObservables = new ArrayList<>(); - for (Manga manga : localMangas) { - Observable tempObs = selectedSource.pullMangaFromNetwork(manga.url) - .subscribeOn(Schedulers.io()) - .flatMap(networkManga -> { - Manga.copyFromNetwork(manga, networkManga); - db.insertManga(manga).executeAsBlocking(); - return Observable.just(manga); - }); - mangaObservables.add(tempObs); - } - return Observable.merge(mangaObservables); - }) + .window(3) + .concatMap(pack -> pack.concatMap(this::getMangaDetails)) .filter(manga -> manga.initialized) .onBackpressureBuffer() .observeOn(AndroidSchedulers.mainThread()) @@ -171,12 +157,22 @@ public class CataloguePresenter extends BasePresenter { })); } + private Observable getMangaDetails(final Manga manga) { + return selectedSource.pullMangaFromNetwork(manga.url) + .subscribeOn(Schedulers.io()) + .flatMap(networkManga -> { + Manga.copyFromNetwork(manga, networkManga); + db.insertManga(manga).executeAsBlocking(); + return Observable.just(manga); + }); + } + public void onSearchEvent(String query, boolean now) { // If the query is empty or not debounced, resolve it instantly if (now || query.equals("")) queryFromSearch(query); else if (mQueryDebouncerSubject != null) - mQueryDebouncerSubject.onNext(Observable.just(query)); + mQueryDebouncerSubject.onNext(query); } private void queryFromSearch(String query) { diff --git a/app/src/main/java/eu/kanade/mangafeed/sources/Mangafox.java b/app/src/main/java/eu/kanade/mangafeed/sources/Mangafox.java new file mode 100644 index 0000000000..01e001181c --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/sources/Mangafox.java @@ -0,0 +1,179 @@ +package eu.kanade.mangafeed.sources; + +import android.content.Context; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.util.ArrayList; +import java.util.List; + +import eu.kanade.mangafeed.data.helpers.SourceManager; +import eu.kanade.mangafeed.data.models.Chapter; +import eu.kanade.mangafeed.data.models.Manga; +import eu.kanade.mangafeed.sources.base.Source; + +public class Mangafox extends Source { + + public static final String NAME = "Mangafox (EN)"; + + private static final String INITIAL_UPDATE_URL = "http://mangafox.me/directory/"; + private static final String INITIAL_SEARCH_URL = + "http://mangafox.me/search.php?name_method=cw&advopts=1&order=az&sort=name"; + + public Mangafox(Context context) { + super(context); + } + + @Override + public String getName() { + return NAME; + } + + @Override + public int getSourceId() { + return SourceManager.MANGAFOX; + } + + @Override + public boolean isLoginRequired() { + return false; + } + + @Override + protected String getUrlFromPageNumber(int page) { + return INITIAL_UPDATE_URL + page + ".htm"; + } + + @Override + public List parsePopularMangasFromHtml(String unparsedHtml) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + List mangaList = new ArrayList<>(); + + Elements mangaHtmlBlocks = parsedDocument.select("div#mangalist > ul.list > li"); + for (Element currentHtmlBlock : mangaHtmlBlocks) { + Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); + mangaList.add(currentManga); + } + + return mangaList; + } + + private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { + Manga mangaFromHtmlBlock = new Manga(); + mangaFromHtmlBlock.source = getSourceId(); + + Element urlElement = htmlBlock.select("a.title").first(); + + if (urlElement != null) { + mangaFromHtmlBlock.url = urlElement.attr("href"); + mangaFromHtmlBlock.title = urlElement.text(); + } + + return mangaFromHtmlBlock; + } + + @Override + protected String getSearchUrl(String query, int page) { + return INITIAL_SEARCH_URL + "&name=" + query + "&page=" + page; + } + + @Override + protected List parseSearchFromHtml(String unparsedHtml) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + List mangaList = new ArrayList<>(); + + Elements mangaHtmlBlocks = parsedDocument.select("table#listing > tbody > tr:gt(0)"); + for (Element currentHtmlBlock : mangaHtmlBlocks) { + Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); + mangaList.add(currentManga); + } + + return mangaList; + } + + private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) { + Manga mangaFromHtmlBlock = new Manga(); + mangaFromHtmlBlock.source = getSourceId(); + + Element urlElement = htmlBlock.select("a.series_preview").first(); + + if (urlElement != null) { + mangaFromHtmlBlock.url = urlElement.attr("href"); + mangaFromHtmlBlock.title = urlElement.text(); + } + + return mangaFromHtmlBlock; + } + + @Override + protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + Element infoElement = parsedDocument.select("div#title").first(); + Element titleElement = infoElement.select("h2 > a").first(); + Element rowElement = infoElement.select("table > tbody > tr:eq(1)").first(); + Element authorElement = rowElement.select("td:eq(1)").first(); + Element artistElement = rowElement.select("td:eq(2)").first(); + Element genreElement = rowElement.select("td:eq(3)").first(); + Element descriptionElement = infoElement.select("p.summary").first(); + Element thumbnailUrlElement = parsedDocument.select("div.cover > img").first(); + + Manga newManga = new Manga(); + newManga.url = mangaUrl; + + if (titleElement != null) { + String title = titleElement.text(); + // Strip the last word + title = title.substring(0, title.lastIndexOf(" ")); + newManga.title = title; + } + if (artistElement != null) { + String fieldArtist = artistElement.text(); + newManga.artist = fieldArtist; + } + if (authorElement != null) { + String fieldAuthor = authorElement.text(); + newManga.author = fieldAuthor; + } + if (descriptionElement != null) { + String fieldDescription = descriptionElement.text(); + newManga.description = fieldDescription; + } + if (genreElement != null) { + String fieldGenre = genreElement.text(); + newManga.genre = fieldGenre; + } + if (thumbnailUrlElement != null) { + String fieldThumbnailUrl = thumbnailUrlElement.attr("src"); + newManga.thumbnail_url = fieldThumbnailUrl; + } +// if (statusElement != null) { +// boolean fieldCompleted = statusElement.text().contains("Completed"); +// newManga.status = fieldCompleted + ""; +// } + + newManga.initialized = true; + + return newManga; + } + + @Override + protected List parseHtmlToChapters(String unparsedHtml) { + return null; + } + + @Override + protected List parseHtmlToPageUrls(String unparsedHtml) { + return null; + } + + @Override + protected String parseHtmlToImageUrl(String unparsedHtml) { + return null; + } +} diff --git a/app/src/main/java/eu/kanade/mangafeed/sources/MangaHere.java b/app/src/main/java/eu/kanade/mangafeed/sources/Mangahere.java similarity index 87% rename from app/src/main/java/eu/kanade/mangafeed/sources/MangaHere.java rename to app/src/main/java/eu/kanade/mangafeed/sources/Mangahere.java index db8f0e4195..02a3cf2e02 100644 --- a/app/src/main/java/eu/kanade/mangafeed/sources/MangaHere.java +++ b/app/src/main/java/eu/kanade/mangafeed/sources/Mangahere.java @@ -21,15 +21,15 @@ import eu.kanade.mangafeed.data.models.Manga; import eu.kanade.mangafeed.sources.base.Source; import rx.Observable; -public class MangaHere extends Source { +public class Mangahere extends Source { - public static final String NAME = "MangaHere (EN)"; + public static final String NAME = "Mangahere (EN)"; public static final String BASE_URL = "www.mangahere.co"; - private static final String INITIAL_UPDATE_URL = "http://www.mangahere.co/latest/"; + private static final String INITIAL_UPDATE_URL = "http://www.mangahere.co/directory/"; private static final String INITIAL_SEARCH_URL = "http://www.mangahere.co/search.php?"; - public MangaHere(Context context) { + public Mangahere(Context context) { super(context); } @@ -45,7 +45,7 @@ public class MangaHere extends Source { @Override protected String getUrlFromPageNumber(int page) { - return INITIAL_UPDATE_URL + page + "/"; + return INITIAL_UPDATE_URL + page + ".htm?views.za"; } @Override @@ -101,42 +101,55 @@ public class MangaHere extends Source { public List parsePopularMangasFromHtml(String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); - List updatedMangaList = new ArrayList<>(); + List mangaList = new ArrayList<>(); - Elements updatedHtmlBlocks = parsedDocument.select("div.manga_updates dl"); - for (Element currentHtmlBlock : updatedHtmlBlocks) { - Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock); - - updatedMangaList.add(currentlyUpdatedManga); + Elements mangaHtmlBlocks = parsedDocument.select("div.directory_list > ul > li"); + for (Element currentHtmlBlock : mangaHtmlBlocks) { + Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); + mangaList.add(currentManga); } - return updatedMangaList; + return mangaList; + } + + private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { + Manga mangaFromHtmlBlock = new Manga(); + mangaFromHtmlBlock.source = getSourceId(); + + Element urlElement = htmlBlock.select("div.title > a").first(); + + if (urlElement != null) { + mangaFromHtmlBlock.url = urlElement.attr("href"); + mangaFromHtmlBlock.title = urlElement.attr("title"); + } + + return mangaFromHtmlBlock; } @Override protected List parseSearchFromHtml(String unparsedHtml) { - return null; + Document parsedDocument = Jsoup.parse(unparsedHtml); + + List mangaList = new ArrayList<>(); + + Elements mangaHtmlBlocks = parsedDocument.select("div.result_search > dl"); + for (Element currentHtmlBlock : mangaHtmlBlocks) { + Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); + mangaList.add(currentManga); + } + + return mangaList; } - private Manga constructMangaFromHtmlBlock(Element htmlBlock) { + private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) { Manga mangaFromHtmlBlock = new Manga(); mangaFromHtmlBlock.source = getSourceId(); Element urlElement = htmlBlock.select("a.manga_info").first(); - Element nameElement = htmlBlock.select("a.manga_info").first(); - Element updateElement = htmlBlock.select("span.time").first(); if (urlElement != null) { - String fieldUrl = urlElement.attr("href"); - mangaFromHtmlBlock.url = fieldUrl; - } - if (nameElement != null) { - String fieldName = nameElement.text(); - mangaFromHtmlBlock.title = fieldName; - } - if (updateElement != null) { - long fieldUpdate = parseUpdateFromElement(updateElement); - mangaFromHtmlBlock.last_update = fieldUpdate; + mangaFromHtmlBlock.url = urlElement.attr("href"); + mangaFromHtmlBlock.title = urlElement.text(); } return mangaFromHtmlBlock; diff --git a/app/src/main/java/eu/kanade/mangafeed/sources/base/Source.java b/app/src/main/java/eu/kanade/mangafeed/sources/base/Source.java index 82183dbceb..f8040875fe 100644 --- a/app/src/main/java/eu/kanade/mangafeed/sources/base/Source.java +++ b/app/src/main/java/eu/kanade/mangafeed/sources/base/Source.java @@ -85,9 +85,7 @@ public abstract class Source extends BaseSource { return Observable.from(pages) .filter(page -> page.getImageUrl() == null) .window(overrideNumberOfConcurrentPageDownloads()) - .concatMap(batchedPages -> - batchedPages.concatMap(this::getImageUrlFromPage) - ); + .concatMap(batchedPages -> batchedPages.concatMap(this::getImageUrlFromPage)); } private Observable getImageUrlFromPage(final Page page) { diff --git a/app/src/main/java/eu/kanade/mangafeed/ui/activity/MainActivity.java b/app/src/main/java/eu/kanade/mangafeed/ui/activity/MainActivity.java index a1035f7782..92aebe0fc5 100644 --- a/app/src/main/java/eu/kanade/mangafeed/ui/activity/MainActivity.java +++ b/app/src/main/java/eu/kanade/mangafeed/ui/activity/MainActivity.java @@ -46,9 +46,9 @@ public class MainActivity extends BaseActivity { new PrimaryDrawerItem() .withName(R.string.library_title) .withIdentifier(R.id.nav_drawer_library), - new PrimaryDrawerItem() - .withName(R.string.recent_updates_title) - .withIdentifier(R.id.nav_drawer_recent_updates), +// new PrimaryDrawerItem() +// .withName(R.string.recent_updates_title) +// .withIdentifier(R.id.nav_drawer_recent_updates), new PrimaryDrawerItem() .withName(R.string.catalogues_title) .withIdentifier(R.id.nav_drawer_catalogues),