From dd5692bb2d1cc7e3e26e0dde2e70e4cf297e68a6 Mon Sep 17 00:00:00 2001 From: len Date: Tue, 22 Mar 2016 19:00:05 +0100 Subject: [PATCH] Rewrite sources. Implement Batoto and Kissmanga --- app/build.gradle | 3 + .../data/download/DownloadManager.kt | 15 +- .../data/download/model/Download.java | 6 +- .../tachiyomi/data/glide/MangaModelLoader.kt | 8 +- .../data/library/LibraryUpdateService.kt | 6 +- ...areScraper.kt => CloudflareInterceptor.kt} | 4 +- .../tachiyomi/data/network/NetworkHelper.kt | 6 +- .../eu/kanade/tachiyomi/data/network/Req.kt | 4 +- .../tachiyomi/data/source/SourceManager.kt | 59 +-- .../data/source/base/BaseSource.java | 99 ---- .../data/source/base/LoginSource.java | 15 - .../data/source/base/OnlineSource.kt | 448 ++++++++++++++++++ .../data/source/base/ParsedOnlineSource.kt | 189 ++++++++ .../tachiyomi/data/source/base/Source.kt | 251 ++-------- .../data/source/base/YamlOnlineSource.kt | 166 +++++++ .../source/base/YamlOnlineSourceMappings.kt | 214 +++++++++ .../data/source/online/english/Batoto.java | 393 --------------- .../data/source/online/english/Batoto.kt | 270 +++++++++++ .../data/source/online/english/Kissmanga.kt | 209 +++----- .../data/source/online/english/Mangafox.java | 245 ---------- .../data/source/online/english/Mangahere.java | 313 ------------ .../source/online/english/ReadMangaToday.java | 290 ------------ .../data/source/online/russian/Mangachan.java | 240 ---------- .../data/source/online/russian/Mintmanga.java | 225 --------- .../data/source/online/russian/Readmanga.java | 225 --------- .../injection/component/AppComponent.kt | 3 + .../ui/catalogue/CataloguePresenter.kt | 27 +- .../ui/manga/chapter/ChaptersPresenter.kt | 2 +- .../ui/manga/info/MangaInfoFragment.kt | 6 +- .../ui/manga/info/MangaInfoPresenter.kt | 2 +- .../tachiyomi/ui/reader/ReaderPresenter.kt | 60 ++- .../ui/setting/SettingsSourcesFragment.kt | 6 +- .../tachiyomi/util/ChapterSourceSync.kt | 5 +- .../java/eu/kanade/tachiyomi/util/Parser.java | 52 -- .../widget/preference/SourceLoginDialog.kt | 7 +- .../library/LibraryUpdateServiceTest.java | 14 +- 36 files changed, 1516 insertions(+), 2571 deletions(-) rename app/src/main/java/eu/kanade/tachiyomi/data/network/{CloudflareScraper.kt => CloudflareInterceptor.kt} (94%) delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/base/BaseSource.java delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/base/LoginSource.java create mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/base/OnlineSource.kt create mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/base/ParsedOnlineSource.kt create mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSource.kt create mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSourceMappings.kt delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.java create mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.kt delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangafox.java delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangahere.java delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/ReadMangaToday.java delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mangachan.java delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mintmanga.java delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Readmanga.java delete mode 100644 app/src/main/java/eu/kanade/tachiyomi/util/Parser.java diff --git a/app/build.gradle b/app/build.gradle index c98daf7f33..705a276fcc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -131,6 +131,9 @@ dependencies { // JSON compile 'com.google.code.gson:gson:2.6.2' + // YAML + compile 'org.yaml:snakeyaml:1.17' + // JavaScript engine compile 'com.squareup.duktape:duktape-android:0.9.5' diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt index 7df77e3518..1350218add 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt @@ -13,6 +13,7 @@ import eu.kanade.tachiyomi.data.download.model.DownloadQueue import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.source.SourceManager +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.source.base.Source import eu.kanade.tachiyomi.data.source.model.Page import eu.kanade.tachiyomi.util.DiskUtils @@ -108,7 +109,7 @@ class DownloadManager(private val context: Context, private val sourceManager: S // Create a download object for every chapter and add them to the downloads queue fun downloadChapters(manga: Manga, chapters: List) { - val source = sourceManager.get(manga.source) + val source = sourceManager.get(manga.source) as? OnlineSource ?: return // Used to avoid downloading chapters with the same name val addedChapters = ArrayList() @@ -182,8 +183,8 @@ class DownloadManager(private val context: Context, private val sourceManager: S DiskUtils.createDirectory(download.directory) val pageListObservable = if (download.pages == null) - // Pull page list from network and add them to download object - download.source.pullPageListFromNetwork(download.chapter.url) + // Pull page list from network and add them to download object + download.source.fetchPageListFromNetwork(download.chapter) .doOnNext { pages -> download.pages = pages savePageList(download) @@ -199,7 +200,7 @@ class DownloadManager(private val context: Context, private val sourceManager: S download.status = Download.DOWNLOADING } // Get all the URLs to the source images, fetch pages if necessary - .flatMap { download.source.getAllImageUrlsFromPageList(it) } + .flatMap { download.source.fetchAllImageUrlsFromPageList(it) } // Start downloading images, consider we can have downloaded images already .concatMap { page -> getOrDownloadImage(page, download) } // Do when page is downloaded. @@ -251,9 +252,9 @@ class DownloadManager(private val context: Context, private val sourceManager: S } // Save image on disk - private fun downloadImage(page: Page, source: Source, directory: File, filename: String): Observable { + private fun downloadImage(page: Page, source: OnlineSource, directory: File, filename: String): Observable { page.status = Page.DOWNLOAD_IMAGE - return source.getImageProgressResponse(page) + return source.imageResponse(page) .flatMap { try { val file = File(directory, filename) @@ -376,7 +377,7 @@ class DownloadManager(private val context: Context, private val sourceManager: S } fun getAbsoluteMangaDirectory(source: Source, manga: Manga): File { - val mangaRelativePath = source.visibleName + + val mangaRelativePath = source.toString() + File.separator + manga.title.replace("[^\\sa-zA-Z0-9.-]".toRegex(), "_") diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/download/model/Download.java b/app/src/main/java/eu/kanade/tachiyomi/data/download/model/Download.java index b0891edee0..8bd19fbf48 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/download/model/Download.java +++ b/app/src/main/java/eu/kanade/tachiyomi/data/download/model/Download.java @@ -5,12 +5,12 @@ import java.util.List; import eu.kanade.tachiyomi.data.database.models.Chapter; import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.base.Source; +import eu.kanade.tachiyomi.data.source.base.OnlineSource; import eu.kanade.tachiyomi.data.source.model.Page; import rx.subjects.PublishSubject; public class Download { - public Source source; + public OnlineSource source; public Manga manga; public Chapter chapter; public List pages; @@ -29,7 +29,7 @@ public class Download { public static final int ERROR = 4; - public Download(Source source, Manga manga, Chapter chapter) { + public Download(OnlineSource source, Manga manga, Chapter chapter) { this.source = source; this.manga = manga; this.chapter = chapter; diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/glide/MangaModelLoader.kt b/app/src/main/java/eu/kanade/tachiyomi/data/glide/MangaModelLoader.kt index 9402c8fabc..c5fa164139 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/glide/MangaModelLoader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/glide/MangaModelLoader.kt @@ -9,6 +9,7 @@ import eu.kanade.tachiyomi.App import eu.kanade.tachiyomi.data.cache.CoverCache import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.source.SourceManager +import eu.kanade.tachiyomi.data.source.base.OnlineSource import java.io.File import java.io.InputStream import javax.inject.Inject @@ -103,12 +104,11 @@ class MangaModelLoader(context: Context) : StreamModelLoader { * * @param manga the model. */ - fun getHeaders(manga: Manga): LazyHeaders { + fun getHeaders(manga: Manga): Headers { + val source = sourceManager.get(manga.source) as? OnlineSource ?: return LazyHeaders.DEFAULT return cachedHeaders.getOrPut(manga.source) { - val source = sourceManager.get(manga.source)!! - LazyHeaders.Builder().apply { - for ((key, value) in source.requestHeaders.toMultimap()) { + for ((key, value) in source.headers.toMultimap()) { addHeader(key, value[0]) } }.build() diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt index 00af24da90..c942c08fa2 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt @@ -18,6 +18,7 @@ import eu.kanade.tachiyomi.data.database.models.Category import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.source.SourceManager +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.ui.main.MainActivity import eu.kanade.tachiyomi.util.* import rx.Observable @@ -288,9 +289,8 @@ class LibraryUpdateService : Service() { * @return a pair of the inserted and removed chapters. */ fun updateManga(manga: Manga): Observable> { - val source = sourceManager.get(manga.source) - return source!! - .pullChaptersFromNetwork(manga.url) + val source = sourceManager.get(manga.source) as? OnlineSource ?: return Observable.empty() + return source.fetchChapterList(manga) .map { syncChaptersWithSource(db, it, manga, source) } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareScraper.kt b/app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareInterceptor.kt similarity index 94% rename from app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareScraper.kt rename to app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareInterceptor.kt index 21cca0f1a7..2fb74ef9d1 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareScraper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/network/CloudflareInterceptor.kt @@ -6,7 +6,7 @@ import okhttp3.Interceptor import okhttp3.Request import okhttp3.Response -object CloudflareScraper { +class CloudflareInterceptor(private val cookies: PersistentCookieStore) : Interceptor { //language=RegExp private val operationPattern = Regex("""setTimeout\(function\(\)\{\s+(var t,r,a,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n""") @@ -17,7 +17,7 @@ object CloudflareScraper { //language=RegExp private val challengePattern = Regex("""name="jschl_vc" value="(\w+)"""") - fun request(chain: Interceptor.Chain, cookies: PersistentCookieStore): Response { + override fun intercept(chain: Interceptor.Chain): Response { val response = chain.proceed(chain.request()) // Check if we already solved a challenge diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/network/NetworkHelper.kt b/app/src/main/java/eu/kanade/tachiyomi/data/network/NetworkHelper.kt index 33eed5ffa1..1b61f14697 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/network/NetworkHelper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/network/NetworkHelper.kt @@ -32,20 +32,18 @@ class NetworkHelper(context: Context) { .build() val cloudflareClient = defaultClient.newBuilder() - .addInterceptor { CloudflareScraper.request(it, cookies) } + .addInterceptor(CloudflareInterceptor(cookies)) .build() val cookies: PersistentCookieStore get() = cookieManager.store - @JvmOverloads fun request(request: Request, client: OkHttpClient = defaultClient): Observable { return Observable.fromCallable { - client.newCall(request).execute().apply { body().close() } + client.newCall(request).execute() } } - @JvmOverloads fun requestBody(request: Request, client: OkHttpClient = defaultClient): Observable { return Observable.fromCallable { client.newCall(request).execute().body().string() diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/network/Req.kt b/app/src/main/java/eu/kanade/tachiyomi/data/network/Req.kt index 0cedb2e970..65dfac2254 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/network/Req.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/network/Req.kt @@ -1,9 +1,9 @@ package eu.kanade.tachiyomi.data.network import okhttp3.* -import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeUnit.MINUTES -private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, TimeUnit.MINUTES).build() +private val DEFAULT_CACHE_CONTROL = CacheControl.Builder().maxAge(10, MINUTES).build() private val DEFAULT_HEADERS = Headers.Builder().build() private val DEFAULT_BODY: RequestBody = FormBody.Builder().build() diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/SourceManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/SourceManager.kt index 9499326668..cef43935df 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/SourceManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/SourceManager.kt @@ -1,17 +1,19 @@ package eu.kanade.tachiyomi.data.source import android.content.Context +import android.os.Environment +import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.source.base.Source -import eu.kanade.tachiyomi.data.source.online.english.* -import eu.kanade.tachiyomi.data.source.online.russian.Mangachan -import eu.kanade.tachiyomi.data.source.online.russian.Mintmanga -import eu.kanade.tachiyomi.data.source.online.russian.Readmanga -import java.util.* +import eu.kanade.tachiyomi.data.source.base.YamlOnlineSource +import eu.kanade.tachiyomi.data.source.online.english.Batoto +import eu.kanade.tachiyomi.data.source.online.english.Kissmanga +import org.yaml.snakeyaml.Yaml +import timber.log.Timber +import java.io.File open class SourceManager(private val context: Context) { - val sourcesMap: HashMap - val BATOTO = 1 val MANGAHERE = 2 val MANGAFOX = 3 @@ -23,38 +25,39 @@ open class SourceManager(private val context: Context) { val LAST_SOURCE = 8 - init { - sourcesMap = createSourcesMap() - } + val sourcesMap = createSources() open fun get(sourceKey: Int): Source? { return sourcesMap[sourceKey] } - private fun createSource(sourceKey: Int): Source? = when (sourceKey) { - BATOTO -> Batoto(context) - MANGAHERE -> Mangahere(context) - MANGAFOX -> Mangafox(context) - KISSMANGA -> Kissmanga(context) - READMANGA -> Readmanga(context) - MINTMANGA -> Mintmanga(context) - MANGACHAN -> Mangachan(context) - READMANGATODAY -> ReadMangaToday(context) + fun getOnlineSources() = sourcesMap.values.filterIsInstance(OnlineSource::class.java) + + private fun createSource(id: Int): Source? = when (id) { + BATOTO -> Batoto(context, id) + KISSMANGA -> Kissmanga(context, id) else -> null } - private fun createSourcesMap(): HashMap { - val map = HashMap() + private fun createSources(): Map = hashMapOf().apply { for (i in 1..LAST_SOURCE) { - val source = createSource(i) - if (source != null) { - source.id = i - map.put(i, source) + createSource(i)?.let { put(i, it) } + } + + val parsersDir = File(Environment.getExternalStorageDirectory().absolutePath + + File.separator + context.getString(R.string.app_name), "parsers") + + if (parsersDir.exists()) { + val yaml = Yaml() + for (file in parsersDir.listFiles().filter { it.extension == "yml" }) { + try { + val map = file.inputStream().use { yaml.loadAs(it, Map::class.java) } + YamlOnlineSource(context, map).let { put(it.id, it) } + } catch (e: Exception) { + Timber.e("Error loading source from file. Bad format?") + } } } - return map } - fun getSources(): List = ArrayList(sourcesMap.values) - } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/BaseSource.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/BaseSource.java deleted file mode 100644 index 6817c84e15..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/BaseSource.java +++ /dev/null @@ -1,99 +0,0 @@ -package eu.kanade.tachiyomi.data.source.base; - -import org.jsoup.nodes.Document; - -import java.util.List; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import okhttp3.Headers; -import okhttp3.Response; -import rx.Observable; - -public abstract class BaseSource { - - private int id; - - // Id of the source - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } - - public abstract Language getLang(); - - // Name of the source to display - public abstract String getName(); - - // Name of the source to display with the language - public String getVisibleName() { - return getName() + " (" + getLang().getCode() + ")"; - } - - // Base url of the source, like: http://example.com - public abstract String getBaseUrl(); - - // True if the source requires a login - public abstract boolean isLoginRequired(); - - // Return the initial popular mangas URL - protected abstract String getInitialPopularMangasUrl(); - - // Return the initial search url given a query - protected abstract String getInitialSearchUrl(String query); - - // Get the popular list of mangas from the source's parsed document - protected abstract List parsePopularMangasFromHtml(Document parsedHtml); - - // Get the next popular page URL or null if it's the last - protected abstract String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page); - - // Get the searched list of mangas from the source's parsed document - protected abstract List parseSearchFromHtml(Document parsedHtml); - - // Get the next search page URL or null if it's the last - protected abstract String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query); - - // Given the URL of a manga and the result of the request, return the details of the manga - protected abstract Manga parseHtmlToManga(String mangaUrl, String unparsedHtml); - - // Given the result of the request to mangas' chapters, return a list of chapters - protected abstract List parseHtmlToChapters(String unparsedHtml); - - // Given the result of the request to a chapter, return the list of URLs of the chapter - protected abstract List parseHtmlToPageUrls(String unparsedHtml); - - // Given the result of the request to a chapter's page, return the URL of the image of the page - protected abstract String parseHtmlToImageUrl(String unparsedHtml); - - - // Login related methods, shouldn't be overriden if the source doesn't require it - public Observable login(String username, String password) { - throw new UnsupportedOperationException("Not implemented"); - } - - public boolean isLogged() { - throw new UnsupportedOperationException("Not implemented"); - } - - protected boolean isAuthenticationSuccessful(Response response) { - throw new UnsupportedOperationException("Not implemented"); - } - - // Default headers, it can be overriden by children or just add new keys - protected Headers.Builder headersBuilder() { - Headers.Builder builder = new Headers.Builder(); - builder.add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)"); - return builder; - } - - @Override - public String toString() { - return getVisibleName(); - } -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/LoginSource.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/LoginSource.java deleted file mode 100644 index 3865373a0c..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/LoginSource.java +++ /dev/null @@ -1,15 +0,0 @@ -package eu.kanade.tachiyomi.data.source.base; - -import android.content.Context; - -public abstract class LoginSource extends Source { - - public LoginSource(Context context) { - super(context); - } - - @Override - public boolean isLoginRequired() { - return true; - } -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/OnlineSource.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/OnlineSource.kt new file mode 100644 index 0000000000..83e135472e --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/OnlineSource.kt @@ -0,0 +1,448 @@ +package eu.kanade.tachiyomi.data.source.base + +import android.content.Context +import eu.kanade.tachiyomi.App +import eu.kanade.tachiyomi.data.cache.ChapterCache +import eu.kanade.tachiyomi.data.database.models.Chapter +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.network.NetworkHelper +import eu.kanade.tachiyomi.data.network.get +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.source.Language +import eu.kanade.tachiyomi.data.source.model.MangasPage +import eu.kanade.tachiyomi.data.source.model.Page +import okhttp3.* +import rx.Observable +import javax.inject.Inject + +/** + * A simple implementation for sources from a website. + * + * @param context the application context. + */ +abstract class OnlineSource(context: Context) : Source { + + /** + * Network service. + */ + @Inject lateinit var network: NetworkHelper + + /** + * Chapter cache. + */ + @Inject lateinit var chapterCache: ChapterCache + + /** + * Preferences helper. + */ + @Inject lateinit var preferences: PreferencesHelper + + /** + * Base url of the website without the trailing slash, like: http://mysite.com + */ + abstract val baseUrl: String + + /** + * Language of the source. + */ + abstract val lang: Language + + /** + * Headers used for requests. + */ + val headers by lazy { headersBuilder().build() } + + /** + * Default network client for doing requests. + */ + open val client: OkHttpClient + get() = network.defaultClient + + init { + // Inject dependencies. + App.get(context).component.inject(this) + } + + /** + * Headers builder for requests. Implementations can override this method for custom headers. + */ + open protected fun headersBuilder() = Headers.Builder().apply { + add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)") + } + + /** + * Visible name of the source. + */ + override fun toString() = "$name (${lang.code})" + + // Login source + + open fun isLoginRequired() = false + + open fun isLogged(): Boolean = throw Exception("Not implemented") + + open fun login(username: String, password: String): Observable + = throw Exception("Not implemented") + + open fun isAuthenticationSuccessful(response: Response): Boolean + = throw Exception("Not implemented") + + /** + * Returns an observable containing a page with a list of manga. Normally it's not needed to + * override this method. + * + * @param page the page object where the information will be saved, like the list of manga, + * the current page and the next page url. + */ + open fun fetchPopularManga(page: MangasPage): Observable = network + .request(popularMangaRequest(page), client) + .map { response -> + page.apply { + mangas = mutableListOf() + popularMangaParse(response, this) + } + } + + /** + * Returns the request for the popular manga given the page. Override only if it's needed to + * send different headers or request method like POST. + * + * @param page the page object. + */ + open protected fun popularMangaRequest(page: MangasPage): Request { + if (page.page == 1) { + page.url = popularMangaInitialUrl() + } + return get(page.url, headers) + } + + /** + * Returns the absolute url of the first page to popular manga. + */ + abstract protected fun popularMangaInitialUrl(): String + + /** + * Parse the response from the site. It should add a list of manga and the absolute url to the + * next page (if it has a next one) to [page]. + * + * @param response the response from the site. + * @param page the page object to be filled. + */ + abstract protected fun popularMangaParse(response: Response, page: MangasPage) + + /** + * Returns an observable containing a page with a list of manga. Normally it's not needed to + * override this method. + * + * @param page the page object where the information will be saved, like the list of manga, + * the current page and the next page url. + * @param query the search query. + */ + open fun fetchSearchManga(page: MangasPage, query: String): Observable = network + .request(searchMangaRequest(page, query), client) + .map { response -> + page.apply { + mangas = mutableListOf() + searchMangaParse(response, this, query) + } + } + + /** + * Returns the request for the search manga given the page. Override only if it's needed to + * send different headers or request method like POST. + * + * @param page the page object. + * @param query the search query. + */ + open protected fun searchMangaRequest(page: MangasPage, query: String): Request { + if (page.page == 1) { + page.url = searchMangaInitialUrl(query) + } + return get(page.url, headers) + } + + /** + * Returns the absolute url of the first page to popular manga. + * + * @param query the search query. + */ + abstract protected fun searchMangaInitialUrl(query: String): String + + /** + * Parse the response from the site. It should add a list of manga and the absolute url to the + * next page (if it has a next one) to [page]. + * + * @param response the response from the site. + * @param page the page object to be filled. + * @param query the search query. + */ + abstract protected fun searchMangaParse(response: Response, page: MangasPage, query: String) + + /** + * Returns an observable with the updated details for a manga. Normally it's not needed to + * override this method. + * + * @param manga the manga to be updated. + */ + override fun fetchMangaDetails(manga: Manga): Observable = network + .request(mangaDetailsRequest(manga), client) + .map { response -> + Manga.create(manga.url, id).apply { + mangaDetailsParse(response, this) + initialized = true + } + } + + /** + * Returns the request for updating a manga. Override only if it's needed to override the url, + * send different headers or request method like POST. + * + * @param manga the manga to be updated. + */ + open protected fun mangaDetailsRequest(manga: Manga): Request { + return get(baseUrl + manga.url, headers) + } + + /** + * Parse the response from the site. It should fill [manga]. + * + * @param response the response from the site. + * @param manga the manga whose fields have to be filled. + */ + abstract protected fun mangaDetailsParse(response: Response, manga: Manga) + + /** + * Returns an observable with the updated chapter list for a manga. Normally it's not needed to + * override this method. + * + * @param manga the manga to look for chapters. + */ + override fun fetchChapterList(manga: Manga): Observable> = network + .request(chapterListRequest(manga), client) + .map { response -> + mutableListOf().apply { + chapterListParse(response, this) + if (isEmpty()) { + throw Exception("No chapters found") + } + } + } + + /** + * Returns the request for updating the chapter list. Override only if it's needed to override + * the url, send different headers or request method like POST. + * + * @param manga the manga to look for chapters. + */ + open protected fun chapterListRequest(manga: Manga): Request { + return get(baseUrl + manga.url, headers) + } + + /** + * Parse the response from the site. It should fill [chapters]. + * + * @param response the response from the site. + * @param chapters the chapter list to be filled. + */ + abstract protected fun chapterListParse(response: Response, chapters: MutableList) + + /** + * Returns an observable with the page list for a chapter. It tries to return the page list from + * the local cache, otherwise fallbacks to network calling [fetchPageListFromNetwork]. + * + * @param chapter the chapter whose page list has to be fetched. + */ + final override fun fetchPageList(chapter: Chapter): Observable> = chapterCache + .getPageListFromCache(getChapterCacheKey(chapter)) + .onErrorResumeNext { fetchPageListFromNetwork(chapter) } + + /** + * Returns an observable with the page list for a chapter. Normally it's not needed to override + * this method. + * + * @param chapter the chapter whose page list has to be fetched. + */ + open fun fetchPageListFromNetwork(chapter: Chapter): Observable> = network + .request(pageListRequest(chapter), client) + .map { response -> + mutableListOf().apply { + pageListParse(response, this) + if (isEmpty()) { + throw Exception("Page list is empty") + } + } + } + + /** + * Returns the request for getting the page list. Override only if it's needed to override the + * url, send different headers or request method like POST. + * + * @param chapter the chapter whose page list has to be fetched + */ + open protected fun pageListRequest(chapter: Chapter): Request { + return get(baseUrl + chapter.url, headers) + } + + /** + * Parse the response from the site. It should fill [pages]. + * + * @param response the response from the site. + * @param pages the page list to be filled. + */ + abstract protected fun pageListParse(response: Response, pages: MutableList) + + /** + * Returns the key for the page list to be stored in [ChapterCache]. + */ + private fun getChapterCacheKey(chapter: Chapter) = "$id${chapter.url}" + + /** + * Returns an observable with the page containing the source url of the image. If there's any + * error, it will return null instead of throwing an exception. + * + * @param page the page whose source image has to be fetched. + */ + open protected fun fetchImageUrl(page: Page): Observable { + page.status = Page.LOAD_PAGE + return network + .request(imageUrlRequest(page), client) + .map { imageUrlParse(it) } + .doOnError { page.status = Page.ERROR } + .onErrorReturn { null } + .doOnNext { page.imageUrl = it } + .map { page } + } + + /** + * Returns the request for getting the url to the source image. Override only if it's needed to + * override the url, send different headers or request method like POST. + * + * @param page the chapter whose page list has to be fetched + */ + open protected fun imageUrlRequest(page: Page): Request { + return get(page.url, headers) + } + + /** + * Parse the response from the site. It should return the absolute url to the source image. + * + * @param response the response from the site. + */ + abstract protected fun imageUrlParse(response: Response): String + + /** + * Returns an observable of the page with the downloaded image. + * + * @param page the page whose source image has to be downloaded. + */ + final override fun fetchImage(page: Page): Observable = + if (page.imageUrl.isNullOrEmpty()) + fetchImageUrl(page).flatMap { getCachedImage(it) } + else + getCachedImage(page) + + /** + * Returns an observable with the response of the source image. + * + * @param page the page whose source image has to be downloaded. + */ + fun imageResponse(page: Page): Observable = network + .requestBodyProgress(imageRequest(page), page) + .doOnNext { + if (!it.isSuccessful) { + it.body().close() + throw RuntimeException("Not a valid response") + } + } + + /** + * Returns the request for getting the source image. Override only if it's needed to override + * the url, send different headers or request method like POST. + * + * @param page the chapter whose page list has to be fetched + */ + open protected fun imageRequest(page: Page): Request { + return get(page.imageUrl, headers) + } + + /** + * Returns an observable of the page that gets the image from the chapter or fallbacks to + * network and copies it to the cache calling [cacheImage]. + * + * @param page the page. + */ + fun getCachedImage(page: Page): Observable { + val pageObservable = Observable.just(page) + if (page.imageUrl.isNullOrEmpty()) + return pageObservable + + return pageObservable + .flatMap { + if (!chapterCache.isImageInCache(page.imageUrl)) { + cacheImage(page) + } else { + Observable.just(page) + } + } + .doOnNext { + page.imagePath = chapterCache.getImagePath(page.imageUrl) + page.status = Page.READY + } + .doOnError { page.status = Page.ERROR } + .onErrorReturn { page } + } + + /** + * Returns an observable of the page that downloads the image to [ChapterCache]. + * + * @param page the page. + */ + private fun cacheImage(page: Page): Observable { + page.status = Page.DOWNLOAD_IMAGE + return imageResponse(page) + .doOnNext { chapterCache.putImageToCache(page.imageUrl, it, preferences.reencodeImage()) } + .map { page } + } + + + // Utility methods + + /** + * Returns an absolute url from a href. + * + * Ex: + * href="http://example.com/foo" url="http://example.com" -> http://example.com/foo + * href="/mypath" url="http://example.com/foo" -> http://example.com/mypath + * href="bar" url="http://example.com/foo" -> http://example.com/bar + * href="bar" url="http://example.com/foo/" -> http://example.com/foo/bar + * + * @param href the href attribute from the html. + * @param url the requested url. + */ + fun getAbsoluteUrl(href: String, url: HttpUrl) = when { + href.startsWith("http://") || href.startsWith("https://") -> href + href.startsWith("/") -> url.newBuilder().encodedPath("/").fragment(null).query(null) + .toString() + href.substring(1) + else -> url.toString().substringBeforeLast('/') + "/$href" + } + + fun fetchAllImageUrlsFromPageList(pages: List) = Observable.from(pages) + .filter { !it.imageUrl.isNullOrEmpty() } + .mergeWith(fetchRemainingImageUrlsFromPageList(pages)) + + fun fetchRemainingImageUrlsFromPageList(pages: List) = Observable.from(pages) + .filter { it.imageUrl.isNullOrEmpty() } + .concatMap { fetchImageUrl(it) } + + fun savePageList(chapter: Chapter, pages: List?) { + if (pages != null) { + chapterCache.putPageListToCache(getChapterCacheKey(chapter), pages) + } + } + + // Overridable method to allow custom parsing. + open fun parseChapterNumber(chapter: Chapter) { + + } + +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/ParsedOnlineSource.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/ParsedOnlineSource.kt new file mode 100644 index 0000000000..8a17878bd9 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/ParsedOnlineSource.kt @@ -0,0 +1,189 @@ +package eu.kanade.tachiyomi.data.source.base + +import android.content.Context +import eu.kanade.tachiyomi.data.database.models.Chapter +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.source.model.MangasPage +import eu.kanade.tachiyomi.data.source.model.Page +import okhttp3.Response +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element + +/** + * A simple implementation for sources from a website using Jsoup, an HTML parser. + * + * @param context the application context. + */ +abstract class ParsedOnlineSource(context: Context) : OnlineSource(context) { + + /** + * Parse the response from the site and fills [page]. + * + * @param response the response from the site. + * @param page the page object to be filled. + */ + override fun popularMangaParse(response: Response, page: MangasPage) { + val document = Jsoup.parse(response.body().string()) + for (element in document.select(popularMangaSelector())) { + Manga().apply { + source = this@ParsedOnlineSource.id + popularMangaFromElement(element, this) + page.mangas.add(this) + } + } + + popularMangaNextPageSelector()?.let { selector -> + page.nextPageUrl = document.select(selector).first()?.attr("href")?.let { + getAbsoluteUrl(it, response.request().url()) + } + } + } + + /** + * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga. + */ + abstract protected fun popularMangaSelector(): String + + /** + * Fills [manga] with the given [element]. Most sites only show the title and the url, it's + * totally safe to fill only those two values. + * + * @param element an element obtained from [popularMangaSelector]. + * @param manga the manga to fill. + */ + abstract protected fun popularMangaFromElement(element: Element, manga: Manga) + + /** + * Returns the Jsoup selector that returns the tag linking to the next page, or null if + * there's no next page. + */ + abstract protected fun popularMangaNextPageSelector(): String? + + /** + * Parse the response from the site and fills [page]. + * + * @param response the response from the site. + * @param page the page object to be filled. + * @param query the search query. + */ + override fun searchMangaParse(response: Response, page: MangasPage, query: String) { + val document = Jsoup.parse(response.body().string()) + for (element in document.select(searchMangaSelector())) { + Manga().apply { + source = this@ParsedOnlineSource.id + searchMangaFromElement(element, this) + page.mangas.add(this) + } + } + + searchMangaNextPageSelector()?.let { selector -> + page.nextPageUrl = document.select(selector).first()?.attr("href")?.let { + getAbsoluteUrl(it, response.request().url()) + } + } + } + + /** + * Returns the Jsoup selector that returns a list of [Element] corresponding to each manga. + */ + abstract protected fun searchMangaSelector(): String + + /** + * Fills [manga] with the given [element]. Most sites only show the title and the url, it's + * totally safe to fill only those two values. + * + * @param element an element obtained from [searchMangaSelector]. + * @param manga the manga to fill. + */ + abstract protected fun searchMangaFromElement(element: Element, manga: Manga) + + /** + * Returns the Jsoup selector that returns the tag linking to the next page, or null if + * there's no next page. + */ + abstract protected fun searchMangaNextPageSelector(): String? + + /** + * Parse the response from the site and fills the details of [manga]. + * + * @param response the response from the site. + * @param manga the manga to fill. + */ + override fun mangaDetailsParse(response: Response, manga: Manga) { + mangaDetailsParse(Jsoup.parse(response.body().string()), manga) + } + + /** + * Fills the details of [manga] from the given [document]. + * + * @param document the parsed document. + * @param manga the manga to fill. + */ + abstract protected fun mangaDetailsParse(document: Document, manga: Manga) + + /** + * Parse the response from the site and fills the chapter list. + * + * @param response the response from the site. + * @param chapters the list of chapters to fill. + */ + override fun chapterListParse(response: Response, chapters: MutableList) { + val document = Jsoup.parse(response.body().string()) + + for (element in document.select(chapterListSelector())) { + Chapter.create().apply { + chapterFromElement(element, this) + chapters.add(this) + } + } + } + + /** + * Returns the Jsoup selector that returns a list of [Element] corresponding to each chapter. + */ + abstract protected fun chapterListSelector(): String + + /** + * Fills [chapter] with the given [element]. + * + * @param element an element obtained from [chapterListSelector]. + * @param chapter the chapter to fill. + */ + abstract protected fun chapterFromElement(element: Element, chapter: Chapter) + + /** + * Parse the response from the site and fills the page list. + * + * @param response the response from the site. + * @param pages the list of pages to fill. + */ + override fun pageListParse(response: Response, pages: MutableList) { + pageListParse(Jsoup.parse(response.body().string()), pages) + } + + /** + * Fills [pages] from the given [document]. + * + * @param document the parsed document. + * @param pages the list of pages to fill. + */ + abstract protected fun pageListParse(document: Document, pages: MutableList) + + /** + * Parse the response from the site and returns the absolute url to the source image. + * + * @param response the response from the site. + */ + override fun imageUrlParse(response: Response): String { + return imageUrlParse(Jsoup.parse(response.body().string())) + } + + /** + * Returns the absolute url to the source image from the document. + * + * @param document the parsed document. + */ + abstract protected fun imageUrlParse(document: Document): String + +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/Source.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/Source.kt index cdb54a3535..d9a7225c31 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/Source.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/Source.kt @@ -1,228 +1,51 @@ package eu.kanade.tachiyomi.data.source.base -import android.content.Context -import eu.kanade.tachiyomi.App -import eu.kanade.tachiyomi.data.cache.ChapterCache import eu.kanade.tachiyomi.data.database.models.Chapter import eu.kanade.tachiyomi.data.database.models.Manga -import eu.kanade.tachiyomi.data.network.NetworkHelper -import eu.kanade.tachiyomi.data.network.get -import eu.kanade.tachiyomi.data.preference.PreferencesHelper -import eu.kanade.tachiyomi.data.source.model.MangasPage import eu.kanade.tachiyomi.data.source.model.Page -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response -import org.jsoup.Jsoup import rx.Observable -import rx.schedulers.Schedulers -import java.util.* -import javax.inject.Inject -abstract class Source(context: Context) : BaseSource() { +/** + * A basic interface for creating a source. It could be an online source, a local source, etc... + */ +interface Source { - @Inject protected lateinit var networkService: NetworkHelper - @Inject protected lateinit var chapterCache: ChapterCache - @Inject protected lateinit var prefs: PreferencesHelper + /** + * Id for the source. Must be unique. + */ + val id: Int - val requestHeaders by lazy { headersBuilder().build() } + /** + * Name of the source. + */ + val name: String - init { - App.get(context).component.inject(this) - } + /** + * Returns an observable with the updated details for a manga. + * + * @param manga the manga to update. + */ + fun fetchMangaDetails(manga: Manga): Observable - open val networkClient: OkHttpClient - get() = networkService.defaultClient + /** + * Returns an observable with all the available chapters for a manga. + * + * @param manga the manga to update. + */ + fun fetchChapterList(manga: Manga): Observable> - override fun isLoginRequired(): Boolean { - return false - } + /** + * Returns an observable with the list of pages a chapter has. + * + * @param chapter the chapter. + */ + fun fetchPageList(chapter: Chapter): Observable> - protected fun popularMangaRequest(page: MangasPage): Request { - if (page.page == 1) { - page.url = initialPopularMangasUrl - } + /** + * Returns an observable with the path of the image. + * + * @param page the page. + */ + fun fetchImage(page: Page): Observable - return get(page.url, requestHeaders) - } - - protected open fun searchMangaRequest(page: MangasPage, query: String): Request { - if (page.page == 1) { - page.url = getInitialSearchUrl(query) - } - - return get(page.url, requestHeaders) - } - - protected open fun mangaDetailsRequest(mangaUrl: String): Request { - return get(baseUrl + mangaUrl, requestHeaders) - } - - protected fun chapterListRequest(mangaUrl: String): Request { - return get(baseUrl + mangaUrl, requestHeaders) - } - - protected open fun pageListRequest(chapterUrl: String): Request { - return get(baseUrl + chapterUrl, requestHeaders) - } - - protected open fun imageUrlRequest(page: Page): Request { - return get(page.url, requestHeaders) - } - - protected open fun imageRequest(page: Page): Request { - return get(page.imageUrl, requestHeaders) - } - - // Get the most popular mangas from the source - open fun pullPopularMangasFromNetwork(page: MangasPage): Observable { - return networkService.requestBody(popularMangaRequest(page), networkClient) - .map { Jsoup.parse(it) } - .doOnNext { doc -> page.mangas = parsePopularMangasFromHtml(doc) } - .doOnNext { doc -> page.nextPageUrl = parseNextPopularMangasUrl(doc, page) } - .map { response -> page } - } - - // Get mangas from the source with a query - open fun searchMangasFromNetwork(page: MangasPage, query: String): Observable { - return networkService.requestBody(searchMangaRequest(page, query), networkClient) - .map { Jsoup.parse(it) } - .doOnNext { doc -> page.mangas = parseSearchFromHtml(doc) } - .doOnNext { doc -> page.nextPageUrl = parseNextSearchUrl(doc, page, query) } - .map { response -> page } - } - - // Get manga details from the source - open fun pullMangaFromNetwork(mangaUrl: String): Observable { - return networkService.requestBody(mangaDetailsRequest(mangaUrl), networkClient) - .flatMap { Observable.just(parseHtmlToManga(mangaUrl, it)) } - } - - // Get chapter list of a manga from the source - open fun pullChaptersFromNetwork(mangaUrl: String): Observable> { - return networkService.requestBody(chapterListRequest(mangaUrl), networkClient) - .flatMap { unparsedHtml -> - val chapters = parseHtmlToChapters(unparsedHtml) - if (!chapters.isEmpty()) - Observable.just(chapters) - else - Observable.error(Exception("No chapters found")) - } - } - - open fun getCachedPageListOrPullFromNetwork(chapterUrl: String): Observable> { - return chapterCache.getPageListFromCache(getChapterCacheKey(chapterUrl)) - .onErrorResumeNext { pullPageListFromNetwork(chapterUrl) } - .onBackpressureBuffer() - } - - open fun pullPageListFromNetwork(chapterUrl: String): Observable> { - return networkService.requestBody(pageListRequest(chapterUrl), networkClient) - .flatMap { unparsedHtml -> - val pages = convertToPages(parseHtmlToPageUrls(unparsedHtml)) - if (!pages.isEmpty()) - Observable.just(parseFirstPage(pages, unparsedHtml)) - else - Observable.error(Exception("Page list is empty")) - } - } - - open fun getAllImageUrlsFromPageList(pages: List): Observable { - return Observable.from(pages) - .filter { page -> page.imageUrl != null } - .mergeWith(getRemainingImageUrlsFromPageList(pages)) - } - - // Get the URLs of the images of a chapter - open fun getRemainingImageUrlsFromPageList(pages: List): Observable { - return Observable.from(pages) - .filter { page -> page.imageUrl == null } - .concatMap { getImageUrlFromPage(it) } - } - - open fun getImageUrlFromPage(page: Page): Observable { - page.status = Page.LOAD_PAGE - return networkService.requestBody(imageUrlRequest(page), networkClient) - .flatMap { unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml)) } - .onErrorResumeNext { e -> - page.status = Page.ERROR - Observable.just(null) - } - .flatMap { imageUrl -> - page.imageUrl = imageUrl - Observable.just(page) - } - .subscribeOn(Schedulers.io()) - } - - open fun getCachedImage(page: Page): Observable { - val pageObservable = Observable.just(page) - if (page.imageUrl == null) - return pageObservable - - return pageObservable - .flatMap { p -> - if (!chapterCache.isImageInCache(page.imageUrl)) { - return@flatMap cacheImage(page) - } - Observable.just(page) - } - .flatMap { p -> - page.imagePath = chapterCache.getImagePath(page.imageUrl) - page.status = Page.READY - Observable.just(page) - } - .onErrorResumeNext { e -> - page.status = Page.ERROR - Observable.just(page) - } - } - - private fun cacheImage(page: Page): Observable { - page.status = Page.DOWNLOAD_IMAGE - return getImageProgressResponse(page) - .flatMap { resp -> - chapterCache.putImageToCache(page.imageUrl, resp, prefs.reencodeImage()) - Observable.just(page) - } - } - - open fun getImageProgressResponse(page: Page): Observable { - return networkService.requestBodyProgress(imageRequest(page), page) - .doOnNext { - if (!it.isSuccessful) { - it.body().close() - throw RuntimeException("Not a valid response") - } - } - } - - fun savePageList(chapterUrl: String, pages: List?) { - if (pages != null) - chapterCache.putPageListToCache(getChapterCacheKey(chapterUrl), pages) - } - - protected open fun convertToPages(pageUrls: List): List { - val pages = ArrayList() - for (i in pageUrls.indices) { - pages.add(Page(i, pageUrls[i])) - } - return pages - } - - protected open fun parseFirstPage(pages: List, unparsedHtml: String): List { - val firstImage = parseHtmlToImageUrl(unparsedHtml) - pages[0].imageUrl = firstImage - return pages - } - - protected fun getChapterCacheKey(chapterUrl: String): String { - return "$id$chapterUrl" - } - - // Overridable method to allow custom parsing. - open fun parseChapterNumber(chapter: Chapter) { - - } - -} +} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSource.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSource.kt new file mode 100644 index 0000000000..0fcc6fe2c0 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSource.kt @@ -0,0 +1,166 @@ +package eu.kanade.tachiyomi.data.source.base + +import android.content.Context +import eu.kanade.tachiyomi.data.database.models.Chapter +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.network.get +import eu.kanade.tachiyomi.data.network.post +import eu.kanade.tachiyomi.data.source.getLanguages +import eu.kanade.tachiyomi.data.source.model.MangasPage +import eu.kanade.tachiyomi.data.source.model.Page +import okhttp3.Request +import okhttp3.Response +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.* + +class YamlOnlineSource(context: Context, mappings: Map<*, *>) : OnlineSource(context) { + + val map = YamlSourceNode(mappings) + + override val name: String + get() = map.name + + override val baseUrl = map.host.let { + if (it.endsWith("/")) it.dropLast(1) else it + } + + override val lang = map.lang.toUpperCase().let { code -> + getLanguages().find { code == it.code }!! + } + + override val client = when(map.client) { + "cloudflare" -> network.cloudflareClient + else -> network.defaultClient + } + + override val id = map.id.let { + if (it is Int) it else (lang.code.hashCode() + 31 * it.hashCode()) and 0x7fffffff + } + + override fun popularMangaRequest(page: MangasPage): Request { + if (page.page == 1) { + page.url = popularMangaInitialUrl() + } + return when (map.popular.method?.toLowerCase()) { + "post" -> post(page.url, headers, map.popular.createForm()) + else -> get(page.url, headers) + } + } + + override fun popularMangaInitialUrl() = map.popular.url + + override fun popularMangaParse(response: Response, page: MangasPage) { + val document = Jsoup.parse(response.body().string()) + for (element in document.select(map.popular.manga_css)) { + Manga().apply { + source = this@YamlOnlineSource.id + title = element.text() + setUrl(element.attr("href")) + page.mangas.add(this) + } + } + + map.popular.next_url_css?.let { selector -> + page.nextPageUrl = document.select(selector).first()?.attr("href")?.let { + getAbsoluteUrl(it, response.request().url()) + } + } + } + + override fun searchMangaRequest(page: MangasPage, query: String): Request { + if (page.page == 1) { + page.url = searchMangaInitialUrl(query) + } + return when (map.search.method?.toLowerCase()) { + "post" -> post(page.url, headers, map.search.createForm()) + else -> get(page.url, headers) + } + } + + override fun searchMangaInitialUrl(query: String) = map.search.url.replace("\$query", query) + + override fun searchMangaParse(response: Response, page: MangasPage, query: String) { + val document = Jsoup.parse(response.body().string()) + for (element in document.select(map.search.manga_css)) { + Manga().apply { + source = this@YamlOnlineSource.id + title = element.text() + setUrl(element.attr("href")) + page.mangas.add(this) + } + } + + map.search.next_url_css?.let { selector -> + page.nextPageUrl = document.select(selector).first()?.attr("href")?.let { + getAbsoluteUrl(it, response.request().url()) + } + } + } + + override fun mangaDetailsParse(response: Response, manga: Manga) { + val document = Jsoup.parse(response.body().string()) + with(map.manga) { + val pool = parts.get(document) + + manga.author = author?.process(document, pool) + manga.artist = artist?.process(document, pool) + manga.description = summary?.process(document, pool) + manga.thumbnail_url = cover?.process(document, pool) + manga.genre = genres?.process(document, pool) + manga.status = status?.getStatus(document, pool) ?: Manga.UNKNOWN + } + } + + override fun chapterListParse(response: Response, chapters: MutableList) { + val document = Jsoup.parse(response.body().string()) + with(map.chapters) { + val pool = emptyMap() + val dateFormat = SimpleDateFormat(date?.format, Locale.ENGLISH) + + for (element in document.select(chapter_css)) { + val chapter = Chapter.create() + element.select(title).first().let { + chapter.name = it.text() + chapter.setUrl(it.attr("href")) + } + val dateElement = element.select(date?.select).first() + chapter.date_upload = date?.getDate(dateElement, pool, dateFormat)?.time ?: 0 + chapters.add(chapter) + } + } + } + + override fun pageListParse(response: Response, pages: MutableList) { + val document = Jsoup.parse(response.body().string()) + with(map.pages) { + val url = response.request().url().toString() + pages_css?.let { + for (element in document.select(it)) { + val value = element.attr(pages_attr) + val pageUrl = replace?.let { url.replace(it.toRegex(), replacement!!.replace("\$value", value)) } ?: value + pages.add(Page(pages.size, pageUrl)) + } + } + + for ((i, element) in document.select(image_css).withIndex()) { + val page = pages.getOrElse(i) { Page(i, "").apply { pages.add(this) } } + page.imageUrl = element.attr(image_attr).let { + getAbsoluteUrl(it, response.request().url()) + } + } + } + + } + + override fun imageUrlParse(response: Response): String { + val document = Jsoup.parse(response.body().string()) + return with(map.pages) { + document.select(image_css).first().attr(image_attr).let { + getAbsoluteUrl(it, response.request().url()) + } + } + } + +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSourceMappings.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSourceMappings.kt new file mode 100644 index 0000000000..5e1d3b9ae1 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/base/YamlOnlineSourceMappings.kt @@ -0,0 +1,214 @@ +@file:Suppress("UNCHECKED_CAST") + +package eu.kanade.tachiyomi.data.source.base + +import eu.kanade.tachiyomi.data.database.models.Manga +import okhttp3.FormBody +import okhttp3.RequestBody +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.ParseException +import java.text.SimpleDateFormat +import java.util.* + +private fun toMap(map: Any?) = map as? Map + +class YamlSourceNode(uncheckedMap: Map<*, *>) { + + val map = toMap(uncheckedMap)!! + + val id: Any by map + + val name: String by map + + val host: String by map + + val lang: String by map + + val client: String? + get() = map["client"] as? String + + val popular = PopularNode(toMap(map["popular"])!!) + + val search = SearchNode(toMap(map["search"])!!) + + val manga = MangaNode(toMap(map["manga"])!!) + + val chapters = ChaptersNode(toMap(map["chapters"])!!) + + val pages = PagesNode(toMap(map["pages"])!!) +} + +interface RequestableNode { + + val map: Map + + val url: String + get() = map["url"] as String + + val method: String? + get() = map["method"] as? String + + val payload: Map? + get() = map["payload"] as? Map + + fun createForm(): RequestBody { + return FormBody.Builder().apply { + payload?.let { + for ((key, value) in it) { + add(key, value) + } + } + }.build() + } + +} + +class PopularNode(override val map: Map): RequestableNode { + + val manga_css: String by map + + val next_url_css: String? + get() = map["next_url_css"] as? String + +} + +class SearchNode(override val map: Map): RequestableNode { + + val manga_css: String by map + + val next_url_css: String? + get() = map["next_url_css"] as? String +} + +class MangaNode(private val map: Map) { + + val parts = CacheNode(toMap(map["parts"]) ?: emptyMap()) + + val artist = toMap(map["artist"])?.let { SelectableNode(it) } + + val author = toMap(map["author"])?.let { SelectableNode(it) } + + val summary = toMap(map["summary"])?.let { SelectableNode(it) } + + val status = toMap(map["status"])?.let { StatusNode(it) } + + val genres = toMap(map["genres"])?.let { SelectableNode(it) } + + val cover = toMap(map["cover"])?.let { CoverNode(it) } + +} + +class ChaptersNode(private val map: Map) { + + val chapter_css: String by map + + val title: String by map + + val date = toMap(toMap(map["date"]))?.let { DateNode(it) } +} + +class CacheNode(private val map: Map) { + + fun get(document: Document) = map.mapValues { document.select(it.value as String).first() } +} + +open class SelectableNode(private val map: Map) { + + val select: String by map + + val from: String? + get() = map["from"] as? String + + open val attr: String? + get() = map["attr"] as? String + + val capture: String? + get() = map["capture"] as? String + + fun process(document: Element, cache: Map): String { + val parent = from?.let { cache[it] } ?: document + val node = parent.select(select).first() + var text = attr?.let { node.attr(it) } ?: node.text() + capture?.let { + text = Regex(it).find(text)?.groupValues?.get(1) ?: text + } + return text + } +} + +class StatusNode(private val map: Map) : SelectableNode(map) { + + val complete: String? + get() = map["complete"] as? String + + val ongoing: String? + get() = map["ongoing"] as? String + + val licensed: String? + get() = map["licensed"] as? String + + fun getStatus(document: Element, cache: Map): Int { + val text = process(document, cache) + complete?.let { + if (text.contains(it)) return Manga.COMPLETED + } + ongoing?.let { + if (text.contains(it)) return Manga.ONGOING + } + licensed?.let { + if (text.contains(it)) return Manga.LICENSED + } + return Manga.UNKNOWN + } +} + +class CoverNode(private val map: Map) : SelectableNode(map) { + + override val attr: String? + get() = map["attr"] as? String ?: "src" +} + +class DateNode(private val map: Map) : SelectableNode(map) { + + val format: String by map + + fun getDate(document: Element, cache: Map, formatter: SimpleDateFormat): Date { + val text = process(document, cache) + try { + return formatter.parse(text) + } catch (exception: ParseException) {} + + for (i in 0..7) { + (map["day$i"] as? List)?.let { + it.find { it.toRegex().containsMatchIn(text) }?.let { + return Calendar.getInstance().apply { add(Calendar.DATE, -i) }.time + } + } + } + + return Date(0) + } + +} + +class PagesNode(private val map: Map) { + + val pages_css: String? + get() = map["pages_css"] as? String + + val pages_attr: String? + get() = map["pages_attr"] as? String ?: "value" + + val replace: String? + get() = map["url_replace"] as? String + + val replacement: String? + get() = map["url_replacement"] as? String + + val image_css: String by map + + val image_attr: String + get() = map["image_attr"] as? String ?: "src" + +} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.java deleted file mode 100644 index bf7ac21a36..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.java +++ /dev/null @@ -1,393 +0,0 @@ -package eu.kanade.tachiyomi.data.source.online.english; - -import android.content.Context; -import android.net.Uri; -import android.text.Html; -import android.text.TextUtils; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.net.URI; -import java.net.URISyntaxException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.network.ReqKt; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.LanguageKt; -import eu.kanade.tachiyomi.data.source.base.LoginSource; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import eu.kanade.tachiyomi.data.source.model.Page; -import eu.kanade.tachiyomi.util.Parser; -import okhttp3.Cookie; -import okhttp3.FormBody; -import okhttp3.Headers; -import okhttp3.Request; -import okhttp3.Response; -import rx.Observable; -import rx.functions.Func1; - -public class Batoto extends LoginSource { - - public static final String NAME = "Batoto"; - public static final String BASE_URL = "http://bato.to"; - public static final String POPULAR_MANGAS_URL = BASE_URL + "/search_ajax?order_cond=views&order=desc&p=%s"; - public static final String SEARCH_URL = BASE_URL + "/search_ajax?name=%s&p=%s"; - public static final String CHAPTER_URL = BASE_URL + "/areader?id=%s&p=1"; - public static final String PAGE_URL = BASE_URL + "/areader?id=%s&p=%s"; - public static final String MANGA_URL = BASE_URL + "/comic_pop?id=%s"; - public static final String LOGIN_URL = BASE_URL + "/forums/index.php?app=core&module=global§ion=login"; - - public static final Pattern staffNotice = Pattern.compile("=+Batoto Staff Notice=+([^=]+)==+", Pattern.CASE_INSENSITIVE); - - private final Pattern datePattern; - private final Map dateFields; - - public Batoto(Context context) { - super(context); - - datePattern = Pattern.compile("(\\d+|A|An)\\s+(.*?)s? ago.*"); - dateFields = new HashMap() {{ - put("second", Calendar.SECOND); - put("minute", Calendar.MINUTE); - put("hour", Calendar.HOUR); - put("day", Calendar.DATE); - put("week", Calendar.WEEK_OF_YEAR); - put("month", Calendar.MONTH); - put("year", Calendar.YEAR); - }}; - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getBaseUrl() { - return BASE_URL; - } - - public Language getLang() { - return LanguageKt.getEN(); - } - - @Override - protected Headers.Builder headersBuilder() { - Headers.Builder builder = super.headersBuilder(); - builder.add("Cookie", "lang_option=English"); - builder.add("Referer", "http://bato.to/reader"); - return builder; - } - - @Override - public String getInitialPopularMangasUrl() { - return String.format(POPULAR_MANGAS_URL, 1); - } - - @Override - public String getInitialSearchUrl(String query) { - return String.format(SEARCH_URL, Uri.encode(query), 1); - } - - @Override - protected Request mangaDetailsRequest(String mangaUrl) { - String mangaId = mangaUrl.substring(mangaUrl.lastIndexOf('r') + 1); - return ReqKt.get(String.format(MANGA_URL, mangaId), getRequestHeaders()); - } - - @Override - protected Request pageListRequest(String pageUrl) { - String id = pageUrl.substring(pageUrl.indexOf('#') + 1); - return ReqKt.get(String.format(CHAPTER_URL, id), getRequestHeaders()); - } - - @Override - protected Request imageUrlRequest(Page page) { - String pageUrl = page.getUrl(); - int start = pageUrl.indexOf('#') + 1; - int end = pageUrl.indexOf('_', start); - String id = pageUrl.substring(start, end); - return ReqKt.get(String.format(PAGE_URL, id, pageUrl.substring(end+1)), getRequestHeaders()); - } - - private List parseMangasFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - if (!parsedHtml.text().contains("No (more) comics found!")) { - for (Element currentHtmlBlock : parsedHtml.select("tr:not([id]):not([class])")) { - Manga manga = constructMangaFromHtmlBlock(currentHtmlBlock); - mangaList.add(manga); - } - } - return mangaList; - } - - @Override - protected List parsePopularMangasFromHtml(Document parsedHtml) { - return parseMangasFromHtml(parsedHtml); - } - - @Override - protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = Parser.element(parsedHtml, "#show_more_row"); - return next != null ? String.format(POPULAR_MANGAS_URL, page.page + 1) : null; - } - - @Override - protected List parseSearchFromHtml(Document parsedHtml) { - return parseMangasFromHtml(parsedHtml); - } - - private Manga constructMangaFromHtmlBlock(Element htmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = Parser.element(htmlBlock, "a[href^=http://bato.to]"); - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.text().trim(); - } - return manga; - } - - @Override - protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - Element next = Parser.element(parsedHtml, "#show_more_row"); - return next != null ? String.format(SEARCH_URL, query, page.page + 1) : null; - } - - @Override - protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - Element tbody = parsedDocument.select("tbody").first(); - Element artistElement = tbody.select("tr:contains(Author/Artist:)").first(); - Elements genreElements = tbody.select("tr:contains(Genres:) img"); - - Manga manga = Manga.create(mangaUrl); - manga.author = Parser.text(artistElement, "td:eq(1)"); - manga.artist = Parser.text(artistElement, "td:eq(2)", manga.author); - manga.description = Parser.text(tbody, "tr:contains(Description:) > td:eq(1)"); - manga.thumbnail_url = Parser.src(parsedDocument, "img[src^=http://img.bato.to/forums/uploads/]"); - manga.status = parseStatus(Parser.text(parsedDocument, "tr:contains(Status:) > td:eq(1)")); - - if (!genreElements.isEmpty()) { - List genres = new ArrayList<>(); - for (Element element : genreElements) { - genres.add(element.attr("alt")); - } - manga.genre = TextUtils.join(", ", genres); - } - - manga.initialized = true; - return manga; - } - - private int parseStatus(String status) { - switch (status) { - case "Ongoing": - return Manga.ONGOING; - case "Complete": - return Manga.COMPLETED; - default: - return Manga.UNKNOWN; - } - } - - @Override - protected List parseHtmlToChapters(String unparsedHtml) { - Matcher matcher = staffNotice.matcher(unparsedHtml); - if (matcher.find()) { - String notice = Html.fromHtml(matcher.group(1)).toString().trim(); - throw new RuntimeException(notice); - } - - Document parsedDocument = Jsoup.parse(unparsedHtml); - - List chapterList = new ArrayList<>(); - - Elements chapterElements = parsedDocument.select("tr.row.lang_English.chapter_row"); - for (Element chapterElement : chapterElements) { - Chapter chapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(chapter); - } - return chapterList; - - } - - private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter chapter = Chapter.create(); - - Element urlElement = chapterElement.select("a[href^=http://bato.to/reader").first(); - Element dateElement = chapterElement.select("td").get(4); - - if (urlElement != null) { - String fieldUrl = urlElement.attr("href"); - chapter.setUrl(fieldUrl); - chapter.name = urlElement.text().trim(); - } - if (dateElement != null) { - chapter.date_upload = parseDateFromElement(dateElement); - } - return chapter; - } - - @SuppressWarnings("WrongConstant") - private long parseDateFromElement(Element dateElement) { - String dateAsString = dateElement.text(); - - Date date; - try { - date = new SimpleDateFormat("dd MMMMM yyyy - hh:mm a", Locale.ENGLISH).parse(dateAsString); - } catch (ParseException e) { - Matcher m = datePattern.matcher(dateAsString); - - if (m.matches()) { - String number = m.group(1); - int amount = number.contains("A") ? 1 : Integer.parseInt(m.group(1)); - String unit = m.group(2); - - Calendar cal = Calendar.getInstance(); - cal.add(dateFields.get(unit), -amount); - date = cal.getTime(); - } else { - return 0; - } - } - return date.getTime(); - } - - @Override - protected List parseHtmlToPageUrls(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - List pageUrlList = new ArrayList<>(); - - Element selectElement = Parser.element(parsedDocument, "#page_select"); - if (selectElement != null) { - for (Element pageUrlElement : selectElement.select("option")) { - pageUrlList.add(pageUrlElement.attr("value")); - } - } else { - // For webtoons in one page - for (int i = 0; i < parsedDocument.select("div > img").size(); i++) { - pageUrlList.add(""); - } - } - - return pageUrlList; - } - - @Override - protected List parseFirstPage(List pages, String unparsedHtml) { - if (!unparsedHtml.contains("Want to see this chapter per page instead?")) { - String firstImage = parseHtmlToImageUrl(unparsedHtml); - pages.get(0).setImageUrl(firstImage); - } else { - // For webtoons in one page - Document parsedDocument = Jsoup.parse(unparsedHtml); - Elements imageUrls = parsedDocument.select("div > img"); - for (int i = 0; i < pages.size(); i++) { - pages.get(i).setImageUrl(imageUrls.get(i).attr("src")); - } - } - return (List) pages; - } - - @Override - protected String parseHtmlToImageUrl(String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - Element imageElement = parsedDocument.getElementById("comic_page"); - return imageElement.attr("src"); - } - - @Override - public Observable login(final String username, final String password) { - return getNetworkService().requestBody(ReqKt.get(LOGIN_URL, getRequestHeaders())) - .flatMap(new Func1>() { - @Override - public Observable call(String response) {return doLogin(response, username, password);} - }) - .map(new Func1() { - @Override - public Boolean call(Response resp) {return isAuthenticationSuccessful(resp);} - }); - } - - private Observable doLogin(String response, String username, String password) { - Document doc = Jsoup.parse(response); - Element form = doc.select("#login").first(); - String postUrl = form.attr("action"); - - FormBody.Builder formBody = new FormBody.Builder(); - Element authKey = form.select("input[name=auth_key]").first(); - - formBody.add(authKey.attr("name"), authKey.attr("value")); - formBody.add("ips_username", username); - formBody.add("ips_password", password); - formBody.add("invisible", "1"); - formBody.add("rememberMe", "1"); - - return getNetworkService().request(ReqKt.post(postUrl, getRequestHeaders(), formBody.build())); - } - - @Override - protected boolean isAuthenticationSuccessful(Response response) { - return response.priorResponse() != null && response.priorResponse().code() == 302; - } - - @Override - public boolean isLogged() { - try { - for (Cookie cookie : getNetworkService().getCookies().get(new URI(BASE_URL))) { - if (cookie.name().equals("pass_hash")) - return true; - } - - } catch (URISyntaxException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public Observable> pullChaptersFromNetwork(final String mangaUrl) { - Observable> observable; - String username = getPrefs().sourceUsername(this); - String password = getPrefs().sourcePassword(this); - if (username.isEmpty() && password.isEmpty()) { - observable = Observable.error(new Exception("User not logged")); - } - else if (!isLogged()) { - observable = login(username, password) - .flatMap(new Func1>>() { - @Override - public Observable> call(Boolean result) {return Batoto.super.pullChaptersFromNetwork(mangaUrl);} - }); - } - else { - observable = super.pullChaptersFromNetwork(mangaUrl); - } - return observable; - } - -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.kt new file mode 100644 index 0000000000..c376350463 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Batoto.kt @@ -0,0 +1,270 @@ +package eu.kanade.tachiyomi.data.source.online.english + +import android.content.Context +import android.net.Uri +import android.text.Html +import eu.kanade.tachiyomi.data.database.models.Chapter +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.network.get +import eu.kanade.tachiyomi.data.network.post +import eu.kanade.tachiyomi.data.source.EN +import eu.kanade.tachiyomi.data.source.Language +import eu.kanade.tachiyomi.data.source.base.ParsedOnlineSource +import eu.kanade.tachiyomi.data.source.model.MangasPage +import eu.kanade.tachiyomi.data.source.model.Page +import okhttp3.FormBody +import okhttp3.Request +import okhttp3.Response +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import rx.Observable +import java.net.URI +import java.net.URISyntaxException +import java.text.ParseException +import java.text.SimpleDateFormat +import java.util.* +import java.util.regex.Pattern + +class Batoto(context: Context, override val id: Int) : ParsedOnlineSource(context) { + + override val name = "Batoto" + + override val baseUrl = "http://bato.to" + + override val lang: Language get() = EN + + private val datePattern = Pattern.compile("(\\d+|A|An)\\s+(.*?)s? ago.*") + + private val dateFields = HashMap().apply { + put("second", Calendar.SECOND) + put("minute", Calendar.MINUTE) + put("hour", Calendar.HOUR) + put("day", Calendar.DATE) + put("week", Calendar.WEEK_OF_YEAR) + put("month", Calendar.MONTH) + put("year", Calendar.YEAR) + } + + private val staffNotice = Pattern.compile("=+Batoto Staff Notice=+([^=]+)==+", Pattern.CASE_INSENSITIVE) + + override fun headersBuilder() = super.headersBuilder() + .add("Cookie", "lang_option=English") + .add("Referer", "http://bato.to/reader") + + override fun popularMangaInitialUrl() = "$baseUrl/search_ajax?order_cond=views&order=desc&p=1" + + override fun searchMangaInitialUrl(query: String) = "$baseUrl/search_ajax?name=${Uri.encode(query)}&p=1" + + override fun mangaDetailsRequest(manga: Manga): Request { + val mangaId = manga.url.substringAfterLast("r") + return get("$baseUrl/comic_pop?id=$mangaId", headers) + } + + override fun pageListRequest(chapter: Chapter): Request { + val id = chapter.url.substringAfterLast("#") + return get("$baseUrl/areader?id=$id&p=1", headers) + } + + override fun imageUrlRequest(page: Page): Request { + val pageUrl = page.url + val start = pageUrl.indexOf("#") + 1 + val end = pageUrl.indexOf("_", start) + val id = pageUrl.substring(start, end) + return get("$baseUrl/areader?id=$id&p=${pageUrl.substring(end+1)}", headers) + } + + override fun popularMangaParse(response: Response, page: MangasPage) { + val document = Jsoup.parse(response.body().string()) + for (element in document.select(popularMangaSelector())) { + Manga().apply { + source = this@Batoto.id + popularMangaFromElement(element, this) + page.mangas.add(this) + } + } + + page.nextPageUrl = document.select(popularMangaNextPageSelector()).first()?.let { + "$baseUrl/search_ajax?order_cond=views&order=desc&p=${page.page + 1}" + } + } + + override fun popularMangaSelector() = "tr:not([id]):not([class])" + + override fun popularMangaFromElement(element: Element, manga: Manga) { + element.select("a[href^=http://bato.to]").first().let { + manga.setUrl(it.attr("href")) + manga.title = it.text().trim() + } + } + + override fun popularMangaNextPageSelector() = "#show_more_row" + + override fun searchMangaParse(response: Response, page: MangasPage, query: String) { + val document = Jsoup.parse(response.body().string()) + for (element in document.select(searchMangaSelector())) { + Manga().apply { + source = this@Batoto.id + searchMangaFromElement(element, this) + page.mangas.add(this) + } + } + + page.nextPageUrl = document.select(searchMangaNextPageSelector()).first()?.let { + "$baseUrl/search_ajax?name=${Uri.encode(query)}&p=${page.page + 1}" + } + } + + override fun searchMangaSelector() = popularMangaSelector() + + override fun searchMangaFromElement(element: Element, manga: Manga) { + popularMangaFromElement(element, manga) + } + + override fun searchMangaNextPageSelector() = popularMangaNextPageSelector() + + override fun mangaDetailsParse(document: Document, manga: Manga) { + val tbody = document.select("tbody").first() + val artistElement = tbody.select("tr:contains(Author/Artist:)").first() + + manga.author = artistElement.select("td:eq(1)").first()?.text() + manga.artist = artistElement.select("td:eq(2)").first()?.text() ?: manga.author + manga.description = tbody.select("tr:contains(Description:) > td:eq(1)").first()?.text() + manga.thumbnail_url = document.select("img[src^=http://img.bato.to/forums/uploads/]").first()?.attr("src") + manga.status = parseStatus(document.select("tr:contains(Status:) > td:eq(1)").first()?.text()) + manga.genre = tbody.select("tr:contains(Genres:) img").map { it.attr("alt") }.joinToString(", ") + } + + private fun parseStatus(status: String?) = when (status) { + "Ongoing" -> Manga.ONGOING + "Complete" -> Manga.COMPLETED + else -> Manga.UNKNOWN + } + + override fun chapterListParse(response: Response, chapters: MutableList) { + val body = response.body().string() + val matcher = staffNotice.matcher(body) + if (matcher.find()) { + val notice = Html.fromHtml(matcher.group(1)).toString().trim() + throw RuntimeException(notice) + } + + val document = Jsoup.parse(body) + + for (element in document.select(chapterListSelector())) { + Chapter.create().apply { + chapterFromElement(element, this) + chapters.add(this) + } + } + } + + override fun chapterListSelector() = "tr.row.lang_English.chapter_row" + + override fun chapterFromElement(element: Element, chapter: Chapter) { + val urlElement = element.select("a[href^=http://bato.to/reader").first() + + chapter.setUrl(urlElement.attr("href")) + chapter.name = urlElement.text() + chapter.date_upload = element.select("td").getOrNull(4)?.let { + parseDateFromElement(it) + } ?: 0 + } + + private fun parseDateFromElement(dateElement: Element): Long { + val dateAsString = dateElement.text() + + val date: Date + try { + date = SimpleDateFormat("dd MMMMM yyyy - hh:mm a", Locale.ENGLISH).parse(dateAsString) + } catch (e: ParseException) { + val m = datePattern.matcher(dateAsString) + + if (m.matches()) { + val number = m.group(1) + val amount = if (number.contains("A")) 1 else Integer.parseInt(m.group(1)) + val unit = m.group(2) + + date = Calendar.getInstance().apply { + add(dateFields[unit]!!, -amount) + }.time + } else { + return 0 + } + } + + return date.time + } + + override fun pageListParse(document: Document, pages: MutableList) { + val selectElement = document.select("#page_select").first() + if (selectElement != null) { + for ((i, element) in selectElement.select("option").withIndex()) { + pages.add(Page(i, element.attr("value"))) + } + } else { + // For webtoons in one page + for ((i, element) in document.select("div > img").withIndex()) { + pages.add(Page(i, "", element.attr("src"))) + } + } + } + + override fun imageUrlParse(document: Document): String { + return document.select("#comic_page").first().attr("src") + } + + override fun login(username: String, password: String) = + network.request(get("$baseUrl/forums/index.php?app=core&module=global§ion=login", headers)) + .map { it.body().string() } + .flatMap { doLogin(it, username, password) } + .map { isAuthenticationSuccessful(it) } + + private fun doLogin(response: String, username: String, password: String): Observable { + val doc = Jsoup.parse(response) + val form = doc.select("#login").first() + val url = form.attr("action") + val authKey = form.select("input[name=auth_key]").first() + + val payload = FormBody.Builder().apply { + add(authKey.attr("name"), authKey.attr("value")) + add("ips_username", username) + add("ips_password", password) + add("invisible", "1") + add("rememberMe", "1") + }.build() + + return network.request(post(url, headers, payload)) + } + + override fun isLoginRequired() = true + + override fun isAuthenticationSuccessful(response: Response) = + response.priorResponse() != null && response.priorResponse().code() == 302 + + override fun isLogged(): Boolean { + try { + return network.cookies.get(URI(baseUrl)).find { it.name() == "pass_hash" } != null + } catch (e: URISyntaxException) { + // Ignore + } + return false + } + + override fun fetchChapterList(manga: Manga): Observable> { + if (!isLogged()) { + val username = preferences.sourceUsername(this) + val password = preferences.sourcePassword(this) + + if (username.isNullOrEmpty() || password.isNullOrEmpty()) { + return Observable.error(Exception("User not logged")) + } else { + return login(username, password).flatMap { super.fetchChapterList(manga) } + } + + } else { + return super.fetchChapterList(manga) + } + } + +} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Kissmanga.kt b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Kissmanga.kt index fd89f6c018..75e9780443 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Kissmanga.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Kissmanga.kt @@ -6,195 +6,116 @@ import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.network.get import eu.kanade.tachiyomi.data.network.post import eu.kanade.tachiyomi.data.source.EN -import eu.kanade.tachiyomi.data.source.base.Source +import eu.kanade.tachiyomi.data.source.Language +import eu.kanade.tachiyomi.data.source.base.ParsedOnlineSource import eu.kanade.tachiyomi.data.source.model.MangasPage import eu.kanade.tachiyomi.data.source.model.Page -import eu.kanade.tachiyomi.util.Parser import okhttp3.FormBody import okhttp3.OkHttpClient import okhttp3.Request -import org.jsoup.Jsoup +import okhttp3.Response import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import java.text.ParseException import java.text.SimpleDateFormat -import java.util.* import java.util.regex.Pattern -class Kissmanga(context: Context) : Source(context) { +class Kissmanga(context: Context, override val id: Int) : ParsedOnlineSource(context) { - override fun getName() = NAME + override val name = "Kissmanga" - override fun getBaseUrl() = BASE_URL + override val baseUrl = "http://kissmanga.com" - override fun getLang() = EN + override val lang: Language get() = EN - override val networkClient: OkHttpClient - get() = networkService.cloudflareClient + override val client: OkHttpClient get() = network.cloudflareClient - override fun getInitialPopularMangasUrl(): String { - return String.format(POPULAR_MANGAS_URL, 1) - } + override fun popularMangaInitialUrl() = "$baseUrl/MangaList/MostPopular" - override fun getInitialSearchUrl(query: String): String { - return SEARCH_URL + override fun popularMangaSelector() = "table.listing tr:gt(1)" + + override fun popularMangaFromElement(element: Element, manga: Manga) { + element.select("td a:eq(0)").first().let { + manga.setUrl(it.attr("href")) + manga.title = it.text() + } } override fun searchMangaRequest(page: MangasPage, query: String): Request { if (page.page == 1) { - page.url = getInitialSearchUrl(query) + page.url = searchMangaInitialUrl(query) } - val form = FormBody.Builder() - form.add("authorArtist", "") - form.add("mangaName", query) - form.add("status", "") - form.add("genres", "") + val form = FormBody.Builder().apply { + add("authorArtist", "") + add("mangaName", query) + add("status", "") + add("genres", "") + }.build() - return post(page.url, requestHeaders, form.build()) + return post(page.url, headers, form) } - override fun pageListRequest(chapterUrl: String): Request { - return post(baseUrl + chapterUrl, requestHeaders) + override fun popularMangaNextPageSelector() = "li > a:contains(› Next)" + + override fun searchMangaSelector() = popularMangaSelector() + + override fun searchMangaFromElement(element: Element, manga: Manga) { + popularMangaFromElement(element, manga) } - override fun imageRequest(page: Page): Request { - return get(page.imageUrl) + override fun searchMangaNextPageSelector() = null + + override fun searchMangaInitialUrl(query: String) = "$baseUrl/AdvanceSearch" + + override fun mangaDetailsParse(document: Document, manga: Manga) { + val infoElement = document.select("div.barContent").first() + + manga.author = infoElement.select("p:has(span:contains(Author:)) > a").first()?.text() + manga.genre = infoElement.select("p:has(span:contains(Genres:)) > *:gt(0)").text() + manga.description = infoElement.select("p:has(span:contains(Summary:)) ~ p").text() + manga.status = parseStatus(infoElement.select("p:has(span:contains(Status:))").first()?.text()) + manga.thumbnail_url = document.select(".rightBox:eq(0) img").first()?.attr("src") } - override fun parsePopularMangasFromHtml(parsedHtml: Document): List { - val mangaList = ArrayList() - - for (currentHtmlBlock in parsedHtml.select("table.listing tr:gt(1)")) { - val manga = constructPopularMangaFromHtml(currentHtmlBlock) - mangaList.add(manga) - } - - return mangaList - } - - private fun constructPopularMangaFromHtml(htmlBlock: Element): Manga { - val manga = Manga() - manga.source = id - - val urlElement = Parser.element(htmlBlock, "td a:eq(0)") - - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")) - manga.title = urlElement.text() - } - - return manga - } - - override fun parseNextPopularMangasUrl(parsedHtml: Document, page: MangasPage): String? { - val path = Parser.href(parsedHtml, "li > a:contains(› Next)") - return if (path != null) BASE_URL + path else null - } - - override fun parseSearchFromHtml(parsedHtml: Document): List { - return parsePopularMangasFromHtml(parsedHtml) - } - - override fun parseNextSearchUrl(parsedHtml: Document, page: MangasPage, query: String): String? { - return null - } - - override fun parseHtmlToManga(mangaUrl: String, unparsedHtml: String): Manga { - val parsedDocument = Jsoup.parse(unparsedHtml) - val infoElement = parsedDocument.select("div.barContent").first() - - val manga = Manga.create(mangaUrl) - manga.title = Parser.text(infoElement, "a.bigChar") - manga.author = Parser.text(infoElement, "p:has(span:contains(Author:)) > a") - manga.genre = Parser.allText(infoElement, "p:has(span:contains(Genres:)) > *:gt(0)") - manga.description = Parser.allText(infoElement, "p:has(span:contains(Summary:)) ~ p") - manga.status = parseStatus(Parser.text(infoElement, "p:has(span:contains(Status:))")!!) - - val thumbnail = Parser.src(parsedDocument, ".rightBox:eq(0) img") - if (thumbnail != null) { - manga.thumbnail_url = thumbnail - } - - manga.initialized = true - return manga - } - - private fun parseStatus(status: String): Int { - if (status.contains("Ongoing")) { - return Manga.ONGOING - } - if (status.contains("Completed")) { - return Manga.COMPLETED + fun parseStatus(status: String?): Int { + if (status != null) { + when { + status.contains("Ongoing") -> return Manga.ONGOING + status.contains("Completed") -> return Manga.COMPLETED + } } return Manga.UNKNOWN } - override fun parseHtmlToChapters(unparsedHtml: String): List { - val parsedDocument = Jsoup.parse(unparsedHtml) - val chapterList = ArrayList() + override fun chapterListSelector() = "table.listing tr:gt(1)" - for (chapterElement in parsedDocument.select("table.listing tr:gt(1)")) { - val chapter = constructChapterFromHtmlBlock(chapterElement) - chapterList.add(chapter) - } + override fun chapterFromElement(element: Element, chapter: Chapter) { + val urlElement = element.select("a").first() - return chapterList + chapter.setUrl(urlElement.attr("href")) + chapter.name = urlElement.text() + chapter.date_upload = element.select("td:eq(1)").first()?.text()?.let { + SimpleDateFormat("MM/dd/yyyy").parse(it).time + } ?: 0 } - private fun constructChapterFromHtmlBlock(chapterElement: Element): Chapter { - val chapter = Chapter.create() + override fun pageListRequest(chapter: Chapter) = post(baseUrl + chapter.url, headers) - val urlElement = Parser.element(chapterElement, "a") - val date = Parser.text(chapterElement, "td:eq(1)") - - if (urlElement != null) { - chapter.setUrl(urlElement.attr("href")) - chapter.name = urlElement.text() - } - if (date != null) { - try { - chapter.date_upload = SimpleDateFormat("MM/dd/yyyy", Locale.ENGLISH).parse(date).time - } catch (e: ParseException) { /* Ignore */ - } - - } - return chapter - } - - override fun parseHtmlToPageUrls(unparsedHtml: String): List { - val parsedDocument = Jsoup.parse(unparsedHtml) - val pageUrlList = ArrayList() - - val numImages = parsedDocument.select("#divImage img").size - - for (i in 0..numImages - 1) { - pageUrlList.add("") - } - return pageUrlList - } - - override fun parseFirstPage(pages: List, unparsedHtml: String): List { + override fun pageListParse(response: Response, pages: MutableList) { val p = Pattern.compile("lstImages.push\\(\"(.+?)\"") - val m = p.matcher(unparsedHtml) + val m = p.matcher(response.body().string()) var i = 0 while (m.find()) { - pages[i++].imageUrl = m.group(1) + pages.add(Page(i++, "", m.group(1))) } - return pages } - override fun parseHtmlToImageUrl(unparsedHtml: String): String? { - return null - } + // Not used + override fun pageListParse(document: Document, pages: MutableList) {} - companion object { + override fun imageUrlRequest(page: Page) = get(page.url) - val NAME = "Kissmanga" - val BASE_URL = "http://kissmanga.com" - val POPULAR_MANGAS_URL = BASE_URL + "/MangaList/MostPopular?page=%s" - val SEARCH_URL = BASE_URL + "/AdvanceSearch" - } + override fun imageUrlParse(document: Document) = "" -} +} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangafox.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangafox.java deleted file mode 100644 index d92efb985b..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangafox.java +++ /dev/null @@ -1,245 +0,0 @@ -package eu.kanade.tachiyomi.data.source.online.english; - -import android.content.Context; -import android.net.Uri; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Locale; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.LanguageKt; -import eu.kanade.tachiyomi.data.source.base.Source; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import eu.kanade.tachiyomi.util.Parser; - -public class Mangafox extends Source { - - public static final String NAME = "Mangafox"; - public static final String BASE_URL = "http://mangafox.me"; - public static final String POPULAR_MANGAS_URL = BASE_URL + "/directory/%s"; - public static final String SEARCH_URL = - BASE_URL + "/search.php?name_method=cw&advopts=1&order=za&sort=views&name=%s&page=%s"; - - public Mangafox(Context context) { - super(context); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getBaseUrl() { - return BASE_URL; - } - - public Language getLang() { - return LanguageKt.getEN(); - } - - @Override - protected String getInitialPopularMangasUrl() { - return String.format(POPULAR_MANGAS_URL, ""); - } - - @Override - protected String getInitialSearchUrl(String query) { - return String.format(SEARCH_URL, Uri.encode(query), 1); - } - - @Override - protected List parsePopularMangasFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - for (Element currentHtmlBlock : parsedHtml.select("div#mangalist > ul.list > li")) { - Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); - mangaList.add(currentManga); - } - return mangaList; - } - - private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = Parser.element(htmlBlock, "a.title"); - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.text(); - } - return manga; - } - - @Override - protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = Parser.element(parsedHtml, "a:has(span.next)"); - return next != null ? String.format(POPULAR_MANGAS_URL, next.attr("href")) : null; - } - - @Override - protected List parseSearchFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - for (Element currentHtmlBlock : parsedHtml.select("table#listing > tbody > tr:gt(0)")) { - Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); - mangaList.add(currentManga); - } - return mangaList; - } - - private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) { - Manga mangaFromHtmlBlock = new Manga(); - mangaFromHtmlBlock.source = getId(); - - Element urlElement = Parser.element(htmlBlock, "a.series_preview"); - if (urlElement != null) { - mangaFromHtmlBlock.setUrl(urlElement.attr("href")); - mangaFromHtmlBlock.title = urlElement.text(); - } - return mangaFromHtmlBlock; - } - - @Override - protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - Element next = Parser.element(parsedHtml, "a:has(span.next)"); - return next != null ? BASE_URL + next.attr("href") : null; - } - - @Override - protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - Element infoElement = parsedDocument.select("div#title").first(); - Element rowElement = infoElement.select("table > tbody > tr:eq(1)").first(); - Element sideInfoElement = parsedDocument.select("#series_info").first(); - - Manga manga = Manga.create(mangaUrl); - manga.author = Parser.text(rowElement, "td:eq(1)"); - manga.artist = Parser.text(rowElement, "td:eq(2)"); - manga.description = Parser.text(infoElement, "p.summary"); - manga.genre = Parser.text(rowElement, "td:eq(3)"); - manga.thumbnail_url = Parser.src(sideInfoElement, "div.cover > img"); - manga.status = parseStatus(Parser.text(sideInfoElement, ".data")); - - manga.initialized = true; - return manga; - } - - private int parseStatus(String status) { - if (status.contains("Ongoing")) { - return Manga.ONGOING; - } - if (status.contains("Completed")) { - return Manga.COMPLETED; - } - return Manga.UNKNOWN; - } - - @Override - protected List parseHtmlToChapters(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - List chapterList = new ArrayList<>(); - - for (Element chapterElement : parsedDocument.select("div#chapters li div")) { - Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(currentChapter); - } - return chapterList; - } - - private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter chapter = Chapter.create(); - - Element urlElement = chapterElement.select("a.tips").first(); - Element dateElement = chapterElement.select("span.date").first(); - - if (urlElement != null) { - chapter.setUrl(urlElement.attr("href")); - chapter.name = urlElement.text(); - } - if (dateElement != null) { - chapter.date_upload = parseUpdateFromElement(dateElement); - } - return chapter; - } - - private long parseUpdateFromElement(Element updateElement) { - String updatedDateAsString = updateElement.text(); - - if (updatedDateAsString.contains("Today")) { - Calendar today = Calendar.getInstance(); - today.set(Calendar.HOUR_OF_DAY, 0); - today.set(Calendar.MINUTE, 0); - today.set(Calendar.SECOND, 0); - today.set(Calendar.MILLISECOND, 0); - - try { - Date withoutDay = new SimpleDateFormat("h:mm a", Locale.ENGLISH).parse(updatedDateAsString.replace("Today", "")); - return today.getTimeInMillis() + withoutDay.getTime(); - } catch (ParseException e) { - return today.getTimeInMillis(); - } - } else if (updatedDateAsString.contains("Yesterday")) { - Calendar yesterday = Calendar.getInstance(); - yesterday.add(Calendar.DATE, -1); - yesterday.set(Calendar.HOUR_OF_DAY, 0); - yesterday.set(Calendar.MINUTE, 0); - yesterday.set(Calendar.SECOND, 0); - yesterday.set(Calendar.MILLISECOND, 0); - - try { - Date withoutDay = new SimpleDateFormat("h:mm a", Locale.ENGLISH).parse(updatedDateAsString.replace("Yesterday", "")); - return yesterday.getTimeInMillis() + withoutDay.getTime(); - } catch (ParseException e) { - return yesterday.getTimeInMillis(); - } - } else { - try { - Date specificDate = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(updatedDateAsString); - - return specificDate.getTime(); - } catch (ParseException e) { - // Do Nothing. - } - } - - return 0; - } - - @Override - protected List parseHtmlToPageUrls(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - List pageUrlList = new ArrayList<>(); - - Elements pageUrlElements = parsedDocument.select("select.m").first().select("option:not([value=0])"); - String baseUrl = parsedDocument.select("div#series a").first().attr("href").replace("1.html", ""); - for (Element pageUrlElement : pageUrlElements) { - pageUrlList.add(baseUrl + pageUrlElement.attr("value") + ".html"); - } - - return pageUrlList; - } - - @Override - protected String parseHtmlToImageUrl(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - Element imageElement = parsedDocument.getElementById("image"); - return imageElement.attr("src"); - } -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangahere.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangahere.java deleted file mode 100644 index 43aa3ee4ce..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/Mangahere.java +++ /dev/null @@ -1,313 +0,0 @@ -package eu.kanade.tachiyomi.data.source.online.english; - -import android.content.Context; -import android.net.Uri; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Locale; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.LanguageKt; -import eu.kanade.tachiyomi.data.source.base.Source; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import eu.kanade.tachiyomi.util.Parser; - -public class Mangahere extends Source { - - public static final String NAME = "Mangahere"; - public static final String BASE_URL = "http://www.mangahere.co"; - public static final String POPULAR_MANGAS_URL = BASE_URL + "/directory/%s"; - public static final String SEARCH_URL = BASE_URL + "/search.php?name=%s&page=%s&sort=views&order=za"; - - public Mangahere(Context context) { - super(context); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getBaseUrl() { - return BASE_URL; - } - - public Language getLang() { - return LanguageKt.getEN(); - } - - @Override - protected String getInitialPopularMangasUrl() { - return String.format(POPULAR_MANGAS_URL, ""); - } - - @Override - protected String getInitialSearchUrl(String query) { - return String.format(SEARCH_URL, Uri.encode(query), 1); - } - - @Override - public List parsePopularMangasFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - for (Element currentHtmlBlock : parsedHtml.select("div.directory_list > ul > li")) { - Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); - mangaList.add(currentManga); - } - return mangaList; - } - - private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = Parser.element(htmlBlock, "div.title > a"); - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.attr("title"); - } - return manga; - } - - @Override - protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = Parser.element(parsedHtml, "div.next-page > a.next"); - return next != null ? String.format(POPULAR_MANGAS_URL, next.attr("href")) : null; - } - - @Override - protected List parseSearchFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - Elements mangaHtmlBlocks = parsedHtml.select("div.result_search > dl"); - for (Element currentHtmlBlock : mangaHtmlBlocks) { - Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock); - mangaList.add(currentManga); - } - return mangaList; - } - - private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = Parser.element(htmlBlock, "a.manga_info"); - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.text(); - } - return manga; - } - - @Override - protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - Element next = Parser.element(parsedHtml, "div.next-page > a.next"); - return next != null ? BASE_URL + next.attr("href") : null; - } - - private long parseUpdateFromElement(Element updateElement) { - String updatedDateAsString = updateElement.text(); - - if (updatedDateAsString.contains("Today")) { - Calendar today = Calendar.getInstance(); - today.set(Calendar.HOUR_OF_DAY, 0); - today.set(Calendar.MINUTE, 0); - today.set(Calendar.SECOND, 0); - today.set(Calendar.MILLISECOND, 0); - - try { - Date withoutDay = new SimpleDateFormat("MMM d, yyyy h:mma", Locale.ENGLISH).parse(updatedDateAsString.replace("Today", "")); - return today.getTimeInMillis() + withoutDay.getTime(); - } catch (ParseException e) { - return today.getTimeInMillis(); - } - } else if (updatedDateAsString.contains("Yesterday")) { - Calendar yesterday = Calendar.getInstance(); - yesterday.add(Calendar.DATE, -1); - yesterday.set(Calendar.HOUR_OF_DAY, 0); - yesterday.set(Calendar.MINUTE, 0); - yesterday.set(Calendar.SECOND, 0); - yesterday.set(Calendar.MILLISECOND, 0); - - try { - Date withoutDay = new SimpleDateFormat("MMM d, yyyy h:mma", Locale.ENGLISH).parse(updatedDateAsString.replace("Yesterday", "")); - return yesterday.getTimeInMillis() + withoutDay.getTime(); - } catch (ParseException e) { - return yesterday.getTimeInMillis(); - } - } else { - try { - Date specificDate = new SimpleDateFormat("MMM d, yyyy h:mma", Locale.ENGLISH).parse(updatedDateAsString); - - return specificDate.getTime(); - } catch (ParseException e) { - // Do Nothing. - } - } - - return 0; - } - - public Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("
    "); - int endIndex = unparsedHtml.indexOf("
", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - Element detailElement = parsedDocument.select("ul.detail_topText").first(); - - Manga manga = Manga.create(mangaUrl); - manga.author = Parser.text(parsedDocument, "a[href^=http://www.mangahere.co/author/]"); - manga.artist = Parser.text(parsedDocument, "a[href^=http://www.mangahere.co/artist/]"); - - String description = Parser.text(detailElement, "#show"); - if (description != null) { - manga.description = description.substring(0, description.length() - "Show less".length()); - } - String genres = Parser.text(detailElement, "li:eq(3)"); - if (genres != null) { - manga.genre = genres.substring("Genre(s):".length()); - } - manga.status = parseStatus(Parser.text(detailElement, "li:eq(6)")); - - beginIndex = unparsedHtml.indexOf("", beginIndex); - trimmedHtml = unparsedHtml.substring(beginIndex, endIndex + 2); - - parsedDocument = Jsoup.parse(trimmedHtml); - manga.thumbnail_url = Parser.src(parsedDocument, "img"); - - manga.initialized = true; - return manga; - } - - private int parseStatus(String status) { - if (status.contains("Ongoing")) { - return Manga.ONGOING; - } - if (status.contains("Completed")) { - return Manga.COMPLETED; - } - return Manga.UNKNOWN; - } - - @Override - public List parseHtmlToChapters(String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("
    "); - int endIndex = unparsedHtml.indexOf("
", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - - List chapterList = new ArrayList<>(); - - for (Element chapterElement : parsedDocument.getElementsByTag("li")) { - Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(currentChapter); - } - return chapterList; - } - - private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter chapter = Chapter.create(); - - Element urlElement = chapterElement.select("a").first(); - Element dateElement = chapterElement.select("span.right").first(); - - if (urlElement != null) { - chapter.setUrl(urlElement.attr("href")); - chapter.name = urlElement.text(); - } - if (dateElement != null) { - chapter.date_upload = parseDateFromElement(dateElement); - } - return chapter; - } - - private long parseDateFromElement(Element dateElement) { - String dateAsString = dateElement.text(); - - if (dateAsString.contains("Today")) { - Calendar today = Calendar.getInstance(); - today.set(Calendar.HOUR_OF_DAY, 0); - today.set(Calendar.MINUTE, 0); - today.set(Calendar.SECOND, 0); - today.set(Calendar.MILLISECOND, 0); - - try { - Date withoutDay = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(dateAsString.replace("Today", "")); - return today.getTimeInMillis() + withoutDay.getTime(); - } catch (ParseException e) { - return today.getTimeInMillis(); - } - } else if (dateAsString.contains("Yesterday")) { - Calendar yesterday = Calendar.getInstance(); - yesterday.add(Calendar.DATE, -1); - yesterday.set(Calendar.HOUR_OF_DAY, 0); - yesterday.set(Calendar.MINUTE, 0); - yesterday.set(Calendar.SECOND, 0); - yesterday.set(Calendar.MILLISECOND, 0); - - try { - Date withoutDay = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(dateAsString.replace("Yesterday", "")); - return yesterday.getTimeInMillis() + withoutDay.getTime(); - } catch (ParseException e) { - return yesterday.getTimeInMillis(); - } - } else { - try { - Date date = new SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parse(dateAsString); - - return date.getTime(); - } catch (ParseException e) { - // Do Nothing. - } - } - return 0; - } - - @Override - public List parseHtmlToPageUrls(String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("
"); - int endIndex = unparsedHtml.indexOf("
", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - - List pageUrlList = new ArrayList<>(); - - Elements pageUrlElements = parsedDocument.select("select.wid60").first().getElementsByTag("option"); - for (Element pageUrlElement : pageUrlElements) { - pageUrlList.add(pageUrlElement.attr("value")); - } - - return pageUrlList; - } - - @Override - public String parseHtmlToImageUrl(String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("
"); - int endIndex = unparsedHtml.indexOf("
", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - - Element imageElement = parsedDocument.getElementById("image"); - - return imageElement.attr("src"); - } - -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/ReadMangaToday.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/ReadMangaToday.java deleted file mode 100644 index afac46c721..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/english/ReadMangaToday.java +++ /dev/null @@ -1,290 +0,0 @@ -package eu.kanade.tachiyomi.data.source.online.english; - -import android.content.Context; -import android.net.Uri; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.LanguageKt; -import eu.kanade.tachiyomi.data.source.base.Source; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import eu.kanade.tachiyomi.util.Parser; -import okhttp3.Headers; -import rx.Observable; -import rx.functions.Action1; -import rx.functions.Func1; - -public class ReadMangaToday extends Source { - public static final String NAME = "ReadMangaToday"; - public static final String BASE_URL = "http://www.readmanga.today"; - public static final String POPULAR_MANGAS_URL = BASE_URL + "/hot-manga/%s"; - public static final String SEARCH_URL = BASE_URL + "/service/search?q=%s"; - - private static JsonParser parser = new JsonParser(); - private static Gson gson = new Gson(); - - public ReadMangaToday(Context context) { - super(context); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getBaseUrl() { - return BASE_URL; - } - - @Override - protected String getInitialPopularMangasUrl() { - return String.format(POPULAR_MANGAS_URL, ""); - } - - @Override - protected String getInitialSearchUrl(String query) { - return String.format(SEARCH_URL, Uri.encode(query), 1); - } - - @Override - public Language getLang() { - return LanguageKt.getEN(); - } - - @Override - public List parsePopularMangasFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - for (Element currentHtmlBlock : parsedHtml.select("div.hot-manga > div.style-list > div.box")) { - Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock); - mangaList.add(currentManga); - } - return mangaList; - } - - private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = Parser.element(htmlBlock, "div.title > h2 > a"); - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.attr("title"); - } - return manga; - } - - @Override - protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - Element next = Parser.element(parsedHtml, "div.hot-manga > ul.pagination > li > a:contains(»)"); - return next != null ? next.attr("href") : null; - } - - @Override - public Observable searchMangasFromNetwork(final MangasPage page, String query) { - return networkService - .requestBody(searchMangaRequest(page, query), networkService.getDefaultClient()) - .doOnNext(new Action1() { - @Override - public void call(String doc) { - page.mangas = ReadMangaToday.this.parseSearchFromJson(doc); - } - }) - .map(new Func1() { - @Override - public MangasPage call(String response) { - return page; - } - }); - } - - @Override - protected Headers.Builder headersBuilder() { - return super.headersBuilder().add("X-Requested-With", "XMLHttpRequest"); - } - - protected List parseSearchFromJson(String unparsedJson) { - List mangaList = new ArrayList<>(); - - JsonArray mangasArray = parser.parse(unparsedJson).getAsJsonArray(); - - for (JsonElement mangaElement : mangasArray) { - Manga currentManga = constructSearchMangaFromJsonObject(mangaElement.getAsJsonObject()); - mangaList.add(currentManga); - } - return mangaList; - } - - private Manga constructSearchMangaFromJsonObject(JsonObject jsonObject) { - Manga manga = new Manga(); - manga.source = getId(); - - manga.setUrl(gson.fromJson(jsonObject.get("url"), String.class)); - manga.title = gson.fromJson(jsonObject.get("title"), String.class); - - return manga; - } - - @Override - protected List parseSearchFromHtml(Document parsedHtml) { - return null; - } - - @Override - protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - return null; - } - - public Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf(""); - int endIndex = unparsedHtml.indexOf("", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - Element detailElement = parsedDocument.select("div.movie-meta").first(); - - Manga manga = Manga.create(mangaUrl); - for (Element castHtmlBlock : parsedDocument.select("div.cast ul.cast-list > li")) { - String name = Parser.text(castHtmlBlock, "ul > li > a"); - String role = Parser.text(castHtmlBlock, "ul > li:eq(1)"); - if (role.equals("Author")) { - manga.author = name; - } else if (role.equals("Artist")) { - manga.artist = name; - } - } - - String description = Parser.text(detailElement, "li.movie-detail"); - if (description != null) { - manga.description = description; - } - String genres = Parser.text(detailElement, "dl.dl-horizontal > dd:eq(5)"); - if (genres != null) { - manga.genre = genres; - } - manga.status = parseStatus(Parser.text(detailElement, "dl.dl-horizontal > dd:eq(3)")); - manga.thumbnail_url = Parser.src(detailElement, "img.img-responsive"); - - manga.initialized = true; - return manga; - } - - private int parseStatus(String status) { - if (status.contains("Ongoing")) { - return Manga.ONGOING; - } else if (status.contains("Completed")) { - return Manga.COMPLETED; - } - return Manga.UNKNOWN; - } - - @Override - public List parseHtmlToChapters(String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf(""); - int endIndex = unparsedHtml.indexOf("", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - - List chapterList = new ArrayList<>(); - - for (Element chapterElement : parsedDocument.select("ul.chp_lst > li")) { - Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(currentChapter); - } - return chapterList; - } - - private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter chapter = Chapter.create(); - - Element urlElement = chapterElement.select("a").first(); - Element dateElement = chapterElement.select("span.dte").first(); - - if (urlElement != null) { - chapter.setUrl(urlElement.attr("href")); - chapter.name = urlElement.select("span.val").text(); - } - if (dateElement != null) { - chapter.date_upload = parseDateFromElement(dateElement); - } - return chapter; - } - - private long parseDateFromElement(Element dateElement) { - String dateAsString = dateElement.text(); - String[] dateWords = dateAsString.split(" "); - - if (dateWords.length == 3) { - int timeAgo = Integer.parseInt(dateWords[0]); - Calendar date = Calendar.getInstance(); - - if (dateWords[1].contains("Minute")) { - date.add(Calendar.MINUTE, - timeAgo); - } else if (dateWords[1].contains("Hour")) { - date.add(Calendar.HOUR_OF_DAY, - timeAgo); - } else if (dateWords[1].contains("Day")) { - date.add(Calendar.DAY_OF_YEAR, -timeAgo); - } else if (dateWords[1].contains("Week")) { - date.add(Calendar.WEEK_OF_YEAR, -timeAgo); - } else if (dateWords[1].contains("Month")) { - date.add(Calendar.MONTH, -timeAgo); - } else if (dateWords[1].contains("Year")) { - date.add(Calendar.YEAR, -timeAgo); - } - - return date.getTimeInMillis(); - } - - return 0; - } - - @Override - public List parseHtmlToPageUrls(String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf(""); - int endIndex = unparsedHtml.indexOf("", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - - List pageUrlList = new ArrayList<>(); - - Elements pageUrlElements = parsedDocument.select("ul.list-switcher-2 > li > select.jump-menu").first().getElementsByTag("option"); - for (Element pageUrlElement : pageUrlElements) { - pageUrlList.add(pageUrlElement.attr("value")); - } - - return pageUrlList; - } - - @Override - public String parseHtmlToImageUrl(String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf(""); - int endIndex = unparsedHtml.indexOf("", beginIndex); - String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); - - Document parsedDocument = Jsoup.parse(trimmedHtml); - - Element imageElement = Parser.element(parsedDocument, "img.img-responsive-2"); - - return imageElement.attr("src"); - } - -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mangachan.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mangachan.java deleted file mode 100644 index 4fa8c8c96c..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mangachan.java +++ /dev/null @@ -1,240 +0,0 @@ -package eu.kanade.tachiyomi.data.source.online.russian; - -import android.content.Context; -import android.net.Uri; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.LanguageKt; -import eu.kanade.tachiyomi.data.source.base.Source; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import eu.kanade.tachiyomi.data.source.model.Page; -import eu.kanade.tachiyomi.util.Parser; - -public class Mangachan extends Source { - - public static final String NAME = "Mangachan"; - public static final String BASE_URL = "http://mangachan.ru"; - public static final String POPULAR_MANGAS_URL = BASE_URL + "/mostfavorites"; - public static final String SEARCH_URL = BASE_URL + "/?do=search&subaction=search&story=%s"; - - public Mangachan(Context context) { - super(context); - } - - @Override - public Language getLang() { - return LanguageKt.getRU(); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getBaseUrl() { - return BASE_URL; - } - - @Override - protected String getInitialPopularMangasUrl() { - return POPULAR_MANGAS_URL; - } - - @Override - protected String getInitialSearchUrl(String query) { - return String.format(SEARCH_URL, Uri.encode(query)); - } - - @Override - protected List parsePopularMangasFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - for (Element currentHtmlBlock : parsedHtml.select("div.content_row")) { - Manga manga = constructPopularMangaFromHtml(currentHtmlBlock); - mangaList.add(manga); - } - - return mangaList; - } - - private Manga constructPopularMangaFromHtml(Element currentHtmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = currentHtmlBlock.getElementsByTag("h2").select("a").first(); - Element imgElement = currentHtmlBlock.getElementsByClass("manga_images").select("img").first(); - - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.text(); - } - - if (imgElement != null) { - manga.thumbnail_url = BASE_URL + imgElement.attr("src"); - } - - return manga; - } - - @Override - protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - String path = Parser.href(parsedHtml, "a:contains(Вперед)"); - return path != null ? POPULAR_MANGAS_URL + path : null; - } - - @Override - protected List parseSearchFromHtml(Document parsedHtml) { - return parsePopularMangasFromHtml(parsedHtml); - } - - @Override - protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - return null; - } - - @Override - protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - Element infoElement = parsedDocument.getElementsByClass("mangatitle").first(); - String description = parsedDocument.getElementById("description").text(); - - Manga manga = Manga.create(mangaUrl); - - manga.author = infoElement.select("tr:eq(2) td:eq(1)").text(); - manga.genre = infoElement.select("tr:eq(5) td:eq(1)").text(); - manga.status = parseStatus(infoElement.select("tr:eq(4) td:eq(1)").text()); - - manga.description = description.replaceAll("Прислать описание", ""); - - manga.initialized = true; - return manga; - } - - private int parseStatus(String status) { - if (status.contains("перевод продолжается")) { - return Manga.ONGOING; - } else if (status.contains("перевод завершен")) { - return Manga.COMPLETED; - } else return Manga.UNKNOWN; - } - - @Override - protected List parseHtmlToChapters(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - List chapterList = new ArrayList<>(); - - for (Element chapterElement : parsedDocument.select("table.table_cha tr:gt(1)")) { - Chapter chapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(chapter); - } - return chapterList; - } - - private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter chapter = Chapter.create(); - - Element urlElement = chapterElement.select("a").first(); - String date = Parser.text(chapterElement, "div.date"); - - if (urlElement != null) { - chapter.name = urlElement.text(); - chapter.url = urlElement.attr("href"); - } - - if (date != null) { - try { - chapter.date_upload = new SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH).parse(date).getTime(); - } catch (ParseException e) { /* Ignore */ } - } - return chapter; - } - - // Without this extra chapters are in the wrong place in the list - @Override - public void parseChapterNumber(Chapter chapter) { - // For chapters with url like /online/254903-fairy-tail_v56_ch474.html - String url = chapter.url.replace(".html", ""); - Pattern pattern = Pattern.compile("\\d+_ch[\\d.]+"); - Matcher matcher = pattern.matcher(url); - - if (matcher.find()) { - String[] parts = matcher.group().split("_ch"); - chapter.chapter_number = Float.parseFloat(parts[0] + "." + AddZero(parts[1])); - } else { // For chapters with url like /online/61216-3298.html - String name = chapter.name; - name = name.replaceAll("[\\s\\d\\w\\W]+v", ""); - String volume = name.substring(0, name.indexOf(" - ")); - String[] parts = name.replaceFirst("\\d+ - ", "").split(" "); - - chapter.chapter_number = Float.parseFloat(volume + "." + AddZero(parts[0])); - } - } - - private String AddZero(String num) { - if (Float.parseFloat(num) < 1000f) { - num = "0" + num.replace(".", ""); - } - if (Float.parseFloat(num) < 100f) { - num = "0" + num.replace(".", ""); - } - if (Float.parseFloat(num) < 10f) { - num = "0" + num.replace(".", ""); - } - return num; - } - - @Override - protected List parseHtmlToPageUrls(String unparsedHtml) { - ArrayList pages = new ArrayList<>(); - - int beginIndex = unparsedHtml.indexOf("fullimg\":["); - int endIndex = unparsedHtml.indexOf(']', beginIndex); - - String trimmedHtml = unparsedHtml.substring(beginIndex + 10, endIndex); - trimmedHtml = trimmedHtml.replaceAll("\"", ""); - - String[] pageUrls = trimmedHtml.split(","); - for (int i = 0; i < pageUrls.length; i++) { - pages.add(""); - } - - return pages; - } - - @Override - protected List parseFirstPage(List pages, String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("fullimg\":["); - int endIndex = unparsedHtml.indexOf(']', beginIndex); - - String trimmedHtml = unparsedHtml.substring(beginIndex + 10, endIndex); - trimmedHtml = trimmedHtml.replaceAll("\"", ""); - - String[] pageUrls = trimmedHtml.split(","); - for (int i = 0; i < pageUrls.length; i++) { - pages.get(i).setImageUrl(pageUrls[i].replaceAll("im.?\\.", "")); - } - - return (List) pages; - } - - @Override - protected String parseHtmlToImageUrl(String unparsedHtml) { - return null; - } -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mintmanga.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mintmanga.java deleted file mode 100644 index 2a9b4ef5d4..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Mintmanga.java +++ /dev/null @@ -1,225 +0,0 @@ -package eu.kanade.tachiyomi.data.source.online.russian; - -import android.content.Context; -import android.net.Uri; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.LanguageKt; -import eu.kanade.tachiyomi.data.source.base.Source; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import eu.kanade.tachiyomi.data.source.model.Page; -import eu.kanade.tachiyomi.util.Parser; - -public class Mintmanga extends Source { - - public static final String NAME = "Mintmanga"; - public static final String BASE_URL = "http://mintmanga.com"; - public static final String POPULAR_MANGAS_URL = BASE_URL + "/list?sortType=rate"; - public static final String SEARCH_URL = BASE_URL + "/search?q=%s"; - - public Mintmanga(Context context) { - super(context); - } - - @Override - public Language getLang() { - return LanguageKt.getRU(); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getBaseUrl() { - return BASE_URL; - } - - @Override - protected String getInitialPopularMangasUrl() { - return POPULAR_MANGAS_URL; - } - - @Override - protected String getInitialSearchUrl(String query) { - return String.format(SEARCH_URL, Uri.encode(query)); - } - - @Override - protected List parsePopularMangasFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - for (Element currentHtmlBlock : parsedHtml.select("div.desc")) { - Manga manga = constructPopularMangaFromHtml(currentHtmlBlock); - mangaList.add(manga); - } - - return mangaList; - } - - private Manga constructPopularMangaFromHtml(Element currentHtmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = currentHtmlBlock.getElementsByTag("h3").select("a").first(); - - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.text(); - } - - return manga; - } - - @Override - protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - String path = Parser.href(parsedHtml, "a:contains(→)"); - return path != null ? BASE_URL + path : null; - } - - @Override - protected List parseSearchFromHtml(Document parsedHtml) { - return parsePopularMangasFromHtml(parsedHtml); - } - - @Override - protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - return null; - } - - @Override - protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - Element infoElement = parsedDocument.select("div.leftContent").first(); - - Manga manga = Manga.create(mangaUrl); - manga.title = Parser.text(infoElement, "span.eng-name"); - manga.author = Parser.text(infoElement, "span.elem_author "); - manga.genre = Parser.allText(infoElement, "span.elem_genre ").replaceAll(" ,", ","); - manga.description = Parser.allText(infoElement, "div.manga-description"); - if (Parser.text(infoElement, "h1.names:contains(Сингл)") != null) { - manga.status = Manga.COMPLETED; - } else { - manga.status = parseStatus(Parser.text(infoElement, "p:has(b:contains(Перевод:))")); - } - - String thumbnail = Parser.element(infoElement, "img").attr("data-full"); - if (thumbnail != null) { - manga.thumbnail_url = thumbnail; - } - - manga.initialized = true; - return manga; - } - - private int parseStatus(String status) { - if (status.contains("продолжается")) { - return Manga.ONGOING; - } - if (status.contains("завершен")) { - return Manga.COMPLETED; - } - return Manga.UNKNOWN; - } - - @Override - protected List parseHtmlToChapters(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - List chapterList = new ArrayList<>(); - - for (Element chapterElement : parsedDocument.select("div.chapters-link tbody tr")) { - Chapter chapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(chapter); - } - - return chapterList; - } - - private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter chapter = Chapter.create(); - - Element urlElement = Parser.element(chapterElement, "a"); - String date = Parser.text(chapterElement, "td:eq(1)"); - - if (urlElement != null) { - chapter.setUrl(urlElement.attr("href") + "?mature=1"); - chapter.name = urlElement.text().replaceAll(" новое", ""); - } - - if (date != null) { - try { - chapter.date_upload = new SimpleDateFormat("dd/MM/yy", Locale.ENGLISH).parse(date).getTime(); - } catch (ParseException e) { /* Ignore */ } - } - return chapter; - } - - // Without this extra chapters are in the wrong place in the list - @Override - public void parseChapterNumber(Chapter chapter) { - String url = chapter.url.replace("?mature=1", ""); - - String[] urlParts = url.replaceAll("/[\\w\\d]+/vol", "").split("/"); - if (Float.parseFloat(urlParts[1]) < 1000f) { - urlParts[1] = "0" + urlParts[1]; - } - if (Float.parseFloat(urlParts[1]) < 100f) { - urlParts[1] = "0" + urlParts[1]; - } - if (Float.parseFloat(urlParts[1]) < 10f) { - urlParts[1] = "0" + urlParts[1]; - } - - chapter.chapter_number = Float.parseFloat(urlParts[0] + "." + urlParts[1]); - } - - @Override - protected List parseHtmlToPageUrls(String unparsedHtml) { - ArrayList pages = new ArrayList<>(); - - int beginIndex = unparsedHtml.indexOf("rm_h.init( ["); - int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex); - - String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex); - trimmedHtml = trimmedHtml.replaceAll("[\"']", ""); - String[] pageUrls = trimmedHtml.split("],\\["); - for (int i = 0; i < pageUrls.length; i++) { - pages.add(""); - } - return pages; - } - - @Override - protected List parseFirstPage(List pages, String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("rm_h.init( ["); - int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex); - - String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex); - trimmedHtml = trimmedHtml.replaceAll("[\"']", ""); - String[] pageUrls = trimmedHtml.split("],\\["); - for (int i = 0; i < pageUrls.length; i++) { - String[] urlParts = pageUrls[i].split(","); // auto/06/35,http://e4.adultmanga.me/,/55/01.png - String page = urlParts[1] + urlParts[0] + urlParts[2]; - pages.get(i).setImageUrl(page); - } - return (List) pages; - } - - @Override - protected String parseHtmlToImageUrl(String unparsedHtml) { - return null; - } -} \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Readmanga.java b/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Readmanga.java deleted file mode 100644 index 408eca9cc9..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/data/source/online/russian/Readmanga.java +++ /dev/null @@ -1,225 +0,0 @@ -package eu.kanade.tachiyomi.data.source.online.russian; - -import android.content.Context; -import android.net.Uri; - -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import eu.kanade.tachiyomi.data.database.models.Chapter; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.Language; -import eu.kanade.tachiyomi.data.source.LanguageKt; -import eu.kanade.tachiyomi.data.source.base.Source; -import eu.kanade.tachiyomi.data.source.model.MangasPage; -import eu.kanade.tachiyomi.data.source.model.Page; -import eu.kanade.tachiyomi.util.Parser; - -public class Readmanga extends Source { - - public static final String NAME = "Readmanga"; - public static final String BASE_URL = "http://readmanga.me"; - public static final String POPULAR_MANGAS_URL = BASE_URL + "/list?sortType=rate"; - public static final String SEARCH_URL = BASE_URL + "/search?q=%s"; - - public Readmanga(Context context) { - super(context); - } - - @Override - public Language getLang() { - return LanguageKt.getRU(); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public String getBaseUrl() { - return BASE_URL; - } - - @Override - protected String getInitialPopularMangasUrl() { - return POPULAR_MANGAS_URL; - } - - @Override - protected String getInitialSearchUrl(String query) { - return String.format(SEARCH_URL, Uri.encode(query)); - } - - @Override - protected List parsePopularMangasFromHtml(Document parsedHtml) { - List mangaList = new ArrayList<>(); - - for (Element currentHtmlBlock : parsedHtml.select("div.desc")) { - Manga manga = constructPopularMangaFromHtml(currentHtmlBlock); - mangaList.add(manga); - } - - return mangaList; - } - - private Manga constructPopularMangaFromHtml(Element currentHtmlBlock) { - Manga manga = new Manga(); - manga.source = getId(); - - Element urlElement = currentHtmlBlock.getElementsByTag("h3").select("a").first(); - - if (urlElement != null) { - manga.setUrl(urlElement.attr("href")); - manga.title = urlElement.text(); - } - - return manga; - } - - @Override - protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) { - String path = Parser.href(parsedHtml, "a:contains(→)"); - return path != null ? BASE_URL + path : null; - } - - @Override - protected List parseSearchFromHtml(Document parsedHtml) { - return parsePopularMangasFromHtml(parsedHtml); - } - - @Override - protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) { - return null; - } - - @Override - protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - Element infoElement = parsedDocument.select("div.leftContent").first(); - - Manga manga = Manga.create(mangaUrl); - manga.title = Parser.text(infoElement, "span.eng-name"); - manga.author = Parser.text(infoElement, "span.elem_author "); - manga.genre = Parser.allText(infoElement, "span.elem_genre ").replaceAll(" ,", ","); - manga.description = Parser.allText(infoElement, "div.manga-description"); - if (Parser.text(infoElement, "h1.names:contains(Сингл)") != null) { - manga.status = Manga.COMPLETED; - } else { - manga.status = parseStatus(Parser.text(infoElement, "p:has(b:contains(Перевод:))")); - } - - String thumbnail = Parser.element(infoElement, "img").attr("data-full"); - if (thumbnail != null) { - manga.thumbnail_url = thumbnail; - } - - manga.initialized = true; - return manga; - } - - private int parseStatus(String status) { - if (status.contains("продолжается")) { - return Manga.ONGOING; - } - if (status.contains("завершен")) { - return Manga.COMPLETED; - } - return Manga.UNKNOWN; - } - - @Override - protected List parseHtmlToChapters(String unparsedHtml) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - List chapterList = new ArrayList<>(); - - for (Element chapterElement : parsedDocument.select("div.chapters-link tbody tr")) { - Chapter chapter = constructChapterFromHtmlBlock(chapterElement); - chapterList.add(chapter); - } - - return chapterList; - } - - private Chapter constructChapterFromHtmlBlock(Element chapterElement) { - Chapter chapter = Chapter.create(); - - Element urlElement = Parser.element(chapterElement, "a"); - String date = Parser.text(chapterElement, "td:eq(1)"); - - if (urlElement != null) { - chapter.setUrl(urlElement.attr("href") + "?mature=1"); - chapter.name = urlElement.text().replaceAll(" новое", ""); - } - - if (date != null) { - try { - chapter.date_upload = new SimpleDateFormat("dd/MM/yy", Locale.ENGLISH).parse(date).getTime(); - } catch (ParseException e) { /* Ignore */ } - } - return chapter; - } - - // Without this extra chapters are in the wrong place in the list - @Override - public void parseChapterNumber(Chapter chapter) { - String url = chapter.url.replace("?mature=1", ""); - - String[] urlParts = url.replaceAll("/[\\w\\d]+/vol", "").split("/"); - if (Float.parseFloat(urlParts[1]) < 1000f) { - urlParts[1] = "0" + urlParts[1]; - } - if (Float.parseFloat(urlParts[1]) < 100f) { - urlParts[1] = "0" + urlParts[1]; - } - if (Float.parseFloat(urlParts[1]) < 10f) { - urlParts[1] = "0" + urlParts[1]; - } - - chapter.chapter_number = Float.parseFloat(urlParts[0] + "." + urlParts[1]); - } - - @Override - protected List parseHtmlToPageUrls(String unparsedHtml) { - ArrayList pages = new ArrayList<>(); - - int beginIndex = unparsedHtml.indexOf("rm_h.init( ["); - int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex); - - String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex); - trimmedHtml = trimmedHtml.replaceAll("[\"']", ""); - String[] pageUrls = trimmedHtml.split("],\\["); - for (int i = 0; i < pageUrls.length; i++) { - pages.add(""); - } - return pages; - } - - @Override - protected List parseFirstPage(List pages, String unparsedHtml) { - int beginIndex = unparsedHtml.indexOf("rm_h.init( ["); - int endIndex = unparsedHtml.indexOf("], 0, false);", beginIndex); - - String trimmedHtml = unparsedHtml.substring(beginIndex + 13, endIndex); - trimmedHtml = trimmedHtml.replaceAll("[\"']", ""); - String[] pageUrls = trimmedHtml.split("],\\["); - for (int i = 0; i < pageUrls.length; i++) { - String[] urlParts = pageUrls[i].split(","); // auto/12/56,http://e7.postfact.ru/,/51/01.jpg_res.jpg - String page = urlParts[1] + urlParts[0] + urlParts[2]; - pages.get(i).setImageUrl(page); - } - return (List) pages; - } - - @Override - protected String parseHtmlToImageUrl(String unparsedHtml) { - return null; - } -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/injection/component/AppComponent.kt b/app/src/main/java/eu/kanade/tachiyomi/injection/component/AppComponent.kt index 889e0bc118..8c9e8f91bf 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/injection/component/AppComponent.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/injection/component/AppComponent.kt @@ -9,6 +9,7 @@ import eu.kanade.tachiyomi.data.library.LibraryUpdateService import eu.kanade.tachiyomi.data.mangasync.UpdateMangaSyncService import eu.kanade.tachiyomi.data.mangasync.base.MangaSyncService import eu.kanade.tachiyomi.data.source.base.Source +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.updater.UpdateDownloader import eu.kanade.tachiyomi.injection.module.AppModule import eu.kanade.tachiyomi.injection.module.DataModule @@ -49,6 +50,8 @@ interface AppComponent { fun inject(source: Source) fun inject(mangaSyncService: MangaSyncService) + fun inject(onlineSource: OnlineSource) + fun inject(libraryUpdateService: LibraryUpdateService) fun inject(downloadService: DownloadService) fun inject(updateMangaSyncService: UpdateMangaSyncService) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CataloguePresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CataloguePresenter.kt index 601ce8d92d..cc9f43098b 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CataloguePresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/CataloguePresenter.kt @@ -8,6 +8,7 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.source.EN import eu.kanade.tachiyomi.data.source.SourceManager +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.source.base.Source import eu.kanade.tachiyomi.data.source.model.MangasPage import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter @@ -52,7 +53,7 @@ class CataloguePresenter : BasePresenter() { /** * Active source. */ - lateinit var source: Source + lateinit var source: OnlineSource private set /** @@ -163,7 +164,7 @@ class CataloguePresenter : BasePresenter() { * * @param source the new active source. */ - fun setActiveSource(source: Source) { + fun setActiveSource(source: OnlineSource) { prefs.lastUsedCatalogueSource().set(source.id) this.source = source restartPager() @@ -222,9 +223,9 @@ class CataloguePresenter : BasePresenter() { } val observable = if (query.isEmpty()) - source.pullPopularMangasFromNetwork(nextMangasPage) + source.fetchPopularManga(nextMangasPage) else - source.searchMangasFromNetwork(nextMangasPage, query) + source.fetchSearchManga(nextMangasPage, query) return observable.subscribeOn(Schedulers.io()) .doOnNext { lastMangasPage = it } @@ -268,7 +269,7 @@ class CataloguePresenter : BasePresenter() { * @return an observable of the manga to initialize */ private fun getMangaDetailsObservable(manga: Manga): Observable { - return source.pullMangaFromNetwork(manga.url) + return source.fetchMangaDetails(manga) .flatMap { networkManga -> manga.copyFrom(networkManga) db.insertManga(manga).executeAsBlocking() @@ -282,13 +283,13 @@ class CataloguePresenter : BasePresenter() { * * @return a source. */ - fun getLastUsedSource(): Source { + fun getLastUsedSource(): OnlineSource { val id = prefs.lastUsedCatalogueSource().get() ?: -1 val source = sourceManager.get(id) if (!isValidSource(source)) { return findFirstValidSource() } - return source!! + return source as OnlineSource } /** @@ -298,10 +299,10 @@ class CataloguePresenter : BasePresenter() { * @return true if the source is valid, false otherwise. */ fun isValidSource(source: Source?): Boolean { - if (source == null) return false + if (source == null || source !is OnlineSource) return false return with(source) { - if (!isLoginRequired || isLogged) { + if (!isLoginRequired() || isLogged()) { true } else { prefs.sourceUsername(this) != "" && prefs.sourcePassword(this) != "" @@ -314,14 +315,14 @@ class CataloguePresenter : BasePresenter() { * * @return the index of the first valid source. */ - fun findFirstValidSource(): Source { - return sources.find { isValidSource(it) }!! + fun findFirstValidSource(): OnlineSource { + return sources.first { isValidSource(it) } } /** * Returns a list of enabled sources ordered by language and name. */ - private fun getEnabledSources(): List { + private fun getEnabledSources(): List { val languages = prefs.enabledLanguages().getOrDefault() // Ensure at least one language @@ -329,7 +330,7 @@ class CataloguePresenter : BasePresenter() { languages.add(EN.code) } - return sourceManager.getSources() + return sourceManager.getOnlineSources() .filter { it.lang.code in languages } .sortedBy { "(${it.lang.code}) ${it.name}" } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt index 7439f59f7b..eb50341e8d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt @@ -96,7 +96,7 @@ class ChaptersPresenter : BasePresenter() { } fun getOnlineChaptersObs(): Observable> { - return source.pullChaptersFromNetwork(manga.url) + return source.fetchChapterList(manga) .subscribeOn(Schedulers.io()) .map { syncChaptersWithSource(db, it, manga, source) } .observeOn(AndroidSchedulers.mainThread()) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoFragment.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoFragment.kt index c8cad099f5..c79688d2a0 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoFragment.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoFragment.kt @@ -8,6 +8,7 @@ import com.bumptech.glide.Glide import com.bumptech.glide.load.engine.DiskCacheStrategy import eu.kanade.tachiyomi.R import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.source.base.Source import eu.kanade.tachiyomi.ui.base.fragment.BaseRxFragment import eu.kanade.tachiyomi.util.getResourceColor @@ -96,7 +97,7 @@ class MangaInfoFragment : BaseRxFragment() { // If manga source is known update source TextView. if (source != null) { - manga_source.text = source.visibleName + manga_source.text = source.toString() } // Update genres TextView. @@ -140,8 +141,9 @@ class MangaInfoFragment : BaseRxFragment() { * Open the manga in browser. */ fun openInBrowser() { + val source = presenter.source as? OnlineSource ?: return try { - val url = Uri.parse(presenter.source.baseUrl + presenter.manga.url) + val url = Uri.parse(source.baseUrl + presenter.manga.url) val intent = CustomTabsIntent.Builder() .setToolbarColor(context.theme.getResourceColor(R.attr.colorPrimary)) .build() diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoPresenter.kt index 73ffe02e03..cb04354ec3 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoPresenter.kt @@ -99,7 +99,7 @@ class MangaInfoPresenter : BasePresenter() { * @return manga information. */ private fun fetchMangaObs(): Observable { - return source.pullMangaFromNetwork(manga.url) + return source.fetchMangaDetails(manga) .flatMap { networkManga -> manga.copyFrom(networkManga) db.insertManga(manga).executeAsBlocking() diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt index 1e93f2200b..c21217683f 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt @@ -12,6 +12,7 @@ import eu.kanade.tachiyomi.data.mangasync.MangaSyncManager import eu.kanade.tachiyomi.data.mangasync.UpdateMangaSyncService import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.source.SourceManager +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.source.base.Source import eu.kanade.tachiyomi.data.source.model.Page import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter @@ -126,9 +127,16 @@ class ReaderPresenter : BasePresenter() { observable = Observable.from(ch.pages) .flatMap { downloadManager.getDownloadedImage(it, chapterDir) } } else { - observable = source.getAllImageUrlsFromPageList(ch.pages) - .flatMap({ source.getCachedImage(it) }, 2) - .doOnCompleted { source.savePageList(ch.url, ch.pages) } + observable = source.let { source -> + if (source is OnlineSource) { + source.fetchAllImageUrlsFromPageList(ch.pages) + .flatMap({ source.getCachedImage(it) }, 2) + .doOnCompleted { source.savePageList(ch, ch.pages) } + } else { + Observable.from(ch.pages) + .flatMap { source.fetchImage(it) } + } + } } observable.doOnCompleted { if (!isSeamlessMode && chapter === ch) { @@ -139,13 +147,7 @@ class ReaderPresenter : BasePresenter() { // Listen por retry events add(retryPageSubject.observeOn(Schedulers.io()) - .flatMap { page -> - if (page.imageUrl == null) - source.getImageUrlFromPage(page) - else - Observable.just(page) - } - .flatMap { source.getCachedImage(it) } + .flatMap { source.fetchImage(it) } .subscribe()) } @@ -156,7 +158,7 @@ class ReaderPresenter : BasePresenter() { Observable.just(downloadManager.getSavedPageList(source, manga, chapter)!!) else // Fetch the page list from cache or fallback to network - source.getCachedPageListOrPullFromNetwork(chapter.url) + source.fetchPageList(chapter) .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) @@ -200,26 +202,15 @@ class ReaderPresenter : BasePresenter() { // Preload the first pages of the next chapter. Only for non seamless mode private fun getPreloadNextChapterObservable(): Observable { - return source.getCachedPageListOrPullFromNetwork(nextChapter!!.url) + val nextChapter = nextChapter ?: return Observable.error(Exception("No next chapter")) + return source.fetchPageList(nextChapter) .flatMap { pages -> - nextChapter!!.pages = pages + nextChapter.pages = pages val pagesToPreload = Math.min(pages.size, 5) Observable.from(pages).take(pagesToPreload) } // Preload up to 5 images - .concatMap { page -> - if (page.imageUrl == null) - source.getImageUrlFromPage(page) - else - Observable.just(page) - } - // Download the first image - .concatMap { page -> - if (page.pageNumber == 0) - source.getCachedImage(page) - else - Observable.just(page) - } + .concatMap { source.fetchImage(it) } .subscribeOn(Schedulers.io()) .observeOn(AndroidSchedulers.mainThread()) .doOnCompleted { stopPreloadingNextChapter() } @@ -324,7 +315,7 @@ class ReaderPresenter : BasePresenter() { // Cache current page list progress for online chapters to allow a faster reopen if (!chapter.isDownloaded) { - source.savePageList(chapter.url, pages) + source.let { if (it is OnlineSource) it.savePageList(chapter, pages) } } // Save current progress of the chapter. Mark as read if the chapter is finished @@ -382,7 +373,7 @@ class ReaderPresenter : BasePresenter() { } fun updateMangaSyncLastChapterRead() { - for (mangaSync in mangaSyncList!!) { + for (mangaSync in mangaSyncList ?: emptyList()) { val service = syncManager.getService(mangaSync.sync_id) if (service.isLogged && mangaSync.update) { UpdateMangaSyncService.start(context, mangaSync) @@ -417,16 +408,21 @@ class ReaderPresenter : BasePresenter() { } private fun preloadNextChapter() { - if (hasNextChapter() && !isChapterDownloaded(nextChapter!!)) { - start(PRELOAD_NEXT_CHAPTER) + nextChapter?.let { + if (!isChapterDownloaded(it)) { + start(PRELOAD_NEXT_CHAPTER) + } } } private fun stopPreloadingNextChapter() { if (!isUnsubscribed(PRELOAD_NEXT_CHAPTER)) { stop(PRELOAD_NEXT_CHAPTER) - if (nextChapter!!.pages != null) - source.savePageList(nextChapter!!.url, nextChapter!!.pages) + nextChapter?.let { chapter -> + if (chapter.pages != null) { + source.let { if (it is OnlineSource) it.savePageList(chapter, chapter.pages) } + } + } } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsSourcesFragment.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsSourcesFragment.kt index 16716a3c24..9d236af08c 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsSourcesFragment.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsSourcesFragment.kt @@ -42,11 +42,11 @@ class SettingsSourcesFragment : SettingsNestedFragment() { .subscribe { languages -> sourcesPref.removeAll() - val enabledSources = settingsActivity.sourceManager.getSources() + val enabledSources = settingsActivity.sourceManager.getOnlineSources() .filter { it.lang.code in languages } for (source in enabledSources) { - if (source.isLoginRequired) { + if (source.isLoginRequired()) { val pref = createSource(source) sourcesPref.addPreference(pref) } @@ -65,7 +65,7 @@ class SettingsSourcesFragment : SettingsNestedFragment() { fun createSource(source: Source): Preference { return LoginPreference(preferenceManager.context).apply { key = preferences.keys.sourceUsername(source.id) - title = source.visibleName + title = source.toString() setOnPreferenceClickListener { val fragment = SourceLoginDialog.newInstance(source) diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/ChapterSourceSync.kt b/app/src/main/java/eu/kanade/tachiyomi/util/ChapterSourceSync.kt index 5d83832337..490e3deb7b 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/util/ChapterSourceSync.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/util/ChapterSourceSync.kt @@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.util import eu.kanade.tachiyomi.data.database.DatabaseHelper import eu.kanade.tachiyomi.data.database.models.Chapter import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.source.base.Source import java.util.* @@ -34,7 +35,9 @@ fun syncChaptersWithSource(db: DatabaseHelper, // Recognize number for new chapters. toAdd.forEach { - source.parseChapterNumber(it) + if (source is OnlineSource) { + source.parseChapterNumber(it) + } ChapterRecognition.parseChapterNumber(it, manga) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/Parser.java b/app/src/main/java/eu/kanade/tachiyomi/util/Parser.java deleted file mode 100644 index 282663c57a..0000000000 --- a/app/src/main/java/eu/kanade/tachiyomi/util/Parser.java +++ /dev/null @@ -1,52 +0,0 @@ -package eu.kanade.tachiyomi.util; - -import android.support.annotation.Nullable; - -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -public final class Parser { - - private Parser() throws InstantiationException { - throw new InstantiationException("This class is not for instantiation"); - } - - @Nullable - public static Element element(Element container, String pattern) { - return container.select(pattern).first(); - } - - @Nullable - public static String text(Element container, String pattern) { - return text(container, pattern, null); - } - - @Nullable - public static String text(Element container, String pattern, String defValue) { - Element element = container.select(pattern).first(); - return element != null ? element.text() : defValue; - } - - @Nullable - public static String allText(Element container, String pattern) { - Elements elements = container.select(pattern); - return !elements.isEmpty() ? elements.text() : null; - } - - @Nullable - public static String attr(Element container, String pattern, String attr) { - Element element = container.select(pattern).first(); - return element != null ? element.attr(attr) : null; - } - - @Nullable - public static String href(Element container, String pattern) { - return attr(container, pattern, "href"); - } - - @Nullable - public static String src(Element container, String pattern) { - return attr(container, pattern, "src"); - } - -} diff --git a/app/src/main/java/eu/kanade/tachiyomi/widget/preference/SourceLoginDialog.kt b/app/src/main/java/eu/kanade/tachiyomi/widget/preference/SourceLoginDialog.kt index 5196efba8e..88c4018692 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/widget/preference/SourceLoginDialog.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/widget/preference/SourceLoginDialog.kt @@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.widget.preference import android.os.Bundle import android.view.View import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.data.source.base.OnlineSource import eu.kanade.tachiyomi.data.source.base.Source import eu.kanade.tachiyomi.ui.setting.SettingsActivity import eu.kanade.tachiyomi.util.toast @@ -23,17 +24,17 @@ class SourceLoginDialog : LoginDialogPreference() { } } - lateinit var source: Source + lateinit var source: OnlineSource override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) val sourceId = arguments.getInt("key") - source = (activity as SettingsActivity).sourceManager.get(sourceId)!! + source = (activity as SettingsActivity).sourceManager.get(sourceId) as OnlineSource } override fun setCredentialsOnView(view: View) = with(view) { - dialog_title.text = getString(R.string.login_title, source.visibleName) + dialog_title.text = getString(R.string.login_title, source.toString()) username.setText(preferences.sourceUsername(source)) password.setText(preferences.sourcePassword(source)) } diff --git a/app/src/test/java/eu/kanade/tachiyomi/data/library/LibraryUpdateServiceTest.java b/app/src/test/java/eu/kanade/tachiyomi/data/library/LibraryUpdateServiceTest.java index 94496c683a..324ac3bae5 100644 --- a/app/src/test/java/eu/kanade/tachiyomi/data/library/LibraryUpdateServiceTest.java +++ b/app/src/test/java/eu/kanade/tachiyomi/data/library/LibraryUpdateServiceTest.java @@ -17,7 +17,7 @@ import eu.kanade.tachiyomi.BuildConfig; import eu.kanade.tachiyomi.CustomRobolectricGradleTestRunner; import eu.kanade.tachiyomi.data.database.models.Chapter; import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.source.base.Source; +import eu.kanade.tachiyomi.data.source.base.OnlineSource; import rx.Observable; import static org.assertj.core.api.Assertions.assertThat; @@ -32,14 +32,14 @@ public class LibraryUpdateServiceTest { ShadowApplication app; Context context; LibraryUpdateService service; - Source source; + OnlineSource source; @Before public void setup() { app = ShadowApplication.getInstance(); context = app.getApplicationContext(); service = Robolectric.setupService(LibraryUpdateService.class); - source = mock(Source.class); + source = mock(OnlineSource.class); when(service.sourceManager.get(anyInt())).thenReturn(source); } @@ -62,7 +62,7 @@ public class LibraryUpdateServiceTest { List sourceChapters = createChapters("/chapter1", "/chapter2"); - when(source.pullChaptersFromNetwork(manga.url)).thenReturn(Observable.just(sourceChapters)); + when(source.fetchChapterList(manga)).thenReturn(Observable.just(sourceChapters)); service.updateManga(manga).subscribe(); @@ -79,9 +79,9 @@ public class LibraryUpdateServiceTest { List chapters3 = createChapters("/achapter1", "/achapter2"); // One of the updates will fail - when(source.pullChaptersFromNetwork("/manga1")).thenReturn(Observable.just(chapters)); - when(source.pullChaptersFromNetwork("/manga2")).thenReturn(Observable.>error(new Exception())); - when(source.pullChaptersFromNetwork("/manga3")).thenReturn(Observable.just(chapters3)); + when(source.fetchChapterList(favManga.get(0))).thenReturn(Observable.just(chapters)); + when(source.fetchChapterList(favManga.get(1))).thenReturn(Observable.>error(new Exception())); + when(source.fetchChapterList(favManga.get(2))).thenReturn(Observable.just(chapters3)); service.updateMangaList(service.getMangaToUpdate(null)).subscribe();