From 2f2b0fae7f3c076ab057b539343bc9d6f64a8365 Mon Sep 17 00:00:00 2001 From: Aria Moradi Date: Fri, 25 Dec 2020 05:35:21 +0330 Subject: [PATCH] save sources list --- .../network/CloudflareInterceptor.kt | 177 ++++++++++++++++++ .../kanade/tachiyomi/network/NetworkHelper.kt | 10 +- .../main/kotlin/ir/armor/tachidesk/Main.kt | 68 ++++++- .../database/model/ExtensionTable.kt | 17 +- .../tachidesk/database/model/SourcesTable.kt | 54 ++++-- 5 files changed, 296 insertions(+), 30 deletions(-) create mode 100644 server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt new file mode 100644 index 0000000..2f326a7 --- /dev/null +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt @@ -0,0 +1,177 @@ +package eu.kanade.tachiyomi.network + +//import android.annotation.SuppressLint +//import android.content.Context +//import android.os.Build +//import android.os.Handler +//import android.os.Looper +//import android.webkit.WebSettings +//import android.webkit.WebView +//import android.widget.Toast +//import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.source.online.HttpSource +//import eu.kanade.tachiyomi.util.lang.launchUI +//import eu.kanade.tachiyomi.util.system.WebViewClientCompat +//import eu.kanade.tachiyomi.util.system.WebViewUtil +//import eu.kanade.tachiyomi.util.system.isOutdated +//import eu.kanade.tachiyomi.util.system.setDefaultSettings +//import eu.kanade.tachiyomi.util.system.toast +import okhttp3.Cookie +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Request +import okhttp3.Response +//import uy.kohesive.injekt.injectLazy +import java.io.IOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class CloudflareInterceptor() : Interceptor { + +// private val handler = Handler(Looper.getMainLooper()) + +// private val networkHelper = NetworkHelper() + + /** + * When this is called, it initializes the WebView if it wasn't already. We use this to avoid + * blocking the main thread too much. If used too often we could consider moving it to the + * Application class. + */ +// private val initWebView by lazy { +// WebSettings.getDefaultUserAgent(context) +// } + + @Synchronized + override fun intercept(chain: Interceptor.Chain): Response { + val originalRequest = chain.request() + return chain.proceed(originalRequest) + +// if (!WebViewUtil.supportsWebView(context)) { +// launchUI { +// context.toast(R.string.information_webview_required, Toast.LENGTH_LONG) +// } +// return chain.proceed(originalRequest) +// } +// +// initWebView +// +// val response = chain.proceed(originalRequest) +// +// // Check if Cloudflare anti-bot is on +// if (response.code != 503 || response.header("Server") !in SERVER_CHECK) { +// return response +// } +// +// try { +// response.close() +// networkHelper.cookieManager.remove(originalRequest.url, COOKIE_NAMES, 0) +// val oldCookie = networkHelper.cookieManager.get(originalRequest.url) +// .firstOrNull { it.name == "cf_clearance" } +// resolveWithWebView(originalRequest, oldCookie) +// +// return chain.proceed(originalRequest) +// } catch (e: Exception) { +// // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that +// // we don't crash the entire app +// throw IOException(e) +// } + } +// +//// @SuppressLint("SetJavaScriptEnabled") +// private fun resolveWithWebView(request: Request, oldCookie: Cookie?) { +// // We need to lock this thread until the WebView finds the challenge solution url, because +// // OkHttp doesn't support asynchronous interceptors. +// val latch = CountDownLatch(1) +// +// var webView: WebView? = null +// +// var challengeFound = false +// var cloudflareBypassed = false +// var isWebViewOutdated = false +// +// val origRequestUrl = request.url.toString() +// val headers = request.headers.toMultimap().mapValues { it.value.getOrNull(0) ?: "" }.toMutableMap() +// headers["X-Requested-With"] = WebViewUtil.REQUESTED_WITH +// +// handler.post { +// val webview = WebView(context) +// webView = webview +// webview.setDefaultSettings() +// +// // Avoid sending empty User-Agent, Chromium WebView will reset to default if empty +// webview.settings.userAgentString = request.header("User-Agent") +// ?: HttpSource.DEFAULT_USERAGENT +// +// webview.webViewClient = object : WebViewClientCompat() { +// override fun onPageFinished(view: WebView, url: String) { +// fun isCloudFlareBypassed(): Boolean { +// return networkHelper.cookieManager.get(origRequestUrl.toHttpUrl()) +// .firstOrNull { it.name == "cf_clearance" } +// .let { it != null && it != oldCookie } +// } +// +// if (isCloudFlareBypassed()) { +// cloudflareBypassed = true +// latch.countDown() +// } +// +// // HTTP error codes are only received since M +// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && +// url == origRequestUrl && !challengeFound +// ) { +// // The first request didn't return the challenge, abort. +// latch.countDown() +// } +// } +// +// override fun onReceivedErrorCompat( +// view: WebView, +// errorCode: Int, +// description: String?, +// failingUrl: String, +// isMainFrame: Boolean +// ) { +// if (isMainFrame) { +// if (errorCode == 503) { +// // Found the Cloudflare challenge page. +// challengeFound = true +// } else { +// // Unlock thread, the challenge wasn't found. +// latch.countDown() +// } +// } +// } +// } +// +// webView?.loadUrl(origRequestUrl, headers) +// } +// +// // Wait a reasonable amount of time to retrieve the solution. The minimum should be +// // around 4 seconds but it can take more due to slow networks or server issues. +// latch.await(12, TimeUnit.SECONDS) +// +// handler.post { +// if (!cloudflareBypassed) { +// isWebViewOutdated = webView?.isOutdated() == true +// } +// +// webView?.stopLoading() +// webView?.destroy() +// } +// +// // Throw exception if we failed to bypass Cloudflare +// if (!cloudflareBypassed) { +// // Prompt user to update WebView if it seems too outdated +// if (isWebViewOutdated) { +// context.toast(R.string.information_webview_outdated, Toast.LENGTH_LONG) +// } +// +// throw Exception(context.getString(R.string.information_cloudflare_bypass_failure)) +// } +// } +// +// companion object { +// private val SERVER_CHECK = arrayOf("cloudflare-nginx", "cloudflare") +// private val COOKIE_NAMES = listOf("__cfduid", "cf_clearance") +// } +} diff --git a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt index a91e237..2d2fced 100644 --- a/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt +++ b/server/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt @@ -62,9 +62,9 @@ class NetworkHelper() { builder.build() } -// val cloudflareClient by lazy { -// client.newBuilder() -// .addInterceptor(CloudflareInterceptor(context)) -// .build() -// } + val cloudflareClient by lazy { + client.newBuilder() + .addInterceptor(CloudflareInterceptor()) + .build() + } } diff --git a/server/src/main/kotlin/ir/armor/tachidesk/Main.kt b/server/src/main/kotlin/ir/armor/tachidesk/Main.kt index dbd71ca..bc941a3 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/Main.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/Main.kt @@ -4,12 +4,12 @@ import com.googlecode.dex2jar.tools.Dex2jarCmd import eu.kanade.tachiyomi.extension.api.ExtensionGithubApi import eu.kanade.tachiyomi.extension.model.Extension import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.source.SourceFactory import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.online.HttpSource import io.javalin.Javalin import ir.armor.tachidesk.database.makeDataBaseTables -import ir.armor.tachidesk.database.model.ExtensionDataClass -import ir.armor.tachidesk.database.model.ExtensionsTable +import ir.armor.tachidesk.database.model.* import kotlinx.coroutines.runBlocking import okhttp3.Request import okio.buffer @@ -144,7 +144,7 @@ class Main { } fun downloadApk(apkName: String): Int { - val extension = getExtensionList(true).first { it.apkName == apkName } + val extensionRecord = getExtensionList(true).first { it.apkName == apkName } val fileNameWithoutType = apkName.substringBefore(".apk") val dirPathWithoutType = "${Config.extensionsRoot}/$apkName" @@ -153,7 +153,7 @@ class Main { if (!File(dexPath).exists()) { runBlocking { val api = ExtensionGithubApi() - val apkToDownload = api.getApkUrl(extension) + val apkToDownload = api.getApkUrl(extensionRecord) val apkFilePath = "$dirPathWithoutType.apk" val jarFilePath = "$dirPathWithoutType.jar" @@ -164,14 +164,67 @@ class Main { val className: String = APKExtractor.extract_dex_and_read_className(apkFilePath, dexFilePath) - + println(className) // dex -> jar Dex2jarCmd.main(dexFilePath, "-o", jarFilePath, "--force") File(apkFilePath).delete() + // update sources of the extension + val child = URLClassLoader(arrayOf(URL("file:$jarFilePath")), this::class.java.classLoader) + val classToLoad = Class.forName(className, true, child) + val instance = classToLoad.newInstance() + + val extensionId = transaction { + return@transaction ExtensionsTable.select { ExtensionsTable.name eq extensionRecord.name }.first()[ExtensionsTable.id] + } + + if (instance is HttpSource) {// single source + val httpSource = instance as HttpSource + transaction { +// SourceEntity.new { +// sourceId = httpSource.id +// name = httpSource.name +// this.extension = ExtensionEntity.find { ExtensionsTable.name eq extension.name }.first().id +// } + if (SourcesTable.select { SourcesTable.sourceId eq httpSource.id }.count() == 0L) { + SourcesTable.insert { + it[this.sourceId] = httpSource.id + it[name] = httpSource.name + it[this.lang] = httpSource.lang + it[extension] = extensionId + } + } +// println(httpSource.id) +// println(httpSource.name) +// println() + } + + } else { // multi source + val sourceFactory = instance as SourceFactory + transaction { + sourceFactory.createSources().forEachIndexed { index, source -> + val httpSource = source as HttpSource + if (SourcesTable.select { SourcesTable.sourceId eq httpSource.id }.count() == 0L) { + SourcesTable.insert { + it[this.sourceId] = httpSource.id + it[name] = httpSource.name + it[this.lang] = httpSource.lang + it[extension] = extensionId + it[partOfFactorySource] = true + it[positionInFactorySource] = index + } + } +// println(httpSource.id) +// println(httpSource.name) +// println() + } + } + } + + // update extension info transaction { - ExtensionsTable.update({ ExtensionsTable.name eq extension.name }) { + ExtensionsTable.update({ ExtensionsTable.name eq extensionRecord.name }) { it[installed] = true it[classFQName] = className } @@ -179,8 +232,7 @@ class Main { } return 201 // we downloaded successfully - } - else { + } else { return 302 } } diff --git a/server/src/main/kotlin/ir/armor/tachidesk/database/model/ExtensionTable.kt b/server/src/main/kotlin/ir/armor/tachidesk/database/model/ExtensionTable.kt index 9cd3a05..9df365e 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/database/model/ExtensionTable.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/database/model/ExtensionTable.kt @@ -31,7 +31,22 @@ data class ExtensionDataClass( val lang: String, val isNsfw: Boolean, val apkName: String, - val iconUrl : String, + val iconUrl: String, val installed: Boolean, val classFQName: String, ) + +class ExtensionEntity(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(ExtensionsTable) + + var name by ExtensionsTable.name + var pkgName by ExtensionsTable.pkgName + var versionName by ExtensionsTable.versionName + var versionCode by ExtensionsTable.versionCode + var lang by ExtensionsTable.lang + var isNsfw by ExtensionsTable.isNsfw + var apkName by ExtensionsTable.apkName + var iconUrl by ExtensionsTable.iconUrl + var installed by ExtensionsTable.installed + var classFQName by ExtensionsTable.classFQName +} diff --git a/server/src/main/kotlin/ir/armor/tachidesk/database/model/SourcesTable.kt b/server/src/main/kotlin/ir/armor/tachidesk/database/model/SourcesTable.kt index c83cbac..03fdf13 100644 --- a/server/src/main/kotlin/ir/armor/tachidesk/database/model/SourcesTable.kt +++ b/server/src/main/kotlin/ir/armor/tachidesk/database/model/SourcesTable.kt @@ -1,26 +1,48 @@ package ir.armor.tachidesk.database.model -import org.jetbrains.exposed.dao.Entity -import org.jetbrains.exposed.dao.IntEntity -import org.jetbrains.exposed.dao.IntEntityClass +import org.jetbrains.exposed.dao.* import org.jetbrains.exposed.dao.id.EntityID +import org.jetbrains.exposed.dao.id.IdTable +import org.jetbrains.exposed.dao.id.IntIdTable import org.jetbrains.exposed.sql.Column import org.jetbrains.exposed.sql.Table -object SourcesTable : Table() { - val id: Column = long("id") - val name: Column = varchar("name", 128) +object SourcesTable : IntIdTable() { + val sourceId = long("source_id") + val name= varchar("name", 128) + val lang= varchar("lang", 5) val extension = reference("extension", ExtensionsTable) - override val primaryKey = PrimaryKey(id) + val partOfFactorySource = bool("part_of_factory_source").default(false) + val positionInFactorySource = integer("position_in_factory_source").nullable() } -//class Source : Entity() { -// companion object : Entity(SourcesTable) +class SourceEntity(id: EntityID) : IntEntity(id) { + companion object : IntEntityClass(SourcesTable) + + var sourceId by SourcesTable.sourceId + var name by SourcesTable.name + var lang by SourcesTable.lang + var extension by ExtensionEntity referencedOn SourcesTable.extension + var partOfFactorySource by SourcesTable.partOfFactorySource + var positionInFactorySource by SourcesTable.positionInFactorySource +} + + +//object SourcesTable : IdTable() { +// override val id = long("id").entityId() +// val name= varchar("name", 128) +// val extension = reference("extension", ExtensionsTable) +// val partOfFactorySource = bool("part_of_factory_source").default(false) +// val positionInFactorySource = integer("position_in_factory_source").nullable() // -// val name by ExtensionsTable.name -// val pkgName by ExtensionsTable.pkgName -// val versionName by ExtensionsTable.versionName -// val versionCode by ExtensionsTable.versionCode -// val lang by ExtensionsTable.lang -// val isNsfw by ExtensionsTable.isNsfw -//} \ No newline at end of file +// override val primaryKey = PrimaryKey(id) +//} +// +//class SourceEntity(id: EntityID) : LongEntity(id) { +// companion object : LongEntityClass(SourcesTable) +// +// var name by SourcesTable.name +// var extension by SourcesTable.extension +// var partOfFactorySource by SourcesTable.partOfFactorySource +// var positionInFactorySource by SourcesTable.positionInFactorySource +//}