Test solving Cloudflare's challenge with WebView

This commit is contained in:
inorichi 2019-04-01 17:20:13 +02:00
parent 55bf1c31a6
commit f1f6a2b341
8 changed files with 250 additions and 194 deletions

View File

@ -7,9 +7,9 @@ import eu.kanade.tachiyomi.data.database.models.Track
import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.data.track.TrackService import eu.kanade.tachiyomi.data.track.TrackService
import eu.kanade.tachiyomi.data.track.model.TrackSearch import eu.kanade.tachiyomi.data.track.model.TrackSearch
import okhttp3.HttpUrl
import rx.Completable import rx.Completable
import rx.Observable import rx.Observable
import java.net.URI
class Myanimelist(private val context: Context, id: Int) : TrackService(id) { class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
@ -114,23 +114,23 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
override fun logout() { override fun logout() {
super.logout() super.logout()
preferences.trackToken(this).delete() preferences.trackToken(this).delete()
networkService.cookies.remove(URI(BASE_URL)) networkService.cookieManager.remove(HttpUrl.parse(BASE_URL)!!)
} }
override val isLogged: Boolean override val isLogged: Boolean
get() = !getUsername().isEmpty() && get() = !getUsername().isEmpty() &&
!getPassword().isEmpty() && !getPassword().isEmpty() &&
checkCookies(URI(BASE_URL)) && checkCookies() &&
!getCSRF().isEmpty() !getCSRF().isEmpty()
private fun getCSRF(): String = preferences.trackToken(this).getOrDefault() private fun getCSRF(): String = preferences.trackToken(this).getOrDefault()
private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf) private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf)
private fun checkCookies(uri: URI): Boolean { private fun checkCookies(): Boolean {
var ckCount = 0 var ckCount = 0
val url = HttpUrl.parse(BASE_URL)!!
for (ck in networkService.cookies.get(uri)) { for (ck in networkService.cookieManager.get(url)) {
if (ck.name() == USER_SESSION_COOKIE || ck.name() == LOGGED_IN_COOKIE) if (ck.name() == USER_SESSION_COOKIE || ck.name() == LOGGED_IN_COOKIE)
ckCount++ ckCount++
} }

View File

@ -0,0 +1,63 @@
package eu.kanade.tachiyomi.network
import android.content.Context
import android.os.Build
import android.webkit.CookieManager
import android.webkit.CookieSyncManager
import okhttp3.Cookie
import okhttp3.CookieJar
import okhttp3.HttpUrl
class AndroidCookieJar(context: Context) : CookieJar {
private val manager = CookieManager.getInstance()
private val syncManager by lazy { CookieSyncManager.createInstance(context) }
override fun saveFromResponse(url: HttpUrl, cookies: MutableList<Cookie>) {
val urlString = url.toString()
for (cookie in cookies) {
manager.setCookie(urlString, cookie.toString())
}
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
syncManager.sync()
}
}
override fun loadForRequest(url: HttpUrl): List<Cookie> {
return get(url)
}
fun get(url: HttpUrl): List<Cookie> {
val cookies = manager.getCookie(url.toString())
return if (cookies != null && !cookies.isEmpty()) {
cookies.split(";").mapNotNull { Cookie.parse(url, it) }
} else {
emptyList()
}
}
fun remove(url: HttpUrl) {
val cookies = manager.getCookie(url.toString()) ?: return
val domain = ".${url.host()}"
cookies.split(";")
.map { it.substringBefore("=") }
.onEach { manager.setCookie(domain, "$it=;Max-Age=-1") }
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
syncManager.sync()
}
}
fun removeAll() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
manager.removeAllCookies {}
} else {
manager.removeAllCookie()
syncManager.sync()
}
}
}

View File

@ -1,31 +1,32 @@
package eu.kanade.tachiyomi.network package eu.kanade.tachiyomi.network
import com.squareup.duktape.Duktape import android.annotation.SuppressLint
import okhttp3.* import android.content.Context
import android.os.Handler
import android.os.HandlerThread
import android.webkit.WebResourceResponse
import android.webkit.WebView
import eu.kanade.tachiyomi.util.WebViewClientCompat
import okhttp3.Interceptor
import okhttp3.Request
import okhttp3.Response
import timber.log.Timber
import java.io.IOException import java.io.IOException
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
class CloudflareInterceptor : Interceptor { class CloudflareInterceptor(private val context: Context) : Interceptor {
private val operationPattern = Regex("""setTimeout\(function\(\)\{\s+(var (?:\w,)+f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n""")
private val passPattern = Regex("""name="pass" value="(.+?)"""")
private val challengePattern = Regex("""name="jschl_vc" value="(\w+)"""")
private val sPattern = Regex("""name="s" value="([^"]+)""")
private val kPattern = Regex("""k\s+=\s+'([^']+)';""")
private val serverCheck = arrayOf("cloudflare-nginx", "cloudflare") private val serverCheck = arrayOf("cloudflare-nginx", "cloudflare")
private interface IBase64 { private val handler by lazy {
fun decode(input: String): String val thread = HandlerThread("WebViewThread").apply {
} uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, e ->
Timber.e(e)
private val b64: IBase64 = object : IBase64 { }
override fun decode(input: String): String { start()
return okio.ByteString.decodeBase64(input)!!.utf8()
} }
Handler(thread.looper)
} }
@Synchronized @Synchronized
@ -34,8 +35,14 @@ class CloudflareInterceptor : Interceptor {
// Check if Cloudflare anti-bot is on // Check if Cloudflare anti-bot is on
if (response.code() == 503 && response.header("Server") in serverCheck) { if (response.code() == 503 && response.header("Server") in serverCheck) {
return try { try {
chain.proceed(resolveChallenge(response)) response.close()
if (resolveWithWebView(chain.request())) {
// Retry original request
return chain.proceed(chain.request())
} else {
throw Exception("Failed resolving Cloudflare challenge")
}
} catch (e: Exception) { } catch (e: Exception) {
// Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
// we don't crash the entire app // we don't crash the entire app
@ -46,65 +53,76 @@ class CloudflareInterceptor : Interceptor {
return response return response
} }
private fun resolveChallenge(response: Response): Request { private fun isChallengeResolverUrl(url: String): Boolean {
Duktape.create().use { duktape -> return "chk_jschl" in url
val originalRequest = response.request() }
val url = originalRequest.url()
val domain = url.host()
val content = response.body()!!.string()
// CloudFlare requires waiting 4 seconds before resolving the challenge @SuppressLint("SetJavaScriptEnabled")
Thread.sleep(4000) private fun resolveWithWebView(request: Request): Boolean {
val latch = CountDownLatch(1)
val operation = operationPattern.find(content)?.groups?.get(1)?.value var result = false
val challenge = challengePattern.find(content)?.groups?.get(1)?.value var isResolvingChallenge = false
val pass = passPattern.find(content)?.groups?.get(1)?.value
val s = sPattern.find(content)?.groups?.get(1)?.value
// If `k` is null, it uses old methods. val requestUrl = request.url().toString()
val k = kPattern.find(content)?.groups?.get(1)?.value ?: "" val headers = request.headers().toMultimap().mapValues { it.value.getOrNull(0) ?: "" }
val innerHTMLValue = Regex("""<div(.*)id="$k"(.*)>(.*)</div>""")
.find(content)?.groups?.get(3)?.value ?: ""
if (operation == null || challenge == null || pass == null || s == null) { handler.post {
throw Exception("Failed resolving Cloudflare challenge") val view = WebView(context)
view.settings.javaScriptEnabled = true
view.settings.userAgentString = request.header("User-Agent")
view.webViewClient = object : WebViewClientCompat() {
override fun shouldInterceptRequestCompat(
view: WebView,
url: String
): WebResourceResponse? {
val isChallengeResolverUrl = isChallengeResolverUrl(url)
if (requestUrl != url && !isChallengeResolverUrl) {
return WebResourceResponse("text/plain", "UTF-8", null)
}
if (isChallengeResolverUrl) {
isResolvingChallenge = true
}
return null
}
override fun onPageFinished(view: WebView, url: String) {
super.onPageFinished(view, url)
if (isResolvingChallenge && url == requestUrl) {
setResultAndFinish(true)
}
}
override fun onReceivedErrorCompat(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
isMainFrame: Boolean
) {
if ((errorCode != 503 && requestUrl == failingUrl) ||
isChallengeResolverUrl(failingUrl)
) {
setResultAndFinish(false)
}
}
private fun setResultAndFinish(resolved: Boolean) {
result = resolved
latch.countDown()
view.stopLoading()
view.destroy()
}
} }
// Export native Base64 decode function to js object. view.loadUrl(requestUrl, headers)
duktape.set("b64", IBase64::class.java, b64)
// Return simulated innerHTML when call DOM.
val simulatedDocumentJS = """var document = { getElementById: function (x) { return { innerHTML: "$innerHTMLValue" }; } }"""
val js = operation
.replace(Regex("""a\.value = (.+\.toFixed\(10\);).+"""), "$1")
.replace(Regex("""\s{3,}[a-z](?: = |\.).+"""), "")
.replace("t.length", "${domain.length}")
.replace("\n", "")
val result = duktape.evaluate("""$simulatedDocumentJS;$ATOB_JS;var t="$domain";$js""") as String
val cloudflareUrl = HttpUrl.parse("${url.scheme()}://$domain/cdn-cgi/l/chk_jschl")!!
.newBuilder()
.addQueryParameter("jschl_vc", challenge)
.addQueryParameter("pass", pass)
.addQueryParameter("s", s)
.addQueryParameter("jschl_answer", result)
.toString()
val cloudflareHeaders = originalRequest.headers()
.newBuilder()
.add("Referer", url.toString())
.add("Accept", "text/html,application/xhtml+xml,application/xml")
.add("Accept-Language", "en")
.build()
return GET(cloudflareUrl, cloudflareHeaders, cache = CacheControl.Builder().build())
} }
latch.await(12, TimeUnit.SECONDS)
return result
} }
companion object {
// atob() is browser API, Using Android's own function. (java.util.Base64 can't be used because of min API level)
private const val ATOB_JS = """var atob = function (input) { return b64.decode(input) }"""
}
} }

View File

@ -2,11 +2,7 @@ package eu.kanade.tachiyomi.network
import android.content.Context import android.content.Context
import android.os.Build import android.os.Build
import okhttp3.Cache import okhttp3.*
import okhttp3.CipherSuite
import okhttp3.ConnectionSpec
import okhttp3.OkHttpClient
import okhttp3.TlsVersion
import java.io.File import java.io.File
import java.io.IOException import java.io.IOException
import java.net.InetAddress import java.net.InetAddress
@ -15,11 +11,7 @@ import java.net.UnknownHostException
import java.security.KeyManagementException import java.security.KeyManagementException
import java.security.KeyStore import java.security.KeyStore
import java.security.NoSuchAlgorithmException import java.security.NoSuchAlgorithmException
import javax.net.ssl.SSLContext import javax.net.ssl.*
import javax.net.ssl.SSLSocket
import javax.net.ssl.SSLSocketFactory
import javax.net.ssl.TrustManagerFactory
import javax.net.ssl.X509TrustManager
class NetworkHelper(context: Context) { class NetworkHelper(context: Context) {
@ -27,7 +19,7 @@ class NetworkHelper(context: Context) {
private val cacheSize = 5L * 1024 * 1024 // 5 MiB private val cacheSize = 5L * 1024 * 1024 // 5 MiB
private val cookieManager = PersistentCookieJar(context) val cookieManager = AndroidCookieJar(context)
val client = OkHttpClient.Builder() val client = OkHttpClient.Builder()
.cookieJar(cookieManager) .cookieJar(cookieManager)
@ -36,12 +28,9 @@ class NetworkHelper(context: Context) {
.build() .build()
val cloudflareClient = client.newBuilder() val cloudflareClient = client.newBuilder()
.addInterceptor(CloudflareInterceptor()) .addInterceptor(CloudflareInterceptor(context))
.build() .build()
val cookies: PersistentCookieStore
get() = cookieManager.store
private fun OkHttpClient.Builder.enableTLS12(): OkHttpClient.Builder { private fun OkHttpClient.Builder.enableTLS12(): OkHttpClient.Builder {
if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) { if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) {
return this return this

View File

@ -1,19 +0,0 @@
package eu.kanade.tachiyomi.network
import android.content.Context
import okhttp3.Cookie
import okhttp3.CookieJar
import okhttp3.HttpUrl
class PersistentCookieJar(context: Context) : CookieJar {
val store = PersistentCookieStore(context)
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
store.addAll(url, cookies)
}
override fun loadForRequest(url: HttpUrl): List<Cookie> {
return store.get(url)
}
}

View File

@ -1,78 +0,0 @@
package eu.kanade.tachiyomi.network
import android.content.Context
import okhttp3.Cookie
import okhttp3.HttpUrl
import java.net.URI
import java.util.concurrent.ConcurrentHashMap
class PersistentCookieStore(context: Context) {
private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
init {
for ((key, value) in prefs.all) {
@Suppress("UNCHECKED_CAST")
val cookies = value as? Set<String>
if (cookies != null) {
try {
val url = HttpUrl.parse("http://$key") ?: continue
val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) }
.filter { !it.hasExpired() }
cookieMap.put(key, nonExpiredCookies)
} catch (e: Exception) {
// Ignore
}
}
}
}
@Synchronized
fun addAll(url: HttpUrl, cookies: List<Cookie>) {
val key = url.uri().host
// Append or replace the cookies for this domain.
val cookiesForDomain = cookieMap[key].orEmpty().toMutableList()
for (cookie in cookies) {
// Find a cookie with the same name. Replace it if found, otherwise add a new one.
val pos = cookiesForDomain.indexOfFirst { it.name() == cookie.name() }
if (pos == -1) {
cookiesForDomain.add(cookie)
} else {
cookiesForDomain[pos] = cookie
}
}
cookieMap.put(key, cookiesForDomain)
// Get cookies to be stored in disk
val newValues = cookiesForDomain.asSequence()
.filter { it.persistent() && !it.hasExpired() }
.map(Cookie::toString)
.toSet()
prefs.edit().putStringSet(key, newValues).apply()
}
@Synchronized
fun removeAll() {
prefs.edit().clear().apply()
cookieMap.clear()
}
fun remove(uri: URI) {
prefs.edit().remove(uri.host).apply()
cookieMap.remove(uri.host)
}
fun get(url: HttpUrl) = get(url.uri().host)
fun get(uri: URI) = get(uri.host)
private fun get(url: String): List<Cookie> {
return cookieMap[url].orEmpty().filter { !it.hasExpired() }
}
private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt()
}

View File

@ -43,7 +43,7 @@ class SettingsAdvancedController : SettingsController() {
titleRes = R.string.pref_clear_cookies titleRes = R.string.pref_clear_cookies
onClick { onClick {
network.cookies.removeAll() network.cookieManager.removeAll()
activity?.toast(R.string.cookies_cleared) activity?.toast(R.string.cookies_cleared)
} }
} }

View File

@ -0,0 +1,83 @@
package eu.kanade.tachiyomi.util
import android.annotation.TargetApi
import android.os.Build
import android.webkit.*
@Suppress("OverridingDeprecatedMember")
abstract class WebViewClientCompat : WebViewClient() {
open fun shouldOverrideUrlCompat(view: WebView, url: String): Boolean {
return false
}
open fun shouldInterceptRequestCompat(view: WebView, url: String): WebResourceResponse? {
return null
}
open fun onReceivedErrorCompat(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String,
isMainFrame: Boolean) {
}
@TargetApi(Build.VERSION_CODES.N)
final override fun shouldOverrideUrlLoading(
view: WebView,
request: WebResourceRequest
): Boolean {
return shouldOverrideUrlCompat(view, request.url.toString())
}
final override fun shouldOverrideUrlLoading(view: WebView, url: String): Boolean {
return shouldOverrideUrlCompat(view, url)
}
@TargetApi(Build.VERSION_CODES.LOLLIPOP)
final override fun shouldInterceptRequest(
view: WebView,
request: WebResourceRequest
): WebResourceResponse? {
return shouldInterceptRequestCompat(view, request.url.toString())
}
final override fun shouldInterceptRequest(
view: WebView,
url: String
): WebResourceResponse? {
return shouldInterceptRequestCompat(view, url)
}
@TargetApi(Build.VERSION_CODES.M)
final override fun onReceivedError(
view: WebView,
request: WebResourceRequest,
error: WebResourceError
) {
onReceivedErrorCompat(view, error.errorCode, error.description?.toString(),
request.url.toString(), request.isForMainFrame)
}
final override fun onReceivedError(
view: WebView,
errorCode: Int,
description: String?,
failingUrl: String
) {
onReceivedErrorCompat(view, errorCode, description, failingUrl, failingUrl == view.url)
}
@TargetApi(Build.VERSION_CODES.M)
final override fun onReceivedHttpError(
view: WebView,
request: WebResourceRequest,
error: WebResourceResponse
) {
onReceivedErrorCompat(view, error.statusCode, error.reasonPhrase, request.url
.toString(), request.isForMainFrame)
}
}