Test solving Cloudflare's challenge with WebView

app/src/main/java/eu/kanade/tachiyomi/data/track/myanimelist/MyAnimeList.kt

@@ -7,9 +7,9 @@ import eu.kanade.tachiyomi.data.database.models.Track
 import eu.kanade.tachiyomi.data.preference.getOrDefault
 import eu.kanade.tachiyomi.data.track.TrackService
 import eu.kanade.tachiyomi.data.track.model.TrackSearch
+import okhttp3.HttpUrl
 import rx.Completable
 import rx.Observable
-import java.net.URI
 
 class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
 
@@ -114,23 +114,23 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
     override fun logout() {
         super.logout()
         preferences.trackToken(this).delete()
-        networkService.cookies.remove(URI(BASE_URL))
+        networkService.cookieManager.remove(HttpUrl.parse(BASE_URL)!!)
     }
 
     override val isLogged: Boolean
         get() = !getUsername().isEmpty() &&
                 !getPassword().isEmpty() &&
-                checkCookies(URI(BASE_URL)) &&
+                checkCookies() &&
                 !getCSRF().isEmpty()
 
     private fun getCSRF(): String = preferences.trackToken(this).getOrDefault()
 
     private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf)
 
-    private fun checkCookies(uri: URI): Boolean {
+    private fun checkCookies(): Boolean {
         var ckCount = 0
-
+        val url = HttpUrl.parse(BASE_URL)!!
-        for (ck in networkService.cookies.get(uri)) {
+        for (ck in networkService.cookieManager.get(url)) {
             if (ck.name() == USER_SESSION_COOKIE || ck.name() == LOGGED_IN_COOKIE)
                 ckCount++
         }

app/src/main/java/eu/kanade/tachiyomi/network/AndroidCookieJar.kt

@@ -0,0 +1,63 @@
+package eu.kanade.tachiyomi.network
+
+import android.content.Context
+import android.os.Build
+import android.webkit.CookieManager
+import android.webkit.CookieSyncManager
+import okhttp3.Cookie
+import okhttp3.CookieJar
+import okhttp3.HttpUrl
+
+class AndroidCookieJar(context: Context) : CookieJar {
+
+    private val manager = CookieManager.getInstance()
+
+    private val syncManager by lazy { CookieSyncManager.createInstance(context) }
+
+    override fun saveFromResponse(url: HttpUrl, cookies: MutableList<Cookie>) {
+        val urlString = url.toString()
+
+        for (cookie in cookies) {
+            manager.setCookie(urlString, cookie.toString())
+        }
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
+            syncManager.sync()
+        }
+    }
+
+    override fun loadForRequest(url: HttpUrl): List<Cookie> {
+        return get(url)
+    }
+
+    fun get(url: HttpUrl): List<Cookie> {
+        val cookies = manager.getCookie(url.toString())
+
+        return if (cookies != null && !cookies.isEmpty()) {
+            cookies.split(";").mapNotNull { Cookie.parse(url, it) }
+        } else {
+            emptyList()
+        }
+    }
+
+    fun remove(url: HttpUrl) {
+        val cookies = manager.getCookie(url.toString()) ?: return
+        val domain = ".${url.host()}"
+        cookies.split(";")
+            .map { it.substringBefore("=") }
+            .onEach { manager.setCookie(domain, "$it=;Max-Age=-1") }
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
+            syncManager.sync()
+        }
+    }
+
+    fun removeAll() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            manager.removeAllCookies {}
+        } else {
+            manager.removeAllCookie()
+            syncManager.sync()
+        }
+    }
+
+}

app/src/main/java/eu/kanade/tachiyomi/network/CloudflareInterceptor.kt

@@ -1,31 +1,32 @@
 package eu.kanade.tachiyomi.network
 
-import com.squareup.duktape.Duktape
+import android.annotation.SuppressLint
-import okhttp3.*
+import android.content.Context
+import android.os.Handler
+import android.os.HandlerThread
+import android.webkit.WebResourceResponse
+import android.webkit.WebView
+import eu.kanade.tachiyomi.util.WebViewClientCompat
+import okhttp3.Interceptor
+import okhttp3.Request
+import okhttp3.Response
+import timber.log.Timber
 import java.io.IOException
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
 
-class CloudflareInterceptor : Interceptor {
+class CloudflareInterceptor(private val context: Context) : Interceptor {
-
-    private val operationPattern = Regex("""setTimeout\(function\(\)\{\s+(var (?:\w,)+f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n""")
-    
-    private val passPattern = Regex("""name="pass" value="(.+?)"""")
-
-    private val challengePattern = Regex("""name="jschl_vc" value="(\w+)"""")
-
-    private val sPattern = Regex("""name="s" value="([^"]+)""")
-
-    private val kPattern = Regex("""k\s+=\s+'([^']+)';""")
 
     private val serverCheck = arrayOf("cloudflare-nginx", "cloudflare")
 
-    private interface IBase64 {
+    private val handler by lazy {
-        fun decode(input: String): String
+        val thread = HandlerThread("WebViewThread").apply {
-    }
+            uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, e ->
-
+                Timber.e(e)
-    private val b64: IBase64 = object : IBase64 {
+            }
-        override fun decode(input: String): String {
+            start()
-            return okio.ByteString.decodeBase64(input)!!.utf8()
         }
+        Handler(thread.looper)
     }
 
     @Synchronized
@@ -34,8 +35,14 @@ class CloudflareInterceptor : Interceptor {
 
         // Check if Cloudflare anti-bot is on
         if (response.code() == 503 && response.header("Server") in serverCheck) {
-            return try {
+            try {
-                chain.proceed(resolveChallenge(response))
+                response.close()
+                if (resolveWithWebView(chain.request())) {
+                    // Retry original request
+                    return chain.proceed(chain.request())
+                } else {
+                    throw Exception("Failed resolving Cloudflare challenge")
+                }
             } catch (e: Exception) {
                 // Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
                 // we don't crash the entire app
@@ -46,65 +53,76 @@ class CloudflareInterceptor : Interceptor {
         return response
     }
 
-    private fun resolveChallenge(response: Response): Request {
+    private fun isChallengeResolverUrl(url: String): Boolean {
-        Duktape.create().use { duktape ->
+        return "chk_jschl" in url
-            val originalRequest = response.request()
+    }
-            val url = originalRequest.url()
-            val domain = url.host()
-            val content = response.body()!!.string()
 
-            // CloudFlare requires waiting 4 seconds before resolving the challenge
+    @SuppressLint("SetJavaScriptEnabled")
-            Thread.sleep(4000)
+    private fun resolveWithWebView(request: Request): Boolean {
+        val latch = CountDownLatch(1)
 
-            val operation = operationPattern.find(content)?.groups?.get(1)?.value
+        var result = false
-            val challenge = challengePattern.find(content)?.groups?.get(1)?.value
+        var isResolvingChallenge = false
-            val pass = passPattern.find(content)?.groups?.get(1)?.value
-            val s = sPattern.find(content)?.groups?.get(1)?.value
 
-            // If `k` is null, it uses old methods.
+        val requestUrl = request.url().toString()
-            val k = kPattern.find(content)?.groups?.get(1)?.value ?: ""
+        val headers = request.headers().toMultimap().mapValues { it.value.getOrNull(0) ?: "" }
-            val innerHTMLValue = Regex("""<div(.*)id="$k"(.*)>(.*)</div>""")
-                    .find(content)?.groups?.get(3)?.value ?: ""
 
-            if (operation == null || challenge == null || pass == null || s == null) {
+        handler.post {
-                throw Exception("Failed resolving Cloudflare challenge")
+            val view = WebView(context)
+            view.settings.javaScriptEnabled = true
+            view.settings.userAgentString = request.header("User-Agent")
+            view.webViewClient = object : WebViewClientCompat() {
+
+                override fun shouldInterceptRequestCompat(
+                        view: WebView,
+                        url: String
+                ): WebResourceResponse? {
+                    val isChallengeResolverUrl = isChallengeResolverUrl(url)
+                    if (requestUrl != url && !isChallengeResolverUrl) {
+                        return WebResourceResponse("text/plain", "UTF-8", null)
+                    }
+
+                    if (isChallengeResolverUrl) {
+                        isResolvingChallenge = true
+                    }
+                    return null
+                }
+
+                override fun onPageFinished(view: WebView, url: String) {
+                    super.onPageFinished(view, url)
+                    if (isResolvingChallenge && url == requestUrl) {
+                        setResultAndFinish(true)
+                    }
+                }
+
+                override fun onReceivedErrorCompat(
+                        view: WebView,
+                        errorCode: Int,
+                        description: String?,
+                        failingUrl: String,
+                        isMainFrame: Boolean
+                ) {
+                    if ((errorCode != 503 && requestUrl == failingUrl) ||
+                        isChallengeResolverUrl(failingUrl)
+                    ) {
+                        setResultAndFinish(false)
+                    }
+                }
+
+                private fun setResultAndFinish(resolved: Boolean) {
+                    result = resolved
+                    latch.countDown()
+                    view.stopLoading()
+                    view.destroy()
+                }
             }
 
-            // Export native Base64 decode function to js object.
+            view.loadUrl(requestUrl, headers)
-            duktape.set("b64", IBase64::class.java, b64)
-
-            // Return simulated innerHTML when call DOM.
-            val simulatedDocumentJS = """var document = { getElementById: function (x) { return { innerHTML: "$innerHTMLValue" }; } }"""
-
-            val js = operation
-                    .replace(Regex("""a\.value = (.+\.toFixed\(10\);).+"""), "$1")
-                    .replace(Regex("""\s{3,}[a-z](?: = |\.).+"""), "")
-                    .replace("t.length", "${domain.length}")
-                    .replace("\n", "")
-
-            val result = duktape.evaluate("""$simulatedDocumentJS;$ATOB_JS;var t="$domain";$js""") as String
-
-            val cloudflareUrl = HttpUrl.parse("${url.scheme()}://$domain/cdn-cgi/l/chk_jschl")!!
-                    .newBuilder()
-                    .addQueryParameter("jschl_vc", challenge)
-                    .addQueryParameter("pass", pass)
-                    .addQueryParameter("s", s)
-                    .addQueryParameter("jschl_answer", result)
-                    .toString()
-
-            val cloudflareHeaders = originalRequest.headers()
-                    .newBuilder()
-                    .add("Referer", url.toString())
-                    .add("Accept", "text/html,application/xhtml+xml,application/xml")
-                    .add("Accept-Language", "en")
-                    .build()
-
-            return GET(cloudflareUrl, cloudflareHeaders, cache = CacheControl.Builder().build())
         }
+
+        latch.await(12, TimeUnit.SECONDS)
+
+        return result
     }
 
-    companion object {
+}
-        // atob() is browser API, Using Android's own function. (java.util.Base64 can't be used because of min API level)
-        private const val ATOB_JS = """var atob = function (input) { return b64.decode(input) }"""
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/network/NetworkHelper.kt

@@ -2,11 +2,7 @@ package eu.kanade.tachiyomi.network
 
 import android.content.Context
 import android.os.Build
-import okhttp3.Cache
+import okhttp3.*
-import okhttp3.CipherSuite
-import okhttp3.ConnectionSpec
-import okhttp3.OkHttpClient
-import okhttp3.TlsVersion
 import java.io.File
 import java.io.IOException
 import java.net.InetAddress
@@ -15,11 +11,7 @@ import java.net.UnknownHostException
 import java.security.KeyManagementException
 import java.security.KeyStore
 import java.security.NoSuchAlgorithmException
-import javax.net.ssl.SSLContext
+import javax.net.ssl.*
-import javax.net.ssl.SSLSocket
-import javax.net.ssl.SSLSocketFactory
-import javax.net.ssl.TrustManagerFactory
-import javax.net.ssl.X509TrustManager
 
 class NetworkHelper(context: Context) {
 
@@ -27,7 +19,7 @@ class NetworkHelper(context: Context) {
 
     private val cacheSize = 5L * 1024 * 1024 // 5 MiB
 
-    private val cookieManager = PersistentCookieJar(context)
+    val cookieManager = AndroidCookieJar(context)
 
     val client = OkHttpClient.Builder()
             .cookieJar(cookieManager)
@@ -36,12 +28,9 @@ class NetworkHelper(context: Context) {
             .build()
 
     val cloudflareClient = client.newBuilder()
-            .addInterceptor(CloudflareInterceptor())
+            .addInterceptor(CloudflareInterceptor(context))
             .build()
 
-    val cookies: PersistentCookieStore
-        get() = cookieManager.store
-
     private fun OkHttpClient.Builder.enableTLS12(): OkHttpClient.Builder {
         if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) {
             return this

app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieJar.kt

@@ -1,19 +0,0 @@
-package eu.kanade.tachiyomi.network
-
-import android.content.Context
-import okhttp3.Cookie
-import okhttp3.CookieJar
-import okhttp3.HttpUrl
-
-class PersistentCookieJar(context: Context) : CookieJar {
-
-    val store = PersistentCookieStore(context)
-
-    override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
-        store.addAll(url, cookies)
-    }
-
-    override fun loadForRequest(url: HttpUrl): List<Cookie> {
-        return store.get(url)
-    }
-}

app/src/main/java/eu/kanade/tachiyomi/network/PersistentCookieStore.kt

@@ -1,78 +0,0 @@
-package eu.kanade.tachiyomi.network
-
-import android.content.Context
-import okhttp3.Cookie
-import okhttp3.HttpUrl
-import java.net.URI
-import java.util.concurrent.ConcurrentHashMap
-
-class PersistentCookieStore(context: Context) {
-
-    private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
-    private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
-
-    init {
-        for ((key, value) in prefs.all) {
-            @Suppress("UNCHECKED_CAST")
-            val cookies = value as? Set<String>
-            if (cookies != null) {
-                try {
-                    val url = HttpUrl.parse("http://$key") ?: continue
-                    val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) }
-                            .filter { !it.hasExpired() }
-                    cookieMap.put(key, nonExpiredCookies)
-                } catch (e: Exception) {
-                    // Ignore
-                }
-            }
-        }
-    }
-
-    @Synchronized
-    fun addAll(url: HttpUrl, cookies: List<Cookie>) {
-        val key = url.uri().host
-
-        // Append or replace the cookies for this domain.
-        val cookiesForDomain = cookieMap[key].orEmpty().toMutableList()
-        for (cookie in cookies) {
-            // Find a cookie with the same name. Replace it if found, otherwise add a new one.
-            val pos = cookiesForDomain.indexOfFirst { it.name() == cookie.name() }
-            if (pos == -1) {
-                cookiesForDomain.add(cookie)
-            } else {
-                cookiesForDomain[pos] = cookie
-            }
-        }
-        cookieMap.put(key, cookiesForDomain)
-
-        // Get cookies to be stored in disk
-        val newValues = cookiesForDomain.asSequence()
-                .filter { it.persistent() && !it.hasExpired() }
-                .map(Cookie::toString)
-                .toSet()
-
-        prefs.edit().putStringSet(key, newValues).apply()
-    }
-
-    @Synchronized
-    fun removeAll() {
-        prefs.edit().clear().apply()
-        cookieMap.clear()
-    }
-
-    fun remove(uri: URI) {
-        prefs.edit().remove(uri.host).apply()
-        cookieMap.remove(uri.host)
-    }
-
-    fun get(url: HttpUrl) = get(url.uri().host)
-
-    fun get(uri: URI) = get(uri.host)
-
-    private fun get(url: String): List<Cookie> {
-        return cookieMap[url].orEmpty().filter { !it.hasExpired() }
-    }
-
-    private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt()
-
-}

app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt

@@ -43,7 +43,7 @@ class SettingsAdvancedController : SettingsController() {
             titleRes = R.string.pref_clear_cookies
 
             onClick {
-                network.cookies.removeAll()
+                network.cookieManager.removeAll()
                 activity?.toast(R.string.cookies_cleared)
             }
         }

app/src/main/java/eu/kanade/tachiyomi/util/WebViewClientCompat.kt

@@ -0,0 +1,83 @@
+package eu.kanade.tachiyomi.util
+
+import android.annotation.TargetApi
+import android.os.Build
+import android.webkit.*
+
+@Suppress("OverridingDeprecatedMember")
+abstract class WebViewClientCompat : WebViewClient() {
+
+    open fun shouldOverrideUrlCompat(view: WebView, url: String): Boolean {
+        return false
+    }
+
+    open fun shouldInterceptRequestCompat(view: WebView, url: String): WebResourceResponse? {
+        return null
+    }
+
+    open fun onReceivedErrorCompat(
+            view: WebView,
+            errorCode: Int,
+            description: String?,
+            failingUrl: String,
+            isMainFrame: Boolean) {
+
+    }
+
+    @TargetApi(Build.VERSION_CODES.N)
+    final override fun shouldOverrideUrlLoading(
+            view: WebView,
+            request: WebResourceRequest
+    ): Boolean {
+        return shouldOverrideUrlCompat(view, request.url.toString())
+    }
+
+    final override fun shouldOverrideUrlLoading(view: WebView, url: String): Boolean {
+        return shouldOverrideUrlCompat(view, url)
+    }
+
+    @TargetApi(Build.VERSION_CODES.LOLLIPOP)
+    final override fun shouldInterceptRequest(
+            view: WebView,
+            request: WebResourceRequest
+    ): WebResourceResponse? {
+        return shouldInterceptRequestCompat(view, request.url.toString())
+    }
+
+    final override fun shouldInterceptRequest(
+            view: WebView,
+            url: String
+    ): WebResourceResponse? {
+        return shouldInterceptRequestCompat(view, url)
+    }
+
+    @TargetApi(Build.VERSION_CODES.M)
+    final override fun onReceivedError(
+            view: WebView,
+            request: WebResourceRequest,
+            error: WebResourceError
+    ) {
+        onReceivedErrorCompat(view, error.errorCode, error.description?.toString(),
+                request.url.toString(), request.isForMainFrame)
+    }
+
+    final override fun onReceivedError(
+            view: WebView,
+            errorCode: Int,
+            description: String?,
+            failingUrl: String
+    ) {
+        onReceivedErrorCompat(view, errorCode, description, failingUrl, failingUrl == view.url)
+    }
+
+    @TargetApi(Build.VERSION_CODES.M)
+    final override fun onReceivedHttpError(
+            view: WebView,
+            request: WebResourceRequest,
+            error: WebResourceResponse
+    ) {
+        onReceivedErrorCompat(view, error.statusCode, error.reasonPhrase, request.url
+            .toString(), request.isForMainFrame)
+    }
+
+}