From e57a999c9ce7653b735293a60dd990bea65d95c1 Mon Sep 17 00:00:00 2001 From: Gauthier Date: Mon, 24 May 2021 23:29:20 +0800 Subject: [PATCH] pull the rate limit interceptors from the extensions repo (#5163) apply a rate limit to anilist, current limit is 90 per minute --- .../data/track/anilist/AnilistApi.kt | 7 +- .../tachiyomi/network/RateLimitInterceptor.kt | 59 +++++++++++++++++ .../SpecificHostRateLimitInterceptor.kt | 65 +++++++++++++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 app/src/main/java/eu/kanade/tachiyomi/network/RateLimitInterceptor.kt create mode 100644 app/src/main/java/eu/kanade/tachiyomi/network/SpecificHostRateLimitInterceptor.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/track/anilist/AnilistApi.kt b/app/src/main/java/eu/kanade/tachiyomi/data/track/anilist/AnilistApi.kt index 18da573a68..6b0ae76fa1 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/track/anilist/AnilistApi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/track/anilist/AnilistApi.kt @@ -8,6 +8,7 @@ import com.afollestad.date.year import eu.kanade.tachiyomi.data.database.models.Track import eu.kanade.tachiyomi.data.track.model.TrackSearch import eu.kanade.tachiyomi.network.POST +import eu.kanade.tachiyomi.network.RateLimitInterceptor import eu.kanade.tachiyomi.network.await import eu.kanade.tachiyomi.network.jsonMime import eu.kanade.tachiyomi.network.parseAs @@ -27,10 +28,14 @@ import kotlinx.serialization.json.putJsonObject import okhttp3.OkHttpClient import okhttp3.RequestBody.Companion.toRequestBody import java.util.Calendar +import java.util.concurrent.TimeUnit.MINUTES class AnilistApi(val client: OkHttpClient, interceptor: AnilistInterceptor) { - private val authClient = client.newBuilder().addInterceptor(interceptor).build() + private val authClient = client.newBuilder() + .addInterceptor(interceptor) + .addInterceptor(RateLimitInterceptor(85, 1, MINUTES)) + .build() suspend fun addLibManga(track: Track): Track { return withIOContext { diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/RateLimitInterceptor.kt b/app/src/main/java/eu/kanade/tachiyomi/network/RateLimitInterceptor.kt new file mode 100644 index 0000000000..f17f4bcb46 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/network/RateLimitInterceptor.kt @@ -0,0 +1,59 @@ +package eu.kanade.tachiyomi.network + +import android.os.SystemClock +import okhttp3.Interceptor +import okhttp3.Response +import timber.log.Timber +import java.util.concurrent.TimeUnit + +/** + * An OkHttp interceptor that handles rate limiting. + * + * Examples: + * + * permits = 5, period = 1, unit = seconds => 5 requests per second + * permits = 10, period = 2, unit = minutes => 10 requests per 2 minutes + * + * @param permits {Int} Number of requests allowed within a period of units. + * @param period {Long} The limiting duration. Defaults to 1. + * @param unit {TimeUnit} The unit of time for the period. Defaults to seconds. + */ +class RateLimitInterceptor( + private val permits: Int, + private val period: Long = 1, + private val unit: TimeUnit = TimeUnit.SECONDS +) : Interceptor { + + private val requestQueue = ArrayList(permits) + private val rateLimitMillis = unit.toMillis(period) + + override fun intercept(chain: Interceptor.Chain): Response { + synchronized(requestQueue) { + val now = SystemClock.elapsedRealtime() + val waitTime = if (requestQueue.size < permits) { + 0 + } else { + val oldestReq = requestQueue[0] + val newestReq = requestQueue[permits - 1] + + if (newestReq - oldestReq > rateLimitMillis) { + 0 + } else { + oldestReq + rateLimitMillis - now // Remaining time + } + } + + if (requestQueue.size == permits) { + requestQueue.removeAt(0) + } + if (waitTime > 0) { + requestQueue.add(now + waitTime) + Thread.sleep(waitTime) // Sleep inside synchronized to pause queued requests + } else { + requestQueue.add(now) + } + } + + return chain.proceed(chain.request()) + } +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/network/SpecificHostRateLimitInterceptor.kt b/app/src/main/java/eu/kanade/tachiyomi/network/SpecificHostRateLimitInterceptor.kt new file mode 100644 index 0000000000..58920f6247 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/network/SpecificHostRateLimitInterceptor.kt @@ -0,0 +1,65 @@ +package eu.kanade.tachiyomi.network + +import android.os.SystemClock +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import java.util.concurrent.TimeUnit + +/** + * An OkHttp interceptor that handles given url host's rate limiting. + * + * Examples: + * + * httpUrl = "api.manga.com".toHttpUrlOrNull(), permits = 5, period = 1, unit = seconds => 5 requests per second to api.manga.com + * httpUrl = "imagecdn.manga.com".toHttpUrlOrNull(), permits = 10, period = 2, unit = minutes => 10 requests per 2 minutes to imagecdn.manga.com + * + * @param httpUrl {HttpUrl} The url host that this interceptor should handle. Will get url's host by using HttpUrl.host() + * @param permits {Int} Number of requests allowed within a period of units. + * @param period {Long} The limiting duration. Defaults to 1. + * @param unit {TimeUnit} The unit of time for the period. Defaults to seconds. + */ +class SpecificHostRateLimitInterceptor( + private val httpUrl: HttpUrl, + private val permits: Int, + private val period: Long = 1, + private val unit: TimeUnit = TimeUnit.SECONDS +) : Interceptor { + + private val requestQueue = ArrayList(permits) + private val rateLimitMillis = unit.toMillis(period) + private val host = httpUrl.host + + override fun intercept(chain: Interceptor.Chain): Response { + if (chain.request().url.host != host) { + return chain.proceed(chain.request()) + } + synchronized(requestQueue) { + val now = SystemClock.elapsedRealtime() + val waitTime = if (requestQueue.size < permits) { + 0 + } else { + val oldestReq = requestQueue[0] + val newestReq = requestQueue[permits - 1] + + if (newestReq - oldestReq > rateLimitMillis) { + 0 + } else { + oldestReq + rateLimitMillis - now // Remaining time + } + } + + if (requestQueue.size == permits) { + requestQueue.removeAt(0) + } + if (waitTime > 0) { + requestQueue.add(now + waitTime) + Thread.sleep(waitTime) // Sleep inside synchronized to pause queued requests + } else { + requestQueue.add(now) + } + } + + return chain.proceed(chain.request()) + } +}