80 lines
2.9 KiB
Scala
80 lines
2.9 KiB
Scala
package com.twitter.timelines.prediction.common.aggregates.real_time
|
|
|
|
import com.twitter.conversions.DurationOps._
|
|
import com.twitter.finagle.stats.StatsReceiver
|
|
import com.twitter.storehaus.ReplicatedReadableStore
|
|
import com.twitter.storehaus.Store
|
|
import com.twitter.timelines.clients.memcache_common._
|
|
import com.twitter.timelines.util.FailOpenHandler
|
|
import com.twitter.util.Future
|
|
|
|
object ServedFeaturesMemcacheConfigBuilder {
|
|
def getTwCacheDestination(cluster: String, isProd: Boolean = false): String =
|
|
if (!isProd) {
|
|
s"/srv#/test/$cluster/cache//twemcache_timelines_served_features_cache"
|
|
} else {
|
|
s"/srv#/prod/$cluster/cache/timelines_served_features"
|
|
}
|
|
|
|
/**
|
|
* @cluster The DC of the cache that this client will send requests to. This
|
|
* can be different to the DC where the summingbird job is running in.
|
|
* @isProd Define if this client is part of a production summingbird job as
|
|
* different accesspoints will need to be chosen.
|
|
*/
|
|
def build(cluster: String, isProd: Boolean = false): StorehausMemcacheConfig =
|
|
StorehausMemcacheConfig(
|
|
destName = getTwCacheDestination(cluster, isProd),
|
|
keyPrefix = "",
|
|
requestTimeout = 200.milliseconds,
|
|
numTries = 2,
|
|
globalTimeout = 400.milliseconds,
|
|
tcpConnectTimeout = 200.milliseconds,
|
|
connectionAcquisitionTimeout = 200.milliseconds,
|
|
numPendingRequests = 1000,
|
|
isReadOnly = false
|
|
)
|
|
}
|
|
|
|
/**
|
|
* If lookup key does not exist locally, make a call to the replicated store(s).
|
|
* If value exists remotely, write the first returned value to the local store
|
|
* and return it. Map any exceptions to None so that the subsequent operations
|
|
* may proceed.
|
|
*/
|
|
class LocallyReplicatedStore[-K, V](
|
|
localStore: Store[K, V],
|
|
remoteStore: ReplicatedReadableStore[K, V],
|
|
scopedStatsReceiver: StatsReceiver)
|
|
extends Store[K, V] {
|
|
private[this] val failOpenHandler = new FailOpenHandler(scopedStatsReceiver.scope("failOpen"))
|
|
private[this] val localFailsCounter = scopedStatsReceiver.counter("localFails")
|
|
private[this] val localWritesCounter = scopedStatsReceiver.counter("localWrites")
|
|
private[this] val remoteFailsCounter = scopedStatsReceiver.counter("remoteFails")
|
|
|
|
override def get(k: K): Future[Option[V]] =
|
|
failOpenHandler {
|
|
localStore
|
|
.get(k)
|
|
.flatMap {
|
|
case Some(v) => Future.value(Some(v))
|
|
case _ => {
|
|
localFailsCounter.incr()
|
|
val replicatedOptFu = remoteStore.get(k)
|
|
// async write if result is not empty
|
|
replicatedOptFu.onSuccess {
|
|
case Some(v) => {
|
|
localWritesCounter.incr()
|
|
localStore.put((k, Some(v)))
|
|
}
|
|
case _ => {
|
|
remoteFailsCounter.incr()
|
|
Unit
|
|
}
|
|
}
|
|
replicatedOptFu
|
|
}
|
|
}
|
|
} { _: Throwable => Future.None }
|
|
}
|