[docx] split commit for file 6200
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
dbcd08179c
commit
4e32fcb29f
|
|
@ -1,60 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-incompatible-scaladoc", # see http://go/bazel-incompatible-scaladoc
|
||||
],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind",
|
||||
"3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala",
|
||||
"3rdparty/jvm/com/twitter/bijection:core",
|
||||
"3rdparty/jvm/com/twitter/bijection:thrift",
|
||||
"3rdparty/jvm/org/apache/thrift:libthrift",
|
||||
"diffshow",
|
||||
"fanoutservice/thrift/src/main/thrift:thrift-scala",
|
||||
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
|
||||
"finagle/finagle-core/src/main",
|
||||
"flock-client/src/main/scala",
|
||||
"mediaservices/commons/src/main/thrift:thrift-scala",
|
||||
"scrooge/scrooge-core",
|
||||
"tweetypie/servo/repo",
|
||||
"tweetypie/servo/repo/src/main/thrift:thrift-scala",
|
||||
"tweetypie/servo/util",
|
||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||
"src/scala/com/twitter/takedown/util",
|
||||
"src/thrift/com/twitter/context:feature-context-scala",
|
||||
"src/thrift/com/twitter/context:twitter-context-scala",
|
||||
"src/thrift/com/twitter/escherbird:media-annotation-structs-scala",
|
||||
"src/thrift/com/twitter/expandodo:cards-scala",
|
||||
"src/thrift/com/twitter/geoduck:geoduck-scala",
|
||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||
"src/thrift/com/twitter/gizmoduck:user-thrift-scala",
|
||||
"src/thrift/com/twitter/guano:guano-scala",
|
||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:audit-scala",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:events-scala",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:media-entity-scala",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"stitch/stitch-core",
|
||||
"stitch/stitch-core/src/main/scala/com/twitter/stitch",
|
||||
"tweetypie/server/src/main/scala/com/twitter/tweetypie",
|
||||
"tweetypie/server/src/main/scala/com/twitter/tweetypie/backends",
|
||||
"tweetypie/server/src/main/scala/com/twitter/tweetypie/core",
|
||||
"tweetypie/server/src/main/scala/com/twitter/tweetypie/media",
|
||||
"tweetypie/server/src/main/scala/com/twitter/tweetypie/repository",
|
||||
"tweetypie/server/src/main/scala/com/twitter/tweetypie/serverutil",
|
||||
"tweetypie/server/src/main/thrift:compiled-scala",
|
||||
"tweetypie/common/src/scala/com/twitter/tweetypie/additionalfields",
|
||||
"tweetypie/common/src/scala/com/twitter/tweetypie/client_id",
|
||||
"tweetypie/common/src/scala/com/twitter/tweetypie/media",
|
||||
"tweetypie/common/src/scala/com/twitter/tweetypie/storage",
|
||||
"tweetypie/common/src/scala/com/twitter/tweetypie/tflock",
|
||||
"tweetypie/common/src/scala/com/twitter/tweetypie/util",
|
||||
"twitter-context",
|
||||
"util/util-slf4j-api/src/main/scala/com/twitter/util/logging",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,420 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import com.fasterxml.jackson.module.scala.DefaultScalaModule
|
||||
import com.twitter.scrooge.TFieldBlob
|
||||
import com.twitter.servo.cache.LockingCache._
|
||||
import com.twitter.servo.cache._
|
||||
import com.twitter.tweetypie.additionalfields.AdditionalFields
|
||||
import com.twitter.tweetypie.repository.CachedBounceDeleted.isBounceDeleted
|
||||
import com.twitter.tweetypie.repository.CachedBounceDeleted.toBounceDeletedCachedTweet
|
||||
import com.twitter.tweetypie.repository._
|
||||
import com.twitter.tweetypie.store.TweetUpdate._
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
import com.twitter.util.Time
|
||||
import diffshow.DiffShow
|
||||
|
||||
trait CachingTweetStore
|
||||
extends TweetStoreBase[CachingTweetStore]
|
||||
with InsertTweet.Store
|
||||
with ReplicatedInsertTweet.Store
|
||||
with DeleteTweet.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with ReplicatedDeleteTweet.Store
|
||||
with UndeleteTweet.Store
|
||||
with AsyncUndeleteTweet.Store
|
||||
with ReplicatedUndeleteTweet.Store
|
||||
with SetAdditionalFields.Store
|
||||
with ReplicatedSetAdditionalFields.Store
|
||||
with DeleteAdditionalFields.Store
|
||||
with AsyncDeleteAdditionalFields.Store
|
||||
with ReplicatedDeleteAdditionalFields.Store
|
||||
with ScrubGeo.Store
|
||||
with ReplicatedScrubGeo.Store
|
||||
with Takedown.Store
|
||||
with ReplicatedTakedown.Store
|
||||
with Flush.Store
|
||||
with UpdatePossiblySensitiveTweet.Store
|
||||
with AsyncUpdatePossiblySensitiveTweet.Store
|
||||
with ReplicatedUpdatePossiblySensitiveTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): CachingTweetStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with CachingTweetStore
|
||||
with InsertTweet.StoreWrapper
|
||||
with ReplicatedInsertTweet.StoreWrapper
|
||||
with DeleteTweet.StoreWrapper
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with ReplicatedDeleteTweet.StoreWrapper
|
||||
with UndeleteTweet.StoreWrapper
|
||||
with AsyncUndeleteTweet.StoreWrapper
|
||||
with ReplicatedUndeleteTweet.StoreWrapper
|
||||
with SetAdditionalFields.StoreWrapper
|
||||
with ReplicatedSetAdditionalFields.StoreWrapper
|
||||
with DeleteAdditionalFields.StoreWrapper
|
||||
with AsyncDeleteAdditionalFields.StoreWrapper
|
||||
with ReplicatedDeleteAdditionalFields.StoreWrapper
|
||||
with ScrubGeo.StoreWrapper
|
||||
with ReplicatedScrubGeo.StoreWrapper
|
||||
with Takedown.StoreWrapper
|
||||
with ReplicatedTakedown.StoreWrapper
|
||||
with Flush.StoreWrapper
|
||||
with UpdatePossiblySensitiveTweet.StoreWrapper
|
||||
with AsyncUpdatePossiblySensitiveTweet.StoreWrapper
|
||||
with ReplicatedUpdatePossiblySensitiveTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object CachingTweetStore {
|
||||
val Action: AsyncWriteAction.CacheUpdate.type = AsyncWriteAction.CacheUpdate
|
||||
|
||||
def apply(
|
||||
tweetCache: LockingCache[TweetKey, Cached[CachedTweet]],
|
||||
tweetKeyFactory: TweetKeyFactory,
|
||||
stats: StatsReceiver
|
||||
): CachingTweetStore = {
|
||||
val ops =
|
||||
new CachingTweetStoreOps(
|
||||
tweetCache,
|
||||
tweetKeyFactory,
|
||||
stats
|
||||
)
|
||||
|
||||
new CachingTweetStore {
|
||||
override val insertTweet: FutureEffect[InsertTweet.Event] = {
|
||||
FutureEffect[InsertTweet.Event](e =>
|
||||
ops.insertTweet(e.internalTweet, e.initialTweetUpdateRequest))
|
||||
}
|
||||
|
||||
override val replicatedInsertTweet: FutureEffect[ReplicatedInsertTweet.Event] =
|
||||
FutureEffect[ReplicatedInsertTweet.Event](e =>
|
||||
ops.insertTweet(e.cachedTweet, e.initialTweetUpdateRequest))
|
||||
|
||||
override val deleteTweet: FutureEffect[DeleteTweet.Event] =
|
||||
FutureEffect[DeleteTweet.Event](e =>
|
||||
ops.deleteTweet(e.tweet.id, updateOnly = true, isBounceDelete = e.isBounceDelete))
|
||||
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
FutureEffect[AsyncDeleteTweet.Event](e =>
|
||||
ops.deleteTweet(e.tweet.id, updateOnly = true, isBounceDelete = e.isBounceDelete))
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
|
||||
override val replicatedDeleteTweet: FutureEffect[ReplicatedDeleteTweet.Event] =
|
||||
FutureEffect[ReplicatedDeleteTweet.Event](e =>
|
||||
ops.deleteTweet(
|
||||
tweetId = e.tweet.id,
|
||||
updateOnly = e.isErasure,
|
||||
isBounceDelete = e.isBounceDelete
|
||||
))
|
||||
|
||||
override val undeleteTweet: FutureEffect[UndeleteTweet.Event] =
|
||||
FutureEffect[UndeleteTweet.Event](e => ops.undeleteTweet(e.internalTweet))
|
||||
|
||||
override val asyncUndeleteTweet: FutureEffect[AsyncUndeleteTweet.Event] =
|
||||
FutureEffect[AsyncUndeleteTweet.Event](e => ops.undeleteTweet(e.cachedTweet))
|
||||
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUndeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUndeleteTweet)
|
||||
|
||||
override val replicatedUndeleteTweet: FutureEffect[ReplicatedUndeleteTweet.Event] =
|
||||
FutureEffect[ReplicatedUndeleteTweet.Event](e => ops.undeleteTweet(e.cachedTweet))
|
||||
|
||||
override val setAdditionalFields: FutureEffect[SetAdditionalFields.Event] =
|
||||
FutureEffect[SetAdditionalFields.Event](e => ops.setAdditionalFields(e.additionalFields))
|
||||
|
||||
override val replicatedSetAdditionalFields: FutureEffect[
|
||||
ReplicatedSetAdditionalFields.Event
|
||||
] =
|
||||
FutureEffect[ReplicatedSetAdditionalFields.Event](e =>
|
||||
ops.setAdditionalFields(e.additionalFields))
|
||||
|
||||
override val deleteAdditionalFields: FutureEffect[DeleteAdditionalFields.Event] =
|
||||
FutureEffect[DeleteAdditionalFields.Event](e =>
|
||||
ops.deleteAdditionalFields(e.tweetId, e.fieldIds))
|
||||
|
||||
override val asyncDeleteAdditionalFields: FutureEffect[AsyncDeleteAdditionalFields.Event] =
|
||||
FutureEffect[AsyncDeleteAdditionalFields.Event](e =>
|
||||
ops.deleteAdditionalFields(e.tweetId, e.fieldIds))
|
||||
|
||||
override val retryAsyncDeleteAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteAdditionalFields.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteAdditionalFields)
|
||||
|
||||
override val replicatedDeleteAdditionalFields: FutureEffect[
|
||||
ReplicatedDeleteAdditionalFields.Event
|
||||
] =
|
||||
FutureEffect[ReplicatedDeleteAdditionalFields.Event](e =>
|
||||
ops.deleteAdditionalFields(e.tweetId, e.fieldIds))
|
||||
|
||||
override val scrubGeo: FutureEffect[ScrubGeo.Event] =
|
||||
FutureEffect[ScrubGeo.Event](e => ops.scrubGeo(e.tweetIds))
|
||||
|
||||
override val replicatedScrubGeo: FutureEffect[ReplicatedScrubGeo.Event] =
|
||||
FutureEffect[ReplicatedScrubGeo.Event](e => ops.scrubGeo(e.tweetIds))
|
||||
|
||||
override val takedown: FutureEffect[Takedown.Event] =
|
||||
FutureEffect[Takedown.Event](e => ops.takedown(e.tweet))
|
||||
|
||||
override val replicatedTakedown: FutureEffect[ReplicatedTakedown.Event] =
|
||||
FutureEffect[ReplicatedTakedown.Event](e => ops.takedown(e.tweet))
|
||||
|
||||
override val flush: FutureEffect[Flush.Event] =
|
||||
FutureEffect[Flush.Event](e => ops.flushTweets(e.tweetIds, logExisting = e.logExisting))
|
||||
.onlyIf(_.flushTweets)
|
||||
|
||||
override val updatePossiblySensitiveTweet: FutureEffect[UpdatePossiblySensitiveTweet.Event] =
|
||||
FutureEffect[UpdatePossiblySensitiveTweet.Event](e => ops.updatePossiblySensitive(e.tweet))
|
||||
|
||||
override val replicatedUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
ReplicatedUpdatePossiblySensitiveTweet.Event
|
||||
] =
|
||||
FutureEffect[ReplicatedUpdatePossiblySensitiveTweet.Event](e =>
|
||||
ops.updatePossiblySensitive(e.tweet))
|
||||
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
AsyncUpdatePossiblySensitiveTweet.Event
|
||||
] =
|
||||
FutureEffect[AsyncUpdatePossiblySensitiveTweet.Event](e =>
|
||||
ops.updatePossiblySensitive(e.tweet))
|
||||
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUpdatePossiblySensitiveTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUpdatePossiblySensitiveTweet)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class CachingTweetStoreOps(
|
||||
tweetCache: LockingCache[TweetKey, Cached[CachedTweet]],
|
||||
tweetKeyFactory: TweetKeyFactory,
|
||||
stats: StatsReceiver,
|
||||
evictionRetries: Int = 3) {
|
||||
type CachedTweetHandler = Handler[Cached[CachedTweet]]
|
||||
|
||||
private val preferNewestPicker = new PreferNewestCached[CachedTweet]
|
||||
|
||||
private val evictionFailedCounter = stats.counter("eviction_failures")
|
||||
|
||||
private val cacheFlushesLog = Logger("com.twitter.tweetypie.store.CacheFlushesLog")
|
||||
|
||||
private[this] val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
|
||||
|
||||
/**
|
||||
* Inserts a tweet into cache, recording all compiled additional fields and all
|
||||
* included passthrough fields. Additionally if the insertion event contains
|
||||
* a 'InitialTweetUpdateRequest` we will update the cache entry for this tweet's
|
||||
* initialTweet.
|
||||
*/
|
||||
def insertTweet(
|
||||
ct: CachedTweet,
|
||||
initialTweetUpdateRequest: Option[InitialTweetUpdateRequest]
|
||||
): Future[Unit] =
|
||||
lockAndSet(
|
||||
ct.tweet.id,
|
||||
insertTweetHandler(ct)
|
||||
).flatMap { _ =>
|
||||
initialTweetUpdateRequest match {
|
||||
case Some(request) =>
|
||||
lockAndSet(
|
||||
request.initialTweetId,
|
||||
updateTweetHandler(tweet => InitialTweetUpdate.updateTweet(tweet, request))
|
||||
)
|
||||
case None =>
|
||||
Future.Unit
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a `deleted` tombstone to cache. If `updateOnly` is true, then we only
|
||||
* write the tombstone if the tweet is already in cache. If `isBounceDelete` we
|
||||
* write a special bounce-deleted CachedTweet record to cache.
|
||||
*/
|
||||
def deleteTweet(tweetId: TweetId, updateOnly: Boolean, isBounceDelete: Boolean): Future[Unit] = {
|
||||
// We only need to store a CachedTweet value the tweet is bounce-deleted to support rendering
|
||||
// timeline tombstones for tweets that violated the Twitter Rules. see go/bounced-tweet
|
||||
val cachedValue = if (isBounceDelete) {
|
||||
found(toBounceDeletedCachedTweet(tweetId))
|
||||
} else {
|
||||
writeThroughCached[CachedTweet](None, CachedValueStatus.Deleted)
|
||||
}
|
||||
|
||||
val pickerHandler =
|
||||
if (updateOnly) {
|
||||
deleteTweetUpdateOnlyHandler(cachedValue)
|
||||
} else {
|
||||
deleteTweetHandler(cachedValue)
|
||||
}
|
||||
|
||||
lockAndSet(tweetId, pickerHandler)
|
||||
}
|
||||
|
||||
def undeleteTweet(ct: CachedTweet): Future[Unit] =
|
||||
lockAndSet(
|
||||
ct.tweet.id,
|
||||
insertTweetHandler(ct)
|
||||
)
|
||||
|
||||
def setAdditionalFields(tweet: Tweet): Future[Unit] =
|
||||
lockAndSet(tweet.id, setFieldsHandler(AdditionalFields.additionalFields(tweet)))
|
||||
|
||||
def deleteAdditionalFields(tweetId: TweetId, fieldIds: Seq[FieldId]): Future[Unit] =
|
||||
lockAndSet(tweetId, deleteFieldsHandler(fieldIds))
|
||||
|
||||
def scrubGeo(tweetIds: Seq[TweetId]): Future[Unit] =
|
||||
Future.join {
|
||||
tweetIds.map { id =>
|
||||
// First, attempt to modify any tweets that are in cache to
|
||||
// avoid having to reload the cached tweet from storage.
|
||||
lockAndSet(id, scrubGeoHandler).unit.rescue {
|
||||
case _: OptimisticLockingCache.LockAndSetFailure =>
|
||||
// If the modification fails, then remove whatever is in
|
||||
// cache. This is much more likely to succeed because it
|
||||
// does not require multiple successful requests to cache.
|
||||
// This will force the tweet to be loaded from storage the
|
||||
// next time it is requested, and the stored tweet will have
|
||||
// the geo information removed.
|
||||
//
|
||||
// This eviction path was added due to frequent failures of
|
||||
// the in-place modification code path, causing geoscrub
|
||||
// daemon tasks to fail.
|
||||
evictOne(tweetKeyFactory.fromId(id), evictionRetries)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def takedown(tweet: Tweet): Future[Unit] =
|
||||
lockAndSet(tweet.id, updateCachedTweetHandler(copyTakedownFieldsForUpdate(tweet)))
|
||||
|
||||
def updatePossiblySensitive(tweet: Tweet): Future[Unit] =
|
||||
lockAndSet(tweet.id, updateTweetHandler(copyNsfwFieldsForUpdate(tweet)))
|
||||
|
||||
def flushTweets(tweetIds: Seq[TweetId], logExisting: Boolean = false): Future[Unit] = {
|
||||
val tweetKeys = tweetIds.map(tweetKeyFactory.fromId)
|
||||
|
||||
Future.when(logExisting) { logExistingValues(tweetKeys) }.ensure {
|
||||
evictAll(tweetKeys)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A LockingCache.Handler that inserts a tweet into cache.
|
||||
*/
|
||||
private def insertTweetHandler(newValue: CachedTweet): Handler[Cached[CachedTweet]] =
|
||||
AlwaysSetHandler(Some(writeThroughCached(Some(newValue), CachedValueStatus.Found)))
|
||||
|
||||
private def foundAndNotBounced(c: Cached[CachedTweet]) =
|
||||
c.status == CachedValueStatus.Found && !isBounceDeleted(c)
|
||||
|
||||
/**
|
||||
* A LockingCache.Handler that updates an existing CachedTweet in cache.
|
||||
*/
|
||||
private def updateTweetHandler(update: Tweet => Tweet): CachedTweetHandler =
|
||||
inCache =>
|
||||
for {
|
||||
cached <- inCache.filter(foundAndNotBounced)
|
||||
cachedTweet <- cached.value
|
||||
updatedTweet = update(cachedTweet.tweet)
|
||||
} yield found(cachedTweet.copy(tweet = updatedTweet))
|
||||
|
||||
/**
|
||||
* A LockingCache.Handler that updates an existing CachedTweet in cache.
|
||||
*/
|
||||
private def updateCachedTweetHandler(update: CachedTweet => CachedTweet): CachedTweetHandler =
|
||||
inCache =>
|
||||
for {
|
||||
cached <- inCache.filter(foundAndNotBounced)
|
||||
cachedTweet <- cached.value
|
||||
updatedCachedTweet = update(cachedTweet)
|
||||
} yield found(updatedCachedTweet)
|
||||
|
||||
private def deleteTweetHandler(value: Cached[CachedTweet]): CachedTweetHandler =
|
||||
PickingHandler(value, preferNewestPicker)
|
||||
|
||||
private def deleteTweetUpdateOnlyHandler(value: Cached[CachedTweet]): CachedTweetHandler =
|
||||
UpdateOnlyPickingHandler(value, preferNewestPicker)
|
||||
|
||||
private def setFieldsHandler(additional: Seq[TFieldBlob]): CachedTweetHandler =
|
||||
inCache =>
|
||||
for {
|
||||
cached <- inCache.filter(foundAndNotBounced)
|
||||
cachedTweet <- cached.value
|
||||
updatedTweet = AdditionalFields.setAdditionalFields(cachedTweet.tweet, additional)
|
||||
updatedCachedTweet = CachedTweet(updatedTweet)
|
||||
} yield found(updatedCachedTweet)
|
||||
|
||||
private def deleteFieldsHandler(fieldIds: Seq[FieldId]): CachedTweetHandler =
|
||||
inCache =>
|
||||
for {
|
||||
cached <- inCache.filter(foundAndNotBounced)
|
||||
cachedTweet <- cached.value
|
||||
updatedTweet = AdditionalFields.unsetFields(cachedTweet.tweet, fieldIds)
|
||||
scrubbedCachedTweet = cachedTweet.copy(tweet = updatedTweet)
|
||||
} yield found(scrubbedCachedTweet)
|
||||
|
||||
private val scrubGeoHandler: CachedTweetHandler =
|
||||
inCache =>
|
||||
for {
|
||||
cached <- inCache.filter(foundAndNotBounced)
|
||||
cachedTweet <- cached.value
|
||||
tweet = cachedTweet.tweet
|
||||
coreData <- tweet.coreData if hasGeo(tweet)
|
||||
scrubbedCoreData = coreData.copy(coordinates = None, placeId = None)
|
||||
scrubbedTweet = tweet.copy(coreData = Some(scrubbedCoreData), place = None)
|
||||
scrubbedCachedTweet = cachedTweet.copy(tweet = scrubbedTweet)
|
||||
} yield found(scrubbedCachedTweet)
|
||||
|
||||
private def evictOne(key: TweetKey, tries: Int): Future[Int] =
|
||||
tweetCache.delete(key).transform {
|
||||
case Throw(_) if tries > 1 => evictOne(key, tries - 1)
|
||||
case Throw(_) => Future.value(1)
|
||||
case Return(_) => Future.value(0)
|
||||
}
|
||||
|
||||
private def evictAll(keys: Seq[TweetKey]): Future[Unit] =
|
||||
Future
|
||||
.collect {
|
||||
keys.map(evictOne(_, evictionRetries))
|
||||
}
|
||||
.onSuccess { (failures: Seq[Int]) => evictionFailedCounter.incr(failures.sum) }
|
||||
.unit
|
||||
|
||||
private def logExistingValues(keys: Seq[TweetKey]): Future[Unit] =
|
||||
tweetCache
|
||||
.get(keys)
|
||||
.map { existing =>
|
||||
for {
|
||||
(key, cached) <- existing.found
|
||||
cachedTweet <- cached.value
|
||||
tweet = cachedTweet.tweet
|
||||
} yield {
|
||||
cacheFlushesLog.info(
|
||||
mapper.writeValueAsString(
|
||||
Map(
|
||||
"key" -> key,
|
||||
"tweet_id" -> tweet.id,
|
||||
"tweet" -> DiffShow.show(tweet)
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
.unit
|
||||
|
||||
private def found(value: CachedTweet): Cached[CachedTweet] =
|
||||
writeThroughCached(Some(value), CachedValueStatus.Found)
|
||||
|
||||
private def writeThroughCached[V](value: Option[V], status: CachedValueStatus): Cached[V] = {
|
||||
val now = Time.now
|
||||
Cached(value, status, now, None, Some(now))
|
||||
}
|
||||
|
||||
private def lockAndSet(tweetId: TweetId, handler: LockingCache.Handler[Cached[CachedTweet]]) =
|
||||
tweetCache.lockAndSet(tweetKeyFactory.fromId(tweetId), handler).unit
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,172 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object DeleteAdditionalFields extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(tweetId: TweetId, fieldIds: Seq[FieldId], userId: UserId, timestamp: Time)
|
||||
extends SyncTweetStoreEvent("delete_additional_fields") {
|
||||
|
||||
def toAsyncRequest: AsyncDeleteAdditionalFieldsRequest =
|
||||
AsyncDeleteAdditionalFieldsRequest(
|
||||
tweetId = tweetId,
|
||||
fieldIds = fieldIds,
|
||||
userId = userId,
|
||||
timestamp = timestamp.inMillis
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val deleteAdditionalFields: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val deleteAdditionalFields: FutureEffect[Event] = wrap(
|
||||
underlying.deleteAdditionalFields)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
asyncEnqueueStore: AsyncEnqueueStore,
|
||||
logLensStore: LogLensStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val deleteAdditionalFields: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
// ignore failures deleting from cache, will be retried in async-path
|
||||
cachingTweetStore.ignoreFailures.deleteAdditionalFields,
|
||||
asyncEnqueueStore.deleteAdditionalFields,
|
||||
logLensStore.deleteAdditionalFields
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncDeleteAdditionalFields extends TweetStore.AsyncModule {
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(
|
||||
request: AsyncDeleteAdditionalFieldsRequest,
|
||||
user: User
|
||||
): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
Event(
|
||||
tweetId = request.tweetId,
|
||||
fieldIds = request.fieldIds,
|
||||
userId = request.userId,
|
||||
optUser = Some(user),
|
||||
timestamp = Time.fromMilliseconds(request.timestamp)
|
||||
),
|
||||
request.retryAction,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class Event(
|
||||
tweetId: TweetId,
|
||||
fieldIds: Seq[FieldId],
|
||||
userId: UserId,
|
||||
optUser: Option[User],
|
||||
timestamp: Time)
|
||||
extends AsyncTweetStoreEvent("async_delete_additional_fields")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
def toAsyncRequest(
|
||||
action: Option[AsyncWriteAction] = None
|
||||
): AsyncDeleteAdditionalFieldsRequest =
|
||||
AsyncDeleteAdditionalFieldsRequest(
|
||||
tweetId = tweetId,
|
||||
fieldIds = fieldIds,
|
||||
userId = userId,
|
||||
timestamp = timestamp.inMillis,
|
||||
retryAction = action
|
||||
)
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.AdditionalFieldDeleteEvent(
|
||||
AdditionalFieldDeleteEvent(
|
||||
deletedFields = Map(tweetId -> fieldIds),
|
||||
userId = optUser.map(_.id)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncDeleteAdditionalFields(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.DeleteAdditionalFields.type =
|
||||
AsyncWriteEventType.DeleteAdditionalFields
|
||||
override val scribedTweetOnFailure: None.type = None
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncDeleteAdditionalFields: FutureEffect[Event]
|
||||
val retryAsyncDeleteAdditionalFields: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncDeleteAdditionalFields: FutureEffect[Event] = wrap(
|
||||
underlying.asyncDeleteAdditionalFields)
|
||||
override val retryAsyncDeleteAdditionalFields: FutureEffect[TweetStoreRetryEvent[Event]] = wrap(
|
||||
underlying.retryAsyncDeleteAdditionalFields
|
||||
)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
replicatingStore: ReplicatingTweetStore,
|
||||
eventBusEnqueueStore: TweetEventBusStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] =
|
||||
Seq(
|
||||
manhattanStore,
|
||||
cachingTweetStore,
|
||||
replicatingStore,
|
||||
eventBusEnqueueStore
|
||||
)
|
||||
|
||||
def build[E <: TweetStoreEvent](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncDeleteAdditionalFields: FutureEffect[Event] = build(
|
||||
_.asyncDeleteAdditionalFields)
|
||||
override val retryAsyncDeleteAdditionalFields: FutureEffect[TweetStoreRetryEvent[Event]] =
|
||||
build(_.retryAsyncDeleteAdditionalFields)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedDeleteAdditionalFields extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(tweetId: TweetId, fieldIds: Seq[FieldId])
|
||||
extends ReplicatedTweetStoreEvent("replicated_delete_additional_fields")
|
||||
|
||||
trait Store {
|
||||
val replicatedDeleteAdditionalFields: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedDeleteAdditionalFields: FutureEffect[Event] =
|
||||
wrap(underlying.replicatedDeleteAdditionalFields)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(cachingTweetStore: CachingTweetStore): Store = {
|
||||
new Store {
|
||||
override val replicatedDeleteAdditionalFields: FutureEffect[Event] =
|
||||
cachingTweetStore.replicatedDeleteAdditionalFields
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,221 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.store.TweetEventDataScrubber.scrub
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object DeleteTweet extends TweetStore.SyncModule {
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
timestamp: Time,
|
||||
user: Option[User] = None,
|
||||
byUserId: Option[UserId] = None,
|
||||
auditPassthrough: Option[AuditDeleteTweet] = None,
|
||||
cascadedFromTweetId: Option[TweetId] = None,
|
||||
isUserErasure: Boolean = false,
|
||||
isBounceDelete: Boolean = false,
|
||||
isLastQuoteOfQuoter: Boolean = false,
|
||||
isAdminDelete: Boolean)
|
||||
extends SyncTweetStoreEvent("delete_tweet") {
|
||||
|
||||
def toAsyncRequest: AsyncDeleteRequest =
|
||||
AsyncDeleteRequest(
|
||||
tweet = tweet,
|
||||
user = user,
|
||||
byUserId = byUserId,
|
||||
timestamp = timestamp.inMillis,
|
||||
auditPassthrough = auditPassthrough,
|
||||
cascadedFromTweetId = cascadedFromTweetId,
|
||||
isUserErasure = isUserErasure,
|
||||
isBounceDelete = isBounceDelete,
|
||||
isLastQuoteOfQuoter = Some(isLastQuoteOfQuoter),
|
||||
isAdminDelete = Some(isAdminDelete)
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val deleteTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val deleteTweet: FutureEffect[Event] = wrap(underlying.deleteTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
asyncEnqueueStore: AsyncEnqueueStore,
|
||||
userCountsUpdatingStore: GizmoduckUserCountsUpdatingStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore,
|
||||
logLensStore: LogLensStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val deleteTweet: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
cachingTweetStore.ignoreFailures.deleteTweet,
|
||||
asyncEnqueueStore.deleteTweet,
|
||||
userCountsUpdatingStore.deleteTweet,
|
||||
tweetCountsUpdatingStore.deleteTweet,
|
||||
logLensStore.deleteTweet
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncDeleteTweet extends TweetStore.AsyncModule {
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(request: AsyncDeleteRequest): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
AsyncDeleteTweet.Event(
|
||||
tweet = request.tweet,
|
||||
timestamp = Time.fromMilliseconds(request.timestamp),
|
||||
optUser = request.user,
|
||||
byUserId = request.byUserId,
|
||||
auditPassthrough = request.auditPassthrough,
|
||||
cascadedFromTweetId = request.cascadedFromTweetId,
|
||||
isUserErasure = request.isUserErasure,
|
||||
isBounceDelete = request.isBounceDelete,
|
||||
isLastQuoteOfQuoter = request.isLastQuoteOfQuoter.getOrElse(false),
|
||||
isAdminDelete = request.isAdminDelete.getOrElse(false)
|
||||
),
|
||||
request.retryAction,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
timestamp: Time,
|
||||
optUser: Option[User] = None,
|
||||
byUserId: Option[UserId] = None,
|
||||
auditPassthrough: Option[AuditDeleteTweet] = None,
|
||||
cascadedFromTweetId: Option[TweetId] = None,
|
||||
isUserErasure: Boolean = false,
|
||||
isBounceDelete: Boolean,
|
||||
isLastQuoteOfQuoter: Boolean = false,
|
||||
isAdminDelete: Boolean)
|
||||
extends AsyncTweetStoreEvent("async_delete_tweet")
|
||||
with TweetStoreTweetEvent {
|
||||
val tweetEventTweetId: TweetId = tweet.id
|
||||
|
||||
def toAsyncRequest(action: Option[AsyncWriteAction] = None): AsyncDeleteRequest =
|
||||
AsyncDeleteRequest(
|
||||
tweet = tweet,
|
||||
user = optUser,
|
||||
byUserId = byUserId,
|
||||
timestamp = timestamp.inMillis,
|
||||
auditPassthrough = auditPassthrough,
|
||||
cascadedFromTweetId = cascadedFromTweetId,
|
||||
retryAction = action,
|
||||
isUserErasure = isUserErasure,
|
||||
isBounceDelete = isBounceDelete,
|
||||
isLastQuoteOfQuoter = Some(isLastQuoteOfQuoter),
|
||||
isAdminDelete = Some(isAdminDelete)
|
||||
)
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.TweetDeleteEvent(
|
||||
TweetDeleteEvent(
|
||||
tweet = scrub(tweet),
|
||||
user = optUser,
|
||||
isUserErasure = Some(isUserErasure),
|
||||
audit = auditPassthrough,
|
||||
byUserId = byUserId,
|
||||
isAdminDelete = Some(isAdminDelete)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncDelete(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.Delete.type = AsyncWriteEventType.Delete
|
||||
override val scribedTweetOnFailure: Option[Tweet] = Some(event.tweet)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncDeleteTweet: FutureEffect[Event]
|
||||
val retryAsyncDeleteTweet: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncDeleteTweet: FutureEffect[Event] = wrap(underlying.asyncDeleteTweet)
|
||||
override val retryAsyncDeleteTweet: FutureEffect[TweetStoreRetryEvent[Event]] = wrap(
|
||||
underlying.retryAsyncDeleteTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
replicatingStore: ReplicatingTweetStore,
|
||||
indexingStore: TweetIndexingStore,
|
||||
eventBusEnqueueStore: TweetEventBusStore,
|
||||
timelineUpdatingStore: TlsTimelineUpdatingStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore,
|
||||
guanoServiceStore: GuanoServiceStore,
|
||||
mediaServiceStore: MediaServiceStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] =
|
||||
Seq(
|
||||
manhattanStore,
|
||||
cachingTweetStore,
|
||||
replicatingStore,
|
||||
indexingStore,
|
||||
eventBusEnqueueStore,
|
||||
timelineUpdatingStore,
|
||||
tweetCountsUpdatingStore,
|
||||
guanoServiceStore,
|
||||
mediaServiceStore
|
||||
)
|
||||
|
||||
def build[E <: TweetStoreEvent](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncDeleteTweet: FutureEffect[Event] = build(_.asyncDeleteTweet)
|
||||
override val retryAsyncDeleteTweet: FutureEffect[TweetStoreRetryEvent[Event]] = build(
|
||||
_.retryAsyncDeleteTweet)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedDeleteTweet extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
isErasure: Boolean,
|
||||
isBounceDelete: Boolean,
|
||||
isLastQuoteOfQuoter: Boolean = false)
|
||||
extends ReplicatedTweetStoreEvent("replicated_delete_tweet")
|
||||
|
||||
trait Store {
|
||||
val replicatedDeleteTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedDeleteTweet: FutureEffect[Event] = wrap(underlying.replicatedDeleteTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore
|
||||
): Store = {
|
||||
new Store {
|
||||
override val replicatedDeleteTweet: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
cachingTweetStore.replicatedDeleteTweet,
|
||||
tweetCountsUpdatingStore.replicatedDeleteTweet.ignoreFailures
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,38 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.timelineservice.fanout.thriftscala.FanoutService
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait FanoutServiceStore extends TweetStoreBase[FanoutServiceStore] with AsyncInsertTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): FanoutServiceStore =
|
||||
new TweetStoreWrapper(w, this) with FanoutServiceStore with AsyncInsertTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object FanoutServiceStore {
|
||||
val Action: AsyncWriteAction.FanoutDelivery.type = AsyncWriteAction.FanoutDelivery
|
||||
|
||||
def apply(
|
||||
fanoutClient: FanoutService.MethodPerEndpoint,
|
||||
stats: StatsReceiver
|
||||
): FanoutServiceStore =
|
||||
new FanoutServiceStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
FutureEffect[AsyncInsertTweet.Event] { event =>
|
||||
fanoutClient.tweetCreateEvent2(
|
||||
TweetCreateEvent(
|
||||
tweet = event.tweet,
|
||||
user = event.user,
|
||||
sourceTweet = event.sourceTweet,
|
||||
sourceUser = event.sourceUser,
|
||||
additionalContext = event.additionalContext,
|
||||
transientContext = event.transientContext
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] = TweetStore.retry(Action, asyncInsertTweet)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,34 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
object Flush extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
tweetIds: Seq[TweetId],
|
||||
flushTweets: Boolean = true,
|
||||
flushCounts: Boolean = true,
|
||||
logExisting: Boolean = true)
|
||||
extends SyncTweetStoreEvent("flush")
|
||||
|
||||
trait Store {
|
||||
val flush: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val flush: FutureEffect[Event] = wrap(underlying.flush)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val flush: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
cachingTweetStore.flush,
|
||||
tweetCountsUpdatingStore.flush
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,72 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.geoduck.backend.relevance.thriftscala.ReportFailure
|
||||
import com.twitter.geoduck.backend.relevance.thriftscala.ReportResult
|
||||
import com.twitter.geoduck.backend.relevance.thriftscala.ConversionReport
|
||||
import com.twitter.geoduck.backend.searchrequestid.thriftscala.SearchRequestID
|
||||
import com.twitter.geoduck.backend.tweetid.thriftscala.TweetID
|
||||
import com.twitter.geoduck.common.thriftscala.GeoduckException
|
||||
import com.twitter.geoduck.service.identifier.thriftscala.PlaceIdentifier
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait GeoSearchRequestIDStore
|
||||
extends TweetStoreBase[GeoSearchRequestIDStore]
|
||||
with AsyncInsertTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): GeoSearchRequestIDStore =
|
||||
new TweetStoreWrapper[GeoSearchRequestIDStore](w, this)
|
||||
with GeoSearchRequestIDStore
|
||||
with AsyncInsertTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object GeoSearchRequestIDStore {
|
||||
type ConversionReporter = FutureArrow[ConversionReport, ReportResult]
|
||||
|
||||
val Action: AsyncWriteAction.GeoSearchRequestId.type = AsyncWriteAction.GeoSearchRequestId
|
||||
private val log = Logger(getClass)
|
||||
|
||||
object FailureHandler {
|
||||
def translateException(failure: ReportResult.Failure): GeoduckException = {
|
||||
failure.failure match {
|
||||
case ReportFailure.Failure(exception) => exception
|
||||
case _ => GeoduckException("Unknown failure: " + failure.toString)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def apply(conversionReporter: ConversionReporter): GeoSearchRequestIDStore =
|
||||
new GeoSearchRequestIDStore {
|
||||
|
||||
val conversionEffect: FutureEffect[ConversionReport] =
|
||||
FutureEffect
|
||||
.fromPartial[ReportResult] {
|
||||
case unionFailure: ReportResult.Failure =>
|
||||
Future.exception(FailureHandler.translateException(unionFailure))
|
||||
}
|
||||
.contramapFuture(conversionReporter)
|
||||
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
conversionEffect.contramapOption[AsyncInsertTweet.Event] { event =>
|
||||
for {
|
||||
isUserProtected <- event.user.safety.map(_.isProtected)
|
||||
geoSearchRequestID <- event.geoSearchRequestId
|
||||
placeType <- event.tweet.place.map(_.`type`)
|
||||
placeId <- event.tweet.coreData.flatMap(_.placeId)
|
||||
placeIdLong <- Try(java.lang.Long.parseUnsignedLong(placeId, 16)).toOption
|
||||
if placeType == PlaceType.Poi && isUserProtected == false
|
||||
} yield {
|
||||
ConversionReport(
|
||||
requestID = SearchRequestID(requestID = geoSearchRequestID),
|
||||
tweetID = TweetID(event.tweet.id),
|
||||
placeID = PlaceIdentifier(placeIdLong)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncInsertTweet)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,48 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.gizmoduck.thriftscala.{CountsUpdateField => Field}
|
||||
import com.twitter.tweetypie.backends.Gizmoduck
|
||||
|
||||
trait GizmoduckUserCountsUpdatingStore
|
||||
extends TweetStoreBase[GizmoduckUserCountsUpdatingStore]
|
||||
with InsertTweet.Store
|
||||
with DeleteTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): GizmoduckUserCountsUpdatingStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with GizmoduckUserCountsUpdatingStore
|
||||
with InsertTweet.StoreWrapper
|
||||
with DeleteTweet.StoreWrapper
|
||||
}
|
||||
|
||||
/**
|
||||
* A TweetStore implementation that sends user-specific count updates to Gizmoduck.
|
||||
*/
|
||||
object GizmoduckUserCountsUpdatingStore {
|
||||
def isUserTweet(tweet: Tweet): Boolean =
|
||||
!TweetLenses.nullcast.get(tweet) && TweetLenses.narrowcast.get(tweet).isEmpty
|
||||
|
||||
def apply(
|
||||
incr: Gizmoduck.IncrCount,
|
||||
hasMedia: Tweet => Boolean
|
||||
): GizmoduckUserCountsUpdatingStore = {
|
||||
def incrField(field: Field, amt: Int): FutureEffect[Tweet] =
|
||||
FutureEffect[Tweet](tweet => incr((getUserId(tweet), field, amt)))
|
||||
|
||||
def incrAll(amt: Int): FutureEffect[Tweet] =
|
||||
FutureEffect.inParallel(
|
||||
incrField(Field.Tweets, amt).onlyIf(isUserTweet),
|
||||
incrField(Field.MediaTweets, amt).onlyIf(t => isUserTweet(t) && hasMedia(t))
|
||||
)
|
||||
|
||||
new GizmoduckUserCountsUpdatingStore {
|
||||
override val insertTweet: FutureEffect[InsertTweet.Event] =
|
||||
incrAll(1).contramap[InsertTweet.Event](_.tweet)
|
||||
|
||||
override val deleteTweet: FutureEffect[DeleteTweet.Event] =
|
||||
incrAll(-1)
|
||||
.contramap[DeleteTweet.Event](_.tweet)
|
||||
.onlyIf(!_.isUserErasure)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,68 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.gizmoduck.thriftscala.LookupContext
|
||||
import com.twitter.gizmoduck.thriftscala.ModifiedAccount
|
||||
import com.twitter.gizmoduck.thriftscala.ModifiedUser
|
||||
import com.twitter.tweetypie.backends.Gizmoduck
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait GizmoduckUserGeotagUpdateStore
|
||||
extends TweetStoreBase[GizmoduckUserGeotagUpdateStore]
|
||||
with AsyncInsertTweet.Store
|
||||
with ScrubGeoUpdateUserTimestamp.Store {
|
||||
def wrap(w: TweetStore.Wrap): GizmoduckUserGeotagUpdateStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with GizmoduckUserGeotagUpdateStore
|
||||
with AsyncInsertTweet.StoreWrapper
|
||||
with ScrubGeoUpdateUserTimestamp.StoreWrapper
|
||||
}
|
||||
|
||||
/**
|
||||
* A TweetStore implementation that updates a Gizmoduck user's user_has_geotagged_status flag.
|
||||
* If a tweet is geotagged and the user's flag is not set, call out to Gizmoduck to update it.
|
||||
*/
|
||||
object GizmoduckUserGeotagUpdateStore {
|
||||
val Action: AsyncWriteAction.UserGeotagUpdate.type = AsyncWriteAction.UserGeotagUpdate
|
||||
|
||||
def apply(
|
||||
modifyAndGet: Gizmoduck.ModifyAndGet,
|
||||
stats: StatsReceiver
|
||||
): GizmoduckUserGeotagUpdateStore = {
|
||||
// Counts the number of times that the scrubGeo actually cleared the
|
||||
// hasGeotaggedStatuses bit for a user.
|
||||
val clearedCounter = stats.counter("has_geotag_cleared")
|
||||
|
||||
// Counts the number of times that asyncInsertTweet actually set the
|
||||
// hasGeotaggedStatuses bit for a user.
|
||||
val setCounter = stats.counter("has_geotag_set")
|
||||
|
||||
def setHasGeotaggedStatuses(value: Boolean): FutureEffect[UserId] = {
|
||||
val modifiedAccount = ModifiedAccount(hasGeotaggedStatuses = Some(value))
|
||||
val modifiedUser = ModifiedUser(account = Some(modifiedAccount))
|
||||
FutureEffect(userId => modifyAndGet((LookupContext(), userId, modifiedUser)).unit)
|
||||
}
|
||||
|
||||
new GizmoduckUserGeotagUpdateStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
setHasGeotaggedStatuses(true)
|
||||
.contramap[AsyncInsertTweet.Event](_.user.id)
|
||||
.onSuccess(_ => setCounter.incr())
|
||||
.onlyIf { e =>
|
||||
// only with geo info and an account that doesn't yet have geotagged statuses flag set
|
||||
hasGeo(e.tweet) && (e.user.account.exists(!_.hasGeotaggedStatuses))
|
||||
}
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncInsertTweet)
|
||||
|
||||
override val scrubGeoUpdateUserTimestamp: FutureEffect[ScrubGeoUpdateUserTimestamp.Event] =
|
||||
setHasGeotaggedStatuses(false)
|
||||
.contramap[ScrubGeoUpdateUserTimestamp.Event](_.userId)
|
||||
.onlyIf(_.mightHaveGeotaggedStatuses)
|
||||
.onSuccess(_ => clearedCounter.incr())
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,144 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.guano.{thriftscala => guano}
|
||||
import com.twitter.servo.util.Scribe
|
||||
import com.twitter.takedown.util.TakedownReasons
|
||||
import com.twitter.tseng.withholding.thriftscala.TakedownReason
|
||||
import com.twitter.tweetypie.thriftscala.AuditDeleteTweet
|
||||
|
||||
object Guano {
|
||||
case class MalwareAttempt(
|
||||
url: String,
|
||||
userId: UserId,
|
||||
clientAppId: Option[Long],
|
||||
remoteHost: Option[String]) {
|
||||
def toScribeMessage: guano.ScribeMessage =
|
||||
guano.ScribeMessage(
|
||||
`type` = guano.ScribeType.MalwareAttempt,
|
||||
malwareAttempt = Some(
|
||||
guano.MalwareAttempt(
|
||||
timestamp = Time.now.inSeconds,
|
||||
host = remoteHost,
|
||||
userId = userId,
|
||||
url = url,
|
||||
`type` = guano.MalwareAttemptType.Status,
|
||||
clientAppId = clientAppId.map(_.toInt) // yikes!
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
case class DestroyTweet(
|
||||
tweet: Tweet,
|
||||
userId: UserId,
|
||||
byUserId: UserId,
|
||||
passthrough: Option[AuditDeleteTweet]) {
|
||||
def toScribeMessage: guano.ScribeMessage =
|
||||
guano.ScribeMessage(
|
||||
`type` = guano.ScribeType.DestroyStatus,
|
||||
destroyStatus = Some(
|
||||
guano.DestroyStatus(
|
||||
`type` = Some(guano.DestroyStatusType.Status),
|
||||
timestamp = Time.now.inSeconds,
|
||||
userId = userId,
|
||||
byUserId = byUserId,
|
||||
statusId = tweet.id,
|
||||
text = "",
|
||||
reason = passthrough
|
||||
.flatMap(_.reason)
|
||||
.flatMap { r => guano.UserActionReason.valueOf(r.name) }
|
||||
.orElse(Some(guano.UserActionReason.Other)),
|
||||
done = passthrough.flatMap(_.done).orElse(Some(true)),
|
||||
host = passthrough.flatMap(_.host),
|
||||
bulkId = passthrough.flatMap(_.bulkId),
|
||||
note = passthrough.flatMap(_.note),
|
||||
runId = passthrough.flatMap(_.runId),
|
||||
clientApplicationId = passthrough.flatMap(_.clientApplicationId),
|
||||
userAgent = passthrough.flatMap(_.userAgent)
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
case class Takedown(
|
||||
tweetId: TweetId,
|
||||
userId: UserId,
|
||||
reason: TakedownReason,
|
||||
takendown: Boolean,
|
||||
note: Option[String],
|
||||
host: Option[String],
|
||||
byUserId: Option[UserId]) {
|
||||
def toScribeMessage: guano.ScribeMessage =
|
||||
guano.ScribeMessage(
|
||||
`type` = guano.ScribeType.PctdAction,
|
||||
pctdAction = Some(
|
||||
guano.PctdAction(
|
||||
`type` = guano.PctdActionType.Status,
|
||||
timestamp = Time.now.inSeconds,
|
||||
tweetId = Some(tweetId),
|
||||
userId = userId,
|
||||
countryCode =
|
||||
TakedownReasons.reasonToCountryCode.applyOrElse(reason, (_: TakedownReason) => ""),
|
||||
takendown = takendown,
|
||||
note = note,
|
||||
host = host,
|
||||
byUserId = byUserId.getOrElse(-1L),
|
||||
reason = Some(reason)
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
case class UpdatePossiblySensitiveTweet(
|
||||
tweetId: TweetId,
|
||||
userId: UserId,
|
||||
byUserId: UserId,
|
||||
action: guano.NsfwTweetActionAction,
|
||||
enabled: Boolean,
|
||||
host: Option[String],
|
||||
note: Option[String]) {
|
||||
def toScribeMessage: guano.ScribeMessage =
|
||||
guano.ScribeMessage(
|
||||
`type` = guano.ScribeType.NsfwTweetAction,
|
||||
nsfwTweetAction = Some(
|
||||
guano.NsfwTweetAction(
|
||||
timestamp = Time.now.inSeconds,
|
||||
host = host,
|
||||
userId = userId,
|
||||
byUserId = byUserId,
|
||||
action = action,
|
||||
enabled = enabled,
|
||||
note = note,
|
||||
tweetId = tweetId
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def apply(
|
||||
scribe: FutureEffect[guano.ScribeMessage] = Scribe(guano.ScribeMessage,
|
||||
Scribe("trust_eng_audit"))
|
||||
): Guano = {
|
||||
new Guano {
|
||||
override val scribeMalwareAttempt: FutureEffect[MalwareAttempt] =
|
||||
scribe.contramap[MalwareAttempt](_.toScribeMessage)
|
||||
|
||||
override val scribeDestroyTweet: FutureEffect[DestroyTweet] =
|
||||
scribe.contramap[DestroyTweet](_.toScribeMessage)
|
||||
|
||||
override val scribeTakedown: FutureEffect[Takedown] =
|
||||
scribe.contramap[Takedown](_.toScribeMessage)
|
||||
|
||||
override val scribeUpdatePossiblySensitiveTweet: FutureEffect[UpdatePossiblySensitiveTweet] =
|
||||
scribe.contramap[UpdatePossiblySensitiveTweet](_.toScribeMessage)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait Guano {
|
||||
val scribeMalwareAttempt: FutureEffect[Guano.MalwareAttempt]
|
||||
val scribeDestroyTweet: FutureEffect[Guano.DestroyTweet]
|
||||
val scribeTakedown: FutureEffect[Guano.Takedown]
|
||||
val scribeUpdatePossiblySensitiveTweet: FutureEffect[Guano.UpdatePossiblySensitiveTweet]
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,120 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.guano.thriftscala.NsfwTweetActionAction
|
||||
import com.twitter.tseng.withholding.thriftscala.TakedownReason
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait GuanoServiceStore
|
||||
extends TweetStoreBase[GuanoServiceStore]
|
||||
with AsyncDeleteTweet.Store
|
||||
with AsyncTakedown.Store
|
||||
with AsyncUpdatePossiblySensitiveTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): GuanoServiceStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with GuanoServiceStore
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with AsyncTakedown.StoreWrapper
|
||||
with AsyncUpdatePossiblySensitiveTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object GuanoServiceStore {
|
||||
val Action: AsyncWriteAction.GuanoScribe.type = AsyncWriteAction.GuanoScribe
|
||||
|
||||
val toGuanoTakedown: (AsyncTakedown.Event, TakedownReason, Boolean) => Guano.Takedown =
|
||||
(event: AsyncTakedown.Event, reason: TakedownReason, takendown: Boolean) =>
|
||||
Guano.Takedown(
|
||||
tweetId = event.tweet.id,
|
||||
userId = getUserId(event.tweet),
|
||||
reason = reason,
|
||||
takendown = takendown,
|
||||
note = event.auditNote,
|
||||
host = event.host,
|
||||
byUserId = event.byUserId
|
||||
)
|
||||
|
||||
val toGuanoUpdatePossiblySensitiveTweet: (
|
||||
AsyncUpdatePossiblySensitiveTweet.Event,
|
||||
Boolean,
|
||||
NsfwTweetActionAction
|
||||
) => Guano.UpdatePossiblySensitiveTweet =
|
||||
(
|
||||
event: AsyncUpdatePossiblySensitiveTweet.Event,
|
||||
updatedValue: Boolean,
|
||||
action: NsfwTweetActionAction
|
||||
) =>
|
||||
Guano.UpdatePossiblySensitiveTweet(
|
||||
tweetId = event.tweet.id,
|
||||
host = event.host.orElse(Some("unknown")),
|
||||
userId = event.user.id,
|
||||
byUserId = event.byUserId,
|
||||
action = action,
|
||||
enabled = updatedValue,
|
||||
note = event.note
|
||||
)
|
||||
|
||||
def apply(guano: Guano, stats: StatsReceiver): GuanoServiceStore = {
|
||||
val deleteByUserIdCounter = stats.counter("deletes_with_by_user_id")
|
||||
val deleteScribeCounter = stats.counter("deletes_resulting_in_scribe")
|
||||
|
||||
new GuanoServiceStore {
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
FutureEffect[AsyncDeleteTweet.Event] { event =>
|
||||
val tweet = event.tweet
|
||||
|
||||
event.byUserId.foreach(_ => deleteByUserIdCounter.incr())
|
||||
|
||||
// Guano the tweet deletion action not initiated from the RetweetsDeletionStore
|
||||
event.byUserId match {
|
||||
case Some(byUserId) =>
|
||||
deleteScribeCounter.incr()
|
||||
guano.scribeDestroyTweet(
|
||||
Guano.DestroyTweet(
|
||||
tweet = tweet,
|
||||
userId = getUserId(tweet),
|
||||
byUserId = byUserId,
|
||||
passthrough = event.auditPassthrough
|
||||
)
|
||||
)
|
||||
case _ =>
|
||||
Future.Unit
|
||||
}
|
||||
}.onlyIf(_.cascadedFromTweetId.isEmpty)
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
|
||||
override val asyncTakedown: FutureEffect[AsyncTakedown.Event] =
|
||||
FutureEffect[AsyncTakedown.Event] { event =>
|
||||
val messages =
|
||||
event.reasonsToAdd.map(toGuanoTakedown(event, _, true)) ++
|
||||
event.reasonsToRemove.map(toGuanoTakedown(event, _, false))
|
||||
Future.join(messages.map(guano.scribeTakedown))
|
||||
}.onlyIf(_.scribeForAudit)
|
||||
|
||||
override val retryAsyncTakedown: FutureEffect[TweetStoreRetryEvent[AsyncTakedown.Event]] =
|
||||
TweetStore.retry(Action, asyncTakedown)
|
||||
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
AsyncUpdatePossiblySensitiveTweet.Event
|
||||
] =
|
||||
FutureEffect[AsyncUpdatePossiblySensitiveTweet.Event] { event =>
|
||||
val messages =
|
||||
event.nsfwAdminChange.map(
|
||||
toGuanoUpdatePossiblySensitiveTweet(event, _, NsfwTweetActionAction.NsfwAdmin)
|
||||
) ++
|
||||
event.nsfwUserChange.map(
|
||||
toGuanoUpdatePossiblySensitiveTweet(event, _, NsfwTweetActionAction.NsfwUser)
|
||||
)
|
||||
Future.join(messages.toSeq.map(guano.scribeUpdatePossiblySensitiveTweet))
|
||||
}
|
||||
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUpdatePossiblySensitiveTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUpdatePossiblySensitiveTweet)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,92 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.store.TweetStoreEvent.NoRetry
|
||||
import com.twitter.tweetypie.store.TweetStoreEvent.RetryStrategy
|
||||
import com.twitter.tweetypie.thriftscala.AsyncIncrBookmarkCountRequest
|
||||
import com.twitter.tweetypie.thriftscala.AsyncWriteAction
|
||||
|
||||
object IncrBookmarkCount extends TweetStore.SyncModule {
|
||||
case class Event(tweetId: TweetId, delta: Int, timestamp: Time)
|
||||
extends SyncTweetStoreEvent("incr_bookmark_count") {
|
||||
val toAsyncRequest: AsyncIncrBookmarkCountRequest =
|
||||
AsyncIncrBookmarkCountRequest(tweetId = tweetId, delta = delta)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val incrBookmarkCount: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val incrBookmarkCount: FutureEffect[Event] = wrap(underlying.incrBookmarkCount)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
asyncEnqueueStore: AsyncEnqueueStore,
|
||||
replicatingStore: ReplicatingTweetStore
|
||||
): Store = {
|
||||
new Store {
|
||||
override val incrBookmarkCount: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
asyncEnqueueStore.incrBookmarkCount,
|
||||
replicatingStore.incrBookmarkCount
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncIncrBookmarkCount extends TweetStore.AsyncModule {
|
||||
case class Event(tweetId: TweetId, delta: Int, timestamp: Time)
|
||||
extends AsyncTweetStoreEvent("async_incr_bookmark_event") {
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
Future.Unit
|
||||
|
||||
override def retryStrategy: RetryStrategy = NoRetry
|
||||
}
|
||||
|
||||
trait Store {
|
||||
def asyncIncrBookmarkCount: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncIncrBookmarkCount: FutureEffect[Event] = wrap(
|
||||
underlying.asyncIncrBookmarkCount)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore): Store = {
|
||||
new Store {
|
||||
override def asyncIncrBookmarkCount: FutureEffect[AsyncIncrBookmarkCount.Event] =
|
||||
tweetCountsUpdatingStore.asyncIncrBookmarkCount
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedIncrBookmarkCount extends TweetStore.ReplicatedModule {
|
||||
case class Event(tweetId: TweetId, delta: Int)
|
||||
extends ReplicatedTweetStoreEvent("replicated_incr_bookmark_count") {
|
||||
override def retryStrategy: RetryStrategy = NoRetry
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val replicatedIncrBookmarkCount: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedIncrBookmarkCount: FutureEffect[Event] = wrap(
|
||||
underlying.replicatedIncrBookmarkCount)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore): Store = {
|
||||
new Store {
|
||||
override val replicatedIncrBookmarkCount: FutureEffect[Event] = {
|
||||
tweetCountsUpdatingStore.replicatedIncrBookmarkCount
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,90 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.store.TweetStoreEvent.NoRetry
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object IncrFavCount extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(tweetId: TweetId, delta: Int, timestamp: Time)
|
||||
extends SyncTweetStoreEvent("incr_fav_count") {
|
||||
val toAsyncRequest: AsyncIncrFavCountRequest = AsyncIncrFavCountRequest(tweetId, delta)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val incrFavCount: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val incrFavCount: FutureEffect[Event] = wrap(underlying.incrFavCount)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
asyncEnqueueStore: AsyncEnqueueStore,
|
||||
replicatingStore: ReplicatingTweetStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val incrFavCount: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
asyncEnqueueStore.incrFavCount,
|
||||
replicatingStore.incrFavCount
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncIncrFavCount extends TweetStore.AsyncModule {
|
||||
|
||||
case class Event(tweetId: TweetId, delta: Int, timestamp: Time)
|
||||
extends AsyncTweetStoreEvent("async_incr_fav_count") {
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
Future.Unit // We need to define this method for TweetStoreEvent.Async but we don't use it
|
||||
|
||||
override def retryStrategy: TweetStoreEvent.RetryStrategy = NoRetry
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncIncrFavCount: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncIncrFavCount: FutureEffect[Event] = wrap(underlying.asyncIncrFavCount)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore): Store = {
|
||||
new Store {
|
||||
override val asyncIncrFavCount: FutureEffect[Event] =
|
||||
tweetCountsUpdatingStore.asyncIncrFavCount
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedIncrFavCount extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(tweetId: TweetId, delta: Int)
|
||||
extends ReplicatedTweetStoreEvent("replicated_incr_fav_count") {
|
||||
override def retryStrategy: TweetStoreEvent.NoRetry.type = NoRetry
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val replicatedIncrFavCount: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedIncrFavCount: FutureEffect[Event] = wrap(
|
||||
underlying.replicatedIncrFavCount)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore): Store = {
|
||||
new Store {
|
||||
override val replicatedIncrFavCount: FutureEffect[Event] =
|
||||
tweetCountsUpdatingStore.replicatedIncrFavCount.ignoreFailures
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.tweetypie.store
|
||||
|
||||
import com.twitter.tweetypie.Tweet
|
||||
import com.twitter.tweetypie.serverutil.ExtendedTweetMetadataBuilder
|
||||
import com.twitter.tweetypie.thriftscala.EditControl
|
||||
import com.twitter.tweetypie.thriftscala.InitialTweetUpdateRequest
|
||||
import com.twitter.tweetypie.util.EditControlUtil
|
||||
|
||||
/* Logic to update the initial tweet with new information when that tweet is edited */
|
||||
object InitialTweetUpdate {
|
||||
|
||||
/* Given the initial tweet and update request, copy updated edit
|
||||
* related fields onto it.
|
||||
*/
|
||||
def updateTweet(initialTweet: Tweet, request: InitialTweetUpdateRequest): Tweet = {
|
||||
|
||||
// compute a new edit control initial with updated list of edit tweet ids
|
||||
val editControl: EditControl.Initial =
|
||||
EditControlUtil.editControlForInitialTweet(initialTweet, request.editTweetId).get()
|
||||
|
||||
// compute the correct extended metadata for a permalink
|
||||
val extendedTweetMetadata =
|
||||
request.selfPermalink.map(link => ExtendedTweetMetadataBuilder(initialTweet, link))
|
||||
|
||||
initialTweet.copy(
|
||||
selfPermalink = initialTweet.selfPermalink.orElse(request.selfPermalink),
|
||||
editControl = Some(editControl),
|
||||
extendedTweetMetadata = initialTweet.extendedTweetMetadata.orElse(extendedTweetMetadata)
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,284 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.context.thriftscala.FeatureContext
|
||||
import com.twitter.tweetypie.core.GeoSearchRequestId
|
||||
import com.twitter.tweetypie.store.TweetEventDataScrubber.scrub
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object InsertTweet extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
user: User,
|
||||
timestamp: Time,
|
||||
_internalTweet: Option[CachedTweet] = None,
|
||||
sourceTweet: Option[Tweet] = None,
|
||||
sourceUser: Option[User] = None,
|
||||
quotedTweet: Option[Tweet] = None,
|
||||
quotedUser: Option[User] = None,
|
||||
parentUserId: Option[UserId] = None,
|
||||
initialTweetUpdateRequest: Option[InitialTweetUpdateRequest] = None,
|
||||
dark: Boolean = false,
|
||||
hydrateOptions: WritePathHydrationOptions = WritePathHydrationOptions(),
|
||||
featureContext: Option[FeatureContext] = None,
|
||||
geoSearchRequestId: Option[GeoSearchRequestId] = None,
|
||||
additionalContext: Option[collection.Map[TweetCreateContextKey, String]] = None,
|
||||
transientContext: Option[TransientCreateContext] = None,
|
||||
quoterHasAlreadyQuotedTweet: Boolean = false,
|
||||
noteTweetMentionedUserIds: Option[Seq[Long]] = None)
|
||||
extends SyncTweetStoreEvent("insert_tweet")
|
||||
with QuotedTweetOps {
|
||||
def internalTweet: CachedTweet =
|
||||
_internalTweet.getOrElse(
|
||||
throw new IllegalStateException(
|
||||
s"internalTweet should have been set in WritePathHydration, ${this}"
|
||||
)
|
||||
)
|
||||
|
||||
def toAsyncRequest(
|
||||
scrubUser: User => User,
|
||||
scrubSourceTweet: Tweet => Tweet,
|
||||
scrubSourceUser: User => User
|
||||
): AsyncInsertRequest =
|
||||
AsyncInsertRequest(
|
||||
tweet = tweet,
|
||||
cachedTweet = internalTweet,
|
||||
user = scrubUser(user),
|
||||
sourceTweet = sourceTweet.map(scrubSourceTweet),
|
||||
sourceUser = sourceUser.map(scrubSourceUser),
|
||||
quotedTweet = quotedTweet.map(scrubSourceTweet),
|
||||
quotedUser = quotedUser.map(scrubSourceUser),
|
||||
parentUserId = parentUserId,
|
||||
featureContext = featureContext,
|
||||
timestamp = timestamp.inMillis,
|
||||
geoSearchRequestId = geoSearchRequestId.map(_.requestID),
|
||||
additionalContext = additionalContext,
|
||||
transientContext = transientContext,
|
||||
quoterHasAlreadyQuotedTweet = Some(quoterHasAlreadyQuotedTweet),
|
||||
initialTweetUpdateRequest = initialTweetUpdateRequest,
|
||||
noteTweetMentionedUserIds = noteTweetMentionedUserIds
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val insertTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val insertTweet: FutureEffect[Event] = wrap(underlying.insertTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
logLensStore: LogLensStore,
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
tweetStatsStore: TweetStatsStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
limiterStore: LimiterStore,
|
||||
asyncEnqueueStore: AsyncEnqueueStore,
|
||||
userCountsUpdatingStore: GizmoduckUserCountsUpdatingStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val insertTweet: FutureEffect[Event] =
|
||||
FutureEffect.sequentially(
|
||||
logLensStore.insertTweet,
|
||||
manhattanStore.insertTweet,
|
||||
tweetStatsStore.insertTweet,
|
||||
FutureEffect.inParallel(
|
||||
// allow write-through caching to fail without failing entire insert
|
||||
cachingTweetStore.ignoreFailures.insertTweet,
|
||||
limiterStore.ignoreFailures.insertTweet,
|
||||
asyncEnqueueStore.insertTweet,
|
||||
userCountsUpdatingStore.insertTweet,
|
||||
tweetCountsUpdatingStore.insertTweet
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncInsertTweet extends TweetStore.AsyncModule {
|
||||
|
||||
private val log = Logger(getClass)
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(request: AsyncInsertRequest): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
Event(
|
||||
tweet = request.tweet,
|
||||
cachedTweet = request.cachedTweet,
|
||||
user = request.user,
|
||||
optUser = Some(request.user),
|
||||
timestamp = Time.fromMilliseconds(request.timestamp),
|
||||
sourceTweet = request.sourceTweet,
|
||||
sourceUser = request.sourceUser,
|
||||
parentUserId = request.parentUserId,
|
||||
featureContext = request.featureContext,
|
||||
quotedTweet = request.quotedTweet,
|
||||
quotedUser = request.quotedUser,
|
||||
geoSearchRequestId = request.geoSearchRequestId,
|
||||
additionalContext = request.additionalContext,
|
||||
transientContext = request.transientContext,
|
||||
quoterHasAlreadyQuotedTweet = request.quoterHasAlreadyQuotedTweet.getOrElse(false),
|
||||
initialTweetUpdateRequest = request.initialTweetUpdateRequest,
|
||||
noteTweetMentionedUserIds = request.noteTweetMentionedUserIds
|
||||
),
|
||||
request.retryAction,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
cachedTweet: CachedTweet,
|
||||
user: User,
|
||||
optUser: Option[User],
|
||||
timestamp: Time,
|
||||
sourceTweet: Option[Tweet] = None,
|
||||
sourceUser: Option[User] = None,
|
||||
parentUserId: Option[UserId] = None,
|
||||
featureContext: Option[FeatureContext] = None,
|
||||
quotedTweet: Option[Tweet] = None,
|
||||
quotedUser: Option[User] = None,
|
||||
geoSearchRequestId: Option[String] = None,
|
||||
additionalContext: Option[collection.Map[TweetCreateContextKey, String]] = None,
|
||||
transientContext: Option[TransientCreateContext] = None,
|
||||
quoterHasAlreadyQuotedTweet: Boolean = false,
|
||||
initialTweetUpdateRequest: Option[InitialTweetUpdateRequest] = None,
|
||||
noteTweetMentionedUserIds: Option[Seq[Long]] = None)
|
||||
extends AsyncTweetStoreEvent("async_insert_tweet")
|
||||
with QuotedTweetOps
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
def toAsyncRequest(action: Option[AsyncWriteAction] = None): AsyncInsertRequest =
|
||||
AsyncInsertRequest(
|
||||
tweet = tweet,
|
||||
cachedTweet = cachedTweet,
|
||||
user = user,
|
||||
sourceTweet = sourceTweet,
|
||||
sourceUser = sourceUser,
|
||||
parentUserId = parentUserId,
|
||||
retryAction = action,
|
||||
featureContext = featureContext,
|
||||
timestamp = timestamp.inMillis,
|
||||
quotedTweet = quotedTweet,
|
||||
quotedUser = quotedUser,
|
||||
geoSearchRequestId = geoSearchRequestId,
|
||||
additionalContext = additionalContext,
|
||||
transientContext = transientContext,
|
||||
quoterHasAlreadyQuotedTweet = Some(quoterHasAlreadyQuotedTweet),
|
||||
initialTweetUpdateRequest = initialTweetUpdateRequest,
|
||||
noteTweetMentionedUserIds = noteTweetMentionedUserIds
|
||||
)
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.TweetCreateEvent(
|
||||
TweetCreateEvent(
|
||||
tweet = scrub(tweet),
|
||||
user = user,
|
||||
sourceUser = sourceUser,
|
||||
sourceTweet = sourceTweet.map(scrub),
|
||||
retweetParentUserId = parentUserId,
|
||||
quotedTweet = publicQuotedTweet.map(scrub),
|
||||
quotedUser = publicQuotedUser,
|
||||
additionalContext = additionalContext,
|
||||
transientContext = transientContext,
|
||||
quoterHasAlreadyQuotedTweet = Some(quoterHasAlreadyQuotedTweet)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncInsert(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.Insert.type = AsyncWriteEventType.Insert
|
||||
override val scribedTweetOnFailure: Option[Tweet] = Some(event.tweet)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncInsertTweet: FutureEffect[Event]
|
||||
val retryAsyncInsertTweet: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncInsertTweet: FutureEffect[Event] = wrap(underlying.asyncInsertTweet)
|
||||
override val retryAsyncInsertTweet: FutureEffect[TweetStoreRetryEvent[Event]] = wrap(
|
||||
underlying.retryAsyncInsertTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
replicatingStore: ReplicatingTweetStore,
|
||||
indexingStore: TweetIndexingStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore,
|
||||
timelineUpdatingStore: TlsTimelineUpdatingStore,
|
||||
eventBusEnqueueStore: TweetEventBusStore,
|
||||
fanoutServiceStore: FanoutServiceStore,
|
||||
scribeMediaTagStore: ScribeMediaTagStore,
|
||||
userGeotagUpdateStore: GizmoduckUserGeotagUpdateStore,
|
||||
geoSearchRequestIDStore: GeoSearchRequestIDStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] =
|
||||
Seq(
|
||||
replicatingStore,
|
||||
indexingStore,
|
||||
timelineUpdatingStore,
|
||||
eventBusEnqueueStore,
|
||||
fanoutServiceStore,
|
||||
userGeotagUpdateStore,
|
||||
tweetCountsUpdatingStore,
|
||||
scribeMediaTagStore,
|
||||
geoSearchRequestIDStore
|
||||
)
|
||||
|
||||
def build[E <: TweetStoreEvent](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncInsertTweet: FutureEffect[Event] = build(_.asyncInsertTweet)
|
||||
override val retryAsyncInsertTweet: FutureEffect[TweetStoreRetryEvent[Event]] = build(
|
||||
_.retryAsyncInsertTweet)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedInsertTweet extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
cachedTweet: CachedTweet,
|
||||
quoterHasAlreadyQuotedTweet: Boolean = false,
|
||||
initialTweetUpdateRequest: Option[InitialTweetUpdateRequest] = None)
|
||||
extends ReplicatedTweetStoreEvent("replicated_insert_tweet")
|
||||
|
||||
trait Store {
|
||||
val replicatedInsertTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedInsertTweet: FutureEffect[Event] = wrap(underlying.replicatedInsertTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore
|
||||
): Store = {
|
||||
new Store {
|
||||
override val replicatedInsertTweet: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
cachingTweetStore.replicatedInsertTweet,
|
||||
tweetCountsUpdatingStore.replicatedInsertTweet.ignoreFailures
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,41 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.backends.LimiterService
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait LimiterStore extends TweetStoreBase[LimiterStore] with InsertTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): LimiterStore =
|
||||
new TweetStoreWrapper(w, this) with LimiterStore with InsertTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object LimiterStore {
|
||||
def apply(
|
||||
incrementCreateSuccess: LimiterService.IncrementByOne,
|
||||
incrementMediaTags: LimiterService.Increment
|
||||
): LimiterStore =
|
||||
new LimiterStore {
|
||||
override val insertTweet: FutureEffect[InsertTweet.Event] =
|
||||
FutureEffect[InsertTweet.Event] { event =>
|
||||
Future.when(!event.dark) {
|
||||
val userId = event.user.id
|
||||
val contributorUserId: Option[UserId] = event.tweet.contributor.map(_.userId)
|
||||
|
||||
val mediaTags = getMediaTagMap(event.tweet)
|
||||
val mediaTagCount = countDistinctUserMediaTags(mediaTags)
|
||||
Future
|
||||
.join(
|
||||
incrementCreateSuccess(userId, contributorUserId),
|
||||
incrementMediaTags(userId, contributorUserId, mediaTagCount)
|
||||
)
|
||||
.unit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def countDistinctUserMediaTags(mediaTags: Map[MediaId, Seq[MediaTag]]): Int =
|
||||
mediaTags.values.flatten.toSeq
|
||||
.collect { case MediaTag(MediaTagType.User, Some(userId), _, _) => userId }
|
||||
.distinct
|
||||
.size
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,169 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import com.fasterxml.jackson.module.scala.DefaultScalaModule
|
||||
import com.twitter.finagle.tracing.Trace
|
||||
import com.twitter.tweetypie.additionalfields.AdditionalFields
|
||||
import com.twitter.tweetypie.client_id.ClientIdHelper
|
||||
import com.twitter.tweetypie.media.Media.ownMedia
|
||||
|
||||
trait LogLensStore
|
||||
extends TweetStoreBase[LogLensStore]
|
||||
with InsertTweet.Store
|
||||
with DeleteTweet.Store
|
||||
with UndeleteTweet.Store
|
||||
with SetAdditionalFields.Store
|
||||
with DeleteAdditionalFields.Store
|
||||
with ScrubGeo.Store
|
||||
with Takedown.Store
|
||||
with UpdatePossiblySensitiveTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): LogLensStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with LogLensStore
|
||||
with InsertTweet.StoreWrapper
|
||||
with DeleteTweet.StoreWrapper
|
||||
with UndeleteTweet.StoreWrapper
|
||||
with SetAdditionalFields.StoreWrapper
|
||||
with DeleteAdditionalFields.StoreWrapper
|
||||
with ScrubGeo.StoreWrapper
|
||||
with Takedown.StoreWrapper
|
||||
with UpdatePossiblySensitiveTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object LogLensStore {
|
||||
def apply(
|
||||
tweetCreationsLogger: Logger,
|
||||
tweetDeletionsLogger: Logger,
|
||||
tweetUndeletionsLogger: Logger,
|
||||
tweetUpdatesLogger: Logger,
|
||||
clientIdHelper: ClientIdHelper,
|
||||
): LogLensStore =
|
||||
new LogLensStore {
|
||||
private[this] val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
|
||||
|
||||
private def logMessage(logger: Logger, data: (String, Any)*): Future[Unit] =
|
||||
Future {
|
||||
val allData = data ++ defaultData
|
||||
val msg = mapper.writeValueAsString(Map(allData: _*))
|
||||
logger.info(msg)
|
||||
}
|
||||
|
||||
// Note: Longs are logged as strings to avoid JSON 53-bit numeric truncation
|
||||
private def defaultData: Seq[(String, Any)] = {
|
||||
val viewer = TwitterContext()
|
||||
Seq(
|
||||
"client_id" -> getOpt(clientIdHelper.effectiveClientId),
|
||||
"service_id" -> getOpt(clientIdHelper.effectiveServiceIdentifier),
|
||||
"trace_id" -> Trace.id.traceId.toString,
|
||||
"audit_ip" -> getOpt(viewer.flatMap(_.auditIp)),
|
||||
"application_id" -> getOpt(viewer.flatMap(_.clientApplicationId).map(_.toString)),
|
||||
"user_agent" -> getOpt(viewer.flatMap(_.userAgent)),
|
||||
"authenticated_user_id" -> getOpt(viewer.flatMap(_.authenticatedUserId).map(_.toString))
|
||||
)
|
||||
}
|
||||
|
||||
private def getOpt[A](opt: Option[A]): Any =
|
||||
opt.getOrElse(null)
|
||||
|
||||
override val insertTweet: FutureEffect[InsertTweet.Event] =
|
||||
FutureEffect[InsertTweet.Event] { event =>
|
||||
logMessage(
|
||||
tweetCreationsLogger,
|
||||
"type" -> "create_tweet",
|
||||
"tweet_id" -> event.tweet.id.toString,
|
||||
"user_id" -> event.user.id.toString,
|
||||
"source_tweet_id" -> getOpt(event.sourceTweet.map(_.id.toString)),
|
||||
"source_user_id" -> getOpt(event.sourceUser.map(_.id.toString)),
|
||||
"directed_at_user_id" -> getOpt(getDirectedAtUser(event.tweet).map(_.userId.toString)),
|
||||
"reply_to_tweet_id" -> getOpt(
|
||||
getReply(event.tweet).flatMap(_.inReplyToStatusId).map(_.toString)),
|
||||
"reply_to_user_id" -> getOpt(getReply(event.tweet).map(_.inReplyToUserId.toString)),
|
||||
"media_ids" -> ownMedia(event.tweet).map(_.mediaId.toString)
|
||||
)
|
||||
}
|
||||
|
||||
override val deleteTweet: FutureEffect[DeleteTweet.Event] =
|
||||
FutureEffect[DeleteTweet.Event] { event =>
|
||||
logMessage(
|
||||
tweetDeletionsLogger,
|
||||
"type" -> "delete_tweet",
|
||||
"tweet_id" -> event.tweet.id.toString,
|
||||
"user_id" -> getOpt(event.user.map(_.id.toString)),
|
||||
"source_tweet_id" -> getOpt(getShare(event.tweet).map(_.sourceStatusId.toString)),
|
||||
"by_user_id" -> getOpt(event.byUserId.map(_.toString)),
|
||||
"passthrough_audit_ip" -> getOpt(event.auditPassthrough.flatMap(_.host)),
|
||||
"media_ids" -> ownMedia(event.tweet).map(_.mediaId.toString),
|
||||
"cascaded_from_tweet_id" -> getOpt(event.cascadedFromTweetId.map(_.toString))
|
||||
)
|
||||
}
|
||||
|
||||
override val undeleteTweet: FutureEffect[UndeleteTweet.Event] =
|
||||
FutureEffect[UndeleteTweet.Event] { event =>
|
||||
logMessage(
|
||||
tweetUndeletionsLogger,
|
||||
"type" -> "undelete_tweet",
|
||||
"tweet_id" -> event.tweet.id.toString,
|
||||
"user_id" -> event.user.id.toString,
|
||||
"source_tweet_id" -> getOpt(getShare(event.tweet).map(_.sourceStatusId.toString)),
|
||||
"media_ids" -> ownMedia(event.tweet).map(_.mediaId.toString)
|
||||
)
|
||||
}
|
||||
|
||||
override val setAdditionalFields: FutureEffect[SetAdditionalFields.Event] =
|
||||
FutureEffect[SetAdditionalFields.Event] { event =>
|
||||
logMessage(
|
||||
tweetUpdatesLogger,
|
||||
"type" -> "set_additional_fields",
|
||||
"tweet_id" -> event.additionalFields.id.toString,
|
||||
"field_ids" -> AdditionalFields.nonEmptyAdditionalFieldIds(event.additionalFields)
|
||||
)
|
||||
}
|
||||
|
||||
override val deleteAdditionalFields: FutureEffect[DeleteAdditionalFields.Event] =
|
||||
FutureEffect[DeleteAdditionalFields.Event] { event =>
|
||||
logMessage(
|
||||
tweetUpdatesLogger,
|
||||
"type" -> "delete_additional_fields",
|
||||
"tweet_id" -> event.tweetId.toString,
|
||||
"field_ids" -> event.fieldIds
|
||||
)
|
||||
}
|
||||
|
||||
override val scrubGeo: FutureEffect[ScrubGeo.Event] =
|
||||
FutureEffect[ScrubGeo.Event] { event =>
|
||||
Future.join(
|
||||
event.tweetIds.map { tweetId =>
|
||||
logMessage(
|
||||
tweetUpdatesLogger,
|
||||
"type" -> "scrub_geo",
|
||||
"tweet_id" -> tweetId.toString,
|
||||
"user_id" -> event.userId.toString
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
override val takedown: FutureEffect[Takedown.Event] =
|
||||
FutureEffect[Takedown.Event] { event =>
|
||||
logMessage(
|
||||
tweetUpdatesLogger,
|
||||
"type" -> "takedown",
|
||||
"tweet_id" -> event.tweet.id.toString,
|
||||
"user_id" -> getUserId(event.tweet).toString,
|
||||
"reasons" -> event.takedownReasons
|
||||
)
|
||||
}
|
||||
|
||||
override val updatePossiblySensitiveTweet: FutureEffect[UpdatePossiblySensitiveTweet.Event] =
|
||||
FutureEffect[UpdatePossiblySensitiveTweet.Event] { event =>
|
||||
logMessage(
|
||||
tweetUpdatesLogger,
|
||||
"type" -> "update_possibly_sensitive_tweet",
|
||||
"tweet_id" -> event.tweet.id.toString,
|
||||
"nsfw_admin" -> TweetLenses.nsfwAdmin(event.tweet),
|
||||
"nsfw_user" -> TweetLenses.nsfwUser(event.tweet)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,231 +0,0 @@
|
|||
/** Copyright 2010 Twitter, Inc. */
|
||||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.tweetypie.additionalfields.AdditionalFields
|
||||
import com.twitter.tweetypie.storage.Field
|
||||
import com.twitter.tweetypie.storage.Response.TweetResponse
|
||||
import com.twitter.tweetypie.storage.Response.TweetResponseCode
|
||||
import com.twitter.tweetypie.storage.TweetStorageClient
|
||||
import com.twitter.tweetypie.storage.TweetStorageClient.GetTweet
|
||||
import com.twitter.tweetypie.storage.TweetStorageException
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class UpdateTweetNotFoundException(tweetId: TweetId) extends Exception
|
||||
|
||||
trait ManhattanTweetStore
|
||||
extends TweetStoreBase[ManhattanTweetStore]
|
||||
with InsertTweet.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with ScrubGeo.Store
|
||||
with SetAdditionalFields.Store
|
||||
with DeleteAdditionalFields.Store
|
||||
with AsyncDeleteAdditionalFields.Store
|
||||
with Takedown.Store
|
||||
with UpdatePossiblySensitiveTweet.Store
|
||||
with AsyncUpdatePossiblySensitiveTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): ManhattanTweetStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with ManhattanTweetStore
|
||||
with InsertTweet.StoreWrapper
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with ScrubGeo.StoreWrapper
|
||||
with SetAdditionalFields.StoreWrapper
|
||||
with DeleteAdditionalFields.StoreWrapper
|
||||
with AsyncDeleteAdditionalFields.StoreWrapper
|
||||
with Takedown.StoreWrapper
|
||||
with UpdatePossiblySensitiveTweet.StoreWrapper
|
||||
with AsyncUpdatePossiblySensitiveTweet.StoreWrapper
|
||||
}
|
||||
|
||||
/**
|
||||
* A TweetStore implementation that writes to Manhattan.
|
||||
*/
|
||||
object ManhattanTweetStore {
|
||||
val Action: AsyncWriteAction.TbirdUpdate.type = AsyncWriteAction.TbirdUpdate
|
||||
|
||||
private val log = Logger(getClass)
|
||||
private val successResponses = Set(TweetResponseCode.Success, TweetResponseCode.Deleted)
|
||||
|
||||
case class AnnotationFailure(message: String) extends Exception(message)
|
||||
|
||||
def apply(tweetStorageClient: TweetStorageClient): ManhattanTweetStore = {
|
||||
|
||||
def handleStorageResponses(
|
||||
responsesStitch: Stitch[Seq[TweetResponse]],
|
||||
action: String
|
||||
): Future[Unit] =
|
||||
Stitch
|
||||
.run(responsesStitch)
|
||||
.onFailure {
|
||||
case ex: TweetStorageException => log.warn("failed on: " + action, ex)
|
||||
case _ =>
|
||||
}
|
||||
.flatMap { responses =>
|
||||
Future.when(responses.exists(resp => !successResponses(resp.overallResponse))) {
|
||||
Future.exception(AnnotationFailure(s"$action gets failure response $responses"))
|
||||
}
|
||||
}
|
||||
|
||||
def updateTweetMediaIds(mutation: Mutation[MediaEntity]): Tweet => Tweet =
|
||||
tweet => tweet.copy(media = tweet.media.map(entities => entities.map(mutation.endo)))
|
||||
|
||||
/**
|
||||
* Does a get and set, and only sets fields that are allowed to be
|
||||
* changed. This also prevents incoming tweets containing incomplete
|
||||
* fields from being saved to Manhattan.
|
||||
*/
|
||||
def updateOneTweetByIdAction(tweetId: TweetId, copyFields: Tweet => Tweet): Future[Unit] = {
|
||||
Stitch.run {
|
||||
tweetStorageClient.getTweet(tweetId).flatMap {
|
||||
case GetTweet.Response.Found(tweet) =>
|
||||
val updatedTweet = copyFields(tweet)
|
||||
|
||||
if (updatedTweet != tweet) {
|
||||
tweetStorageClient.addTweet(updatedTweet)
|
||||
} else {
|
||||
Stitch.Unit
|
||||
}
|
||||
case _ => Stitch.exception(UpdateTweetNotFoundException(tweetId))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This should NOT be used in parallel with other write operations.
|
||||
// A race condition can occur after changes to the storage library to
|
||||
// return all additional fields. The resulting behavior can cause
|
||||
// fields that were modified by other writes to revert to their old value.
|
||||
def updateOneTweetAction(update: Tweet, copyFields: Tweet => Tweet => Tweet): Future[Unit] =
|
||||
updateOneTweetByIdAction(update.id, copyFields(update))
|
||||
|
||||
def tweetStoreUpdateTweet(tweet: Tweet): Future[Unit] = {
|
||||
val setFields = AdditionalFields.nonEmptyAdditionalFieldIds(tweet).map(Field.additionalField)
|
||||
handleStorageResponses(
|
||||
tweetStorageClient.updateTweet(tweet, setFields).map(Seq(_)),
|
||||
s"updateTweet($tweet, $setFields)"
|
||||
)
|
||||
}
|
||||
|
||||
// This is an edit so update the initial Tweet's control
|
||||
def updateInitialTweet(event: InsertTweet.Event): Future[Unit] = {
|
||||
event.initialTweetUpdateRequest match {
|
||||
case Some(request) =>
|
||||
updateOneTweetByIdAction(
|
||||
request.initialTweetId,
|
||||
tweet => InitialTweetUpdate.updateTweet(tweet, request)
|
||||
)
|
||||
case None => Future.Unit
|
||||
}
|
||||
}
|
||||
|
||||
new ManhattanTweetStore {
|
||||
override val insertTweet: FutureEffect[InsertTweet.Event] =
|
||||
FutureEffect[InsertTweet.Event] { event =>
|
||||
Stitch
|
||||
.run(
|
||||
tweetStorageClient.addTweet(event.internalTweet.tweet)
|
||||
).flatMap(_ => updateInitialTweet(event))
|
||||
}
|
||||
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
FutureEffect[AsyncDeleteTweet.Event] { event =>
|
||||
if (event.isBounceDelete) {
|
||||
Stitch.run(tweetStorageClient.bounceDelete(event.tweet.id))
|
||||
} else {
|
||||
Stitch.run(tweetStorageClient.softDelete(event.tweet.id))
|
||||
}
|
||||
}
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
|
||||
override val scrubGeo: FutureEffect[ScrubGeo.Event] =
|
||||
FutureEffect[ScrubGeo.Event] { event =>
|
||||
Stitch.run(tweetStorageClient.scrub(event.tweetIds, Seq(Field.Geo)))
|
||||
}
|
||||
|
||||
override val setAdditionalFields: FutureEffect[SetAdditionalFields.Event] =
|
||||
FutureEffect[SetAdditionalFields.Event] { event =>
|
||||
tweetStoreUpdateTweet(event.additionalFields)
|
||||
}
|
||||
|
||||
override val deleteAdditionalFields: FutureEffect[DeleteAdditionalFields.Event] =
|
||||
FutureEffect[DeleteAdditionalFields.Event] { event =>
|
||||
handleStorageResponses(
|
||||
tweetStorageClient.deleteAdditionalFields(
|
||||
Seq(event.tweetId),
|
||||
event.fieldIds.map(Field.additionalField)
|
||||
),
|
||||
s"deleteAdditionalFields(${event.tweetId}, ${event.fieldIds}})"
|
||||
)
|
||||
}
|
||||
|
||||
override val asyncDeleteAdditionalFields: FutureEffect[AsyncDeleteAdditionalFields.Event] =
|
||||
FutureEffect[AsyncDeleteAdditionalFields.Event] { event =>
|
||||
handleStorageResponses(
|
||||
tweetStorageClient.deleteAdditionalFields(
|
||||
Seq(event.tweetId),
|
||||
event.fieldIds.map(Field.additionalField)
|
||||
),
|
||||
s"deleteAdditionalFields(Seq(${event.tweetId}), ${event.fieldIds}})"
|
||||
)
|
||||
}
|
||||
|
||||
override val retryAsyncDeleteAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteAdditionalFields.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteAdditionalFields)
|
||||
|
||||
override val takedown: FutureEffect[Takedown.Event] =
|
||||
FutureEffect[Takedown.Event] { event =>
|
||||
val (fieldsToUpdate, fieldsToDelete) =
|
||||
Seq(
|
||||
Field.TweetypieOnlyTakedownCountryCodes,
|
||||
Field.TweetypieOnlyTakedownReasons
|
||||
).filter(_ => event.updateCodesAndReasons)
|
||||
.partition(f => event.tweet.getFieldBlob(f.id).isDefined)
|
||||
|
||||
val allFieldsToUpdate = Seq(Field.HasTakedown) ++ fieldsToUpdate
|
||||
|
||||
Future
|
||||
.join(
|
||||
handleStorageResponses(
|
||||
tweetStorageClient
|
||||
.updateTweet(event.tweet, allFieldsToUpdate)
|
||||
.map(Seq(_)),
|
||||
s"updateTweet(${event.tweet}, $allFieldsToUpdate)"
|
||||
),
|
||||
Future.when(fieldsToDelete.nonEmpty) {
|
||||
handleStorageResponses(
|
||||
tweetStorageClient
|
||||
.deleteAdditionalFields(Seq(event.tweet.id), fieldsToDelete),
|
||||
s"deleteAdditionalFields(Seq(${event.tweet.id}), $fieldsToDelete)"
|
||||
)
|
||||
}
|
||||
).unit
|
||||
}
|
||||
|
||||
override val updatePossiblySensitiveTweet: FutureEffect[UpdatePossiblySensitiveTweet.Event] =
|
||||
FutureEffect[UpdatePossiblySensitiveTweet.Event] { event =>
|
||||
updateOneTweetAction(event.tweet, TweetUpdate.copyNsfwFieldsForUpdate)
|
||||
}
|
||||
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
AsyncUpdatePossiblySensitiveTweet.Event
|
||||
] =
|
||||
FutureEffect[AsyncUpdatePossiblySensitiveTweet.Event] { event =>
|
||||
updateOneTweetAction(event.tweet, TweetUpdate.copyNsfwFieldsForUpdate)
|
||||
}
|
||||
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUpdatePossiblySensitiveTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUpdatePossiblySensitiveTweet)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,34 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
import scala.util.matching.Regex
|
||||
|
||||
object MediaIndexHelper {
|
||||
|
||||
/**
|
||||
* Which tweets should we treat as "media" tweets?
|
||||
*
|
||||
* Any tweet that is not a retweet and any of:
|
||||
* - Is explicitly marked as a media tweet.
|
||||
* - Has a media entity.
|
||||
* - Includes a partner media URL.
|
||||
*/
|
||||
def apply(partnerMediaRegexes: Seq[Regex]): Tweet => Boolean = {
|
||||
val isPartnerUrl = partnerUrlMatcher(partnerMediaRegexes)
|
||||
|
||||
tweet =>
|
||||
getShare(tweet).isEmpty &&
|
||||
(hasMediaFlagSet(tweet) ||
|
||||
getMedia(tweet).nonEmpty ||
|
||||
getUrls(tweet).exists(isPartnerUrl))
|
||||
}
|
||||
|
||||
def partnerUrlMatcher(partnerMediaRegexes: Seq[Regex]): UrlEntity => Boolean =
|
||||
_.expanded.exists { expandedUrl =>
|
||||
partnerMediaRegexes.exists(_.findFirstIn(expandedUrl).isDefined)
|
||||
}
|
||||
|
||||
def hasMediaFlagSet(tweet: Tweet): Boolean =
|
||||
tweet.coreData.flatMap(_.hasMedia).getOrElse(false)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,62 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.mediaservices.commons.thriftscala.MediaKey
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.tweetypie.media._
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait MediaServiceStore
|
||||
extends TweetStoreBase[MediaServiceStore]
|
||||
with AsyncDeleteTweet.Store
|
||||
with AsyncUndeleteTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): MediaServiceStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with MediaServiceStore
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with AsyncUndeleteTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object MediaServiceStore {
|
||||
val Action: AsyncWriteAction.MediaDeletion.type = AsyncWriteAction.MediaDeletion
|
||||
|
||||
private def ownMedia(t: Tweet): Seq[(MediaKey, TweetId)] =
|
||||
getMedia(t)
|
||||
.collect {
|
||||
case m if Media.isOwnMedia(t.id, m) => (MediaKeyUtil.get(m), t.id)
|
||||
}
|
||||
|
||||
def apply(
|
||||
deleteMedia: FutureArrow[DeleteMediaRequest, Unit],
|
||||
undeleteMedia: FutureArrow[UndeleteMediaRequest, Unit]
|
||||
): MediaServiceStore =
|
||||
new MediaServiceStore {
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
FutureEffect[AsyncDeleteTweet.Event] { e =>
|
||||
Future.when(!isRetweet(e.tweet)) {
|
||||
val ownMediaKeys: Seq[(MediaKey, TweetId)] = ownMedia(e.tweet)
|
||||
val deleteMediaRequests = ownMediaKeys.map(DeleteMediaRequest.tupled)
|
||||
Future.collect(deleteMediaRequests.map(deleteMedia))
|
||||
}
|
||||
}
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
|
||||
override val asyncUndeleteTweet: FutureEffect[AsyncUndeleteTweet.Event] =
|
||||
FutureEffect[AsyncUndeleteTweet.Event] { e =>
|
||||
Future.when(!isRetweet(e.tweet)) {
|
||||
val ownMediaKeys: Seq[(MediaKey, TweetId)] = ownMedia(e.tweet)
|
||||
val unDeleteMediaRequests = ownMediaKeys.map(UndeleteMediaRequest.tupled)
|
||||
Future.collect(unDeleteMediaRequests.map(undeleteMedia))
|
||||
}
|
||||
}
|
||||
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUndeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUndeleteTweet)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,45 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object QuotedTweetDelete extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
quotingTweetId: TweetId,
|
||||
quotingUserId: UserId,
|
||||
quotedTweetId: TweetId,
|
||||
quotedUserId: UserId,
|
||||
timestamp: Time,
|
||||
optUser: Option[User] = None)
|
||||
extends SyncTweetStoreEvent("quoted_tweet_delete")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.QuotedTweetDeleteEvent(
|
||||
QuotedTweetDeleteEvent(
|
||||
quotingTweetId = quotingTweetId,
|
||||
quotingUserId = quotingUserId,
|
||||
quotedTweetId = quotedTweetId,
|
||||
quotedUserId = quotedUserId
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val quotedTweetDelete: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val quotedTweetDelete: FutureEffect[Event] = wrap(underlying.quotedTweetDelete)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(eventBusEnqueueStore: TweetEventBusStore): Store =
|
||||
new Store {
|
||||
override val quotedTweetDelete: FutureEffect[Event] = eventBusEnqueueStore.quotedTweetDelete
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,33 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
/**
|
||||
* Mixin that implements public quoted tweet and public quoted user
|
||||
* filtering for tweet events that have quoted tweets and users.
|
||||
*/
|
||||
trait QuotedTweetOps {
|
||||
def quotedTweet: Option[Tweet]
|
||||
def quotedUser: Option[User]
|
||||
|
||||
/**
|
||||
* Do we have evidence that the quoted user is unprotected?
|
||||
*/
|
||||
def quotedUserIsPublic: Boolean =
|
||||
// The quoted user should include the `safety` struct, but if it
|
||||
// doesn't for any reason then the quoted tweet and quoted user
|
||||
// should not be included in the events. This is a safety measure to
|
||||
// avoid leaking private information.
|
||||
quotedUser.exists(_.safety.exists(!_.isProtected))
|
||||
|
||||
/**
|
||||
* The quoted tweet, filtered as it should appear through public APIs.
|
||||
*/
|
||||
def publicQuotedTweet: Option[Tweet] =
|
||||
if (quotedUserIsPublic) quotedTweet else None
|
||||
|
||||
/**
|
||||
* The quoted user, filtered as it should appear through public APIs.
|
||||
*/
|
||||
def publicQuotedUser: Option[User] =
|
||||
if (quotedUserIsPublic) quotedUser else None
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,51 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tseng.withholding.thriftscala.TakedownReason
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object QuotedTweetTakedown extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
quotingTweetId: TweetId,
|
||||
quotingUserId: UserId,
|
||||
quotedTweetId: TweetId,
|
||||
quotedUserId: UserId,
|
||||
takedownCountryCodes: Seq[String],
|
||||
takedownReasons: Seq[TakedownReason],
|
||||
timestamp: Time,
|
||||
optUser: Option[User] = None)
|
||||
extends SyncTweetStoreEvent("quoted_tweet_takedown")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.QuotedTweetTakedownEvent(
|
||||
QuotedTweetTakedownEvent(
|
||||
quotingTweetId = quotingTweetId,
|
||||
quotingUserId = quotingUserId,
|
||||
quotedTweetId = quotedTweetId,
|
||||
quotedUserId = quotedUserId,
|
||||
takedownCountryCodes = takedownCountryCodes,
|
||||
takedownReasons = takedownReasons
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val quotedTweetTakedown: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val quotedTweetTakedown: FutureEffect[Event] = wrap(underlying.quotedTweetTakedown)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(eventBusEnqueueStore: TweetEventBusStore): Store =
|
||||
new Store {
|
||||
override val quotedTweetTakedown: FutureEffect[Event] =
|
||||
eventBusEnqueueStore.quotedTweetTakedown
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,180 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
/**
|
||||
* A TweetStore that sends write events to the replication endpoints
|
||||
* of a ThriftTweetService.
|
||||
*
|
||||
* The events that are sent are sufficient to keep the other
|
||||
* instance's caches up to date. The calls contain sufficient data so
|
||||
* that the remote caches can be updated without requiring the remote
|
||||
* Tweetypie to access any other services.
|
||||
*
|
||||
* The replication services two purposes:
|
||||
*
|
||||
* 1. Maintain consistency between caches in different data centers.
|
||||
*
|
||||
* 2. Keep the caches in all data centers warm, protecting backend
|
||||
* services.
|
||||
*
|
||||
* Correctness bugs are worse than bugs that make data less available.
|
||||
* All of these events affect data consistency.
|
||||
*
|
||||
* IncrFavCount.Event and InsertEvents are the least important
|
||||
* from a data consistency standpoint, because the only data
|
||||
* consistency issues are counts, which are cached for a shorter time,
|
||||
* and are not as noticable to end users if they fail to occur.
|
||||
* (Failure to apply them is both less severe and self-correcting.)
|
||||
*
|
||||
* Delete and GeoScrub events are critical, because the cached data
|
||||
* has a long expiration and failure to apply them can result in
|
||||
* violations of user privacy.
|
||||
*
|
||||
* Update events are also important from a legal perspective, since
|
||||
* the update may be updating the per-country take-down status.
|
||||
*
|
||||
* @param svc: The ThriftTweetService implementation that will receive the
|
||||
* replication events. In practice, this will usually be a
|
||||
* deferredrpc service.
|
||||
*/
|
||||
trait ReplicatingTweetStore
|
||||
extends TweetStoreBase[ReplicatingTweetStore]
|
||||
with AsyncInsertTweet.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with AsyncUndeleteTweet.Store
|
||||
with AsyncSetRetweetVisibility.Store
|
||||
with AsyncSetAdditionalFields.Store
|
||||
with AsyncDeleteAdditionalFields.Store
|
||||
with ScrubGeo.Store
|
||||
with IncrFavCount.Store
|
||||
with IncrBookmarkCount.Store
|
||||
with AsyncTakedown.Store
|
||||
with AsyncUpdatePossiblySensitiveTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): ReplicatingTweetStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with ReplicatingTweetStore
|
||||
with AsyncInsertTweet.StoreWrapper
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with AsyncUndeleteTweet.StoreWrapper
|
||||
with AsyncSetRetweetVisibility.StoreWrapper
|
||||
with AsyncSetAdditionalFields.StoreWrapper
|
||||
with AsyncDeleteAdditionalFields.StoreWrapper
|
||||
with ScrubGeo.StoreWrapper
|
||||
with IncrFavCount.StoreWrapper
|
||||
with IncrBookmarkCount.StoreWrapper
|
||||
with AsyncTakedown.StoreWrapper
|
||||
with AsyncUpdatePossiblySensitiveTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object ReplicatingTweetStore {
|
||||
|
||||
val Action: AsyncWriteAction.Replication.type = AsyncWriteAction.Replication
|
||||
|
||||
def apply(
|
||||
svc: ThriftTweetService
|
||||
): ReplicatingTweetStore =
|
||||
new ReplicatingTweetStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
FutureEffect[AsyncInsertTweet.Event] { e =>
|
||||
svc.replicatedInsertTweet2(
|
||||
ReplicatedInsertTweet2Request(
|
||||
e.cachedTweet,
|
||||
initialTweetUpdateRequest = e.initialTweetUpdateRequest
|
||||
))
|
||||
}
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncInsertTweet)
|
||||
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
FutureEffect[AsyncDeleteTweet.Event] { e =>
|
||||
svc.replicatedDeleteTweet2(
|
||||
ReplicatedDeleteTweet2Request(
|
||||
tweet = e.tweet,
|
||||
isErasure = e.isUserErasure,
|
||||
isBounceDelete = e.isBounceDelete
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
|
||||
override val asyncUndeleteTweet: FutureEffect[AsyncUndeleteTweet.Event] =
|
||||
FutureEffect[AsyncUndeleteTweet.Event] { e =>
|
||||
svc.replicatedUndeleteTweet2(ReplicatedUndeleteTweet2Request(e.cachedTweet))
|
||||
}
|
||||
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUndeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUndeleteTweet)
|
||||
|
||||
override val asyncSetAdditionalFields: FutureEffect[AsyncSetAdditionalFields.Event] =
|
||||
FutureEffect[AsyncSetAdditionalFields.Event] { e =>
|
||||
svc.replicatedSetAdditionalFields(SetAdditionalFieldsRequest(e.additionalFields))
|
||||
}
|
||||
|
||||
override val retryAsyncSetAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncSetAdditionalFields.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncSetAdditionalFields)
|
||||
|
||||
override val asyncSetRetweetVisibility: FutureEffect[AsyncSetRetweetVisibility.Event] =
|
||||
FutureEffect[AsyncSetRetweetVisibility.Event] { e =>
|
||||
svc.replicatedSetRetweetVisibility(
|
||||
ReplicatedSetRetweetVisibilityRequest(e.srcId, e.visible)
|
||||
)
|
||||
}
|
||||
|
||||
override val retryAsyncSetRetweetVisibility: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncSetRetweetVisibility.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncSetRetweetVisibility)
|
||||
|
||||
override val asyncDeleteAdditionalFields: FutureEffect[AsyncDeleteAdditionalFields.Event] =
|
||||
FutureEffect[AsyncDeleteAdditionalFields.Event] { e =>
|
||||
svc.replicatedDeleteAdditionalFields(
|
||||
ReplicatedDeleteAdditionalFieldsRequest(Map(e.tweetId -> e.fieldIds))
|
||||
)
|
||||
}
|
||||
|
||||
override val retryAsyncDeleteAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteAdditionalFields.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteAdditionalFields)
|
||||
|
||||
override val scrubGeo: FutureEffect[ScrubGeo.Event] =
|
||||
FutureEffect[ScrubGeo.Event](e => svc.replicatedScrubGeo(e.tweetIds))
|
||||
|
||||
override val incrFavCount: FutureEffect[IncrFavCount.Event] =
|
||||
FutureEffect[IncrFavCount.Event](e => svc.replicatedIncrFavCount(e.tweetId, e.delta))
|
||||
|
||||
override val incrBookmarkCount: FutureEffect[IncrBookmarkCount.Event] =
|
||||
FutureEffect[IncrBookmarkCount.Event](e =>
|
||||
svc.replicatedIncrBookmarkCount(e.tweetId, e.delta))
|
||||
|
||||
override val asyncTakedown: FutureEffect[AsyncTakedown.Event] =
|
||||
FutureEffect[AsyncTakedown.Event](e => svc.replicatedTakedown(e.tweet))
|
||||
|
||||
override val retryAsyncTakedown: FutureEffect[TweetStoreRetryEvent[AsyncTakedown.Event]] =
|
||||
TweetStore.retry(Action, asyncTakedown)
|
||||
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
AsyncUpdatePossiblySensitiveTweet.Event
|
||||
] =
|
||||
FutureEffect[AsyncUpdatePossiblySensitiveTweet.Event](e =>
|
||||
svc.replicatedUpdatePossiblySensitiveTweet(e.tweet))
|
||||
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUpdatePossiblySensitiveTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUpdatePossiblySensitiveTweet)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,38 +0,0 @@
|
|||
package com.twitter.tweetypie.store
|
||||
import com.twitter.tweetypie.FutureEffect
|
||||
import com.twitter.tweetypie.thriftscala.AsyncWriteAction
|
||||
import com.twitter.tweetypie.thriftscala.RetweetArchivalEvent
|
||||
|
||||
trait RetweetArchivalEnqueueStore
|
||||
extends TweetStoreBase[RetweetArchivalEnqueueStore]
|
||||
with AsyncSetRetweetVisibility.Store {
|
||||
def wrap(w: TweetStore.Wrap): RetweetArchivalEnqueueStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with RetweetArchivalEnqueueStore
|
||||
with AsyncSetRetweetVisibility.StoreWrapper
|
||||
}
|
||||
|
||||
object RetweetArchivalEnqueueStore {
|
||||
|
||||
def apply(enqueue: FutureEffect[RetweetArchivalEvent]): RetweetArchivalEnqueueStore =
|
||||
new RetweetArchivalEnqueueStore {
|
||||
override val asyncSetRetweetVisibility: FutureEffect[AsyncSetRetweetVisibility.Event] =
|
||||
FutureEffect[AsyncSetRetweetVisibility.Event] { e =>
|
||||
enqueue(
|
||||
RetweetArchivalEvent(
|
||||
retweetId = e.retweetId,
|
||||
srcTweetId = e.srcId,
|
||||
retweetUserId = e.retweetUserId,
|
||||
srcTweetUserId = e.srcTweetUserId,
|
||||
timestampMs = e.timestamp.inMillis,
|
||||
isArchivingAction = Some(!e.visible)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
override val retryAsyncSetRetweetVisibility: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncSetRetweetVisibility.Event]
|
||||
] =
|
||||
TweetStore.retry(AsyncWriteAction.RetweetArchivalEnqueue, asyncSetRetweetVisibility)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,42 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.servo.util.Scribe
|
||||
import com.twitter.tweetypie.thriftscala.TweetMediaTagEvent
|
||||
|
||||
/**
|
||||
* Scribes thrift-encoded TweetMediaTagEvents (from tweet_events.thrift).
|
||||
*/
|
||||
trait ScribeMediaTagStore extends TweetStoreBase[ScribeMediaTagStore] with AsyncInsertTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): ScribeMediaTagStore =
|
||||
new TweetStoreWrapper(w, this) with ScribeMediaTagStore with AsyncInsertTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object ScribeMediaTagStore {
|
||||
|
||||
private def toMediaTagEvent(event: AsyncInsertTweet.Event): Option[TweetMediaTagEvent] = {
|
||||
val tweet = event.tweet
|
||||
val taggedUserIds = getMediaTagMap(tweet).values.flatten.flatMap(_.userId).toSet
|
||||
val timestamp = Time.now.inMilliseconds
|
||||
if (taggedUserIds.nonEmpty) {
|
||||
Some(TweetMediaTagEvent(tweet.id, getUserId(tweet), taggedUserIds, Some(timestamp)))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
def apply(
|
||||
scribe: FutureEffect[String] = Scribe("tweetypie_media_tag_events")
|
||||
): ScribeMediaTagStore =
|
||||
new ScribeMediaTagStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
Scribe(TweetMediaTagEvent, scribe)
|
||||
.contramapOption[AsyncInsertTweet.Event](toMediaTagEvent)
|
||||
|
||||
// we don't retry this action
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
FutureEffect.unit[TweetStoreRetryEvent[AsyncInsertTweet.Event]]
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,164 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.servo.cache.Cached
|
||||
import com.twitter.servo.cache.CachedValueStatus
|
||||
import com.twitter.servo.cache.LockingCache
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
import com.twitter.tweetypie.backends.GeoScrubEventStore
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
/**
|
||||
* Scrub geo information from Tweets.
|
||||
*/
|
||||
object ScrubGeo extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
tweetIdSet: Set[TweetId],
|
||||
userId: UserId,
|
||||
optUser: Option[User],
|
||||
timestamp: Time,
|
||||
enqueueMax: Boolean)
|
||||
extends SyncTweetStoreEvent("scrub_geo")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
val tweetIds: Seq[TweetId] = tweetIdSet.toSeq
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
tweetIds.map { tweetId =>
|
||||
TweetEventData.TweetScrubGeoEvent(
|
||||
TweetScrubGeoEvent(
|
||||
tweetId = tweetId,
|
||||
userId = userId
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val scrubGeo: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val scrubGeo: FutureEffect[Event] = wrap(underlying.scrubGeo)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
logLensStore: LogLensStore,
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
eventBusEnqueueStore: TweetEventBusStore,
|
||||
replicatingStore: ReplicatingTweetStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val scrubGeo: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
logLensStore.scrubGeo,
|
||||
manhattanStore.scrubGeo,
|
||||
cachingTweetStore.scrubGeo,
|
||||
eventBusEnqueueStore.scrubGeo,
|
||||
replicatingStore.scrubGeo
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedScrubGeo extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(tweetIds: Seq[TweetId]) extends ReplicatedTweetStoreEvent("replicated_scrub_geo")
|
||||
|
||||
trait Store {
|
||||
val replicatedScrubGeo: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedScrubGeo: FutureEffect[Event] = wrap(underlying.replicatedScrubGeo)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(cachingTweetStore: CachingTweetStore): Store = {
|
||||
new Store {
|
||||
override val replicatedScrubGeo: FutureEffect[Event] =
|
||||
cachingTweetStore.replicatedScrubGeo
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the timestamp of the user's most recent request to delete all
|
||||
* location data attached to her tweets. We use the timestamp to ensure
|
||||
* that even if we fail to scrub a particular tweet in storage, we will
|
||||
* not return geo information with that tweet.
|
||||
*
|
||||
* See http://go/geoscrub for more details.
|
||||
*/
|
||||
object ScrubGeoUpdateUserTimestamp extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(userId: UserId, timestamp: Time, optUser: Option[User])
|
||||
extends SyncTweetStoreEvent("scrub_geo_update_user_timestamp")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
def mightHaveGeotaggedStatuses: Boolean =
|
||||
optUser.forall(_.account.forall(_.hasGeotaggedStatuses == true))
|
||||
|
||||
def maxTweetId: TweetId = SnowflakeId.firstIdFor(timestamp + 1.millisecond) - 1
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.UserScrubGeoEvent(
|
||||
UserScrubGeoEvent(
|
||||
userId = userId,
|
||||
maxTweetId = maxTweetId
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
/**
|
||||
* How to update a geo scrub timestamp cache entry. Always prefers
|
||||
* the highest timestamp value that is available, regardless of when
|
||||
* it was added to cache.
|
||||
*/
|
||||
def cacheHandler: LockingCache.Handler[Cached[Time]] = {
|
||||
case Some(c) if c.value.exists(_ >= timestamp) => None
|
||||
case _ => Some(Cached(Some(timestamp), CachedValueStatus.Found, Time.now))
|
||||
}
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val scrubGeoUpdateUserTimestamp: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val scrubGeoUpdateUserTimestamp: FutureEffect[Event] = wrap(
|
||||
underlying.scrubGeoUpdateUserTimestamp)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
geotagUpdateStore: GizmoduckUserGeotagUpdateStore,
|
||||
tweetEventBusStore: TweetEventBusStore,
|
||||
setInManhattan: GeoScrubEventStore.SetGeoScrubTimestamp,
|
||||
cache: LockingCache[UserId, Cached[Time]]
|
||||
): Store = {
|
||||
val manhattanEffect =
|
||||
setInManhattan.asFutureEffect
|
||||
.contramap[Event](e => (e.userId, e.timestamp))
|
||||
|
||||
val cacheEffect =
|
||||
FutureEffect[Event](e => cache.lockAndSet(e.userId, e.cacheHandler).unit)
|
||||
|
||||
new Store {
|
||||
override val scrubGeoUpdateUserTimestamp: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
manhattanEffect,
|
||||
cacheEffect,
|
||||
geotagUpdateStore.scrubGeoUpdateUserTimestamp,
|
||||
tweetEventBusStore.scrubGeoUpdateUserTimestamp
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,155 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object SetAdditionalFields extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(additionalFields: Tweet, userId: UserId, timestamp: Time)
|
||||
extends SyncTweetStoreEvent("set_additional_fields") {
|
||||
|
||||
def toAsyncRequest: AsyncSetAdditionalFieldsRequest =
|
||||
AsyncSetAdditionalFieldsRequest(
|
||||
additionalFields = additionalFields,
|
||||
userId = userId,
|
||||
timestamp = timestamp.inMillis
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val setAdditionalFields: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val setAdditionalFields: FutureEffect[Event] = wrap(underlying.setAdditionalFields)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
asyncEnqueueStore: AsyncEnqueueStore,
|
||||
logLensStore: LogLensStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val setAdditionalFields: FutureEffect[Event] =
|
||||
FutureEffect.sequentially(
|
||||
logLensStore.setAdditionalFields,
|
||||
manhattanStore.setAdditionalFields,
|
||||
// Ignore failures but wait for completion to ensure we attempted to update cache before
|
||||
// running async tasks, in particular publishing an event to EventBus.
|
||||
cachingTweetStore.ignoreFailuresUponCompletion.setAdditionalFields,
|
||||
asyncEnqueueStore.setAdditionalFields
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncSetAdditionalFields extends TweetStore.AsyncModule {
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(
|
||||
request: AsyncSetAdditionalFieldsRequest,
|
||||
user: User
|
||||
): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
Event(
|
||||
additionalFields = request.additionalFields,
|
||||
userId = request.userId,
|
||||
optUser = Some(user),
|
||||
timestamp = Time.fromMilliseconds(request.timestamp)
|
||||
),
|
||||
request.retryAction,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class Event(additionalFields: Tweet, userId: UserId, optUser: Option[User], timestamp: Time)
|
||||
extends AsyncTweetStoreEvent("async_set_additional_fields")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
def toAsyncRequest(action: Option[AsyncWriteAction] = None): AsyncSetAdditionalFieldsRequest =
|
||||
AsyncSetAdditionalFieldsRequest(
|
||||
additionalFields = additionalFields,
|
||||
retryAction = action,
|
||||
userId = userId,
|
||||
timestamp = timestamp.inMillis
|
||||
)
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.AdditionalFieldUpdateEvent(
|
||||
AdditionalFieldUpdateEvent(
|
||||
updatedFields = additionalFields,
|
||||
userId = optUser.map(_.id)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncSetAdditionalFields(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.SetAdditionalFields.type =
|
||||
AsyncWriteEventType.SetAdditionalFields
|
||||
override val scribedTweetOnFailure: None.type = None
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncSetAdditionalFields: FutureEffect[Event]
|
||||
val retryAsyncSetAdditionalFields: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncSetAdditionalFields: FutureEffect[Event] = wrap(
|
||||
underlying.asyncSetAdditionalFields)
|
||||
override val retryAsyncSetAdditionalFields: FutureEffect[TweetStoreRetryEvent[Event]] = wrap(
|
||||
underlying.retryAsyncSetAdditionalFields)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
replicatingStore: ReplicatingTweetStore,
|
||||
eventBusEnqueueStore: TweetEventBusStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] = Seq(replicatingStore, eventBusEnqueueStore)
|
||||
|
||||
def build[E <: TweetStoreEvent](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncSetAdditionalFields: FutureEffect[Event] = build(
|
||||
_.asyncSetAdditionalFields)
|
||||
override val retryAsyncSetAdditionalFields: FutureEffect[TweetStoreRetryEvent[Event]] =
|
||||
build(_.retryAsyncSetAdditionalFields)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedSetAdditionalFields extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(additionalFields: Tweet)
|
||||
extends ReplicatedTweetStoreEvent("replicated_set_additional_fields")
|
||||
|
||||
trait Store {
|
||||
val replicatedSetAdditionalFields: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedSetAdditionalFields: FutureEffect[Event] = wrap(
|
||||
underlying.replicatedSetAdditionalFields)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(cachingTweetStore: CachingTweetStore): Store = {
|
||||
new Store {
|
||||
override val replicatedSetAdditionalFields: FutureEffect[Event] =
|
||||
cachingTweetStore.replicatedSetAdditionalFields
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,172 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object SetRetweetVisibility extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
retweetId: TweetId,
|
||||
visible: Boolean,
|
||||
srcId: TweetId,
|
||||
retweetUserId: UserId,
|
||||
srcTweetUserId: UserId,
|
||||
timestamp: Time)
|
||||
extends SyncTweetStoreEvent("set_retweet_visibility") {
|
||||
def toAsyncRequest: AsyncSetRetweetVisibilityRequest =
|
||||
AsyncSetRetweetVisibilityRequest(
|
||||
retweetId = retweetId,
|
||||
visible = visible,
|
||||
srcId = srcId,
|
||||
retweetUserId = retweetUserId,
|
||||
sourceTweetUserId = srcTweetUserId,
|
||||
timestamp = timestamp.inMillis
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val setRetweetVisibility: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
val setRetweetVisibility: FutureEffect[Event] = wrap(underlying.setRetweetVisibility)
|
||||
}
|
||||
|
||||
object Store {
|
||||
|
||||
/**
|
||||
* [[AsyncEnqueueStore]] - use this store to call the asyncSetRetweetVisibility endpoint.
|
||||
*
|
||||
* @see [[AsyncSetRetweetVisibility.Store.apply]]
|
||||
*/
|
||||
def apply(asyncEnqueueStore: AsyncEnqueueStore): Store =
|
||||
new Store {
|
||||
override val setRetweetVisibility: FutureEffect[Event] =
|
||||
asyncEnqueueStore.setRetweetVisibility
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncSetRetweetVisibility extends TweetStore.AsyncModule {
|
||||
|
||||
case class Event(
|
||||
retweetId: TweetId,
|
||||
visible: Boolean,
|
||||
srcId: TweetId,
|
||||
retweetUserId: UserId,
|
||||
srcTweetUserId: UserId,
|
||||
timestamp: Time)
|
||||
extends AsyncTweetStoreEvent("async_set_retweet_visibility") {
|
||||
def toAsyncRequest(action: Option[AsyncWriteAction] = None): AsyncSetRetweetVisibilityRequest =
|
||||
AsyncSetRetweetVisibilityRequest(
|
||||
retweetId = retweetId,
|
||||
visible = visible,
|
||||
srcId = srcId,
|
||||
retweetUserId = retweetUserId,
|
||||
sourceTweetUserId = srcTweetUserId,
|
||||
retryAction = action,
|
||||
timestamp = timestamp.inMillis
|
||||
)
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncSetRetweetVisibility(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(req: AsyncSetRetweetVisibilityRequest): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
AsyncSetRetweetVisibility.Event(
|
||||
retweetId = req.retweetId,
|
||||
visible = req.visible,
|
||||
srcId = req.srcId,
|
||||
retweetUserId = req.retweetUserId,
|
||||
srcTweetUserId = req.sourceTweetUserId,
|
||||
timestamp = Time.fromMilliseconds(req.timestamp)
|
||||
),
|
||||
req.retryAction,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.SetRetweetVisibility.type =
|
||||
AsyncWriteEventType.SetRetweetVisibility
|
||||
override val scribedTweetOnFailure: None.type = None
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncSetRetweetVisibility: FutureEffect[Event]
|
||||
val retryAsyncSetRetweetVisibility: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
val asyncSetRetweetVisibility: FutureEffect[Event] = wrap(underlying.asyncSetRetweetVisibility)
|
||||
val retryAsyncSetRetweetVisibility: FutureEffect[TweetStoreRetryEvent[Event]] = wrap(
|
||||
underlying.retryAsyncSetRetweetVisibility)
|
||||
}
|
||||
|
||||
object Store {
|
||||
|
||||
/**
|
||||
* [[TweetIndexingStore]] - archive or unarchive a retweet edge in TFlock RetweetGraph
|
||||
* [[TweetCountsCacheUpdatingStore]] - modify the retweet count directly in cache.
|
||||
* [[ReplicatingTweetStore]] - replicate this [[Event]] in the other DC.
|
||||
* [[RetweetArchivalEnqueueStore]] - publish RetweetArchivalEvent to "retweet_archival_events" event stream.
|
||||
*
|
||||
* @see [[ReplicatedSetRetweetVisibility.Store.apply]]
|
||||
*/
|
||||
def apply(
|
||||
tweetIndexingStore: TweetIndexingStore,
|
||||
tweetCountsCacheUpdatingStore: TweetCountsCacheUpdatingStore,
|
||||
replicatingTweetStore: ReplicatingTweetStore,
|
||||
retweetArchivalEnqueueStore: RetweetArchivalEnqueueStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] =
|
||||
Seq(
|
||||
tweetIndexingStore,
|
||||
tweetCountsCacheUpdatingStore,
|
||||
replicatingTweetStore,
|
||||
retweetArchivalEnqueueStore
|
||||
)
|
||||
|
||||
def build[E <: TweetStoreEvent, S](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncSetRetweetVisibility: FutureEffect[Event] = build(
|
||||
_.asyncSetRetweetVisibility)
|
||||
override val retryAsyncSetRetweetVisibility: FutureEffect[TweetStoreRetryEvent[Event]] =
|
||||
build(_.retryAsyncSetRetweetVisibility)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedSetRetweetVisibility extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(srcId: TweetId, visible: Boolean)
|
||||
extends ReplicatedTweetStoreEvent("replicated_set_retweet_visibility")
|
||||
|
||||
trait Store {
|
||||
val replicatedSetRetweetVisibility: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedSetRetweetVisibility: FutureEffect[Event] =
|
||||
wrap(underlying.replicatedSetRetweetVisibility)
|
||||
}
|
||||
|
||||
object Store {
|
||||
|
||||
/**
|
||||
* [[TweetCountsCacheUpdatingStore]] - replicate modifying the retweet count directly in cache.
|
||||
*/
|
||||
def apply(tweetCountsCacheUpdatingStore: TweetCountsCacheUpdatingStore): Store =
|
||||
new Store {
|
||||
override val replicatedSetRetweetVisibility: FutureEffect[Event] =
|
||||
tweetCountsCacheUpdatingStore.replicatedSetRetweetVisibility
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,205 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.takedown.util.TakedownReasons
|
||||
import com.twitter.tseng.withholding.thriftscala.TakedownReason
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object Takedown extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet, // for CachingTweetStore / ManhattanTweetStore / ReplicatedTakedown
|
||||
timestamp: Time,
|
||||
user: Option[User] = None,
|
||||
takedownReasons: Seq[TakedownReason] = Seq(), // for EventBus
|
||||
reasonsToAdd: Seq[TakedownReason] = Seq(), // for Guano
|
||||
reasonsToRemove: Seq[TakedownReason] = Seq(), // for Guano
|
||||
auditNote: Option[String] = None,
|
||||
host: Option[String] = None,
|
||||
byUserId: Option[UserId] = None,
|
||||
eventbusEnqueue: Boolean = true,
|
||||
scribeForAudit: Boolean = true,
|
||||
// If ManhattanTweetStore should update countryCodes and reasons
|
||||
updateCodesAndReasons: Boolean = false)
|
||||
extends SyncTweetStoreEvent("takedown") {
|
||||
def toAsyncRequest(): AsyncTakedownRequest =
|
||||
AsyncTakedownRequest(
|
||||
tweet = tweet,
|
||||
user = user,
|
||||
takedownReasons = takedownReasons,
|
||||
reasonsToAdd = reasonsToAdd,
|
||||
reasonsToRemove = reasonsToRemove,
|
||||
scribeForAudit = scribeForAudit,
|
||||
eventbusEnqueue = eventbusEnqueue,
|
||||
auditNote = auditNote,
|
||||
byUserId = byUserId,
|
||||
host = host,
|
||||
timestamp = timestamp.inMillis
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val takedown: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val takedown: FutureEffect[Event] = wrap(underlying.takedown)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
logLensStore: LogLensStore,
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
asyncEnqueueStore: AsyncEnqueueStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val takedown: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
logLensStore.takedown,
|
||||
FutureEffect.sequentially(
|
||||
manhattanStore.takedown,
|
||||
FutureEffect.inParallel(
|
||||
cachingTweetStore.takedown,
|
||||
asyncEnqueueStore.takedown
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncTakedown extends TweetStore.AsyncModule {
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(request: AsyncTakedownRequest): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
Event(
|
||||
tweet = request.tweet,
|
||||
optUser = request.user,
|
||||
takedownReasons = request.takedownReasons,
|
||||
reasonsToAdd = request.reasonsToAdd,
|
||||
reasonsToRemove = request.reasonsToRemove,
|
||||
auditNote = request.auditNote,
|
||||
host = request.host,
|
||||
byUserId = request.byUserId,
|
||||
eventbusEnqueue = request.eventbusEnqueue,
|
||||
scribeForAudit = request.scribeForAudit,
|
||||
timestamp = Time.fromMilliseconds(request.timestamp)
|
||||
),
|
||||
request.retryAction,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
timestamp: Time,
|
||||
optUser: Option[User],
|
||||
takedownReasons: Seq[TakedownReason], // for EventBus
|
||||
reasonsToAdd: Seq[TakedownReason], // for Guano
|
||||
reasonsToRemove: Seq[TakedownReason], // for Guano
|
||||
auditNote: Option[String], // for Guano
|
||||
host: Option[String], // for Guano
|
||||
byUserId: Option[UserId], // for Guano
|
||||
eventbusEnqueue: Boolean,
|
||||
scribeForAudit: Boolean)
|
||||
extends AsyncTweetStoreEvent("async_takedown")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
def toAsyncRequest(action: Option[AsyncWriteAction] = None): AsyncTakedownRequest =
|
||||
AsyncTakedownRequest(
|
||||
tweet = tweet,
|
||||
user = optUser,
|
||||
takedownReasons = takedownReasons,
|
||||
reasonsToAdd = reasonsToAdd,
|
||||
reasonsToRemove = reasonsToRemove,
|
||||
scribeForAudit = scribeForAudit,
|
||||
eventbusEnqueue = eventbusEnqueue,
|
||||
auditNote = auditNote,
|
||||
byUserId = byUserId,
|
||||
host = host,
|
||||
timestamp = timestamp.inMillis,
|
||||
retryAction = action
|
||||
)
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
optUser.map { user =>
|
||||
TweetEventData.TweetTakedownEvent(
|
||||
TweetTakedownEvent(
|
||||
tweetId = tweet.id,
|
||||
userId = user.id,
|
||||
takedownCountryCodes =
|
||||
takedownReasons.collect(TakedownReasons.reasonToCountryCode).sorted,
|
||||
takedownReasons = takedownReasons
|
||||
)
|
||||
)
|
||||
}.toSeq
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncTakedown(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.Takedown.type = AsyncWriteEventType.Takedown
|
||||
override val scribedTweetOnFailure: Option[Tweet] = Some(event.tweet)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncTakedown: FutureEffect[Event]
|
||||
val retryAsyncTakedown: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncTakedown: FutureEffect[Event] = wrap(underlying.asyncTakedown)
|
||||
override val retryAsyncTakedown: FutureEffect[TweetStoreRetryEvent[Event]] = wrap(
|
||||
underlying.retryAsyncTakedown)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
replicatingStore: ReplicatingTweetStore,
|
||||
guanoStore: GuanoServiceStore,
|
||||
eventBusEnqueueStore: TweetEventBusStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] =
|
||||
Seq(
|
||||
replicatingStore,
|
||||
guanoStore,
|
||||
eventBusEnqueueStore
|
||||
)
|
||||
|
||||
def build[E <: TweetStoreEvent](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncTakedown: FutureEffect[Event] = build(_.asyncTakedown)
|
||||
override val retryAsyncTakedown: FutureEffect[TweetStoreRetryEvent[Event]] = build(
|
||||
_.retryAsyncTakedown)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedTakedown extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(tweet: Tweet) extends ReplicatedTweetStoreEvent("takedown")
|
||||
|
||||
trait Store {
|
||||
val replicatedTakedown: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedTakedown: FutureEffect[Event] = wrap(underlying.replicatedTakedown)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(cachingTweetStore: CachingTweetStore): Store = {
|
||||
new Store {
|
||||
override val replicatedTakedown: FutureEffect[Event] = cachingTweetStore.replicatedTakedown
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,150 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.timelineservice.{thriftscala => tls}
|
||||
import com.twitter.tweetypie.backends.TimelineService
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait TlsTimelineUpdatingStore
|
||||
extends TweetStoreBase[TlsTimelineUpdatingStore]
|
||||
with AsyncInsertTweet.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with AsyncUndeleteTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): TlsTimelineUpdatingStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with TlsTimelineUpdatingStore
|
||||
with AsyncInsertTweet.StoreWrapper
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with AsyncUndeleteTweet.StoreWrapper
|
||||
}
|
||||
|
||||
/**
|
||||
* An implementation of TweetStore that sends update events to
|
||||
* the Timeline Service.
|
||||
*/
|
||||
object TlsTimelineUpdatingStore {
|
||||
val Action: AsyncWriteAction.TimelineUpdate.type = AsyncWriteAction.TimelineUpdate
|
||||
|
||||
/**
|
||||
* Converts a TweetyPie Tweet to tls.Tweet
|
||||
*
|
||||
* @param explicitCreatedAt when Some, overrides the default getTimestamp defined in package
|
||||
* object com.twitter.tweetypie
|
||||
*/
|
||||
def tweetToTLSFullTweet(
|
||||
hasMedia: Tweet => Boolean
|
||||
)(
|
||||
tweet: Tweet,
|
||||
explicitCreatedAt: Option[Time],
|
||||
noteTweetMentionedUserIds: Option[Seq[Long]]
|
||||
): tls.FullTweet =
|
||||
tls.FullTweet(
|
||||
userId = getUserId(tweet),
|
||||
tweetId = tweet.id,
|
||||
mentionedUserIds =
|
||||
noteTweetMentionedUserIds.getOrElse(getMentions(tweet).flatMap(_.userId)).toSet,
|
||||
isNullcasted = TweetLenses.nullcast.get(tweet),
|
||||
conversationId = TweetLenses.conversationId.get(tweet).getOrElse(tweet.id),
|
||||
narrowcastGeos = Set.empty,
|
||||
createdAtMs = explicitCreatedAt.getOrElse(getTimestamp(tweet)).inMillis,
|
||||
hasMedia = hasMedia(tweet),
|
||||
directedAtUserId = TweetLenses.directedAtUser.get(tweet).map(_.userId),
|
||||
retweet = getShare(tweet).map { share =>
|
||||
tls.Retweet(
|
||||
sourceUserId = share.sourceUserId,
|
||||
sourceTweetId = share.sourceStatusId,
|
||||
parentTweetId = Some(share.parentStatusId)
|
||||
)
|
||||
},
|
||||
reply = getReply(tweet).map { reply =>
|
||||
tls.Reply(
|
||||
inReplyToUserId = reply.inReplyToUserId,
|
||||
inReplyToTweetId = reply.inReplyToStatusId
|
||||
)
|
||||
},
|
||||
quote = tweet.quotedTweet.map { qt =>
|
||||
tls.Quote(
|
||||
quotedUserId = qt.userId,
|
||||
quotedTweetId = qt.tweetId
|
||||
)
|
||||
},
|
||||
mediaTags = tweet.mediaTags,
|
||||
text = Some(getText(tweet))
|
||||
)
|
||||
|
||||
val logger: Logger = Logger(getClass)
|
||||
|
||||
def logValidationFailed(stats: StatsReceiver): tls.ProcessEventResult => Unit = {
|
||||
case tls.ProcessEventResult(tls.ProcessEventResultType.ValidationFailed, errors) =>
|
||||
logger.error(s"Validation Failed in processEvent2: $errors")
|
||||
stats.counter("processEvent2_validation_failed").incr()
|
||||
case _ => ()
|
||||
}
|
||||
|
||||
def apply(
|
||||
processEvent2: TimelineService.ProcessEvent2,
|
||||
hasMedia: Tweet => Boolean,
|
||||
stats: StatsReceiver
|
||||
): TlsTimelineUpdatingStore = {
|
||||
val toTlsTweet = tweetToTLSFullTweet(hasMedia) _
|
||||
|
||||
val processAndLog =
|
||||
processEvent2.andThen(FutureArrow.fromFunction(logValidationFailed(stats)))
|
||||
|
||||
new TlsTimelineUpdatingStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
processAndLog
|
||||
.contramap[AsyncInsertTweet.Event] { event =>
|
||||
tls.Event.FullTweetCreate(
|
||||
tls.FullTweetCreateEvent(
|
||||
toTlsTweet(event.tweet, Some(event.timestamp), event.noteTweetMentionedUserIds),
|
||||
event.timestamp.inMillis,
|
||||
featureContext = event.featureContext
|
||||
)
|
||||
)
|
||||
}
|
||||
.asFutureEffect[AsyncInsertTweet.Event]
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncInsertTweet)
|
||||
|
||||
override val asyncUndeleteTweet: FutureEffect[AsyncUndeleteTweet.Event] =
|
||||
processAndLog
|
||||
.contramap[AsyncUndeleteTweet.Event] { event =>
|
||||
tls.Event.FullTweetRestore(
|
||||
tls.FullTweetRestoreEvent(
|
||||
toTlsTweet(event.tweet, None, None),
|
||||
event.deletedAt.map(_.inMillis)
|
||||
)
|
||||
)
|
||||
}
|
||||
.asFutureEffect[AsyncUndeleteTweet.Event]
|
||||
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUndeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUndeleteTweet)
|
||||
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
processAndLog
|
||||
.contramap[AsyncDeleteTweet.Event] { event =>
|
||||
tls.Event.FullTweetDelete(
|
||||
tls.FullTweetDeleteEvent(
|
||||
toTlsTweet(event.tweet, None, None),
|
||||
event.timestamp.inMillis,
|
||||
isUserErasure = Some(event.isUserErasure),
|
||||
isBounceDelete = Some(event.isBounceDelete)
|
||||
)
|
||||
)
|
||||
}
|
||||
.asFutureEffect[AsyncDeleteTweet.Event]
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,358 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.concurrent.Serialized
|
||||
import com.twitter.servo.cache.LockingCache.Handler
|
||||
import com.twitter.servo.cache._
|
||||
import com.twitter.tweetypie.repository.BookmarksKey
|
||||
import com.twitter.tweetypie.repository.FavsKey
|
||||
import com.twitter.tweetypie.repository.QuotesKey
|
||||
import com.twitter.tweetypie.repository.RepliesKey
|
||||
import com.twitter.tweetypie.repository.RetweetsKey
|
||||
import com.twitter.tweetypie.repository.TweetCountKey
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Timer
|
||||
import scala.collection.mutable
|
||||
|
||||
trait TweetCountsCacheUpdatingStore
|
||||
extends TweetStoreBase[TweetCountsCacheUpdatingStore]
|
||||
with InsertTweet.Store
|
||||
with AsyncInsertTweet.Store
|
||||
with ReplicatedInsertTweet.Store
|
||||
with DeleteTweet.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with ReplicatedDeleteTweet.Store
|
||||
with UndeleteTweet.Store
|
||||
with ReplicatedUndeleteTweet.Store
|
||||
with AsyncIncrFavCount.Store
|
||||
with ReplicatedIncrFavCount.Store
|
||||
with AsyncIncrBookmarkCount.Store
|
||||
with ReplicatedIncrBookmarkCount.Store
|
||||
with AsyncSetRetweetVisibility.Store
|
||||
with ReplicatedSetRetweetVisibility.Store
|
||||
with Flush.Store {
|
||||
def wrap(w: TweetStore.Wrap): TweetCountsCacheUpdatingStore = {
|
||||
new TweetStoreWrapper(w, this)
|
||||
with TweetCountsCacheUpdatingStore
|
||||
with InsertTweet.StoreWrapper
|
||||
with AsyncInsertTweet.StoreWrapper
|
||||
with ReplicatedInsertTweet.StoreWrapper
|
||||
with DeleteTweet.StoreWrapper
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with ReplicatedDeleteTweet.StoreWrapper
|
||||
with UndeleteTweet.StoreWrapper
|
||||
with ReplicatedUndeleteTweet.StoreWrapper
|
||||
with AsyncIncrFavCount.StoreWrapper
|
||||
with ReplicatedIncrFavCount.StoreWrapper
|
||||
with AsyncIncrBookmarkCount.StoreWrapper
|
||||
with ReplicatedIncrBookmarkCount.StoreWrapper
|
||||
with AsyncSetRetweetVisibility.StoreWrapper
|
||||
with ReplicatedSetRetweetVisibility.StoreWrapper
|
||||
with Flush.StoreWrapper
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An implementation of TweetStore that updates tweet-specific counts in
|
||||
* the CountsCache.
|
||||
*/
|
||||
object TweetCountsCacheUpdatingStore {
|
||||
private type Action = TweetCountKey => Future[Unit]
|
||||
|
||||
def keys(tweetId: TweetId): Seq[TweetCountKey] =
|
||||
Seq(
|
||||
RetweetsKey(tweetId),
|
||||
RepliesKey(tweetId),
|
||||
FavsKey(tweetId),
|
||||
QuotesKey(tweetId),
|
||||
BookmarksKey(tweetId))
|
||||
|
||||
def relatedKeys(tweet: Tweet): Seq[TweetCountKey] =
|
||||
Seq(
|
||||
getReply(tweet).flatMap(_.inReplyToStatusId).map(RepliesKey(_)),
|
||||
getQuotedTweet(tweet).map(quotedTweet => QuotesKey(quotedTweet.tweetId)),
|
||||
getShare(tweet).map(share => RetweetsKey(share.sourceStatusId))
|
||||
).flatten
|
||||
|
||||
// pick all keys except quotes key
|
||||
def relatedKeysWithoutQuotesKey(tweet: Tweet): Seq[TweetCountKey] =
|
||||
relatedKeys(tweet).filterNot(_.isInstanceOf[QuotesKey])
|
||||
|
||||
def apply(countsStore: CachedCountsStore): TweetCountsCacheUpdatingStore = {
|
||||
val incr: Action = key => countsStore.incr(key, 1)
|
||||
val decr: Action = key => countsStore.incr(key, -1)
|
||||
val init: Action = key => countsStore.add(key, 0)
|
||||
val delete: Action = key => countsStore.delete(key)
|
||||
|
||||
def initCounts(tweetId: TweetId) = Future.join(keys(tweetId).map(init))
|
||||
def incrRelatedCounts(tweet: Tweet, excludeQuotesKey: Boolean = false) = {
|
||||
Future.join {
|
||||
if (excludeQuotesKey) {
|
||||
relatedKeysWithoutQuotesKey(tweet).map(incr)
|
||||
} else {
|
||||
relatedKeys(tweet).map(incr)
|
||||
}
|
||||
}
|
||||
}
|
||||
def deleteCounts(tweetId: TweetId) = Future.join(keys(tweetId).map(delete))
|
||||
|
||||
// Decrement all the counters if is the last quote, otherwise avoid decrementing quote counters
|
||||
def decrRelatedCounts(tweet: Tweet, isLastQuoteOfQuoter: Boolean = false) = {
|
||||
Future.join {
|
||||
if (isLastQuoteOfQuoter) {
|
||||
relatedKeys(tweet).map(decr)
|
||||
} else {
|
||||
relatedKeysWithoutQuotesKey(tweet).map(decr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateFavCount(tweetId: TweetId, delta: Int) =
|
||||
countsStore.incr(FavsKey(tweetId), delta).unit
|
||||
|
||||
def updateBookmarkCount(tweetId: TweetId, delta: Int) =
|
||||
countsStore.incr(BookmarksKey(tweetId), delta).unit
|
||||
|
||||
// these are use specifically for setRetweetVisibility
|
||||
def incrRetweetCount(tweetId: TweetId) = incr(RetweetsKey(tweetId))
|
||||
def decrRetweetCount(tweetId: TweetId) = decr(RetweetsKey(tweetId))
|
||||
|
||||
new TweetCountsCacheUpdatingStore {
|
||||
override val insertTweet: FutureEffect[InsertTweet.Event] =
|
||||
FutureEffect[InsertTweet.Event](e => initCounts(e.tweet.id))
|
||||
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
FutureEffect[AsyncInsertTweet.Event] { e =>
|
||||
incrRelatedCounts(e.cachedTweet.tweet, e.quoterHasAlreadyQuotedTweet)
|
||||
}
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
FutureEffect.unit[TweetStoreRetryEvent[AsyncInsertTweet.Event]]
|
||||
|
||||
override val replicatedInsertTweet: FutureEffect[ReplicatedInsertTweet.Event] =
|
||||
FutureEffect[ReplicatedInsertTweet.Event] { e =>
|
||||
Future
|
||||
.join(
|
||||
initCounts(e.tweet.id),
|
||||
incrRelatedCounts(e.tweet, e.quoterHasAlreadyQuotedTweet)).unit
|
||||
}
|
||||
|
||||
override val deleteTweet: FutureEffect[DeleteTweet.Event] =
|
||||
FutureEffect[DeleteTweet.Event](e => deleteCounts(e.tweet.id))
|
||||
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
FutureEffect[AsyncDeleteTweet.Event](e => decrRelatedCounts(e.tweet, e.isLastQuoteOfQuoter))
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
FutureEffect.unit[TweetStoreRetryEvent[AsyncDeleteTweet.Event]]
|
||||
|
||||
override val replicatedDeleteTweet: FutureEffect[ReplicatedDeleteTweet.Event] =
|
||||
FutureEffect[ReplicatedDeleteTweet.Event] { e =>
|
||||
Future
|
||||
.join(deleteCounts(e.tweet.id), decrRelatedCounts(e.tweet, e.isLastQuoteOfQuoter)).unit
|
||||
}
|
||||
|
||||
override val undeleteTweet: FutureEffect[UndeleteTweet.Event] =
|
||||
FutureEffect[UndeleteTweet.Event] { e =>
|
||||
incrRelatedCounts(e.tweet, e.quoterHasAlreadyQuotedTweet)
|
||||
}
|
||||
|
||||
override val replicatedUndeleteTweet: FutureEffect[ReplicatedUndeleteTweet.Event] =
|
||||
FutureEffect[ReplicatedUndeleteTweet.Event] { e =>
|
||||
incrRelatedCounts(e.tweet, e.quoterHasAlreadyQuotedTweet)
|
||||
}
|
||||
|
||||
override val asyncIncrFavCount: FutureEffect[AsyncIncrFavCount.Event] =
|
||||
FutureEffect[AsyncIncrFavCount.Event](e => updateFavCount(e.tweetId, e.delta))
|
||||
|
||||
override val replicatedIncrFavCount: FutureEffect[ReplicatedIncrFavCount.Event] =
|
||||
FutureEffect[ReplicatedIncrFavCount.Event](e => updateFavCount(e.tweetId, e.delta))
|
||||
|
||||
override val asyncIncrBookmarkCount: FutureEffect[AsyncIncrBookmarkCount.Event] =
|
||||
FutureEffect[AsyncIncrBookmarkCount.Event](e => updateBookmarkCount(e.tweetId, e.delta))
|
||||
|
||||
override val replicatedIncrBookmarkCount: FutureEffect[ReplicatedIncrBookmarkCount.Event] =
|
||||
FutureEffect[ReplicatedIncrBookmarkCount.Event] { e =>
|
||||
updateBookmarkCount(e.tweetId, e.delta)
|
||||
}
|
||||
|
||||
override val asyncSetRetweetVisibility: FutureEffect[AsyncSetRetweetVisibility.Event] =
|
||||
FutureEffect[AsyncSetRetweetVisibility.Event] { e =>
|
||||
if (e.visible) incrRetweetCount(e.srcId) else decrRetweetCount(e.srcId)
|
||||
}
|
||||
|
||||
override val retryAsyncSetRetweetVisibility: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncSetRetweetVisibility.Event]
|
||||
] =
|
||||
FutureEffect.unit[TweetStoreRetryEvent[AsyncSetRetweetVisibility.Event]]
|
||||
|
||||
override val replicatedSetRetweetVisibility: FutureEffect[
|
||||
ReplicatedSetRetweetVisibility.Event
|
||||
] =
|
||||
FutureEffect[ReplicatedSetRetweetVisibility.Event] { e =>
|
||||
if (e.visible) incrRetweetCount(e.srcId) else decrRetweetCount(e.srcId)
|
||||
}
|
||||
|
||||
override val flush: FutureEffect[Flush.Event] =
|
||||
FutureEffect[Flush.Event] { e => Future.collect(e.tweetIds.map(deleteCounts)).unit }
|
||||
.onlyIf(_.flushCounts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A simple trait around the cache operations needed by TweetCountsCacheUpdatingStore.
|
||||
*/
|
||||
trait CachedCountsStore {
|
||||
def add(key: TweetCountKey, count: Count): Future[Unit]
|
||||
def delete(key: TweetCountKey): Future[Unit]
|
||||
def incr(key: TweetCountKey, delta: Count): Future[Unit]
|
||||
}
|
||||
|
||||
object CachedCountsStore {
|
||||
def fromLockingCache(cache: LockingCache[TweetCountKey, Cached[Count]]): CachedCountsStore =
|
||||
new CachedCountsStore {
|
||||
def add(key: TweetCountKey, count: Count): Future[Unit] =
|
||||
cache.add(key, toCached(count)).unit
|
||||
|
||||
def delete(key: TweetCountKey): Future[Unit] =
|
||||
cache.delete(key).unit
|
||||
|
||||
def incr(key: TweetCountKey, delta: Count): Future[Unit] =
|
||||
cache.lockAndSet(key, IncrDecrHandler(delta)).unit
|
||||
}
|
||||
|
||||
def toCached(count: Count): Cached[Count] = {
|
||||
val now = Time.now
|
||||
Cached(Some(count), CachedValueStatus.Found, now, Some(now))
|
||||
}
|
||||
|
||||
case class IncrDecrHandler(delta: Long) extends Handler[Cached[Count]] {
|
||||
override def apply(inCache: Option[Cached[Count]]): Option[Cached[Count]] =
|
||||
inCache.flatMap(incrCount)
|
||||
|
||||
private[this] def incrCount(oldCached: Cached[Count]): Option[Cached[Count]] = {
|
||||
oldCached.value.map { oldCount => oldCached.copy(value = Some(saferIncr(oldCount))) }
|
||||
}
|
||||
|
||||
private[this] def saferIncr(value: Long) = math.max(0, value + delta)
|
||||
|
||||
override lazy val toString: String = "IncrDecrHandler(%s)".format(delta)
|
||||
}
|
||||
|
||||
object QueueIsFullException extends Exception
|
||||
}
|
||||
|
||||
/**
|
||||
* An implementation of CachedCountsStore that can queue and aggregate multiple incr
|
||||
* updates to the same key together. Currently, updates for a key only start to aggregate
|
||||
* after there is a failure to incr on the underlying store, which often indicates contention
|
||||
* due to a high level of updates. After a failure, a key is promoted into a "tracked" state,
|
||||
* and subsequent updates are aggregated together. Periodically, the aggregated updates will
|
||||
* be flushed. If the flush for a key succeeds and no more updates have come in during the flush,
|
||||
* then the key is demoted out of the tracked state. Otherwise, updates continue to aggregate
|
||||
* until the next flush attempt.
|
||||
*/
|
||||
class AggregatingCachedCountsStore(
|
||||
underlying: CachedCountsStore,
|
||||
timer: Timer,
|
||||
flushInterval: Duration,
|
||||
maxSize: Int,
|
||||
stats: StatsReceiver)
|
||||
extends CachedCountsStore
|
||||
with Serialized {
|
||||
private[this] val pendingUpdates: mutable.Map[TweetCountKey, Count] =
|
||||
new mutable.HashMap[TweetCountKey, Count]
|
||||
|
||||
private[this] var trackingCount: Int = 0
|
||||
|
||||
private[this] val promotionCounter = stats.counter("promotions")
|
||||
private[this] val demotionCounter = stats.counter("demotions")
|
||||
private[this] val updateCounter = stats.counter("aggregated_updates")
|
||||
private[this] val overflowCounter = stats.counter("overflows")
|
||||
private[this] val flushFailureCounter = stats.counter("flush_failures")
|
||||
private[this] val trackingCountGauge = stats.addGauge("tracking")(trackingCount.toFloat)
|
||||
|
||||
timer.schedule(flushInterval) { flush() }
|
||||
|
||||
def add(key: TweetCountKey, count: Count): Future[Unit] =
|
||||
underlying.add(key, count)
|
||||
|
||||
def delete(key: TweetCountKey): Future[Unit] =
|
||||
underlying.delete(key)
|
||||
|
||||
def incr(key: TweetCountKey, delta: Count): Future[Unit] =
|
||||
aggregateIfTracked(key, delta).flatMap {
|
||||
case true => Future.Unit
|
||||
case false =>
|
||||
underlying
|
||||
.incr(key, delta)
|
||||
.rescue { case _ => aggregate(key, delta) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues an update to be aggregated and applied to a key at a later time, but only if we are
|
||||
* already aggregating updates for the key.
|
||||
*
|
||||
* @return true the delta was aggregated, false if the key is not being tracked
|
||||
* and the incr should be attempted directly.
|
||||
*/
|
||||
private[this] def aggregateIfTracked(key: TweetCountKey, delta: Count): Future[Boolean] =
|
||||
serialized {
|
||||
pendingUpdates.get(key) match {
|
||||
case None => false
|
||||
case Some(current) =>
|
||||
updateCounter.incr()
|
||||
pendingUpdates(key) = current + delta
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues an update to be aggregated and applied to a key at a later time.
|
||||
*/
|
||||
private[this] def aggregate(key: TweetCountKey, delta: Count): Future[Unit] =
|
||||
serialized {
|
||||
val alreadyTracked = pendingUpdates.contains(key)
|
||||
|
||||
if (!alreadyTracked) {
|
||||
if (pendingUpdates.size < maxSize)
|
||||
promotionCounter.incr()
|
||||
else {
|
||||
overflowCounter.incr()
|
||||
throw CachedCountsStore.QueueIsFullException
|
||||
}
|
||||
}
|
||||
|
||||
(pendingUpdates.get(key).getOrElse(0L) + delta) match {
|
||||
case 0 =>
|
||||
pendingUpdates.remove(key)
|
||||
demotionCounter.incr()
|
||||
|
||||
case aggregatedDelta =>
|
||||
pendingUpdates(key) = aggregatedDelta
|
||||
}
|
||||
|
||||
trackingCount = pendingUpdates.size
|
||||
}
|
||||
|
||||
private[this] def flush(): Future[Unit] = {
|
||||
for {
|
||||
// make a copy of the updates to flush, so that updates can continue to be queued
|
||||
// while the flush is in progress. if an individual flush succeeds, then we
|
||||
// go back and update pendingUpdates.
|
||||
updates <- serialized { pendingUpdates.toSeq.toList }
|
||||
() <- Future.join(for ((key, delta) <- updates) yield flush(key, delta))
|
||||
} yield ()
|
||||
}
|
||||
|
||||
private[this] def flush(key: TweetCountKey, delta: Count): Future[Unit] =
|
||||
underlying
|
||||
.incr(key, delta)
|
||||
.flatMap(_ => aggregate(key, -delta))
|
||||
.handle { case ex => flushFailureCounter.incr() }
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,209 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait TweetEventBusStore
|
||||
extends TweetStoreBase[TweetEventBusStore]
|
||||
with AsyncDeleteAdditionalFields.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with AsyncInsertTweet.Store
|
||||
with AsyncSetAdditionalFields.Store
|
||||
with AsyncTakedown.Store
|
||||
with AsyncUndeleteTweet.Store
|
||||
with AsyncUpdatePossiblySensitiveTweet.Store
|
||||
with QuotedTweetDelete.Store
|
||||
with QuotedTweetTakedown.Store
|
||||
with ScrubGeoUpdateUserTimestamp.Store
|
||||
with ScrubGeo.Store { self =>
|
||||
def wrap(w: TweetStore.Wrap): TweetEventBusStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with TweetEventBusStore
|
||||
with AsyncDeleteAdditionalFields.StoreWrapper
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with AsyncInsertTweet.StoreWrapper
|
||||
with AsyncSetAdditionalFields.StoreWrapper
|
||||
with AsyncTakedown.StoreWrapper
|
||||
with AsyncUndeleteTweet.StoreWrapper
|
||||
with AsyncUpdatePossiblySensitiveTweet.StoreWrapper
|
||||
with QuotedTweetDelete.StoreWrapper
|
||||
with QuotedTweetTakedown.StoreWrapper
|
||||
with ScrubGeo.StoreWrapper
|
||||
with ScrubGeoUpdateUserTimestamp.StoreWrapper
|
||||
|
||||
def inParallel(that: TweetEventBusStore): TweetEventBusStore =
|
||||
new TweetEventBusStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
self.asyncInsertTweet.inParallel(that.asyncInsertTweet)
|
||||
override val asyncDeleteAdditionalFields: FutureEffect[AsyncDeleteAdditionalFields.Event] =
|
||||
self.asyncDeleteAdditionalFields.inParallel(that.asyncDeleteAdditionalFields)
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
self.asyncDeleteTweet.inParallel(that.asyncDeleteTweet)
|
||||
override val asyncSetAdditionalFields: FutureEffect[AsyncSetAdditionalFields.Event] =
|
||||
self.asyncSetAdditionalFields.inParallel(that.asyncSetAdditionalFields)
|
||||
override val asyncTakedown: FutureEffect[AsyncTakedown.Event] =
|
||||
self.asyncTakedown.inParallel(that.asyncTakedown)
|
||||
override val asyncUndeleteTweet: FutureEffect[AsyncUndeleteTweet.Event] =
|
||||
self.asyncUndeleteTweet.inParallel(that.asyncUndeleteTweet)
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
AsyncUpdatePossiblySensitiveTweet.Event
|
||||
] =
|
||||
self.asyncUpdatePossiblySensitiveTweet.inParallel(that.asyncUpdatePossiblySensitiveTweet)
|
||||
override val quotedTweetDelete: FutureEffect[QuotedTweetDelete.Event] =
|
||||
self.quotedTweetDelete.inParallel(that.quotedTweetDelete)
|
||||
override val quotedTweetTakedown: FutureEffect[QuotedTweetTakedown.Event] =
|
||||
self.quotedTweetTakedown.inParallel(that.quotedTweetTakedown)
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
self.retryAsyncInsertTweet.inParallel(that.retryAsyncInsertTweet)
|
||||
override val retryAsyncDeleteAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteAdditionalFields.Event]
|
||||
] =
|
||||
self.retryAsyncDeleteAdditionalFields.inParallel(that.retryAsyncDeleteAdditionalFields)
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
self.retryAsyncDeleteTweet.inParallel(that.retryAsyncDeleteTweet)
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUndeleteTweet.Event]
|
||||
] =
|
||||
self.retryAsyncUndeleteTweet.inParallel(that.retryAsyncUndeleteTweet)
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUpdatePossiblySensitiveTweet.Event]
|
||||
] =
|
||||
self.retryAsyncUpdatePossiblySensitiveTweet.inParallel(
|
||||
that.retryAsyncUpdatePossiblySensitiveTweet
|
||||
)
|
||||
override val retryAsyncSetAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncSetAdditionalFields.Event]
|
||||
] =
|
||||
self.retryAsyncSetAdditionalFields.inParallel(that.retryAsyncSetAdditionalFields)
|
||||
override val retryAsyncTakedown: FutureEffect[TweetStoreRetryEvent[AsyncTakedown.Event]] =
|
||||
self.retryAsyncTakedown.inParallel(that.retryAsyncTakedown)
|
||||
override val scrubGeo: FutureEffect[ScrubGeo.Event] =
|
||||
self.scrubGeo.inParallel(that.scrubGeo)
|
||||
override val scrubGeoUpdateUserTimestamp: FutureEffect[ScrubGeoUpdateUserTimestamp.Event] =
|
||||
self.scrubGeoUpdateUserTimestamp.inParallel(that.scrubGeoUpdateUserTimestamp)
|
||||
}
|
||||
}
|
||||
|
||||
object TweetEventBusStore {
|
||||
val Action: AsyncWriteAction = AsyncWriteAction.EventBusEnqueue
|
||||
|
||||
def safetyTypeForUser(user: User): Option[SafetyType] =
|
||||
user.safety.map(userSafetyToSafetyType)
|
||||
|
||||
def userSafetyToSafetyType(safety: Safety): SafetyType =
|
||||
if (safety.isProtected) {
|
||||
SafetyType.Private
|
||||
} else if (safety.suspended) {
|
||||
SafetyType.Restricted
|
||||
} else {
|
||||
SafetyType.Public
|
||||
}
|
||||
|
||||
def apply(
|
||||
eventStore: FutureEffect[TweetEvent]
|
||||
): TweetEventBusStore = {
|
||||
|
||||
def toTweetEvents(event: TweetStoreTweetEvent): Seq[TweetEvent] =
|
||||
event.toTweetEventData.map { data =>
|
||||
TweetEvent(
|
||||
data,
|
||||
TweetEventFlags(
|
||||
timestampMs = event.timestamp.inMillis,
|
||||
safetyType = event.optUser.flatMap(safetyTypeForUser)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def enqueueEvents[E <: TweetStoreTweetEvent]: FutureEffect[E] =
|
||||
eventStore.liftSeq.contramap[E](toTweetEvents)
|
||||
|
||||
new TweetEventBusStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
enqueueEvents[AsyncInsertTweet.Event]
|
||||
|
||||
override val asyncDeleteAdditionalFields: FutureEffect[AsyncDeleteAdditionalFields.Event] =
|
||||
enqueueEvents[AsyncDeleteAdditionalFields.Event]
|
||||
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
enqueueEvents[AsyncDeleteTweet.Event]
|
||||
|
||||
override val asyncSetAdditionalFields: FutureEffect[AsyncSetAdditionalFields.Event] =
|
||||
enqueueEvents[AsyncSetAdditionalFields.Event]
|
||||
|
||||
override val asyncTakedown: FutureEffect[AsyncTakedown.Event] =
|
||||
enqueueEvents[AsyncTakedown.Event]
|
||||
.onlyIf(_.eventbusEnqueue)
|
||||
|
||||
override val asyncUndeleteTweet: FutureEffect[AsyncUndeleteTweet.Event] =
|
||||
enqueueEvents[AsyncUndeleteTweet.Event]
|
||||
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
AsyncUpdatePossiblySensitiveTweet.Event
|
||||
] =
|
||||
enqueueEvents[AsyncUpdatePossiblySensitiveTweet.Event]
|
||||
|
||||
override val quotedTweetDelete: FutureEffect[QuotedTweetDelete.Event] =
|
||||
enqueueEvents[QuotedTweetDelete.Event]
|
||||
|
||||
override val quotedTweetTakedown: FutureEffect[QuotedTweetTakedown.Event] =
|
||||
enqueueEvents[QuotedTweetTakedown.Event]
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncInsertTweet)
|
||||
|
||||
override val retryAsyncDeleteAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteAdditionalFields.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteAdditionalFields)
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUndeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUndeleteTweet)
|
||||
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUpdatePossiblySensitiveTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUpdatePossiblySensitiveTweet)
|
||||
|
||||
override val retryAsyncSetAdditionalFields: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncSetAdditionalFields.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncSetAdditionalFields)
|
||||
|
||||
override val retryAsyncTakedown: FutureEffect[TweetStoreRetryEvent[AsyncTakedown.Event]] =
|
||||
TweetStore.retry(Action, asyncTakedown)
|
||||
|
||||
override val scrubGeo: FutureEffect[ScrubGeo.Event] =
|
||||
enqueueEvents[ScrubGeo.Event]
|
||||
|
||||
override val scrubGeoUpdateUserTimestamp: FutureEffect[ScrubGeoUpdateUserTimestamp.Event] =
|
||||
enqueueEvents[ScrubGeoUpdateUserTimestamp.Event]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrubs inappropriate fields from tweet events before publishing.
|
||||
*/
|
||||
object TweetEventDataScrubber {
|
||||
def scrub(tweet: Tweet): Tweet =
|
||||
tweet.copy(
|
||||
cards = None,
|
||||
card2 = None,
|
||||
media = tweet.media.map(_.map { mediaEntity => mediaEntity.copy(extensionsReply = None) }),
|
||||
previousCounts = None,
|
||||
editPerspective = None
|
||||
)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,65 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.tflock.TweetIndexer
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
trait TweetIndexingStore
|
||||
extends TweetStoreBase[TweetIndexingStore]
|
||||
with AsyncInsertTweet.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with AsyncUndeleteTweet.Store
|
||||
with AsyncSetRetweetVisibility.Store {
|
||||
def wrap(w: TweetStore.Wrap): TweetIndexingStore =
|
||||
new TweetStoreWrapper(w, this)
|
||||
with TweetIndexingStore
|
||||
with AsyncInsertTweet.StoreWrapper
|
||||
with AsyncDeleteTweet.StoreWrapper
|
||||
with AsyncUndeleteTweet.StoreWrapper
|
||||
with AsyncSetRetweetVisibility.StoreWrapper
|
||||
}
|
||||
|
||||
/**
|
||||
* A TweetStore that sends indexing updates to a TweetIndexer.
|
||||
*/
|
||||
object TweetIndexingStore {
|
||||
val Action: AsyncWriteAction.TweetIndex.type = AsyncWriteAction.TweetIndex
|
||||
|
||||
def apply(indexer: TweetIndexer): TweetIndexingStore =
|
||||
new TweetIndexingStore {
|
||||
override val asyncInsertTweet: FutureEffect[AsyncInsertTweet.Event] =
|
||||
FutureEffect[AsyncInsertTweet.Event](event => indexer.createIndex(event.tweet))
|
||||
|
||||
override val retryAsyncInsertTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncInsertTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncInsertTweet)
|
||||
|
||||
override val asyncDeleteTweet: FutureEffect[AsyncDeleteTweet.Event] =
|
||||
FutureEffect[AsyncDeleteTweet.Event](event =>
|
||||
indexer.deleteIndex(event.tweet, event.isBounceDelete))
|
||||
|
||||
override val retryAsyncDeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncDeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncDeleteTweet)
|
||||
|
||||
override val asyncUndeleteTweet: FutureEffect[AsyncUndeleteTweet.Event] =
|
||||
FutureEffect[AsyncUndeleteTweet.Event](event => indexer.undeleteIndex(event.tweet))
|
||||
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncUndeleteTweet.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncUndeleteTweet)
|
||||
|
||||
override val asyncSetRetweetVisibility: FutureEffect[AsyncSetRetweetVisibility.Event] =
|
||||
FutureEffect[AsyncSetRetweetVisibility.Event] { event =>
|
||||
indexer.setRetweetVisibility(event.retweetId, event.visible)
|
||||
}
|
||||
|
||||
override val retryAsyncSetRetweetVisibility: FutureEffect[
|
||||
TweetStoreRetryEvent[AsyncSetRetweetVisibility.Event]
|
||||
] =
|
||||
TweetStore.retry(Action, asyncSetRetweetVisibility)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,64 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.finagle.stats.RollupStatsReceiver
|
||||
import com.twitter.servo.util.MemoizingStatsReceiver
|
||||
|
||||
/**
|
||||
* Records some stats about inserted tweets. Tweets are currently classified by three criteria:
|
||||
*
|
||||
* - tweet type: "tweet" or "retweet"
|
||||
* - user type: "stresstest", "protected", "restricted", or "public"
|
||||
* - fanout type: "nullcast", "narrowcast", or "usertimeline"
|
||||
*
|
||||
* A counter is incremented for a tweet using those three criteria in order. Counters are
|
||||
* created with a RollupStatsReceiver, so counts are aggregated at each level. Some
|
||||
* example counters are:
|
||||
*
|
||||
* ./insert
|
||||
* ./insert/tweet
|
||||
* ./insert/tweet/public
|
||||
* ./insert/tweet/protected/usertimeline
|
||||
* ./insert/retweet/stresstest
|
||||
* ./insert/retweet/public/nullcast
|
||||
*/
|
||||
trait TweetStatsStore extends TweetStoreBase[TweetStatsStore] with InsertTweet.Store {
|
||||
def wrap(w: TweetStore.Wrap): TweetStatsStore =
|
||||
new TweetStoreWrapper(w, this) with TweetStatsStore with InsertTweet.StoreWrapper
|
||||
}
|
||||
|
||||
object TweetStatsStore {
|
||||
def apply(stats: StatsReceiver): TweetStatsStore = {
|
||||
val rollup = new MemoizingStatsReceiver(new RollupStatsReceiver(stats))
|
||||
val inserts = rollup.scope("insert")
|
||||
|
||||
def tweetType(tweet: Tweet) =
|
||||
if (getShare(tweet).isDefined) "retweet" else "tweet"
|
||||
|
||||
def userType(user: User) =
|
||||
if (user.roles.exists(_.roles.contains("stresstest"))) "stresstest"
|
||||
else if (user.safety.exists(_.isProtected)) "protected"
|
||||
else if (user.safety.exists(_.suspended)) "restricted"
|
||||
else "public"
|
||||
|
||||
def fanoutType(tweet: Tweet) =
|
||||
if (TweetLenses.nullcast(tweet)) "nullcast"
|
||||
else if (TweetLenses.narrowcast(tweet).isDefined) "narrowcast"
|
||||
else "usertimeline"
|
||||
|
||||
new TweetStatsStore {
|
||||
override val insertTweet: FutureEffect[InsertTweet.Event] =
|
||||
FutureEffect[InsertTweet.Event] { event =>
|
||||
inserts
|
||||
.counter(
|
||||
tweetType(event.tweet),
|
||||
userType(event.user),
|
||||
fanoutType(event.tweet)
|
||||
)
|
||||
.incr()
|
||||
|
||||
Future.Unit
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,292 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.finagle.service.RetryPolicy
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.servo.util.RetryHandler
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
import com.twitter.util.Timer
|
||||
|
||||
object TweetStore {
|
||||
// Using the old-school c.t.logging.Logger here as this log is only used by
|
||||
// servo.FutureEffect's trackOutcome method, which needs that kind of logger.
|
||||
val log: com.twitter.logging.Logger = com.twitter.logging.Logger(getClass)
|
||||
|
||||
/**
|
||||
* Adapts a tweet store on a specific TweetStoreEvent type to one that handles
|
||||
* TweetStoreRetryEvents of that type that match the given AsyncWriteAction.
|
||||
*/
|
||||
def retry[T <: AsyncTweetStoreEvent](
|
||||
action: AsyncWriteAction,
|
||||
store: FutureEffect[T]
|
||||
): FutureEffect[TweetStoreRetryEvent[T]] =
|
||||
store.contramap[TweetStoreRetryEvent[T]](_.event).onlyIf(_.action == action)
|
||||
|
||||
/**
|
||||
* Defines an abstract polymorphic operation to be applied to FutureEffects over any
|
||||
* TweetStoreEvent type. The Wrap operation is defined over all possible
|
||||
* FutureEffect[E <: TweetStoreEvent] types.
|
||||
*/
|
||||
trait Wrap {
|
||||
def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E]
|
||||
}
|
||||
|
||||
/**
|
||||
* A Wrap operation that applies standardized metrics collection to the FutureEffect.
|
||||
*/
|
||||
case class Tracked(stats: StatsReceiver) extends Wrap {
|
||||
def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect[E] { event =>
|
||||
Stat.timeFuture(stats.scope(event.name).stat("latency_ms")) {
|
||||
handler(event)
|
||||
}
|
||||
}.trackOutcome(stats, _.name, log)
|
||||
}
|
||||
|
||||
/**
|
||||
* A Wrap operation that makes the FutureEffect enabled according to the given gate.
|
||||
*/
|
||||
case class Gated(gate: Gate[Unit]) extends Wrap {
|
||||
def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E] =
|
||||
handler.enabledBy(gate)
|
||||
}
|
||||
|
||||
/**
|
||||
* A Wrap operation that updates the FutureEffect to ignore failures.
|
||||
*/
|
||||
object IgnoreFailures extends Wrap {
|
||||
def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E] =
|
||||
handler.ignoreFailures
|
||||
}
|
||||
|
||||
/**
|
||||
* A Wrap operation that updates the FutureEffect to ignore failures upon completion.
|
||||
*/
|
||||
object IgnoreFailuresUponCompletion extends Wrap {
|
||||
def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E] =
|
||||
handler.ignoreFailuresUponCompletion
|
||||
}
|
||||
|
||||
/**
|
||||
* A Wrap operation that applies a RetryHandler to FutureEffects.
|
||||
*/
|
||||
case class Retry(retryHandler: RetryHandler[Unit]) extends Wrap {
|
||||
def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E] =
|
||||
handler.retry(retryHandler)
|
||||
}
|
||||
|
||||
/**
|
||||
* A Wrap operation that applies a RetryHandler to FutureEffects.
|
||||
*/
|
||||
case class ReplicatedEventRetry(retryHandler: RetryHandler[Unit]) extends Wrap {
|
||||
def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect[E] { event =>
|
||||
event.retryStrategy match {
|
||||
case TweetStoreEvent.ReplicatedEventLocalRetry => handler.retry(retryHandler)(event)
|
||||
case _ => handler(event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A Wrap operation that configures async-retry behavior to async-write events.
|
||||
*/
|
||||
class AsyncRetry(
|
||||
localRetryPolicy: RetryPolicy[Try[Nothing]],
|
||||
enqueueRetryPolicy: RetryPolicy[Try[Nothing]],
|
||||
timer: Timer,
|
||||
tweetService: ThriftTweetService,
|
||||
scribe: FutureEffect[FailedAsyncWrite]
|
||||
)(
|
||||
stats: StatsReceiver,
|
||||
action: AsyncWriteAction)
|
||||
extends Wrap {
|
||||
|
||||
override def apply[E <: TweetStoreEvent](handler: FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect[E] { event =>
|
||||
event.retryStrategy match {
|
||||
case TweetStoreEvent.EnqueueAsyncRetry(enqueueRetry) =>
|
||||
enqueueAsyncRetry(handler, enqueueRetry)(event)
|
||||
|
||||
case TweetStoreEvent.LocalRetryThenScribeFailure(toFailedAsyncWrite) =>
|
||||
localRetryThenScribeFailure(handler, toFailedAsyncWrite)(event)
|
||||
|
||||
case _ =>
|
||||
handler(event)
|
||||
}
|
||||
}
|
||||
|
||||
private def enqueueAsyncRetry[E <: TweetStoreEvent](
|
||||
handler: FutureEffect[E],
|
||||
enqueueRetry: (ThriftTweetService, AsyncWriteAction) => Future[Unit]
|
||||
): FutureEffect[E] = {
|
||||
val retryInitCounter = stats.counter("retries_initiated")
|
||||
|
||||
// enqueues failed TweetStoreEvents to the deferredrpc-backed tweetService
|
||||
// to be retried. this store uses the enqueueRetryPolicy to retry the enqueue
|
||||
// attempts in the case of deferredrpc application failures.
|
||||
val enqueueRetryHandler =
|
||||
FutureEffect[E](_ => enqueueRetry(tweetService, action))
|
||||
.retry(RetryHandler.failuresOnly(enqueueRetryPolicy, timer, stats.scope("enqueue_retry")))
|
||||
|
||||
handler.rescue {
|
||||
case ex =>
|
||||
TweetStore.log.warning(ex, s"will retry $action")
|
||||
retryInitCounter.incr()
|
||||
enqueueRetryHandler
|
||||
}
|
||||
}
|
||||
|
||||
private def localRetryThenScribeFailure[E <: TweetStoreEvent](
|
||||
handler: FutureEffect[E],
|
||||
toFailedAsyncWrite: AsyncWriteAction => FailedAsyncWrite
|
||||
): FutureEffect[E] = {
|
||||
val exhaustedCounter = stats.counter("retries_exhausted")
|
||||
|
||||
// scribe events that failed after exhausting all retries
|
||||
val scribeEventHandler =
|
||||
FutureEffect[E](_ => scribe(toFailedAsyncWrite(action)))
|
||||
|
||||
// wraps `handle` with a retry policy to retry failures with a backoff. if we exhaust
|
||||
// all retries, then we pass the event to `scribeEventStore` to scribe the failure.
|
||||
handler
|
||||
.retry(RetryHandler.failuresOnly(localRetryPolicy, timer, stats))
|
||||
.rescue {
|
||||
case ex =>
|
||||
TweetStore.log.warning(ex, s"exhausted retries on $action")
|
||||
exhaustedCounter.incr()
|
||||
scribeEventHandler
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parent trait for defining a "module" that defines a TweetStoreEvent type and corresponding
|
||||
* TweetStore and TweetStoreWrapper types.
|
||||
*/
|
||||
sealed trait Module {
|
||||
type Store
|
||||
type StoreWrapper <: Store
|
||||
}
|
||||
|
||||
/**
|
||||
* Parent trait for defining a "module" that defines a sync TweetStoreEvent.
|
||||
*/
|
||||
trait SyncModule extends Module {
|
||||
type Event <: SyncTweetStoreEvent
|
||||
}
|
||||
|
||||
/**
|
||||
* Parent trait for defining a "module" that defines an async TweetStoreEvent and a
|
||||
* TweetStoreRetryEvent.
|
||||
*/
|
||||
trait AsyncModule extends Module {
|
||||
type Event <: AsyncTweetStoreEvent
|
||||
type RetryEvent <: TweetStoreRetryEvent[Event]
|
||||
}
|
||||
|
||||
/**
|
||||
* Parent trait for defining a "module" that defines a replicated TweetStoreEvent.
|
||||
*/
|
||||
trait ReplicatedModule extends Module {
|
||||
type Event <: ReplicatedTweetStoreEvent
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trait for TweetStore implementations that support handler wrapping.
|
||||
*/
|
||||
trait TweetStoreBase[Self] {
|
||||
import TweetStore._
|
||||
|
||||
/**
|
||||
* Returns a new store of type Self with Wrap applied to each event handler in this instance.
|
||||
*/
|
||||
def wrap(w: Wrap): Self
|
||||
|
||||
/**
|
||||
* Applies the Tracked Wrap operation to the store.
|
||||
*/
|
||||
def tracked(stats: StatsReceiver): Self = wrap(Tracked(stats))
|
||||
|
||||
/**
|
||||
* Applies the Gated Wrap operation to the store.
|
||||
*/
|
||||
def enabledBy(gate: Gate[Unit]): Self = wrap(Gated(gate))
|
||||
|
||||
/**
|
||||
* Applies the IgnoreFailures Wrap operation to the store.
|
||||
*/
|
||||
def ignoreFailures: Self = wrap(IgnoreFailures)
|
||||
|
||||
/**
|
||||
* Applies the IgnoreFailuresUponCompletion Wrap operation to the store.
|
||||
*/
|
||||
def ignoreFailuresUponCompletion: Self = wrap(IgnoreFailuresUponCompletion)
|
||||
|
||||
/**
|
||||
* Applies a RetryHandler to each event handler.
|
||||
*/
|
||||
def retry(retryHandler: RetryHandler[Unit]): Self = wrap(Retry(retryHandler))
|
||||
|
||||
/**
|
||||
* Applies a RetryHandler to replicated event handlers.
|
||||
*/
|
||||
def replicatedRetry(retryHandler: RetryHandler[Unit]): Self =
|
||||
wrap(ReplicatedEventRetry(retryHandler))
|
||||
|
||||
/**
|
||||
* Applies the AsyncRetryConfig Wrap operation to the store.
|
||||
*/
|
||||
def asyncRetry(cfg: AsyncRetry): Self = wrap(cfg)
|
||||
}
|
||||
|
||||
/**
|
||||
* An abstract base class for tweet store instances that wrap another tweet store instance.
|
||||
* You can mix event-specific store wrapper traits into this class to automatically
|
||||
* have the event-specific handlers wrapped.
|
||||
*/
|
||||
abstract class TweetStoreWrapper[+T](
|
||||
protected val wrap: TweetStore.Wrap,
|
||||
protected val underlying: T)
|
||||
|
||||
/**
|
||||
* A TweetStore that has a handler for all possible TweetStoreEvents.
|
||||
*/
|
||||
trait TotalTweetStore
|
||||
extends AsyncDeleteAdditionalFields.Store
|
||||
with AsyncDeleteTweet.Store
|
||||
with AsyncIncrBookmarkCount.Store
|
||||
with AsyncIncrFavCount.Store
|
||||
with AsyncInsertTweet.Store
|
||||
with AsyncSetAdditionalFields.Store
|
||||
with AsyncSetRetweetVisibility.Store
|
||||
with AsyncTakedown.Store
|
||||
with AsyncUndeleteTweet.Store
|
||||
with AsyncUpdatePossiblySensitiveTweet.Store
|
||||
with DeleteAdditionalFields.Store
|
||||
with DeleteTweet.Store
|
||||
with Flush.Store
|
||||
with IncrBookmarkCount.Store
|
||||
with IncrFavCount.Store
|
||||
with InsertTweet.Store
|
||||
with QuotedTweetDelete.Store
|
||||
with QuotedTweetTakedown.Store
|
||||
with ReplicatedDeleteAdditionalFields.Store
|
||||
with ReplicatedDeleteTweet.Store
|
||||
with ReplicatedIncrBookmarkCount.Store
|
||||
with ReplicatedIncrFavCount.Store
|
||||
with ReplicatedInsertTweet.Store
|
||||
with ReplicatedScrubGeo.Store
|
||||
with ReplicatedSetAdditionalFields.Store
|
||||
with ReplicatedSetRetweetVisibility.Store
|
||||
with ReplicatedTakedown.Store
|
||||
with ReplicatedUndeleteTweet.Store
|
||||
with ReplicatedUpdatePossiblySensitiveTweet.Store
|
||||
with ScrubGeo.Store
|
||||
with ScrubGeoUpdateUserTimestamp.Store
|
||||
with SetAdditionalFields.Store
|
||||
with SetRetweetVisibility.Store
|
||||
with Takedown.Store
|
||||
with UndeleteTweet.Store
|
||||
with UpdatePossiblySensitiveTweet.Store
|
||||
Binary file not shown.
|
|
@ -1,144 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.finagle.tracing.Trace
|
||||
import com.twitter.tweetypie.store.TweetStoreEvent.RetryStrategy
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object TweetStoreEvent {
|
||||
|
||||
/**
|
||||
* Parent trait for indicating what type of retry strategy to apply to event handlers
|
||||
* for the corresponding event type. Different classes of events use different strategies.
|
||||
*/
|
||||
sealed trait RetryStrategy
|
||||
|
||||
/**
|
||||
* Indicates that the event type doesn't support retries.
|
||||
*/
|
||||
case object NoRetry extends RetryStrategy
|
||||
|
||||
/**
|
||||
* Indicates that if an event handler encounters a failure, it should enqueue a
|
||||
* retry to be performed asynchronously.
|
||||
*/
|
||||
case class EnqueueAsyncRetry(enqueueRetry: (ThriftTweetService, AsyncWriteAction) => Future[Unit])
|
||||
extends RetryStrategy
|
||||
|
||||
/**
|
||||
* Indicates that if an event handler encounters a failure, it should retry
|
||||
* the event locally some number of times, before eventually given up and scribing
|
||||
* the failure.
|
||||
*/
|
||||
case class LocalRetryThenScribeFailure(toFailedAsyncWrite: AsyncWriteAction => FailedAsyncWrite)
|
||||
extends RetryStrategy
|
||||
|
||||
/**
|
||||
* Indicates that if an event handler encounters a failure, it should retry
|
||||
* the event locally some number of times.
|
||||
*/
|
||||
case object ReplicatedEventLocalRetry extends RetryStrategy
|
||||
}
|
||||
|
||||
/**
|
||||
* The abstract parent class for all TweetStoreEvent types.
|
||||
*/
|
||||
sealed trait TweetStoreEvent {
|
||||
val name: String
|
||||
|
||||
val traceId: Long = Trace.id.traceId.toLong
|
||||
|
||||
/**
|
||||
* Indicates a particular retry behavior that should be applied to event handlers for
|
||||
* the corresponding event type. The specifics of the strategy might depend upon the
|
||||
* specific TweetStore implementation.
|
||||
*/
|
||||
def retryStrategy: RetryStrategy
|
||||
}
|
||||
|
||||
abstract class SyncTweetStoreEvent(val name: String) extends TweetStoreEvent {
|
||||
override def retryStrategy: RetryStrategy = TweetStoreEvent.NoRetry
|
||||
}
|
||||
|
||||
abstract class AsyncTweetStoreEvent(val name: String) extends TweetStoreEvent {
|
||||
def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit]
|
||||
|
||||
override def retryStrategy: RetryStrategy = TweetStoreEvent.EnqueueAsyncRetry(enqueueRetry)
|
||||
}
|
||||
|
||||
abstract class ReplicatedTweetStoreEvent(val name: String) extends TweetStoreEvent {
|
||||
override def retryStrategy: RetryStrategy = TweetStoreEvent.ReplicatedEventLocalRetry
|
||||
}
|
||||
|
||||
/**
|
||||
* A trait for all TweetStoreEvents that become TweetEvents.
|
||||
*/
|
||||
trait TweetStoreTweetEvent {
|
||||
val timestamp: Time
|
||||
|
||||
val optUser: Option[User]
|
||||
|
||||
/**
|
||||
* Most TweetStoreTweetEvents map to a single TweetEvent, but some
|
||||
* optionally map to an event and others map to multiple events, so
|
||||
* this method needs to return a Seq of TweetEventData.
|
||||
*/
|
||||
def toTweetEventData: Seq[TweetEventData]
|
||||
}
|
||||
|
||||
/**
|
||||
* The abstract parent class for an event that indicates a particular action
|
||||
* for a particular event that needs to be retried via the async-write-retrying mechanism.
|
||||
*/
|
||||
abstract class TweetStoreRetryEvent[E <: AsyncTweetStoreEvent] extends TweetStoreEvent {
|
||||
override val name = "async_write_retry"
|
||||
|
||||
def action: AsyncWriteAction
|
||||
def event: E
|
||||
|
||||
def eventType: AsyncWriteEventType
|
||||
|
||||
def scribedTweetOnFailure: Option[Tweet]
|
||||
|
||||
override def retryStrategy: RetryStrategy =
|
||||
TweetStoreEvent.LocalRetryThenScribeFailure(action =>
|
||||
FailedAsyncWrite(eventType, action, scribedTweetOnFailure))
|
||||
}
|
||||
|
||||
/**
|
||||
* Functions as a disjunction between an event type E and it's corresonding
|
||||
* retry event type TweetStoreRetryEvent[E]
|
||||
*/
|
||||
case class TweetStoreEventOrRetry[E <: AsyncTweetStoreEvent](
|
||||
event: E,
|
||||
toRetry: Option[TweetStoreRetryEvent[E]]) {
|
||||
def toInitial: Option[E] = if (retryAction.isDefined) None else Some(event)
|
||||
def retryAction: Option[RetryStrategy] = toRetry.map(_.retryStrategy)
|
||||
def hydrate(f: E => Future[E]): Future[TweetStoreEventOrRetry[E]] =
|
||||
f(event).map(e => copy(event = e))
|
||||
}
|
||||
|
||||
object TweetStoreEventOrRetry {
|
||||
def apply[E <: AsyncTweetStoreEvent, R <: TweetStoreRetryEvent[E]](
|
||||
event: E,
|
||||
retryAction: Option[AsyncWriteAction],
|
||||
toRetryEvent: (AsyncWriteAction, E) => R
|
||||
): TweetStoreEventOrRetry[E] =
|
||||
TweetStoreEventOrRetry(event, retryAction.map(action => toRetryEvent(action, event)))
|
||||
|
||||
object First {
|
||||
|
||||
/** matches against TweetStoreEventOrRetry instances for an initial event */
|
||||
def unapply[E <: AsyncTweetStoreEvent](it: TweetStoreEventOrRetry[E]): Option[E] =
|
||||
it.toInitial
|
||||
}
|
||||
|
||||
object Retry {
|
||||
|
||||
/** matches against TweetStoreEventOrRetry instances for a retry event */
|
||||
def unapply[E <: AsyncTweetStoreEvent](
|
||||
it: TweetStoreEventOrRetry[E]
|
||||
): Option[TweetStoreRetryEvent[E]] =
|
||||
it.toRetry
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,41 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object TweetUpdate {
|
||||
|
||||
/**
|
||||
* Copies takedown information from the source [[Tweet]] into [[CachedTweet]].
|
||||
*
|
||||
* Note that this method requires the source [[Tweet]] to have been loaded with the following
|
||||
* additional fields (which happens for all paths that create [[ReplicatedTakedown.Event]], in
|
||||
* both [[TakedownHandler]] and [[UserTakedownHandler]]:
|
||||
* - TweetypieOnlyTakedownReasonsField
|
||||
* - TweetypieOnlyTakedownCountryCodesField
|
||||
* This is done to ensure the remote datacenter of a takedown does not incorrectly try to load
|
||||
* from MH as the data is already cached.
|
||||
*/
|
||||
def copyTakedownFieldsForUpdate(source: Tweet): CachedTweet => CachedTweet =
|
||||
ct => {
|
||||
val newCoreData = source.coreData.get
|
||||
val updatedCoreData = ct.tweet.coreData.map(_.copy(hasTakedown = newCoreData.hasTakedown))
|
||||
ct.copy(
|
||||
tweet = ct.tweet.copy(
|
||||
coreData = updatedCoreData,
|
||||
tweetypieOnlyTakedownCountryCodes = source.tweetypieOnlyTakedownCountryCodes,
|
||||
tweetypieOnlyTakedownReasons = source.tweetypieOnlyTakedownReasons
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
def copyNsfwFieldsForUpdate(source: Tweet): Tweet => Tweet =
|
||||
tweet => {
|
||||
val newCoreData = source.coreData.get
|
||||
val updatedCoreData =
|
||||
tweet.coreData.map { core =>
|
||||
core.copy(nsfwUser = newCoreData.nsfwUser, nsfwAdmin = newCoreData.nsfwAdmin)
|
||||
}
|
||||
tweet.copy(coreData = updatedCoreData)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,237 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.store.TweetEventDataScrubber.scrub
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object UndeleteTweet extends TweetStore.SyncModule {
|
||||
|
||||
/**
|
||||
* A TweetStoreEvent for Undeletion.
|
||||
*/
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
user: User,
|
||||
timestamp: Time,
|
||||
hydrateOptions: WritePathHydrationOptions,
|
||||
_internalTweet: Option[CachedTweet] = None,
|
||||
deletedAt: Option[Time],
|
||||
sourceTweet: Option[Tweet] = None,
|
||||
sourceUser: Option[User] = None,
|
||||
quotedTweet: Option[Tweet] = None,
|
||||
quotedUser: Option[User] = None,
|
||||
parentUserId: Option[UserId] = None,
|
||||
quoterHasAlreadyQuotedTweet: Boolean = false)
|
||||
extends SyncTweetStoreEvent("undelete_tweet")
|
||||
with QuotedTweetOps {
|
||||
def internalTweet: CachedTweet =
|
||||
_internalTweet.getOrElse(
|
||||
throw new IllegalStateException(
|
||||
s"internalTweet should have been set in WritePathHydration, ${this}"
|
||||
)
|
||||
)
|
||||
|
||||
def toAsyncUndeleteTweetRequest: AsyncUndeleteTweetRequest =
|
||||
AsyncUndeleteTweetRequest(
|
||||
tweet = tweet,
|
||||
cachedTweet = internalTweet,
|
||||
user = user,
|
||||
timestamp = timestamp.inMillis,
|
||||
deletedAt = deletedAt.map(_.inMillis),
|
||||
sourceTweet = sourceTweet,
|
||||
sourceUser = sourceUser,
|
||||
quotedTweet = quotedTweet,
|
||||
quotedUser = quotedUser,
|
||||
parentUserId = parentUserId,
|
||||
quoterHasAlreadyQuotedTweet = Some(quoterHasAlreadyQuotedTweet)
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val undeleteTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val undeleteTweet: FutureEffect[Event] = wrap(underlying.undeleteTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
logLensStore: LogLensStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore,
|
||||
asyncEnqueueStore: AsyncEnqueueStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val undeleteTweet: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
logLensStore.undeleteTweet,
|
||||
// ignore failures writing to cache, will be retried in async-path
|
||||
cachingTweetStore.ignoreFailures.undeleteTweet,
|
||||
tweetCountsUpdatingStore.undeleteTweet,
|
||||
asyncEnqueueStore.undeleteTweet
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncUndeleteTweet extends TweetStore.AsyncModule {
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(request: AsyncUndeleteTweetRequest): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
AsyncUndeleteTweet.Event(
|
||||
tweet = request.tweet,
|
||||
cachedTweet = request.cachedTweet,
|
||||
user = request.user,
|
||||
optUser = Some(request.user),
|
||||
timestamp = Time.fromMilliseconds(request.timestamp),
|
||||
deletedAt = request.deletedAt.map(Time.fromMilliseconds),
|
||||
sourceTweet = request.sourceTweet,
|
||||
sourceUser = request.sourceUser,
|
||||
quotedTweet = request.quotedTweet,
|
||||
quotedUser = request.quotedUser,
|
||||
parentUserId = request.parentUserId,
|
||||
quoterHasAlreadyQuotedTweet = request.quoterHasAlreadyQuotedTweet.getOrElse(false)
|
||||
),
|
||||
request.retryAction,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
cachedTweet: CachedTweet,
|
||||
user: User,
|
||||
optUser: Option[User],
|
||||
timestamp: Time,
|
||||
deletedAt: Option[Time],
|
||||
sourceTweet: Option[Tweet],
|
||||
sourceUser: Option[User],
|
||||
quotedTweet: Option[Tweet],
|
||||
quotedUser: Option[User],
|
||||
parentUserId: Option[UserId] = None,
|
||||
quoterHasAlreadyQuotedTweet: Boolean = false)
|
||||
extends AsyncTweetStoreEvent("async_undelete_tweet")
|
||||
with QuotedTweetOps
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
/**
|
||||
* Convert this event into an AsyncUndeleteTweetRequest thrift request object
|
||||
*/
|
||||
def toAsyncRequest(retryAction: Option[AsyncWriteAction] = None): AsyncUndeleteTweetRequest =
|
||||
AsyncUndeleteTweetRequest(
|
||||
tweet = tweet,
|
||||
cachedTweet = cachedTweet,
|
||||
user = user,
|
||||
timestamp = timestamp.inMillis,
|
||||
retryAction = retryAction,
|
||||
deletedAt = deletedAt.map(_.inMillis),
|
||||
sourceTweet = sourceTweet,
|
||||
sourceUser = sourceUser,
|
||||
quotedTweet = quotedTweet,
|
||||
quotedUser = quotedUser,
|
||||
parentUserId = parentUserId,
|
||||
quoterHasAlreadyQuotedTweet = Some(quoterHasAlreadyQuotedTweet)
|
||||
)
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.TweetUndeleteEvent(
|
||||
TweetUndeleteEvent(
|
||||
tweet = scrub(tweet),
|
||||
user = Some(user),
|
||||
sourceTweet = sourceTweet.map(scrub),
|
||||
sourceUser = sourceUser,
|
||||
retweetParentUserId = parentUserId,
|
||||
quotedTweet = publicQuotedTweet.map(scrub),
|
||||
quotedUser = publicQuotedUser,
|
||||
deletedAtMsec = deletedAt.map(_.inMilliseconds)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncUndeleteTweet(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.Undelete.type = AsyncWriteEventType.Undelete
|
||||
override val scribedTweetOnFailure: Option[Tweet] = Some(event.tweet)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncUndeleteTweet: FutureEffect[Event]
|
||||
val retryAsyncUndeleteTweet: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncUndeleteTweet: FutureEffect[Event] = wrap(underlying.asyncUndeleteTweet)
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[TweetStoreRetryEvent[Event]] = wrap(
|
||||
underlying.retryAsyncUndeleteTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
eventBusEnqueueStore: TweetEventBusStore,
|
||||
indexingStore: TweetIndexingStore,
|
||||
replicatingStore: ReplicatingTweetStore,
|
||||
mediaServiceStore: MediaServiceStore,
|
||||
timelineUpdatingStore: TlsTimelineUpdatingStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] =
|
||||
Seq(
|
||||
cachingTweetStore,
|
||||
eventBusEnqueueStore,
|
||||
indexingStore,
|
||||
replicatingStore,
|
||||
mediaServiceStore,
|
||||
timelineUpdatingStore
|
||||
)
|
||||
|
||||
def build[E <: TweetStoreEvent](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncUndeleteTweet: FutureEffect[Event] = build(_.asyncUndeleteTweet)
|
||||
override val retryAsyncUndeleteTweet: FutureEffect[TweetStoreRetryEvent[Event]] = build(
|
||||
_.retryAsyncUndeleteTweet)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedUndeleteTweet extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
cachedTweet: CachedTweet,
|
||||
quoterHasAlreadyQuotedTweet: Boolean = false)
|
||||
extends ReplicatedTweetStoreEvent("replicated_undelete_tweet")
|
||||
|
||||
trait Store {
|
||||
val replicatedUndeleteTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedUndeleteTweet: FutureEffect[Event] = wrap(
|
||||
underlying.replicatedUndeleteTweet)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
tweetCountsUpdatingStore: TweetCountsCacheUpdatingStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val replicatedUndeleteTweet: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
cachingTweetStore.replicatedUndeleteTweet.ignoreFailures,
|
||||
tweetCountsUpdatingStore.replicatedUndeleteTweet.ignoreFailures
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,206 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
package store
|
||||
|
||||
import com.twitter.tweetypie.thriftscala._
|
||||
|
||||
object UpdatePossiblySensitiveTweet extends TweetStore.SyncModule {
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
user: User,
|
||||
timestamp: Time,
|
||||
byUserId: UserId,
|
||||
nsfwAdminChange: Option[Boolean],
|
||||
nsfwUserChange: Option[Boolean],
|
||||
note: Option[String],
|
||||
host: Option[String])
|
||||
extends SyncTweetStoreEvent("update_possibly_sensitive_tweet") {
|
||||
def toAsyncRequest: AsyncUpdatePossiblySensitiveTweetRequest =
|
||||
AsyncUpdatePossiblySensitiveTweetRequest(
|
||||
tweet = tweet,
|
||||
user = user,
|
||||
byUserId = byUserId,
|
||||
timestamp = timestamp.inMillis,
|
||||
nsfwAdminChange = nsfwAdminChange,
|
||||
nsfwUserChange = nsfwUserChange,
|
||||
note = note,
|
||||
host = host
|
||||
)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val updatePossiblySensitiveTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val updatePossiblySensitiveTweet: FutureEffect[Event] = wrap(
|
||||
underlying.updatePossiblySensitiveTweet
|
||||
)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
logLensStore: LogLensStore,
|
||||
asyncEnqueueStore: AsyncEnqueueStore
|
||||
): Store =
|
||||
new Store {
|
||||
override val updatePossiblySensitiveTweet: FutureEffect[Event] =
|
||||
FutureEffect.inParallel(
|
||||
manhattanStore.ignoreFailures.updatePossiblySensitiveTweet,
|
||||
cachingTweetStore.ignoreFailures.updatePossiblySensitiveTweet,
|
||||
logLensStore.updatePossiblySensitiveTweet,
|
||||
asyncEnqueueStore.updatePossiblySensitiveTweet
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object AsyncUpdatePossiblySensitiveTweet extends TweetStore.AsyncModule {
|
||||
|
||||
object Event {
|
||||
def fromAsyncRequest(
|
||||
request: AsyncUpdatePossiblySensitiveTweetRequest
|
||||
): TweetStoreEventOrRetry[Event] =
|
||||
TweetStoreEventOrRetry(
|
||||
AsyncUpdatePossiblySensitiveTweet.Event(
|
||||
tweet = request.tweet,
|
||||
user = request.user,
|
||||
optUser = Some(request.user),
|
||||
timestamp = Time.fromMilliseconds(request.timestamp),
|
||||
byUserId = request.byUserId,
|
||||
nsfwAdminChange = request.nsfwAdminChange,
|
||||
nsfwUserChange = request.nsfwUserChange,
|
||||
note = request.note,
|
||||
host = request.host
|
||||
),
|
||||
request.action,
|
||||
RetryEvent
|
||||
)
|
||||
}
|
||||
|
||||
case class Event(
|
||||
tweet: Tweet,
|
||||
user: User,
|
||||
optUser: Option[User],
|
||||
timestamp: Time,
|
||||
byUserId: UserId,
|
||||
nsfwAdminChange: Option[Boolean],
|
||||
nsfwUserChange: Option[Boolean],
|
||||
note: Option[String],
|
||||
host: Option[String])
|
||||
extends AsyncTweetStoreEvent("async_update_possibly_sensitive_tweet")
|
||||
with TweetStoreTweetEvent {
|
||||
|
||||
def toAsyncRequest(
|
||||
action: Option[AsyncWriteAction] = None
|
||||
): AsyncUpdatePossiblySensitiveTweetRequest =
|
||||
AsyncUpdatePossiblySensitiveTweetRequest(
|
||||
tweet = tweet,
|
||||
user = user,
|
||||
byUserId = byUserId,
|
||||
timestamp = timestamp.inMillis,
|
||||
nsfwAdminChange = nsfwAdminChange,
|
||||
nsfwUserChange = nsfwUserChange,
|
||||
note = note,
|
||||
host = host,
|
||||
action = action
|
||||
)
|
||||
|
||||
override def toTweetEventData: Seq[TweetEventData] =
|
||||
Seq(
|
||||
TweetEventData.TweetPossiblySensitiveUpdateEvent(
|
||||
TweetPossiblySensitiveUpdateEvent(
|
||||
tweetId = tweet.id,
|
||||
userId = user.id,
|
||||
nsfwAdmin = TweetLenses.nsfwAdmin.get(tweet),
|
||||
nsfwUser = TweetLenses.nsfwUser.get(tweet)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
override def enqueueRetry(service: ThriftTweetService, action: AsyncWriteAction): Future[Unit] =
|
||||
service.asyncUpdatePossiblySensitiveTweet(toAsyncRequest(Some(action)))
|
||||
}
|
||||
|
||||
case class RetryEvent(action: AsyncWriteAction, event: Event)
|
||||
extends TweetStoreRetryEvent[Event] {
|
||||
|
||||
override val eventType: AsyncWriteEventType.UpdatePossiblySensitiveTweet.type =
|
||||
AsyncWriteEventType.UpdatePossiblySensitiveTweet
|
||||
override val scribedTweetOnFailure: Option[Tweet] = Some(event.tweet)
|
||||
}
|
||||
|
||||
trait Store {
|
||||
val asyncUpdatePossiblySensitiveTweet: FutureEffect[Event]
|
||||
val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[TweetStoreRetryEvent[Event]]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[Event] = wrap(
|
||||
underlying.asyncUpdatePossiblySensitiveTweet
|
||||
)
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[TweetStoreRetryEvent[Event]] =
|
||||
wrap(
|
||||
underlying.retryAsyncUpdatePossiblySensitiveTweet
|
||||
)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(
|
||||
manhattanStore: ManhattanTweetStore,
|
||||
cachingTweetStore: CachingTweetStore,
|
||||
replicatingStore: ReplicatingTweetStore,
|
||||
guanoStore: GuanoServiceStore,
|
||||
eventBusStore: TweetEventBusStore
|
||||
): Store = {
|
||||
val stores: Seq[Store] =
|
||||
Seq(
|
||||
manhattanStore,
|
||||
cachingTweetStore,
|
||||
replicatingStore,
|
||||
guanoStore,
|
||||
eventBusStore
|
||||
)
|
||||
|
||||
def build[E <: TweetStoreEvent](extract: Store => FutureEffect[E]): FutureEffect[E] =
|
||||
FutureEffect.inParallel[E](stores.map(extract): _*)
|
||||
|
||||
new Store {
|
||||
override val asyncUpdatePossiblySensitiveTweet: FutureEffect[Event] = build(
|
||||
_.asyncUpdatePossiblySensitiveTweet)
|
||||
override val retryAsyncUpdatePossiblySensitiveTweet: FutureEffect[
|
||||
TweetStoreRetryEvent[Event]
|
||||
] = build(
|
||||
_.retryAsyncUpdatePossiblySensitiveTweet
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ReplicatedUpdatePossiblySensitiveTweet extends TweetStore.ReplicatedModule {
|
||||
|
||||
case class Event(tweet: Tweet)
|
||||
extends ReplicatedTweetStoreEvent("replicated_update_possibly_sensitive_tweet")
|
||||
|
||||
trait Store {
|
||||
val replicatedUpdatePossiblySensitiveTweet: FutureEffect[Event]
|
||||
}
|
||||
|
||||
trait StoreWrapper extends Store { self: TweetStoreWrapper[Store] =>
|
||||
override val replicatedUpdatePossiblySensitiveTweet: FutureEffect[Event] = wrap(
|
||||
underlying.replicatedUpdatePossiblySensitiveTweet
|
||||
)
|
||||
}
|
||||
|
||||
object Store {
|
||||
def apply(cachingTweetStore: CachingTweetStore): Store = {
|
||||
new Store {
|
||||
override val replicatedUpdatePossiblySensitiveTweet: FutureEffect[Event] =
|
||||
cachingTweetStore.replicatedUpdatePossiblySensitiveTweet
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,16 +0,0 @@
|
|||
package com.twitter.tweetypie
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator
|
||||
import com.twitter.tweetypie.thriftscala.CachedTweet
|
||||
import com.twitter.context.TwitterContext
|
||||
|
||||
package object store {
|
||||
type JsonGen = JsonGenerator => Unit
|
||||
|
||||
// Bring Tweetypie permitted TwitterContext into scope
|
||||
val TwitterContext: TwitterContext =
|
||||
com.twitter.context.TwitterContext(com.twitter.tweetypie.TwitterContextPermit)
|
||||
|
||||
def cachedTweetFromUnhydratedTweet(tweet: Tweet): CachedTweet =
|
||||
CachedTweet(tweet = tweet)
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
create_thrift_libraries(
|
||||
base_name = "compiled",
|
||||
sources = ["**/*.thrift"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependency_roots = [
|
||||
"mediaservices/commons/src/main/thrift",
|
||||
"tweetypie/servo/repo/src/main/thrift",
|
||||
"src/thrift/com/twitter/context:feature-context",
|
||||
"src/thrift/com/twitter/escherbird:media-annotation-structs",
|
||||
"src/thrift/com/twitter/expandodo:capi",
|
||||
"src/thrift/com/twitter/expandodo:only",
|
||||
"src/thrift/com/twitter/geoduck",
|
||||
"src/thrift/com/twitter/gizmoduck:thrift",
|
||||
"src/thrift/com/twitter/gizmoduck:user-thrift",
|
||||
"src/thrift/com/twitter/servo:servo-exception",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:audit",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:delete_location_data",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:media-entity",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:service",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:stored-tweet-info",
|
||||
"tweetypie/common/src/thrift/com/twitter/tweetypie:tweet",
|
||||
],
|
||||
generate_languages = [
|
||||
"java",
|
||||
"scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,705 +0,0 @@
|
|||
namespace java com.twitter.tweetypie.thriftjava
|
||||
#@namespace scala com.twitter.tweetypie.thriftscala
|
||||
|
||||
include "com/twitter/context/feature_context.thrift"
|
||||
include "com/twitter/expandodo/cards.thrift"
|
||||
include "com/twitter/gizmoduck/user.thrift"
|
||||
include "com/twitter/mediaservices/commons/MediaCommon.thrift"
|
||||
include "com/twitter/mediaservices/commons/MediaInformation.thrift"
|
||||
include "com/twitter/mediaservices/commons/TweetMedia.thrift"
|
||||
include "com/twitter/servo/exceptions.thrift"
|
||||
include "com/twitter/servo/cache/servo_repo.thrift"
|
||||
include "com/twitter/tseng/withholding/withholding.thrift"
|
||||
include "com/twitter/tweetypie/delete_location_data.thrift"
|
||||
include "com/twitter/tweetypie/transient_context.thrift"
|
||||
include "com/twitter/tweetypie/media_entity.thrift"
|
||||
include "com/twitter/tweetypie/tweet.thrift"
|
||||
include "com/twitter/tweetypie/tweet_audit.thrift"
|
||||
include "com/twitter/tweetypie/stored_tweet_info.thrift"
|
||||
include "com/twitter/tweetypie/tweet_service.thrift"
|
||||
|
||||
typedef i16 FieldId
|
||||
|
||||
struct UserIdentity {
|
||||
1: required i64 id
|
||||
2: required string screen_name
|
||||
3: required string real_name
|
||||
# obsolete 4: bool deactivated = 0
|
||||
# obsolete 5: bool suspended = 0
|
||||
}
|
||||
|
||||
enum HydrationType {
|
||||
MENTIONS = 1,
|
||||
URLS = 2,
|
||||
CACHEABLE_MEDIA = 3,
|
||||
QUOTED_TWEET_REF = 4,
|
||||
REPLY_SCREEN_NAME = 5,
|
||||
DIRECTED_AT = 6,
|
||||
CONTRIBUTOR = 7,
|
||||
SELF_THREAD_INFO = 8
|
||||
}
|
||||
|
||||
struct CachedTweet {
|
||||
1: required tweet.Tweet tweet
|
||||
// @obsolete 2: optional set<i16> included_additional_fields
|
||||
3: set<HydrationType> completed_hydrations = []
|
||||
|
||||
// Indicates that a tweet was deleted after being bounced for violating
|
||||
// the Twitter Rules.
|
||||
// When set to true, all other fields in CachedTweet are ignored.
|
||||
4: optional bool is_bounce_deleted
|
||||
|
||||
// Indicates whether this tweet has safety labels stored in Strato.
|
||||
// See com.twitter.tweetypie.core.TweetData.hasSafetyLabels for more details.
|
||||
// @obsolete 5: optional bool has_safety_labels
|
||||
} (persisted='true', hasPersonalData='true')
|
||||
|
||||
struct MediaFaces {
|
||||
1: required map<TweetMedia.MediaSizeType, list<MediaInformation.Face>> faces
|
||||
}
|
||||
|
||||
enum AsyncWriteEventType {
|
||||
INSERT = 1,
|
||||
DELETE = 2,
|
||||
UNDELETE = 3,
|
||||
SET_ADDITIONAL_FIELDS = 4,
|
||||
DELETE_ADDITIONAL_FIELDS = 5,
|
||||
UPDATE_POSSIBLY_SENSITIVE_TWEET = 6,
|
||||
UPDATE_TWEET_MEDIA = 7,
|
||||
TAKEDOWN = 8,
|
||||
SET_RETWEET_VISIBILITY = 9
|
||||
}
|
||||
|
||||
// an enum of actions that could happen in an async-write (insert or delete)
|
||||
enum AsyncWriteAction {
|
||||
HOSEBIRD_ENQUEUE = 1
|
||||
SEARCH_ENQUEUE = 2
|
||||
// obsolete MAIL_ENQUEUE = 3
|
||||
FANOUT_DELIVERY = 4
|
||||
// obsolete FACEBOOK_ENQUEUE = 5
|
||||
TWEET_INDEX = 6
|
||||
TIMELINE_UPDATE = 7
|
||||
CACHE_UPDATE = 8
|
||||
REPLICATION = 9
|
||||
// obsolete MONORAIL_EXPIRY_ENQUEUE = 10
|
||||
USER_GEOTAG_UPDATE = 11
|
||||
// obsolete IBIS_ENQUEUE = 12
|
||||
EVENT_BUS_ENQUEUE = 13
|
||||
// obsolete HOSEBIRD_BINARY_ENQUEUE = 14
|
||||
TBIRD_UPDATE = 15
|
||||
RETWEETS_DELETION = 16
|
||||
GUANO_SCRIBE = 17
|
||||
MEDIA_DELETION = 18
|
||||
GEO_SEARCH_REQUEST_ID = 19
|
||||
SEARCH_THRIFT_ENQUEUE = 20
|
||||
RETWEET_ARCHIVAL_ENQUEUE = 21
|
||||
}
|
||||
|
||||
# This struct is scribed to test_tweetypie_failed_async_write after
|
||||
# an async-write action has failed multiple retries
|
||||
struct FailedAsyncWrite {
|
||||
1: required AsyncWriteEventType event_type
|
||||
2: required AsyncWriteAction action
|
||||
3: optional tweet.Tweet tweet
|
||||
} (persisted='true', hasPersonalData='true')
|
||||
|
||||
# This struct is scribed to test_tweetypie_detached_retweets after
|
||||
# attempting to read a retweet for which the source tweet has been deleted.
|
||||
struct DetachedRetweet {
|
||||
1: required i64 tweet_id (personalDataType='TweetId')
|
||||
2: required i64 user_id (personalDataType='UserId')
|
||||
3: required i64 source_tweet_id (personalDataType='TweetId')
|
||||
} (persisted='true', hasPersonalData='true')
|
||||
|
||||
struct TweetCacheWrite {
|
||||
1: required i64 tweet_id (personalDataType = 'TweetId')
|
||||
// If the tweet id is a snowflake id, this is an offset since tweet creation.
|
||||
// If it is not a snowflake id, then this is a Unix epoch time in
|
||||
// milliseconds. (The idea is that for most tweets, this encoding will make
|
||||
// it easier to see the interval between events and whether it occured soon
|
||||
// acter tweet creation.)
|
||||
2: required i64 timestamp (personalDataType = 'TransactionTimestamp')
|
||||
3: required string action // One of "set", "add", "replace", "cas", "delete"
|
||||
4: required servo_repo.CachedValue cached_value // Contains metadata about the cached value
|
||||
5: optional CachedTweet cached_tweet
|
||||
} (persisted='true', hasPersonalData='true')
|
||||
|
||||
struct AsyncInsertRequest {
|
||||
12: required tweet.Tweet tweet
|
||||
18: required user.User user
|
||||
21: required i64 timestamp
|
||||
// the cacheable version of tweet from field 12
|
||||
29: required CachedTweet cached_tweet
|
||||
# 13: obsolete tweet.Tweet internal_tweet
|
||||
19: optional tweet.Tweet source_tweet
|
||||
20: optional user.User source_user
|
||||
// Used for quote tweet feature
|
||||
22: optional tweet.Tweet quoted_tweet
|
||||
23: optional user.User quoted_user
|
||||
28: optional i64 parent_user_id
|
||||
// Used for delivering the requestId of a geotagged tweet
|
||||
24: optional string geo_search_request_id
|
||||
# 7: obsolete
|
||||
# if not specified, all async insert actions are performed. if specified, only
|
||||
# the specified action is performed; this is used for retrying specific actions
|
||||
# that failed on a previous attempt.
|
||||
10: optional AsyncWriteAction retry_action
|
||||
# 11: obsolete: bool from_monorail = 0
|
||||
# 14: obsolete
|
||||
15: optional feature_context.FeatureContext feature_context
|
||||
# 16: obsolete
|
||||
# 17: obsolete
|
||||
# 26: obsolete: optional tweet.Tweet debug_tweet_copy
|
||||
27: optional map<tweet.TweetCreateContextKey, string> additional_context
|
||||
30: optional transient_context.TransientCreateContext transient_context
|
||||
// Used to check whether the same tweet has been quoted multiple
|
||||
// times by a given user.
|
||||
31: optional bool quoter_has_already_quoted_tweet
|
||||
32: optional InitialTweetUpdateRequest initialTweetUpdateRequest
|
||||
// User ids of users mentioned in note tweet. Used for tls events
|
||||
33: optional list<i64> note_tweet_mentioned_user_ids
|
||||
}
|
||||
|
||||
struct AsyncUpdatePossiblySensitiveTweetRequest {
|
||||
1: required tweet.Tweet tweet
|
||||
2: required user.User user
|
||||
3: required i64 by_user_id
|
||||
4: required i64 timestamp
|
||||
5: optional bool nsfw_admin_change
|
||||
6: optional bool nsfw_user_change
|
||||
7: optional string note
|
||||
8: optional string host
|
||||
9: optional AsyncWriteAction action
|
||||
}
|
||||
|
||||
struct AsyncUpdateTweetMediaRequest {
|
||||
1: required i64 tweet_id
|
||||
2: required list<media_entity.MediaEntity> orphaned_media
|
||||
3: optional AsyncWriteAction retry_action
|
||||
4: optional list<MediaCommon.MediaKey> media_keys
|
||||
}
|
||||
|
||||
struct AsyncSetAdditionalFieldsRequest {
|
||||
1: required tweet.Tweet additional_fields
|
||||
3: required i64 timestamp
|
||||
4: required i64 user_id
|
||||
2: optional AsyncWriteAction retry_action
|
||||
}
|
||||
|
||||
struct AsyncSetRetweetVisibilityRequest {
|
||||
1: required i64 retweet_id
|
||||
// Whether to archive or unarchive(visible=true) the retweet_id edge in the RetweetsGraph.
|
||||
2: required bool visible
|
||||
3: required i64 src_id
|
||||
5: required i64 retweet_user_id
|
||||
6: required i64 source_tweet_user_id
|
||||
7: required i64 timestamp
|
||||
4: optional AsyncWriteAction retry_action
|
||||
}
|
||||
|
||||
struct SetRetweetVisibilityRequest {
|
||||
1: required i64 retweet_id
|
||||
// Whether to archive or unarchive(visible=true) the retweet_id edge in the RetweetsGraph.
|
||||
2: required bool visible
|
||||
}
|
||||
|
||||
struct AsyncEraseUserTweetsRequest {
|
||||
1: required i64 user_id
|
||||
3: required i64 flock_cursor
|
||||
4: required i64 start_timestamp
|
||||
5: required i64 tweet_count
|
||||
}
|
||||
|
||||
struct AsyncDeleteRequest {
|
||||
4: required tweet.Tweet tweet
|
||||
11: required i64 timestamp
|
||||
2: optional user.User user
|
||||
9: optional i64 by_user_id
|
||||
12: optional tweet_audit.AuditDeleteTweet audit_passthrough
|
||||
13: optional i64 cascaded_from_tweet_id
|
||||
# if not specified, all async-delete actions are performed. if specified, only
|
||||
# the specified action is performed; this is used for retrying specific actions
|
||||
# that failed on a previous attempt.
|
||||
3: optional AsyncWriteAction retry_action
|
||||
5: bool delete_media = 1
|
||||
6: bool delete_retweets = 1
|
||||
8: bool scribe_for_audit = 1
|
||||
15: bool is_user_erasure = 0
|
||||
17: bool is_bounce_delete = 0
|
||||
18: optional bool is_last_quote_of_quoter
|
||||
19: optional bool is_admin_delete
|
||||
}
|
||||
|
||||
struct AsyncUndeleteTweetRequest {
|
||||
1: required tweet.Tweet tweet
|
||||
3: required user.User user
|
||||
4: required i64 timestamp
|
||||
// the cacheable version of tweet from field 1
|
||||
12: required CachedTweet cached_tweet
|
||||
# 2: obsolete tweet.Tweet internal_tweet
|
||||
5: optional AsyncWriteAction retry_action
|
||||
6: optional i64 deleted_at
|
||||
7: optional tweet.Tweet source_tweet
|
||||
8: optional user.User source_user
|
||||
9: optional tweet.Tweet quoted_tweet
|
||||
10: optional user.User quoted_user
|
||||
11: optional i64 parent_user_id
|
||||
13: optional bool quoter_has_already_quoted_tweet
|
||||
}
|
||||
|
||||
struct AsyncIncrFavCountRequest {
|
||||
1: required i64 tweet_id
|
||||
2: required i32 delta
|
||||
}
|
||||
|
||||
struct AsyncIncrBookmarkCountRequest {
|
||||
1: required i64 tweet_id
|
||||
2: required i32 delta
|
||||
}
|
||||
|
||||
struct AsyncDeleteAdditionalFieldsRequest {
|
||||
6: required i64 tweet_id
|
||||
7: required list<i16> field_ids
|
||||
4: required i64 timestamp
|
||||
5: required i64 user_id
|
||||
3: optional AsyncWriteAction retry_action
|
||||
}
|
||||
|
||||
// Used for both tweet and user takedowns.
|
||||
// user will be None for user takedowns because user is only used when scribe_for_audit or
|
||||
// eventbus_enqueue are true, which is never the case for user takedown.
|
||||
struct AsyncTakedownRequest {
|
||||
1: required tweet.Tweet tweet
|
||||
|
||||
// Author of the tweet. Used when scribe_for_audit or eventbus_enqueue are true which is the case
|
||||
// for tweet takedown but not user takedown.
|
||||
2: optional user.User user
|
||||
|
||||
// This field is the resulting list of takedown country codes on the tweet after the
|
||||
// countries_to_add and countries_to_remove changes have been applied.
|
||||
13: list<withholding.TakedownReason> takedown_reasons = []
|
||||
|
||||
// This field is the list of takedown reaons to add to the tweet.
|
||||
14: list<withholding.TakedownReason> reasons_to_add = []
|
||||
|
||||
// This field is the list of takedown reasons to remove from the tweet.
|
||||
15: list<withholding.TakedownReason> reasons_to_remove = []
|
||||
|
||||
// This field determines whether or not Tweetypie should write takedown audits
|
||||
// for this request to Guano.
|
||||
6: required bool scribe_for_audit
|
||||
|
||||
// This field determines whether or not Tweetypie should enqueue a
|
||||
// TweetTakedownEvent to EventBus and Hosebird for this request.
|
||||
7: required bool eventbus_enqueue
|
||||
|
||||
// This field is sent as part of the takedown audit that's written to Guano,
|
||||
// and is not persisted with the takedown itself.
|
||||
8: optional string audit_note
|
||||
|
||||
// This field is the ID of the user who initiated the takedown. It is used
|
||||
// when auditing the takedown in Guano. If unset, it will be logged as -1.
|
||||
9: optional i64 by_user_id
|
||||
|
||||
// This field is the host where the request originated or the remote IP that
|
||||
// is associated with the request. It is used when auditing the takedown in
|
||||
// Guano. If unset, it will be logged as "<unknown>".
|
||||
10: optional string host
|
||||
|
||||
11: optional AsyncWriteAction retry_action
|
||||
12: required i64 timestamp
|
||||
}
|
||||
|
||||
struct SetTweetUserTakedownRequest {
|
||||
1: required i64 tweet_id
|
||||
2: required bool has_takedown
|
||||
3: optional i64 user_id
|
||||
}
|
||||
|
||||
enum DataErrorCause {
|
||||
UNKNOWN = 0
|
||||
// Returned on set_tweet_user_takedown when
|
||||
// the SetTweetUserTakedownRequest.user_id does not match the author
|
||||
// of the tweet identified by SetTweetUserTakedownRequest.tweet_id.
|
||||
USER_TWEET_RELATIONSHIP = 1
|
||||
}
|
||||
|
||||
/**
|
||||
* DataError is returned for operations that perform data changes,
|
||||
* but encountered an inconsistency, and the operation cannot
|
||||
* be meaninfully performed.
|
||||
*/
|
||||
exception DataError {
|
||||
1: required string message
|
||||
2: optional DataErrorCause errorCause
|
||||
}
|
||||
|
||||
struct ReplicatedDeleteAdditionalFieldsRequest {
|
||||
/** is a map for backwards compatibility, but will only contain a single tweet id */
|
||||
1: required map<i64, list<i16>> fields_map
|
||||
}
|
||||
|
||||
struct CascadedDeleteTweetRequest {
|
||||
1: required i64 tweet_id
|
||||
2: required i64 cascaded_from_tweet_id
|
||||
3: optional tweet_audit.AuditDeleteTweet audit_passthrough
|
||||
}
|
||||
|
||||
struct QuotedTweetDeleteRequest {
|
||||
1: i64 quoting_tweet_id
|
||||
2: i64 quoted_tweet_id
|
||||
3: i64 quoted_user_id
|
||||
}
|
||||
|
||||
struct QuotedTweetTakedownRequest {
|
||||
1: i64 quoting_tweet_id
|
||||
2: i64 quoted_tweet_id
|
||||
3: i64 quoted_user_id
|
||||
4: list<string> takedown_country_codes = []
|
||||
5: list<withholding.TakedownReason> takedown_reasons = []
|
||||
}
|
||||
|
||||
struct ReplicatedInsertTweet2Request {
|
||||
1: required CachedTweet cached_tweet
|
||||
// Used to check whether the same tweet has been quoted by a user.
|
||||
2: optional bool quoter_has_already_quoted_tweet
|
||||
3: optional InitialTweetUpdateRequest initialTweetUpdateRequest
|
||||
}
|
||||
|
||||
struct ReplicatedDeleteTweet2Request {
|
||||
1: required tweet.Tweet tweet
|
||||
2: required bool is_erasure
|
||||
3: required bool is_bounce_delete
|
||||
4: optional bool is_last_quote_of_quoter
|
||||
}
|
||||
|
||||
struct ReplicatedSetRetweetVisibilityRequest {
|
||||
1: required i64 src_id
|
||||
// Whether to archive or unarchive(visible=true) the retweet_id edge in the RetweetsGraph.
|
||||
2: required bool visible
|
||||
}
|
||||
|
||||
struct ReplicatedUndeleteTweet2Request {
|
||||
1: required CachedTweet cached_tweet
|
||||
2: optional bool quoter_has_already_quoted_tweet
|
||||
}
|
||||
|
||||
struct GetStoredTweetsOptions {
|
||||
1: bool bypass_visibility_filtering = 0
|
||||
2: optional i64 for_user_id
|
||||
3: list<FieldId> additional_field_ids = []
|
||||
}
|
||||
|
||||
struct GetStoredTweetsRequest {
|
||||
1: required list<i64> tweet_ids
|
||||
2: optional GetStoredTweetsOptions options
|
||||
}
|
||||
|
||||
struct GetStoredTweetsResult {
|
||||
1: required stored_tweet_info.StoredTweetInfo stored_tweet
|
||||
}
|
||||
|
||||
struct GetStoredTweetsByUserOptions {
|
||||
1: bool bypass_visibility_filtering = 0
|
||||
2: bool set_for_user_id = 0
|
||||
3: optional i64 start_time_msec
|
||||
4: optional i64 end_time_msec
|
||||
5: optional i64 cursor
|
||||
6: bool start_from_oldest = 0
|
||||
7: list<FieldId> additional_field_ids = []
|
||||
}
|
||||
|
||||
struct GetStoredTweetsByUserRequest {
|
||||
1: required i64 user_id
|
||||
2: optional GetStoredTweetsByUserOptions options
|
||||
}
|
||||
|
||||
struct GetStoredTweetsByUserResult {
|
||||
1: required list<stored_tweet_info.StoredTweetInfo> stored_tweets
|
||||
2: optional i64 cursor
|
||||
}
|
||||
|
||||
/* This is a request to update an initial tweet based on the creation of a edit tweet
|
||||
* initialTweetId: The tweet to be updated
|
||||
* editTweetId: The tweet being created, which is an edit of initialTweetId
|
||||
* selfPermalink: A self permalink for initialTweetId
|
||||
*/
|
||||
struct InitialTweetUpdateRequest {
|
||||
1: required i64 initialTweetId
|
||||
2: required i64 editTweetId
|
||||
3: optional tweet.ShortenedUrl selfPermalink
|
||||
}
|
||||
|
||||
service TweetServiceInternal extends tweet_service.TweetService {
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.erase_user_tweets.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_erase_user_tweets(1: AsyncEraseUserTweetsRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.post_tweet.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_insert(1: AsyncInsertRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.delete_tweets.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_delete(1: AsyncDeleteRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.undelete_tweet.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_undelete_tweet(1: AsyncUndeleteTweetRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.update_possibly_sensitive_tweet.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_update_possibly_sensitive_tweet(1: AsyncUpdatePossiblySensitiveTweetRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.incr_tweet_fav_count.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_incr_fav_count(1: AsyncIncrFavCountRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.incr_tweet_bookmark_count.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_incr_bookmark_count(1: AsyncIncrBookmarkCountRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.set_additional_fields.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_set_additional_fields(1: AsyncSetAdditionalFieldsRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetServiceInternal.set_retweet_visibility.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_set_retweet_visibility(1: AsyncSetRetweetVisibilityRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Set whether the specified retweet ID should be included in its source tweet's retweet count.
|
||||
* This endpoint is invoked from a tweetypie-daemon to adjust retweet counts for all tweets a
|
||||
* suspended or fraudulent (e.g. ROPO-'d) user has retweeted to disincentivize their false engagement.
|
||||
*/
|
||||
void set_retweet_visibility(1: SetRetweetVisibilityRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.delete_additional_fields.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_delete_additional_fields(1: AsyncDeleteAdditionalFieldsRequest field_delete) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Performs the async portion of TweetService.takedown.
|
||||
* Only tweetypie itself can call this.
|
||||
*/
|
||||
void async_takedown(1: AsyncTakedownRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Update the tweet's takedown fields when a user is taken down.
|
||||
* Only tweetypie's UserTakedownChange daemon can call this.
|
||||
*/
|
||||
void set_tweet_user_takedown(1: SetTweetUserTakedownRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error,
|
||||
3: DataError data_error)
|
||||
|
||||
/**
|
||||
* Cascade delete tweet is the logic for removing tweets that are detached
|
||||
* from their dependency which has been deleted. They are already filtered
|
||||
* out from serving, so this operation reconciles storage with the view
|
||||
* presented by Tweetypie.
|
||||
* This RPC call is delegated from daemons or batch jobs. Currently there
|
||||
* are two use-cases when this call is issued:
|
||||
* * Deleting detached retweets after the source tweet was deleted.
|
||||
* This is done through RetweetsDeletion daemon and the
|
||||
* CleanupDetachedRetweets job.
|
||||
* * Deleting edits of an initial tweet that has been deleted.
|
||||
* This is done by CascadedEditedTweetDelete daemon.
|
||||
* Note that, when serving the original delete request for an edit,
|
||||
* the initial tweet is only deleted, which makes all edits hidden.
|
||||
*/
|
||||
void cascaded_delete_tweet(1: CascadedDeleteTweetRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Update the timestamp of the user's most recent request to delete
|
||||
* location data on their tweets. This does not actually remove the
|
||||
* geo information from the user's tweets, but it will prevent the geo
|
||||
* information for this user's tweets from being returned by
|
||||
* Tweetypie.
|
||||
*/
|
||||
void scrub_geo_update_user_timestamp(1: delete_location_data.DeleteLocationData request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Look up tweets quoting a tweet that has been deleted and enqueue a compliance event.
|
||||
* Only tweetypie's QuotedTweetDelete daemon can call this.
|
||||
**/
|
||||
void quoted_tweet_delete(1: QuotedTweetDeleteRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Look up tweets quoting a tweet that has been taken down and enqueue a compliance event.
|
||||
* Only tweetypie's QuotedTweetTakedown daemon can call this.
|
||||
**/
|
||||
void quoted_tweet_takedown(1: QuotedTweetTakedownRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates TweetService.get_tweet_counts from another cluster.
|
||||
*/
|
||||
void replicated_get_tweet_counts(1: tweet_service.GetTweetCountsRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates TweetService.get_tweet_fields from another cluster.
|
||||
*/
|
||||
void replicated_get_tweet_fields(1: tweet_service.GetTweetFieldsRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates TweetService.get_tweets from another cluster.
|
||||
*/
|
||||
void replicated_get_tweets(1: tweet_service.GetTweetsRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.post_tweet InsertTweet event from another cluster.
|
||||
* Note: v1 version of this endpoint previously just took a Tweet which is why it was replaced
|
||||
*/
|
||||
void replicated_insert_tweet2(1: ReplicatedInsertTweet2Request request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.delete_tweets DeleteTweet event from another cluster.
|
||||
*/
|
||||
void replicated_delete_tweet2(1: ReplicatedDeleteTweet2Request request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.incr_tweet_fav_count event from another cluster.
|
||||
*/
|
||||
void replicated_incr_fav_count(1: i64 tweet_id, 2: i32 delta) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.incr_tweet_bookmark_count event from another cluster.
|
||||
*/
|
||||
void replicated_incr_bookmark_count(1: i64 tweet_id, 2: i32 delta) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetServiceInternal.set_retweet_visibility event from another cluster.
|
||||
*/
|
||||
void replicated_set_retweet_visibility(1: ReplicatedSetRetweetVisibilityRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.scrub_geo from another cluster.
|
||||
*/
|
||||
void replicated_scrub_geo(1: list<i64> tweet_ids) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.set_additional_fields event from another cluster.
|
||||
*/
|
||||
void replicated_set_additional_fields(
|
||||
1: tweet_service.SetAdditionalFieldsRequest request
|
||||
) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.delete_additional_fields event from another cluster.
|
||||
*/
|
||||
void replicated_delete_additional_fields(
|
||||
1: ReplicatedDeleteAdditionalFieldsRequest request
|
||||
) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.undelete_tweet event from another cluster.
|
||||
* Note: v1 version of this endpoint previously just took a Tweet which is why it was replaced
|
||||
*/
|
||||
void replicated_undelete_tweet2(1: ReplicatedUndeleteTweet2Request request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.takedown event from another cluster.
|
||||
*/
|
||||
void replicated_takedown(1: tweet.Tweet tweet) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Replicates a TweetService.update_possibly_sensitive_tweet event from another cluster.
|
||||
*/
|
||||
void replicated_update_possibly_sensitive_tweet(1: tweet.Tweet tweet) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Fetches hydrated Tweets and some metadata irrespective of the Tweets' state.
|
||||
*/
|
||||
list<GetStoredTweetsResult> get_stored_tweets(1: GetStoredTweetsRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
|
||||
/**
|
||||
* Fetches hydrated Tweets and some metadata for a particular user, irrespective of the Tweets'
|
||||
* state.
|
||||
*/
|
||||
GetStoredTweetsByUserResult get_stored_tweets_by_user(1: GetStoredTweetsByUserRequest request) throws (
|
||||
1: exceptions.ClientError client_error,
|
||||
2: exceptions.ServerError server_error)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,3 +0,0 @@
|
|||
# Servo
|
||||
|
||||
Servo is a collection of classes and patterns for building services in Scala. It's a grab-bag of code that was deemed useful for service development.
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
target(
|
||||
dependencies = [
|
||||
"tweetypie/servo/decider/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
|
|
@ -1,18 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["**/*.scala"],
|
||||
platform = "java8",
|
||||
provides = scala_artifact(
|
||||
org = "com.twitter",
|
||||
name = "servo-decider",
|
||||
repo = artifactory,
|
||||
),
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"decider",
|
||||
"finagle/finagle-core/src/main",
|
||||
"tweetypie/servo/util",
|
||||
"twitter-server-internal",
|
||||
"twitter-server/server/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,41 +0,0 @@
|
|||
package com.twitter.servo.decider
|
||||
|
||||
import com.twitter.decider.{Decider, Feature}
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.servo.gate.DeciderGate
|
||||
|
||||
/**
|
||||
* Convenience syntax for creating decider gates
|
||||
*/
|
||||
class DeciderGateBuilder(decider: Decider) {
|
||||
|
||||
/**
|
||||
* idGate should be used when the result of the gate needs to be consistent between repeated
|
||||
* invocations, with the condition that consistency is dependent up on passing identical
|
||||
* parameter between the invocations.
|
||||
*/
|
||||
def idGate(key: DeciderKeyName): Gate[Long] =
|
||||
DeciderGate.byId(keyToFeature(key))
|
||||
|
||||
/**
|
||||
* linearGate should be used when the probability of the gate returning true needs to
|
||||
* increase linearly with the availability of feature.
|
||||
*/
|
||||
def linearGate(key: DeciderKeyName): Gate[Unit] =
|
||||
DeciderGate.linear(keyToFeature(key))
|
||||
|
||||
/**
|
||||
* typedLinearGate is a linearGate that conforms to the gate of the specified type.
|
||||
*/
|
||||
def typedLinearGate[T](key: DeciderKeyName): Gate[T] =
|
||||
linearGate(key).contramap[T] { _ => () }
|
||||
|
||||
/**
|
||||
* expGate should be used when the probability of the gate returning true needs to
|
||||
* increase exponentially with the availability of feature.
|
||||
*/
|
||||
def expGate(key: DeciderKeyName, exponent: Int): Gate[Unit] =
|
||||
DeciderGate.exp(keyToFeature(key), exponent)
|
||||
|
||||
def keyToFeature(key: DeciderKeyName): Feature = decider.feature(key.toString)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,3 +0,0 @@
|
|||
package com.twitter.servo.decider
|
||||
|
||||
trait DeciderKeyEnum extends Enumeration
|
||||
Binary file not shown.
|
|
@ -1,5 +0,0 @@
|
|||
package com.twitter.servo
|
||||
|
||||
package object decider {
|
||||
type DeciderKeyName = DeciderKeyEnum#Value
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,34 +0,0 @@
|
|||
package com.twitter.servo.gate
|
||||
|
||||
import com.twitter.decider
|
||||
import com.twitter.servo.util.Gate
|
||||
import scala.annotation.tailrec
|
||||
|
||||
object DeciderGate {
|
||||
|
||||
/**
|
||||
* Create a Gate[Unit] with a probability of returning true
|
||||
* that increases linearly with the availability of feature.
|
||||
*/
|
||||
def linear(feature: decider.Feature): Gate[Unit] =
|
||||
Gate(_ => feature.isAvailable, "DeciderGate.linear(%s)".format(feature))
|
||||
|
||||
/**
|
||||
* Create a Gate[Unit] with a probability of returning true
|
||||
* that increases exponentially with the availability of feature.
|
||||
*/
|
||||
def exp(feature: decider.Feature, exponent: Int): Gate[Unit] = {
|
||||
val gate = if (exponent >= 0) linear(feature) else !linear(feature)
|
||||
|
||||
@tailrec
|
||||
def go(exp: Int): Boolean = if (exp == 0) true else (gate() && go(exp - 1))
|
||||
|
||||
Gate(_ => go(math.abs(exponent)), "DeciderGate.exp(%s, %s)".format(feature, exponent))
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a Gate[Long] that returns true if the given feature is available for an id.
|
||||
*/
|
||||
def byId(feature: decider.Feature): Gate[Long] =
|
||||
Gate(id => feature.isAvailable(id), "DeciderGate.byId(%s)".format(feature))
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue