81 lines
2.4 KiB
Scala
81 lines
2.4 KiB
Scala
package com.twitter.tweetypie.storage
|
|
|
|
import com.twitter.finagle.stats.StatsReceiver
|
|
import com.twitter.stitch.Stitch
|
|
import com.twitter.storage.client.manhattan.kv.ManhattanValue
|
|
import com.twitter.tweetypie.storage.TweetUtils.collectWithRateLimitCheck
|
|
import com.twitter.tweetypie.storage_internal.thriftscala.StoredTweet
|
|
import com.twitter.tweetypie.thriftscala.Tweet
|
|
import com.twitter.util.Time
|
|
|
|
object AddTweetHandler {
|
|
private[storage] type InternalAddTweet = (
|
|
Tweet,
|
|
ManhattanOperations.Insert,
|
|
Scribe,
|
|
StatsReceiver,
|
|
Time
|
|
) => Stitch[Unit]
|
|
|
|
def apply(
|
|
insert: ManhattanOperations.Insert,
|
|
scribe: Scribe,
|
|
stats: StatsReceiver
|
|
): TweetStorageClient.AddTweet =
|
|
tweet => doAddTweet(tweet, insert, scribe, stats, Time.now)
|
|
|
|
def makeRecords(
|
|
storedTweet: StoredTweet,
|
|
timestamp: Time
|
|
): Seq[TweetManhattanRecord] = {
|
|
val core = CoreFieldsCodec.fromTweet(storedTweet)
|
|
val packedCoreFieldsBlob = CoreFieldsCodec.toTFieldBlob(core)
|
|
val coreRecord =
|
|
TweetManhattanRecord(
|
|
TweetKey.coreFieldsKey(storedTweet.id),
|
|
ManhattanValue(TFieldBlobCodec.toByteBuffer(packedCoreFieldsBlob), Some(timestamp))
|
|
)
|
|
|
|
val otherFieldIds =
|
|
TweetFields.nonCoreInternalFields ++ TweetFields.getAdditionalFieldIds(storedTweet)
|
|
|
|
val otherFields =
|
|
storedTweet
|
|
.getFieldBlobs(otherFieldIds)
|
|
.map {
|
|
case (fieldId, tFieldBlob) =>
|
|
TweetManhattanRecord(
|
|
TweetKey.fieldKey(storedTweet.id, fieldId),
|
|
ManhattanValue(TFieldBlobCodec.toByteBuffer(tFieldBlob), Some(timestamp))
|
|
)
|
|
}
|
|
.toSeq
|
|
otherFields :+ coreRecord
|
|
}
|
|
|
|
private[storage] val doAddTweet: InternalAddTweet = (
|
|
tweet: Tweet,
|
|
insert: ManhattanOperations.Insert,
|
|
scribe: Scribe,
|
|
stats: StatsReceiver,
|
|
timestamp: Time
|
|
) => {
|
|
assert(tweet.coreData.isDefined, s"Tweet ${tweet.id} is missing coreData: $tweet")
|
|
|
|
val storedTweet = StorageConversions.toStoredTweet(tweet)
|
|
val records = makeRecords(storedTweet, timestamp)
|
|
val inserts = records.map(insert)
|
|
val insertsWithRateLimitCheck =
|
|
Stitch.collect(inserts.map(_.liftToTry)).map(collectWithRateLimitCheck).lowerFromTry
|
|
|
|
Stats.updatePerFieldQpsCounters(
|
|
"addTweet",
|
|
TweetFields.getAdditionalFieldIds(storedTweet),
|
|
1,
|
|
stats
|
|
)
|
|
|
|
insertsWithRateLimitCheck.unit.onSuccess { _ => scribe.logAdded(storedTweet) }
|
|
}
|
|
}
|