264 lines
11 KiB
Scala
264 lines
11 KiB
Scala
package com.twitter.home_mixer.functional_component.side_effect
|
|
|
|
import com.twitter.home_mixer.model.HomeFeatures._
|
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
|
import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature
|
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
|
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
|
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.ReplaceEntryTimelineInstruction
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.ShowCoverInstruction
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
import com.twitter.stitch.Stitch
|
|
import com.twitter.timelinemixer.clients.persistence.EntryWithItemIds
|
|
import com.twitter.timelinemixer.clients.persistence.ItemIds
|
|
import com.twitter.timelinemixer.clients.persistence.TimelineResponseBatchesClient
|
|
import com.twitter.timelinemixer.clients.persistence.TimelineResponseV3
|
|
import com.twitter.timelines.persistence.thriftscala.TweetScoreV1
|
|
import com.twitter.timelines.persistence.{thriftscala => persistence}
|
|
import com.twitter.timelineservice.model.TimelineQuery
|
|
import com.twitter.timelineservice.model.TimelineQueryOptions
|
|
import com.twitter.timelineservice.model.TweetScore
|
|
import com.twitter.timelineservice.model.core.TimelineKind
|
|
import com.twitter.timelineservice.model.rich.EntityIdType
|
|
import com.twitter.util.Time
|
|
import com.twitter.{timelineservice => tls}
|
|
import javax.inject.Inject
|
|
import javax.inject.Singleton
|
|
|
|
object UpdateTimelinesPersistenceStoreSideEffect {
|
|
val EmptyItemIds = ItemIds(
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None,
|
|
None)
|
|
}
|
|
|
|
/**
|
|
* Side effect that updates the Timelines Persistence Store (Manhattan) with the entries being returned.
|
|
*/
|
|
@Singleton
|
|
class UpdateTimelinesPersistenceStoreSideEffect @Inject() (
|
|
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
|
|
extends PipelineResultSideEffect[PipelineQuery, Timeline] {
|
|
|
|
override val identifier: SideEffectIdentifier =
|
|
SideEffectIdentifier("UpdateTimelinesPersistenceStore")
|
|
|
|
final override def apply(
|
|
inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline]
|
|
): Stitch[Unit] = {
|
|
if (inputs.response.instructions.nonEmpty) {
|
|
val timelineKind = inputs.query.product match {
|
|
case FollowingProduct => TimelineKind.homeLatest
|
|
case ForYouProduct => TimelineKind.home
|
|
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
|
}
|
|
val timelineQuery = TimelineQuery(
|
|
id = inputs.query.getRequiredUserId,
|
|
kind = timelineKind,
|
|
options = TimelineQueryOptions(
|
|
contextualUserId = inputs.query.getOptionalUserId,
|
|
deviceContext = tls.DeviceContext.empty.copy(
|
|
userAgent = inputs.query.clientContext.userAgent,
|
|
clientAppId = inputs.query.clientContext.appId)
|
|
)
|
|
)
|
|
|
|
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
|
inputs.selectedCandidates.flatMap {
|
|
case item: ItemCandidateWithDetails if item.candidate.id.isInstanceOf[Long] =>
|
|
Seq((item.candidateIdLong, item))
|
|
case module: ModuleCandidateWithDetails
|
|
if module.candidates.headOption.exists(_.candidate.id.isInstanceOf[Long]) =>
|
|
module.candidates.map(item => (item.candidateIdLong, item))
|
|
case _ => Seq.empty
|
|
}.toMap
|
|
|
|
val entries = inputs.response.instructions.collect {
|
|
case AddEntriesTimelineInstruction(entries) =>
|
|
entries.collect {
|
|
// includes tweets, tweet previews, and promoted tweets
|
|
case entry: TweetItem if entry.sortIndex.isDefined => {
|
|
Seq(
|
|
buildTweetEntryWithItemIds(
|
|
tweetIdToItemCandidateMap(entry.id),
|
|
entry.sortIndex.get
|
|
))
|
|
}
|
|
// tweet conversation modules are flattened to individual tweets in the persistence store
|
|
case module: TimelineModule
|
|
if module.sortIndex.isDefined && module.items.headOption.exists(
|
|
_.item.isInstanceOf[TweetItem]) =>
|
|
module.items.map { item =>
|
|
buildTweetEntryWithItemIds(
|
|
tweetIdToItemCandidateMap(item.item.id.asInstanceOf[Long]),
|
|
module.sortIndex.get)
|
|
}
|
|
case module: TimelineModule
|
|
if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
|
|
val userIds = module.items
|
|
.map(item =>
|
|
UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
|
|
Some(item.item.id.asInstanceOf[Long])))
|
|
Seq(
|
|
EntryWithItemIds(
|
|
entityIdType = EntityIdType.WhoToFollow,
|
|
sortIndex = module.sortIndex.get,
|
|
size = module.items.size.toShort,
|
|
itemIds = Some(userIds)
|
|
))
|
|
case module: TimelineModule
|
|
if module.sortIndex.isDefined && module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
|
|
val userIds = module.items
|
|
.map(item =>
|
|
UpdateTimelinesPersistenceStoreSideEffect.EmptyItemIds.copy(userId =
|
|
Some(item.item.id.asInstanceOf[Long])))
|
|
Seq(
|
|
EntryWithItemIds(
|
|
entityIdType = EntityIdType.WhoToSubscribe,
|
|
sortIndex = module.sortIndex.get,
|
|
size = module.items.size.toShort,
|
|
itemIds = Some(userIds)
|
|
))
|
|
}.flatten
|
|
case ShowCoverInstruction(cover) =>
|
|
Seq(
|
|
EntryWithItemIds(
|
|
entityIdType = EntityIdType.Prompt,
|
|
sortIndex = cover.sortIndex.get,
|
|
size = 1,
|
|
itemIds = None
|
|
)
|
|
)
|
|
case ReplaceEntryTimelineInstruction(entry) =>
|
|
val namespaceLength = TweetItem.TweetEntryNamespace.toString.length
|
|
Seq(
|
|
EntryWithItemIds(
|
|
entityIdType = EntityIdType.Tweet,
|
|
sortIndex = entry.sortIndex.get,
|
|
size = 1,
|
|
itemIds = Some(
|
|
Seq(
|
|
ItemIds(
|
|
tweetId =
|
|
entry.entryIdToReplace.map(e => e.substring(namespaceLength + 1).toLong),
|
|
sourceTweetId = None,
|
|
quoteTweetId = None,
|
|
sourceAuthorId = None,
|
|
quoteAuthorId = None,
|
|
inReplyToTweetId = None,
|
|
inReplyToAuthorId = None,
|
|
semanticCoreId = None,
|
|
articleId = None,
|
|
hasRelevancePrompt = None,
|
|
promptData = None,
|
|
tweetScore = None,
|
|
entryIdToReplace = entry.entryIdToReplace,
|
|
tweetReactiveData = None,
|
|
userId = None
|
|
)
|
|
))
|
|
)
|
|
)
|
|
|
|
}.flatten
|
|
|
|
val response = TimelineResponseV3(
|
|
clientPlatform = timelineQuery.clientPlatform,
|
|
servedTime = Time.now,
|
|
requestType = requestTypeFromQuery(inputs.query),
|
|
entries = entries)
|
|
|
|
Stitch.callFuture(timelineResponseBatchesClient.insertResponse(timelineQuery, response))
|
|
} else Stitch.Unit
|
|
}
|
|
|
|
override val alerts = Seq(
|
|
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.8)
|
|
)
|
|
|
|
private def buildTweetEntryWithItemIds(
|
|
candidate: ItemCandidateWithDetails,
|
|
sortIndex: Long
|
|
): EntryWithItemIds = {
|
|
val features = candidate.features
|
|
val sourceAuthorId =
|
|
if (features.getOrElse(IsRetweetFeature, false)) features.getOrElse(SourceUserIdFeature, None)
|
|
else features.getOrElse(AuthorIdFeature, None)
|
|
val quoteAuthorId =
|
|
if (features.getOrElse(QuotedTweetIdFeature, None).nonEmpty)
|
|
features.getOrElse(SourceUserIdFeature, None)
|
|
else None
|
|
val tweetScore = features.getOrElse(ScoreFeature, None).map { score =>
|
|
TweetScore.fromThrift(persistence.TweetScore.TweetScoreV1(TweetScoreV1(score)))
|
|
}
|
|
|
|
val itemIds = ItemIds(
|
|
tweetId = Some(candidate.candidateIdLong),
|
|
sourceTweetId = features.getOrElse(SourceTweetIdFeature, None),
|
|
quoteTweetId = features.getOrElse(QuotedTweetIdFeature, None),
|
|
sourceAuthorId = sourceAuthorId,
|
|
quoteAuthorId = quoteAuthorId,
|
|
inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None),
|
|
inReplyToAuthorId = features.getOrElse(DirectedAtUserIdFeature, None),
|
|
semanticCoreId = features.getOrElse(SemanticCoreIdFeature, None),
|
|
articleId = None,
|
|
hasRelevancePrompt = None,
|
|
promptData = None,
|
|
tweetScore = tweetScore,
|
|
entryIdToReplace = None,
|
|
tweetReactiveData = None,
|
|
userId = None
|
|
)
|
|
|
|
val isPreview = features.getOrElse(IsTweetPreviewFeature, default = false)
|
|
val entityType = if (isPreview) EntityIdType.TweetPreview else EntityIdType.Tweet
|
|
|
|
EntryWithItemIds(
|
|
entityIdType = entityType,
|
|
sortIndex = sortIndex,
|
|
size = 1.toShort,
|
|
itemIds = Some(Seq(itemIds))
|
|
)
|
|
}
|
|
|
|
private def requestTypeFromQuery(query: PipelineQuery): persistence.RequestType = {
|
|
val features = query.features.getOrElse(FeatureMap.empty)
|
|
|
|
val featureToRequestType = Seq(
|
|
(PollingFeature, persistence.RequestType.Polling),
|
|
(GetInitialFeature, persistence.RequestType.Initial),
|
|
(GetNewerFeature, persistence.RequestType.Newer),
|
|
(GetMiddleFeature, persistence.RequestType.Middle),
|
|
(GetOlderFeature, persistence.RequestType.Older)
|
|
)
|
|
|
|
featureToRequestType
|
|
.collectFirst {
|
|
case (feature, requestType) if features.getOrElse(feature, false) => requestType
|
|
}.getOrElse(persistence.RequestType.Other)
|
|
}
|
|
}
|