246 lines
12 KiB
Scala
246 lines
12 KiB
Scala
package com.twitter.home_mixer.functional_component.side_effect
|
|
|
|
import com.twitter.finagle.tracing.Trace
|
|
import com.twitter.home_mixer.marshaller.timeline_logging.PromotedTweetDetailsMarshaller
|
|
import com.twitter.home_mixer.marshaller.timeline_logging.TweetDetailsMarshaller
|
|
import com.twitter.home_mixer.marshaller.timeline_logging.WhoToFollowDetailsMarshaller
|
|
import com.twitter.home_mixer.model.HomeFeatures.GetInitialFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.GetMiddleFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.GetNewerFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.GetOlderFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.HasDarkRequestFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.RequestJoinIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature
|
|
import com.twitter.home_mixer.model.request.DeviceContext.RequestContext
|
|
import com.twitter.home_mixer.model.request.HasDeviceContext
|
|
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
|
import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeServedCandidatesFlag
|
|
import com.twitter.home_mixer.param.HomeGlobalParams.EnableScribeServedCandidatesParam
|
|
import com.twitter.home_mixer.service.HomeMixerAlertConfig
|
|
import com.twitter.inject.annotations.Flag
|
|
import com.twitter.logpipeline.client.common.EventPublisher
|
|
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
|
import com.twitter.product_mixer.component_library.model.candidate.BaseUserCandidate
|
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidateDecorator
|
|
import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidateDecorator
|
|
import com.twitter.product_mixer.component_library.side_effect.ScribeLogEventSideEffect
|
|
import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect
|
|
import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier
|
|
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
|
import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails
|
|
import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.AddEntriesTimelineInstruction
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.ModuleItem
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem
|
|
import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem
|
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
import com.twitter.timelines.timeline_logging.{thriftscala => thrift}
|
|
import com.twitter.util.Time
|
|
import javax.inject.Inject
|
|
import javax.inject.Singleton
|
|
|
|
/**
|
|
* Side effect that logs home timeline served candidates to Scribe.
|
|
*/
|
|
@Singleton
|
|
class HomeScribeServedCandidatesSideEffect @Inject() (
|
|
@Flag(ScribeServedCandidatesFlag) enableScribeServedCandidates: Boolean,
|
|
scribeEventPublisher: EventPublisher[thrift.ServedEntry])
|
|
extends ScribeLogEventSideEffect[
|
|
thrift.ServedEntry,
|
|
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
Timeline
|
|
]
|
|
with PipelineResultSideEffect.Conditionally[
|
|
PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
Timeline
|
|
] {
|
|
|
|
override val identifier: SideEffectIdentifier = SideEffectIdentifier("HomeScribeServedCandidates")
|
|
|
|
override def onlyIf(
|
|
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
selectedCandidates: Seq[CandidateWithDetails],
|
|
remainingCandidates: Seq[CandidateWithDetails],
|
|
droppedCandidates: Seq[CandidateWithDetails],
|
|
response: Timeline
|
|
): Boolean = enableScribeServedCandidates && query.params(EnableScribeServedCandidatesParam)
|
|
|
|
override def buildLogEvents(
|
|
query: PipelineQuery with HasSeenTweetIds with HasDeviceContext,
|
|
selectedCandidates: Seq[CandidateWithDetails],
|
|
remainingCandidates: Seq[CandidateWithDetails],
|
|
droppedCandidates: Seq[CandidateWithDetails],
|
|
response: Timeline
|
|
): Seq[thrift.ServedEntry] = {
|
|
val timelineType = query.product match {
|
|
case FollowingProduct => thrift.TimelineType.HomeLatest
|
|
case ForYouProduct => thrift.TimelineType.Home
|
|
case SubscribedProduct => thrift.TimelineType.HomeSubscribed
|
|
case other => throw new UnsupportedOperationException(s"Unknown product: $other")
|
|
}
|
|
val requestProvenance = query.deviceContext.map { deviceContext =>
|
|
deviceContext.requestContextValue match {
|
|
case RequestContext.Foreground => thrift.RequestProvenance.Foreground
|
|
case RequestContext.Launch => thrift.RequestProvenance.Launch
|
|
case RequestContext.PullToRefresh => thrift.RequestProvenance.Ptr
|
|
case _ => thrift.RequestProvenance.Other
|
|
}
|
|
}
|
|
val queryType = query.features.map { featureMap =>
|
|
if (featureMap.getOrElse(GetOlderFeature, false)) thrift.QueryType.GetOlder
|
|
else if (featureMap.getOrElse(GetNewerFeature, false)) thrift.QueryType.GetNewer
|
|
else if (featureMap.getOrElse(GetMiddleFeature, false)) thrift.QueryType.GetMiddle
|
|
else if (featureMap.getOrElse(GetInitialFeature, false)) thrift.QueryType.GetInitial
|
|
else thrift.QueryType.Other
|
|
}
|
|
val requestInfo = thrift.RequestInfo(
|
|
requestTimeMs = query.queryTime.inMilliseconds,
|
|
traceId = Trace.id.traceId.toLong,
|
|
userId = query.getOptionalUserId,
|
|
clientAppId = query.clientContext.appId,
|
|
hasDarkRequest = query.features.flatMap(_.getOrElse(HasDarkRequestFeature, None)),
|
|
parentId = Some(Trace.id.parentId.toLong),
|
|
spanId = Some(Trace.id.spanId.toLong),
|
|
timelineType = Some(timelineType),
|
|
ipAddress = query.clientContext.ipAddress,
|
|
userAgent = query.clientContext.userAgent,
|
|
queryType = queryType,
|
|
requestProvenance = requestProvenance,
|
|
languageCode = query.clientContext.languageCode,
|
|
countryCode = query.clientContext.countryCode,
|
|
requestEndTimeMs = Some(Time.now.inMilliseconds),
|
|
servedRequestId = query.features.flatMap(_.getOrElse(ServedRequestIdFeature, None)),
|
|
requestJoinId = query.features.flatMap(_.getOrElse(RequestJoinIdFeature, None))
|
|
)
|
|
|
|
val tweetIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
|
selectedCandidates.flatMap {
|
|
case item: ItemCandidateWithDetails if item.candidate.isInstanceOf[BaseTweetCandidate] =>
|
|
Seq((item.candidateIdLong, item))
|
|
case module: ModuleCandidateWithDetails
|
|
if module.candidates.headOption.exists(_.candidate.isInstanceOf[BaseTweetCandidate]) =>
|
|
module.candidates.map(item => (item.candidateIdLong, item))
|
|
case _ => Seq.empty
|
|
}.toMap
|
|
|
|
val userIdToItemCandidateMap: Map[Long, ItemCandidateWithDetails] =
|
|
selectedCandidates.flatMap {
|
|
case module: ModuleCandidateWithDetails
|
|
if module.candidates.forall(_.candidate.isInstanceOf[BaseUserCandidate]) =>
|
|
module.candidates.map { item =>
|
|
(item.candidateIdLong, item)
|
|
}
|
|
case _ => Seq.empty
|
|
}.toMap
|
|
|
|
response.instructions.zipWithIndex
|
|
.collect {
|
|
case (AddEntriesTimelineInstruction(entries), index) =>
|
|
entries.collect {
|
|
case entry: TweetItem if entry.promotedMetadata.isDefined =>
|
|
val promotedTweetDetails = PromotedTweetDetailsMarshaller(entry, index)
|
|
Seq(
|
|
thrift.EntryInfo(
|
|
id = entry.id,
|
|
position = index.shortValue(),
|
|
entryId = entry.entryIdentifier,
|
|
entryType = thrift.EntryType.PromotedTweet,
|
|
sortIndex = entry.sortIndex,
|
|
verticalSize = Some(1),
|
|
displayType = Some(entry.displayType.toString),
|
|
details = Some(thrift.ItemDetails.PromotedTweetDetails(promotedTweetDetails))
|
|
)
|
|
)
|
|
case entry: TweetItem =>
|
|
val candidate = tweetIdToItemCandidateMap(entry.id)
|
|
val tweetDetails = TweetDetailsMarshaller(entry, candidate)
|
|
Seq(
|
|
thrift.EntryInfo(
|
|
id = candidate.candidateIdLong,
|
|
position = index.shortValue(),
|
|
entryId = entry.entryIdentifier,
|
|
entryType = thrift.EntryType.Tweet,
|
|
sortIndex = entry.sortIndex,
|
|
verticalSize = Some(1),
|
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
displayType = Some(entry.displayType.toString),
|
|
details = Some(thrift.ItemDetails.TweetDetails(tweetDetails))
|
|
)
|
|
)
|
|
case module: TimelineModule
|
|
if module.entryNamespace.toString == WhoToFollowCandidateDecorator.EntryNamespaceString =>
|
|
module.items.collect {
|
|
case ModuleItem(entry: UserItem, _, _) =>
|
|
val candidate = userIdToItemCandidateMap(entry.id)
|
|
val whoToFollowDetails = WhoToFollowDetailsMarshaller(entry, candidate)
|
|
thrift.EntryInfo(
|
|
id = entry.id,
|
|
position = index.shortValue(),
|
|
entryId = module.entryIdentifier,
|
|
entryType = thrift.EntryType.WhoToFollowModule,
|
|
sortIndex = module.sortIndex,
|
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
displayType = Some(entry.displayType.toString),
|
|
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToFollowDetails))
|
|
)
|
|
}
|
|
case module: TimelineModule
|
|
if module.entryNamespace.toString == WhoToSubscribeCandidateDecorator.EntryNamespaceString =>
|
|
module.items.collect {
|
|
case ModuleItem(entry: UserItem, _, _) =>
|
|
val candidate = userIdToItemCandidateMap(entry.id)
|
|
val whoToSubscribeDetails = WhoToFollowDetailsMarshaller(entry, candidate)
|
|
thrift.EntryInfo(
|
|
id = entry.id,
|
|
position = index.shortValue(),
|
|
entryId = module.entryIdentifier,
|
|
entryType = thrift.EntryType.WhoToSubscribeModule,
|
|
sortIndex = module.sortIndex,
|
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
displayType = Some(entry.displayType.toString),
|
|
details = Some(thrift.ItemDetails.WhoToFollowDetails(whoToSubscribeDetails))
|
|
)
|
|
}
|
|
case module: TimelineModule
|
|
if module.sortIndex.isDefined && module.items.headOption.exists(
|
|
_.item.isInstanceOf[TweetItem]) =>
|
|
module.items.collect {
|
|
case ModuleItem(entry: TweetItem, _, _) =>
|
|
val candidate = tweetIdToItemCandidateMap(entry.id)
|
|
thrift.EntryInfo(
|
|
id = entry.id,
|
|
position = index.shortValue(),
|
|
entryId = module.entryIdentifier,
|
|
entryType = thrift.EntryType.ConversationModule,
|
|
sortIndex = module.sortIndex,
|
|
score = candidate.features.getOrElse(ScoreFeature, None),
|
|
displayType = Some(entry.displayType.toString)
|
|
)
|
|
}
|
|
case _ => Seq.empty
|
|
}.flatten
|
|
// Other instructions
|
|
case _ => Seq.empty[thrift.EntryInfo]
|
|
}.flatten.map { entryInfo =>
|
|
thrift.ServedEntry(
|
|
entry = Some(entryInfo),
|
|
request = requestInfo
|
|
)
|
|
}
|
|
}
|
|
|
|
override val logPipelinePublisher: EventPublisher[thrift.ServedEntry] =
|
|
scribeEventPublisher
|
|
|
|
override val alerts = Seq(
|
|
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert()
|
|
)
|
|
}
|