mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-09-20 22:19:50 +02:00
180 lines
8.1 KiB
Scala
180 lines
8.1 KiB
Scala
package com.twitter.home_mixer.functional_component.feature_hydrator
|
|
|
|
import com.twitter.finagle.stats.StatsReceiver
|
|
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
|
|
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
|
|
import com.twitter.home_mixer.model.request.FollowingProduct
|
|
import com.twitter.home_mixer.model.request.ForYouProduct
|
|
import com.twitter.home_mixer.model.request.ListTweetsProduct
|
|
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
|
import com.twitter.home_mixer.model.request.SubscribedProduct
|
|
import com.twitter.home_mixer.util.tweetypie.RequestFields
|
|
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
|
|
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
|
|
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
|
import com.twitter.product_mixer.core.feature.Feature
|
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
|
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
|
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
|
|
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
|
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
|
import com.twitter.spam.rtf.{thriftscala => rtf}
|
|
import com.twitter.stitch.Stitch
|
|
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
|
import com.twitter.tweetypie.{thriftscala => tp}
|
|
import com.twitter.util.logging.Logging
|
|
import javax.inject.Inject
|
|
import javax.inject.Singleton
|
|
|
|
@Singleton
|
|
class TweetypieFeatureHydrator @Inject() (
|
|
tweetypieStitchClient: TweetypieStitchClient,
|
|
statsReceiver: StatsReceiver)
|
|
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
|
|
with Logging {
|
|
|
|
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
|
|
|
|
override val features: Set[Feature[_, _]] = Set(
|
|
AuthorIdFeature,
|
|
ExclusiveConversationAuthorIdFeature,
|
|
InReplyToTweetIdFeature,
|
|
IsHydratedFeature,
|
|
IsNsfw,
|
|
IsNsfwFeature,
|
|
IsRetweetFeature,
|
|
QuotedTweetDroppedFeature,
|
|
QuotedTweetIdFeature,
|
|
QuotedUserIdFeature,
|
|
SourceTweetIdFeature,
|
|
SourceUserIdFeature,
|
|
TweetTextFeature,
|
|
TweetLanguageFeature,
|
|
VisibilityReason
|
|
)
|
|
|
|
private val DefaultFeatureMap = FeatureMapBuilder()
|
|
.add(IsHydratedFeature, false)
|
|
.add(IsNsfw, None)
|
|
.add(IsNsfwFeature, false)
|
|
.add(QuotedTweetDroppedFeature, false)
|
|
.add(TweetTextFeature, None)
|
|
.add(VisibilityReason, None)
|
|
.build()
|
|
|
|
override def apply(
|
|
query: PipelineQuery,
|
|
candidate: TweetCandidate,
|
|
existingFeatures: FeatureMap
|
|
): Stitch[FeatureMap] = {
|
|
val safetyLevel = query.product match {
|
|
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
|
|
case ForYouProduct =>
|
|
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
|
|
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
|
|
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
|
|
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
|
|
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
|
|
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
|
|
}
|
|
|
|
val tweetFieldsOptions = tp.GetTweetFieldsOptions(
|
|
tweetIncludes = RequestFields.TweetTPHydrationFields,
|
|
includeRetweetedTweet = true,
|
|
includeQuotedTweet = true,
|
|
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
|
|
safetyLevel = Some(safetyLevel),
|
|
forUserId = query.getOptionalUserId
|
|
)
|
|
|
|
val exclusiveAuthorIdOpt =
|
|
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
|
|
|
|
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
|
|
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
|
|
val coreData = found.tweet.coreData
|
|
val isNsfwAdmin = coreData.exists(_.nsfwAdmin)
|
|
val isNsfwUser = coreData.exists(_.nsfwUser)
|
|
|
|
val quotedTweetDropped = quoteOpt.exists {
|
|
case _: tp.TweetFieldsResultState.Filtered => true
|
|
case _: tp.TweetFieldsResultState.NotFound => true
|
|
case _ => false
|
|
}
|
|
val quotedTweetIsNsfw = quoteOpt.exists {
|
|
case quoteTweet: tp.TweetFieldsResultState.Found =>
|
|
quoteTweet.found.tweet.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)
|
|
case _ => false
|
|
}
|
|
|
|
val sourceTweetIsNsfw =
|
|
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
|
|
|
|
val tweetText = coreData.map(_.text)
|
|
val tweetLanguage = found.tweet.language.map(_.language)
|
|
|
|
val tweetAuthorId = coreData.map(_.userId)
|
|
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
|
|
val retweetedTweetId = found.retweetedTweet.map(_.id)
|
|
val quotedTweetId = quoteOpt.flatMap {
|
|
case quoteTweet: tp.TweetFieldsResultState.Found =>
|
|
Some(quoteTweet.found.tweet.id)
|
|
case _ => None
|
|
}
|
|
|
|
val retweetedTweetUserId = found.retweetedTweet.flatMap(_.coreData).map(_.userId)
|
|
val quotedTweetUserId = quoteOpt.flatMap {
|
|
case quoteTweet: tp.TweetFieldsResultState.Found =>
|
|
quoteTweet.found.tweet.coreData.map(_.userId)
|
|
case _ => None
|
|
}
|
|
|
|
val isNsfw = isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw
|
|
|
|
FeatureMapBuilder()
|
|
.add(AuthorIdFeature, tweetAuthorId)
|
|
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
|
|
.add(InReplyToTweetIdFeature, inReplyToTweetId)
|
|
.add(IsHydratedFeature, true)
|
|
.add(IsNsfw, Some(isNsfw))
|
|
.add(IsNsfwFeature, isNsfw)
|
|
.add(IsRetweetFeature, retweetedTweetId.isDefined)
|
|
.add(QuotedTweetDroppedFeature, quotedTweetDropped)
|
|
.add(QuotedTweetIdFeature, quotedTweetId)
|
|
.add(QuotedUserIdFeature, quotedTweetUserId)
|
|
.add(SourceTweetIdFeature, retweetedTweetId)
|
|
.add(SourceUserIdFeature, retweetedTweetUserId)
|
|
.add(TweetLanguageFeature, tweetLanguage)
|
|
.add(TweetTextFeature, tweetText)
|
|
.add(VisibilityReason, found.suppressReason)
|
|
.build()
|
|
|
|
// If no tweet result found, return default and pre-existing features
|
|
case _ =>
|
|
DefaultFeatureMap ++ FeatureMapBuilder()
|
|
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
|
|
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
|
|
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
|
|
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
|
|
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
|
|
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
|
|
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
|
|
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
|
|
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
|
|
.build()
|
|
}
|
|
}
|
|
}
|