mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-06-13 14:48:54 +02:00
f1b5c32734
User Signal Service (USS) is a centralized online platform that supplies comprehensive data on user actions and behaviors on Twitter. This service stores information on both explicit signals, such as Favorites, Retweets, and replies, and implicit signals like Tweet clicks, profile visits, and more.
254 lines
11 KiB
Scala
254 lines
11 KiB
Scala
package com.twitter.usersignalservice.config
|
|
|
|
import com.twitter.conversions.DurationOps._
|
|
import com.twitter.finagle.memcached.{Client => MemcachedClient}
|
|
import com.twitter.finagle.stats.StatsReceiver
|
|
import com.twitter.storehaus.ReadableStore
|
|
import com.twitter.usersignalservice.base.BaseSignalFetcher
|
|
import com.twitter.usersignalservice.base.AggregatedSignalController
|
|
import com.twitter.usersignalservice.base.FilteredSignalFetcherController
|
|
import com.twitter.usersignalservice.base.MemcachedSignalFetcherWrapper
|
|
import com.twitter.usersignalservice.base.Query
|
|
import com.twitter.usersignalservice.base.SignalAggregatedInfo
|
|
import com.twitter.usersignalservice.signals.AccountBlocksFetcher
|
|
import com.twitter.usersignalservice.signals.AccountFollowsFetcher
|
|
import com.twitter.usersignalservice.signals.AccountMutesFetcher
|
|
import com.twitter.usersignalservice.signals.NotificationOpenAndClickFetcher
|
|
import com.twitter.usersignalservice.signals.OriginalTweetsFetcher
|
|
import com.twitter.usersignalservice.signals.ProfileVisitsFetcher
|
|
import com.twitter.usersignalservice.signals.ProfileClickFetcher
|
|
import com.twitter.usersignalservice.signals.RealGraphOonFetcher
|
|
import com.twitter.usersignalservice.signals.ReplyTweetsFetcher
|
|
import com.twitter.usersignalservice.signals.RetweetsFetcher
|
|
import com.twitter.usersignalservice.signals.TweetClickFetcher
|
|
import com.twitter.usersignalservice.signals.TweetFavoritesFetcher
|
|
import com.twitter.usersignalservice.signals.TweetSharesFetcher
|
|
import com.twitter.usersignalservice.signals.VideoTweetsPlayback50Fetcher
|
|
import com.twitter.usersignalservice.signals.VideoTweetsQualityViewFetcher
|
|
import com.twitter.usersignalservice.signals.NegativeEngagedUserFetcher
|
|
import com.twitter.usersignalservice.signals.NegativeEngagedTweetFetcher
|
|
import com.twitter.usersignalservice.thriftscala.Signal
|
|
import com.twitter.usersignalservice.thriftscala.SignalType
|
|
import com.twitter.util.Timer
|
|
import javax.inject.Inject
|
|
import javax.inject.Singleton
|
|
|
|
@Singleton
|
|
class SignalFetcherConfig @Inject() (
|
|
notificationOpenAndClickFetcher: NotificationOpenAndClickFetcher,
|
|
accountFollowsFetcher: AccountFollowsFetcher,
|
|
profileVisitsFetcher: ProfileVisitsFetcher,
|
|
tweetFavoritesFetcher: TweetFavoritesFetcher,
|
|
retweetsFetcher: RetweetsFetcher,
|
|
replyTweetsFetcher: ReplyTweetsFetcher,
|
|
originalTweetsFetcher: OriginalTweetsFetcher,
|
|
tweetSharesFetcher: TweetSharesFetcher,
|
|
memcachedClient: MemcachedClient,
|
|
realGraphOonFetcher: RealGraphOonFetcher,
|
|
tweetClickFetcher: TweetClickFetcher,
|
|
videoTweetsPlayback50Fetcher: VideoTweetsPlayback50Fetcher,
|
|
videoTweetsQualityViewFetcher: VideoTweetsQualityViewFetcher,
|
|
accountMutesFetcher: AccountMutesFetcher,
|
|
accountBlocksFetcher: AccountBlocksFetcher,
|
|
profileClickFetcher: ProfileClickFetcher,
|
|
negativeEngagedTweetFetcher: NegativeEngagedTweetFetcher,
|
|
negativeEngagedUserFetcher: NegativeEngagedUserFetcher,
|
|
statsReceiver: StatsReceiver,
|
|
timer: Timer) {
|
|
|
|
val MemcachedProfileVisitsFetcher: BaseSignalFetcher =
|
|
MemcachedSignalFetcherWrapper(
|
|
memcachedClient,
|
|
profileVisitsFetcher,
|
|
ttl = 8.hours,
|
|
statsReceiver,
|
|
keyPrefix = "uss:pv",
|
|
timer)
|
|
|
|
val MemcachedAccountFollowsFetcher: BaseSignalFetcher = MemcachedSignalFetcherWrapper(
|
|
memcachedClient,
|
|
accountFollowsFetcher,
|
|
ttl = 5.minute,
|
|
statsReceiver,
|
|
keyPrefix = "uss:af",
|
|
timer)
|
|
|
|
val GoodTweetClickDdgFetcher: SignalType => FilteredSignalFetcherController = signalType =>
|
|
FilteredSignalFetcherController(
|
|
tweetClickFetcher,
|
|
signalType,
|
|
statsReceiver,
|
|
timer,
|
|
Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher)
|
|
)
|
|
|
|
val GoodProfileClickDdgFetcher: SignalType => FilteredSignalFetcherController = signalType =>
|
|
FilteredSignalFetcherController(
|
|
profileClickFetcher,
|
|
signalType,
|
|
statsReceiver,
|
|
timer,
|
|
Map(SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher)
|
|
)
|
|
|
|
val GoodProfileClickDdgFetcherWithBlocksMutes: SignalType => FilteredSignalFetcherController =
|
|
signalType =>
|
|
FilteredSignalFetcherController(
|
|
profileClickFetcher,
|
|
signalType,
|
|
statsReceiver,
|
|
timer,
|
|
Map(
|
|
SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher,
|
|
SignalType.AccountMute -> accountMutesFetcher,
|
|
SignalType.AccountBlock -> accountBlocksFetcher
|
|
)
|
|
)
|
|
|
|
val realGraphOonFilteredFetcher: FilteredSignalFetcherController =
|
|
FilteredSignalFetcherController(
|
|
realGraphOonFetcher,
|
|
SignalType.RealGraphOon,
|
|
statsReceiver,
|
|
timer,
|
|
Map(
|
|
SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher
|
|
)
|
|
)
|
|
|
|
val videoTweetsQualityViewFilteredFetcher: FilteredSignalFetcherController =
|
|
FilteredSignalFetcherController(
|
|
videoTweetsQualityViewFetcher,
|
|
SignalType.VideoView90dQualityV1,
|
|
statsReceiver,
|
|
timer,
|
|
Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher)
|
|
)
|
|
|
|
val videoTweetsPlayback50FilteredFetcher: FilteredSignalFetcherController =
|
|
FilteredSignalFetcherController(
|
|
videoTweetsPlayback50Fetcher,
|
|
SignalType.VideoView90dPlayback50V1,
|
|
statsReceiver,
|
|
timer,
|
|
Map(SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher)
|
|
)
|
|
|
|
val uniformTweetSignalInfo: Seq[SignalAggregatedInfo] = Seq(
|
|
SignalAggregatedInfo(SignalType.TweetFavorite, tweetFavoritesFetcher),
|
|
SignalAggregatedInfo(SignalType.Retweet, retweetsFetcher),
|
|
SignalAggregatedInfo(SignalType.Reply, replyTweetsFetcher),
|
|
SignalAggregatedInfo(SignalType.OriginalTweet, originalTweetsFetcher),
|
|
SignalAggregatedInfo(SignalType.TweetShareV1, tweetSharesFetcher),
|
|
SignalAggregatedInfo(SignalType.VideoView90dQualityV1, videoTweetsQualityViewFilteredFetcher),
|
|
)
|
|
|
|
val uniformProducerSignalInfo: Seq[SignalAggregatedInfo] = Seq(
|
|
SignalAggregatedInfo(SignalType.AccountFollow, MemcachedAccountFollowsFetcher),
|
|
SignalAggregatedInfo(
|
|
SignalType.RepeatedProfileVisit90dMinVisit6V1,
|
|
MemcachedProfileVisitsFetcher),
|
|
)
|
|
|
|
val memcachedAccountBlocksFetcher: MemcachedSignalFetcherWrapper = MemcachedSignalFetcherWrapper(
|
|
memcachedClient,
|
|
accountBlocksFetcher,
|
|
ttl = 5.minutes,
|
|
statsReceiver,
|
|
keyPrefix = "uss:ab",
|
|
timer)
|
|
|
|
val memcachedAccountMutesFetcher: MemcachedSignalFetcherWrapper = MemcachedSignalFetcherWrapper(
|
|
memcachedClient,
|
|
accountMutesFetcher,
|
|
ttl = 5.minutes,
|
|
statsReceiver,
|
|
keyPrefix = "uss:am",
|
|
timer)
|
|
|
|
val SignalFetcherMapper: Map[SignalType, ReadableStore[Query, Seq[Signal]]] = Map(
|
|
/* Raw Signals */
|
|
SignalType.AccountFollow -> accountFollowsFetcher,
|
|
SignalType.AccountFollowWithDelay -> MemcachedAccountFollowsFetcher,
|
|
SignalType.GoodProfileClick -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick),
|
|
SignalType.GoodProfileClick20s -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick20s),
|
|
SignalType.GoodProfileClick30s -> GoodProfileClickDdgFetcher(SignalType.GoodProfileClick30s),
|
|
SignalType.GoodProfileClickFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes(
|
|
SignalType.GoodProfileClick),
|
|
SignalType.GoodProfileClick20sFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes(
|
|
SignalType.GoodProfileClick20s),
|
|
SignalType.GoodProfileClick30sFiltered -> GoodProfileClickDdgFetcherWithBlocksMutes(
|
|
SignalType.GoodProfileClick30s),
|
|
SignalType.GoodTweetClick -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick),
|
|
SignalType.GoodTweetClick5s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick5s),
|
|
SignalType.GoodTweetClick10s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick10s),
|
|
SignalType.GoodTweetClick30s -> GoodTweetClickDdgFetcher(SignalType.GoodTweetClick30s),
|
|
SignalType.NegativeEngagedTweetId -> negativeEngagedTweetFetcher,
|
|
SignalType.NegativeEngagedUserId -> negativeEngagedUserFetcher,
|
|
SignalType.NotificationOpenAndClickV1 -> notificationOpenAndClickFetcher,
|
|
SignalType.OriginalTweet -> originalTweetsFetcher,
|
|
SignalType.OriginalTweet90dV2 -> originalTweetsFetcher,
|
|
SignalType.RealGraphOon -> realGraphOonFilteredFetcher,
|
|
SignalType.RepeatedProfileVisit14dMinVisit2V1 -> MemcachedProfileVisitsFetcher,
|
|
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative -> FilteredSignalFetcherController(
|
|
MemcachedProfileVisitsFetcher,
|
|
SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative,
|
|
statsReceiver,
|
|
timer,
|
|
Map(
|
|
SignalType.AccountMute -> accountMutesFetcher,
|
|
SignalType.AccountBlock -> accountBlocksFetcher)
|
|
),
|
|
SignalType.RepeatedProfileVisit90dMinVisit6V1 -> MemcachedProfileVisitsFetcher,
|
|
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative -> FilteredSignalFetcherController(
|
|
MemcachedProfileVisitsFetcher,
|
|
SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative,
|
|
statsReceiver,
|
|
timer,
|
|
Map(
|
|
SignalType.AccountMute -> accountMutesFetcher,
|
|
SignalType.AccountBlock -> accountBlocksFetcher),
|
|
),
|
|
SignalType.RepeatedProfileVisit180dMinVisit6V1 -> MemcachedProfileVisitsFetcher,
|
|
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative -> FilteredSignalFetcherController(
|
|
MemcachedProfileVisitsFetcher,
|
|
SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative,
|
|
statsReceiver,
|
|
timer,
|
|
Map(
|
|
SignalType.AccountMute -> accountMutesFetcher,
|
|
SignalType.AccountBlock -> accountBlocksFetcher),
|
|
),
|
|
SignalType.Reply -> replyTweetsFetcher,
|
|
SignalType.Reply90dV2 -> replyTweetsFetcher,
|
|
SignalType.Retweet -> retweetsFetcher,
|
|
SignalType.Retweet90dV2 -> retweetsFetcher,
|
|
SignalType.TweetFavorite -> tweetFavoritesFetcher,
|
|
SignalType.TweetFavorite90dV2 -> tweetFavoritesFetcher,
|
|
SignalType.TweetShareV1 -> tweetSharesFetcher,
|
|
SignalType.VideoView90dQualityV1 -> videoTweetsQualityViewFilteredFetcher,
|
|
SignalType.VideoView90dPlayback50V1 -> videoTweetsPlayback50FilteredFetcher,
|
|
/* Aggregated Signals */
|
|
SignalType.ProducerBasedUnifiedEngagementWeightedSignal -> AggregatedSignalController(
|
|
uniformProducerSignalInfo,
|
|
uniformProducerSignalEngagementAggregation,
|
|
statsReceiver,
|
|
timer
|
|
),
|
|
SignalType.TweetBasedUnifiedEngagementWeightedSignal -> AggregatedSignalController(
|
|
uniformTweetSignalInfo,
|
|
uniformTweetSignalEngagementAggregation,
|
|
statsReceiver,
|
|
timer
|
|
),
|
|
SignalType.AdFavorite -> tweetFavoritesFetcher,
|
|
/* Negative Signals */
|
|
SignalType.AccountBlock -> memcachedAccountBlocksFetcher,
|
|
SignalType.AccountMute -> memcachedAccountMutesFetcher,
|
|
SignalType.TweetDontLike -> negativeEngagedTweetFetcher,
|
|
SignalType.TweetReport -> negativeEngagedTweetFetcher,
|
|
SignalType.TweetSeeFewer -> negativeEngagedTweetFetcher,
|
|
)
|
|
|
|
}
|