diff --git a/src/thrift/com/twitter/simclusters_v2/top_k_map.docx b/src/thrift/com/twitter/simclusters_v2/top_k_map.docx new file mode 100644 index 000000000..eacb49c50 Binary files /dev/null and b/src/thrift/com/twitter/simclusters_v2/top_k_map.docx differ diff --git a/src/thrift/com/twitter/simclusters_v2/top_k_map.thrift b/src/thrift/com/twitter/simclusters_v2/top_k_map.thrift deleted file mode 100644 index 013215cec..000000000 --- a/src/thrift/com/twitter/simclusters_v2/top_k_map.thrift +++ /dev/null @@ -1,14 +0,0 @@ -namespace java com.twitter.simclusters_v2.thriftjava -namespace py gen.twitter.simclusters_v2.top_k_map -#@namespace scala com.twitter.simclusters_v2.thriftscala -#@namespace strato com.twitter.simclusters_v2 - -include "com/twitter/algebird_internal/algebird.thrift" - -struct TopKClusters { - 1: required map topK(personalDataTypeKey = 'InferredInterests') -}(hasPersonalData = 'true') - -struct TopKTweets { - 1: required map topK(personalDataTypeKey='TweetId') -}(hasPersonalData = 'true') diff --git a/src/thrift/com/twitter/simclusters_v2/tweet_similarity.docx b/src/thrift/com/twitter/simclusters_v2/tweet_similarity.docx new file mode 100644 index 000000000..7eb93cc63 Binary files /dev/null and b/src/thrift/com/twitter/simclusters_v2/tweet_similarity.docx differ diff --git a/src/thrift/com/twitter/simclusters_v2/tweet_similarity.thrift b/src/thrift/com/twitter/simclusters_v2/tweet_similarity.thrift deleted file mode 100644 index 3debd40f0..000000000 --- a/src/thrift/com/twitter/simclusters_v2/tweet_similarity.thrift +++ /dev/null @@ -1,16 +0,0 @@ -namespace java com.twitter.simclusters_v2.thriftjava -namespace py gen.twitter.simclusters_v2.tweet_similarity -#@namespace scala com.twitter.simclusters_v2.thriftscala -#@namespace strato com.twitter.simclusters_v2 - -struct FeaturedTweet { - 1: required i64 tweetId(personalDataType = 'TweetId') - # timestamp when the user engaged or impressed the tweet - 2: required i64 timestamp(personalDataType = 'PrivateTimestamp') -}(persisted = 'true', hasPersonalData = 'true') - -struct LabelledTweetPairs { - 1: required FeaturedTweet queryFeaturedTweet - 2: required FeaturedTweet candidateFeaturedTweet - 3: required bool label -}(persisted = 'true', hasPersonalData = 'true') diff --git a/timelineranker/README.docx b/timelineranker/README.docx new file mode 100644 index 000000000..3256dba4a Binary files /dev/null and b/timelineranker/README.docx differ diff --git a/timelineranker/README.md b/timelineranker/README.md deleted file mode 100644 index 72b9226db..000000000 --- a/timelineranker/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# TimelineRanker - -**TimelineRanker** (TLR) is a legacy service that provides relevance-scored tweets from the Earlybird Search Index and User Tweet Entity Graph (UTEG) service. Despite its name, it no longer performs heavy ranking or model-based ranking itself; it only uses relevance scores from the Search Index for ranked tweet endpoints. - -The following is a list of major services that Timeline Ranker interacts with: - -- **Earlybird-root-superroot (a.k.a Search):** Timeline Ranker calls the Search Index's super root to fetch a list of Tweets. -- **User Tweet Entity Graph (UTEG):** Timeline Ranker calls UTEG to fetch a list of tweets liked by the users you follow. -- **Socialgraph:** Timeline Ranker calls Social Graph Service to obtain the follow graph and user states such as blocked, muted, retweets muted, etc. -- **TweetyPie:** Timeline Ranker hydrates tweets by calling TweetyPie to post-filter tweets based on certain hydrated fields. -- **Manhattan:** Timeline Ranker hydrates some tweet features (e.g., user languages) from Manhattan. - -**Home Mixer** calls Timeline Ranker to fetch tweets from the Earlybird Search Index and User Tweet Entity Graph (UTEG) service to power both the For You and Following Home Timelines. Timeline Ranker performs light ranking based on Earlybird tweet candidate scores and truncates to the number of candidates requested by Home Mixer based on these scores. diff --git a/timelineranker/client/builder/BUILD b/timelineranker/client/builder/BUILD deleted file mode 100644 index 0df429ac5..000000000 --- a/timelineranker/client/builder/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -target( - tags = ["bazel-compatible"], - dependencies = [ - "timelineranker/client/builder/src/main/scala", - ], -) diff --git a/timelineranker/client/builder/BUILD.docx b/timelineranker/client/builder/BUILD.docx new file mode 100644 index 000000000..d54631496 Binary files /dev/null and b/timelineranker/client/builder/BUILD.docx differ diff --git a/timelineranker/client/builder/README.docx b/timelineranker/client/builder/README.docx new file mode 100644 index 000000000..7a690d215 Binary files /dev/null and b/timelineranker/client/builder/README.docx differ diff --git a/timelineranker/client/builder/README.md b/timelineranker/client/builder/README.md deleted file mode 100644 index eb0ee80a6..000000000 --- a/timelineranker/client/builder/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# TimelineRanker client - -Library for creating a client to talk to TLR. It contains a ClientBuilder implementation -with some preferred settings for clients. diff --git a/timelineranker/client/builder/src/main/scala/BUILD b/timelineranker/client/builder/src/main/scala/BUILD deleted file mode 100644 index 66fe81112..000000000 --- a/timelineranker/client/builder/src/main/scala/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["com/twitter/timelineranker/client/*.scala"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-core/src/main", - "finagle/finagle-stats", - "finagle/finagle-thrift/src/main/java", - "servo/client/src/main/scala/com/twitter/servo/client", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelineranker/server/model:thrift-scala", - "timelineranker/common:model", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/client/builder/src/main/scala/BUILD.docx b/timelineranker/client/builder/src/main/scala/BUILD.docx new file mode 100644 index 000000000..91ffebc00 Binary files /dev/null and b/timelineranker/client/builder/src/main/scala/BUILD.docx differ diff --git a/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClient.docx b/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClient.docx new file mode 100644 index 000000000..4aa8b94ec Binary files /dev/null and b/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClient.docx differ diff --git a/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClient.scala b/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClient.scala deleted file mode 100644 index b3cb85e00..000000000 --- a/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClient.scala +++ /dev/null @@ -1,195 +0,0 @@ -package com.twitter.timelineranker.client - -import com.twitter.finagle.SourcedException -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelineranker.model._ -import com.twitter.timelines.util.stats.RequestStats -import com.twitter.timelines.util.stats.RequestStatsReceiver -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -case class TimelineRankerException(message: String) - extends Exception(message) - with SourcedException { - serviceName = "timelineranker" -} - -/** - * A timeline ranker client whose methods accept and produce model object instances - * instead of thrift instances. - */ -class TimelineRankerClient( - private val client: thrift.TimelineRanker.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends RequestStats { - - private[this] val baseScope = statsReceiver.scope("timelineRankerClient") - private[this] val timelinesRequestStats = RequestStatsReceiver(baseScope.scope("timelines")) - private[this] val recycledTweetRequestStats = RequestStatsReceiver( - baseScope.scope("recycledTweet")) - private[this] val recapHydrationRequestStats = RequestStatsReceiver( - baseScope.scope("recapHydration")) - private[this] val recapAuthorRequestStats = RequestStatsReceiver(baseScope.scope("recapAuthor")) - private[this] val entityTweetsRequestStats = RequestStatsReceiver(baseScope.scope("entityTweets")) - private[this] val utegLikedByTweetsRequestStats = RequestStatsReceiver( - baseScope.scope("utegLikedByTweets")) - - private[this] def fetchRecapQueryResultHead( - results: Seq[Try[CandidateTweetsResult]] - ): CandidateTweetsResult = { - results.head match { - case Return(result) => result - case Throw(e) => throw e - } - } - - private[this] def tryResults[Req, Rep]( - reqs: Seq[Req], - stats: RequestStatsReceiver, - findError: Req => Option[thrift.TimelineError], - )( - getRep: (Req, RequestStatsReceiver) => Try[Rep] - ): Seq[Try[Rep]] = { - reqs.map { req => - findError(req) match { - case Some(error) if error.reason.exists { _ == thrift.ErrorReason.OverCapacity } => - // bubble up over capacity error, server shall handle it - stats.onFailure(error) - Throw(error) - case Some(error) => - stats.onFailure(error) - Throw(TimelineRankerException(error.message)) - case None => - getRep(req, stats) - } - } - } - - private[this] def tryCandidateTweetsResults( - responses: Seq[thrift.GetCandidateTweetsResponse], - requestScopedStats: RequestStatsReceiver - ): Seq[Try[CandidateTweetsResult]] = { - def errorInResponse( - response: thrift.GetCandidateTweetsResponse - ): Option[thrift.TimelineError] = { - response.error - } - - tryResults( - responses, - requestScopedStats, - errorInResponse - ) { (response, stats) => - stats.onSuccess() - Return(CandidateTweetsResult.fromThrift(response)) - } - } - - def getTimeline(query: TimelineQuery): Future[Try[Timeline]] = { - getTimelines(Seq(query)).map(_.head) - } - - def getTimelines(queries: Seq[TimelineQuery]): Future[Seq[Try[Timeline]]] = { - def errorInResponse(response: thrift.GetTimelineResponse): Option[thrift.TimelineError] = { - response.error - } - val thriftQueries = queries.map(_.toThrift) - timelinesRequestStats.latency { - client.getTimelines(thriftQueries).map { responses => - tryResults( - responses, - timelinesRequestStats, - errorInResponse - ) { (response, stats) => - response.timeline match { - case Some(timeline) => - stats.onSuccess() - Return(Timeline.fromThrift(timeline)) - // Should not really happen. - case None => - val tlrException = - TimelineRankerException("No timeline returned even when no error occurred.") - stats.onFailure(tlrException) - Throw(tlrException) - } - } - } - } - } - - def getRecycledTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = { - getRecycledTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead) - } - - def getRecycledTweetCandidates( - queries: Seq[RecapQuery] - ): Future[Seq[Try[CandidateTweetsResult]]] = { - val thriftQueries = queries.map(_.toThriftRecapQuery) - recycledTweetRequestStats.latency { - client.getRecycledTweetCandidates(thriftQueries).map { - tryCandidateTweetsResults(_, recycledTweetRequestStats) - } - } - } - - def hydrateTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = { - hydrateTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead) - } - - def hydrateTweetCandidates(queries: Seq[RecapQuery]): Future[Seq[Try[CandidateTweetsResult]]] = { - val thriftQueries = queries.map(_.toThriftRecapHydrationQuery) - recapHydrationRequestStats.latency { - client.hydrateTweetCandidates(thriftQueries).map { - tryCandidateTweetsResults(_, recapHydrationRequestStats) - } - } - } - - def getRecapCandidatesFromAuthors(query: RecapQuery): Future[CandidateTweetsResult] = { - getRecapCandidatesFromAuthors(Seq(query)).map(fetchRecapQueryResultHead) - } - - def getRecapCandidatesFromAuthors( - queries: Seq[RecapQuery] - ): Future[Seq[Try[CandidateTweetsResult]]] = { - val thriftQueries = queries.map(_.toThriftRecapQuery) - recapAuthorRequestStats.latency { - client.getRecapCandidatesFromAuthors(thriftQueries).map { - tryCandidateTweetsResults(_, recapAuthorRequestStats) - } - } - } - - def getEntityTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = { - getEntityTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead) - } - - def getEntityTweetCandidates( - queries: Seq[RecapQuery] - ): Future[Seq[Try[CandidateTweetsResult]]] = { - val thriftQueries = queries.map(_.toThriftEntityTweetsQuery) - entityTweetsRequestStats.latency { - client.getEntityTweetCandidates(thriftQueries).map { - tryCandidateTweetsResults(_, entityTweetsRequestStats) - } - } - } - - def getUtegLikedByTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = { - getUtegLikedByTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead) - } - - def getUtegLikedByTweetCandidates( - queries: Seq[RecapQuery] - ): Future[Seq[Try[CandidateTweetsResult]]] = { - val thriftQueries = queries.map(_.toThriftUtegLikedByTweetsQuery) - utegLikedByTweetsRequestStats.latency { - client.getUtegLikedByTweetCandidates(thriftQueries).map { - tryCandidateTweetsResults(_, utegLikedByTweetsRequestStats) - } - } - } -} diff --git a/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClientBuilder.docx b/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClientBuilder.docx new file mode 100644 index 000000000..380c5527d Binary files /dev/null and b/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClientBuilder.docx differ diff --git a/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClientBuilder.scala b/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClientBuilder.scala deleted file mode 100644 index 2ea321cf2..000000000 --- a/timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClientBuilder.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.timelineranker.client - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.builder.ClientBuilder -import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsClientBuilder._ -import com.twitter.finagle.param.OppTls -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.service.RetryPolicy._ -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.thrift.ThriftClientRequest -import com.twitter.servo.client.Environment.Local -import com.twitter.servo.client.Environment.Staging -import com.twitter.servo.client.Environment.Production -import com.twitter.servo.client.Environment -import com.twitter.servo.client.FinagleClientBuilder -import com.twitter.util.Try -import com.twitter.util.Duration - -sealed trait TimelineRankerClientBuilderBase { - def DefaultName: String = "timelineranker" - - def DefaultProdDest: String - - def DefaultProdRequestTimeout: Duration = 2.seconds - def DefaultProdTimeout: Duration = 3.seconds - def DefaultProdRetryPolicy: RetryPolicy[Try[Nothing]] = - tries(2, TimeoutAndWriteExceptionsOnly orElse ChannelClosedExceptionsOnly) - - def DefaultLocalTcpConnectTimeout: Duration = 1.second - def DefaultLocalConnectTimeout: Duration = 1.second - def DefaultLocalRetryPolicy: RetryPolicy[Try[Nothing]] = tries(2, TimeoutAndWriteExceptionsOnly) - - def apply( - finagleClientBuilder: FinagleClientBuilder, - environment: Environment, - name: String = DefaultName, - serviceIdentifier: ServiceIdentifier = EmptyServiceIdentifier, - opportunisticTlsOpt: Option[OpportunisticTls.Level] = None, - ): ClientBuilder.Complete[ThriftClientRequest, Array[Byte]] = { - val defaultBuilder = finagleClientBuilder.thriftMuxClientBuilder(name) - val destination = getDestOverride(environment) - - val partialClient = environment match { - case Production | Staging => - defaultBuilder - .requestTimeout(DefaultProdRequestTimeout) - .timeout(DefaultProdTimeout) - .retryPolicy(DefaultProdRetryPolicy) - .daemon(daemonize = true) - .dest(destination) - .mutualTls(serviceIdentifier) - case Local => - defaultBuilder - .tcpConnectTimeout(DefaultLocalTcpConnectTimeout) - .connectTimeout(DefaultLocalConnectTimeout) - .retryPolicy(DefaultLocalRetryPolicy) - .failFast(enabled = false) - .daemon(daemonize = false) - .dest(destination) - .mutualTls(serviceIdentifier) - } - - opportunisticTlsOpt match { - case Some(_) => - val opportunisticTlsParam = OppTls(level = opportunisticTlsOpt) - partialClient - .configured(opportunisticTlsParam) - case None => partialClient - } - } - - private def getDestOverride(environment: Environment): String = { - val defaultDest = DefaultProdDest - environment match { - // Allow overriding the target TimelineRanker instance in staging. - // This is typically useful for redline testing of TimelineRanker. - case Staging => - sys.props.getOrElse("target.timelineranker.instance", defaultDest) - case _ => - defaultDest - } - } -} - -object TimelineRankerClientBuilder extends TimelineRankerClientBuilderBase { - override def DefaultProdDest: String = "/s/timelineranker/timelineranker" -} diff --git a/timelineranker/common/BUILD b/timelineranker/common/BUILD deleted file mode 100644 index 35ccfb839..000000000 --- a/timelineranker/common/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -target( - name = "adapter", - dependencies = ["timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter"], -) - -target( - name = "model", - dependencies = ["timelineranker/common/src/main/scala/com/twitter/timelineranker/model"], -) - -target( - tags = ["bazel-compatible"], - dependencies = [ - ":adapter", - ":model", - ], -) diff --git a/timelineranker/common/BUILD.docx b/timelineranker/common/BUILD.docx new file mode 100644 index 000000000..86253d625 Binary files /dev/null and b/timelineranker/common/BUILD.docx differ diff --git a/timelineranker/common/src/main/scala/BUILD b/timelineranker/common/src/main/scala/BUILD deleted file mode 100644 index 6313cf8bd..000000000 --- a/timelineranker/common/src/main/scala/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -target( - tags = ["bazel-compatible"], - dependencies = [ - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - ], -) diff --git a/timelineranker/common/src/main/scala/BUILD.docx b/timelineranker/common/src/main/scala/BUILD.docx new file mode 100644 index 000000000..1349e5abb Binary files /dev/null and b/timelineranker/common/src/main/scala/BUILD.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/BUILD b/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/BUILD deleted file mode 100644 index 208735f15..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "timelineranker/common:model", - "timelines/src/main/scala/com/twitter/timelines/clientconfig", - "timelines/src/main/scala/com/twitter/timelines/model/tweet", - ], -) diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/BUILD.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/BUILD.docx new file mode 100644 index 000000000..5586b6bf2 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/BUILD.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/TimelineServiceAdapter.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/TimelineServiceAdapter.docx new file mode 100644 index 000000000..89291edeb Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/TimelineServiceAdapter.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/TimelineServiceAdapter.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/TimelineServiceAdapter.scala deleted file mode 100644 index e78b117a4..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter/TimelineServiceAdapter.scala +++ /dev/null @@ -1,139 +0,0 @@ -package com.twitter.timelineranker.adapter - -import com.twitter.timelineranker.model._ -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.timelines.model.TweetId -import com.twitter.timelineservice.model.TimelineId -import com.twitter.timelineservice.model.core -import com.twitter.timelineservice.{model => tls} -import com.twitter.timelineservice.{thriftscala => tlsthrift} -import com.twitter.timelineservice.model.core._ -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -/** - * Enables TLR model objects to be converted to/from TLS model/thrift objects. - */ -object TimelineServiceAdapter { - def toTlrQuery( - id: Long, - tlsRange: tls.TimelineRange, - getTweetsFromArchiveIndex: Boolean = true - ): ReverseChronTimelineQuery = { - val timelineId = TimelineId(id, TimelineKind.home) - val maxCount = tlsRange.maxCount - val tweetIdRange = tlsRange.cursor.map { cursor => - TweetIdRange( - fromId = cursor.tweetIdBounds.bottom, - toId = cursor.tweetIdBounds.top - ) - } - val options = ReverseChronTimelineQueryOptions( - getTweetsFromArchiveIndex = getTweetsFromArchiveIndex - ) - ReverseChronTimelineQuery(timelineId, Some(maxCount), tweetIdRange, Some(options)) - } - - def toTlsQuery(query: ReverseChronTimelineQuery): tls.TimelineQuery = { - val tlsRange = toTlsRange(query.range, query.maxCount) - tls.TimelineQuery( - id = query.id.id, - kind = query.id.kind, - range = tlsRange - ) - } - - def toTlsRange(range: Option[TimelineRange], maxCount: Option[Int]): tls.TimelineRange = { - val cursor = range.map { - case tweetIdRange: TweetIdRange => - RequestCursor( - top = tweetIdRange.toId.map(CursorState.fromTweetId), - bottom = tweetIdRange.fromId.map(core.CursorState.fromTweetId) - ) - case _ => - throw new IllegalArgumentException(s"Only TweetIdRange is supported. Found: $range") - } - maxCount - .map { count => tls.TimelineRange(cursor, count) } - .getOrElse(tls.TimelineRange(cursor)) - } - - /** - * Converts TLS timeline to a Try of TLR timeline. - * - * TLS timeline not only contains timeline entries/attributes but also the retrieval state; - * whereas TLR timeline only has entries/attributes. Therefore, the TLS timeline is - * mapped to a Try[Timeline] where the Try part captures retrieval state and - * Timeline captures entries/attributes. - */ - def toTlrTimelineTry(tlsTimeline: tls.Timeline[tls.TimelineEntry]): Try[Timeline] = { - require( - tlsTimeline.kind == TimelineKind.home, - s"Only home timelines are supported. Found: ${tlsTimeline.kind}" - ) - - tlsTimeline.state match { - case Some(TimelineHit) | None => - val tweetEnvelopes = tlsTimeline.entries.map { - case tweet: tls.Tweet => - TimelineEntryEnvelope(Tweet(tweet.tweetId)) - case entry => - throw new Exception(s"Only tweet timelines are supported. Found: $entry") - } - Return(Timeline(TimelineId(tlsTimeline.id, tlsTimeline.kind), tweetEnvelopes)) - case Some(TimelineNotFound) | Some(TimelineUnavailable) => - Throw(new tls.core.TimelineUnavailableException(tlsTimeline.id, Some(tlsTimeline.kind))) - } - } - - def toTlsTimeline(timeline: Timeline): tls.Timeline[tls.Tweet] = { - val entries = timeline.entries.map { entry => - entry.entry match { - case tweet: Tweet => tls.Tweet(tweet.id) - case entry: HydratedTweetEntry => tls.Tweet.fromThrift(entry.tweet) - case _ => - throw new IllegalArgumentException( - s"Only tweet timelines are supported. Found: ${entry.entry}" - ) - } - } - tls.Timeline( - id = timeline.id.id, - kind = timeline.id.kind, - entries = entries - ) - } - - def toTweetIds(timeline: tlsthrift.Timeline): Seq[TweetId] = { - timeline.entries.map { - case tlsthrift.TimelineEntry.Tweet(tweet) => - tweet.statusId - case entry => - throw new IllegalArgumentException(s"Only tweet timelines are supported. Found: ${entry}") - } - } - - def toTweetIds(timeline: Timeline): Seq[TweetId] = { - timeline.entries.map { entry => - entry.entry match { - case tweet: Tweet => tweet.id - case entry: HydratedTweetEntry => entry.tweet.id - case _ => - throw new IllegalArgumentException( - s"Only tweet timelines are supported. Found: ${entry.entry}" - ) - } - } - } - - def toHydratedTweets(timeline: Timeline): Seq[HydratedTweet] = { - timeline.entries.map { entry => - entry.entry match { - case hydratedTweet: HydratedTweet => hydratedTweet - case _ => - throw new IllegalArgumentException(s"Expected hydrated tweet. Found: ${entry.entry}") - } - } - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/BUILD b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/BUILD deleted file mode 100644 index 08f34d64f..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/common/text/language:locale-util", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:features-scala", - "src/thrift/com/twitter/timelineranker/server/model:thrift-scala", - "timelines:config-api-base", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/model/tweet", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - ], - exports = [ - "timelines:config-api-base", - ], -) diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/BUILD.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/BUILD.docx new file mode 100644 index 000000000..c7039b11b Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/BUILD.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweet.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweet.docx new file mode 100644 index 000000000..3f7d30b7e Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweet.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweet.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweet.scala deleted file mode 100644 index ccc31d27b..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweet.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.tweetypie.thriftscala - -object CandidateTweet { - val DefaultFeatures: ThriftTweetFeatures = ThriftTweetFeatures() - - def fromThrift(candidate: thrift.CandidateTweet): CandidateTweet = { - val tweet: thriftscala.Tweet = candidate.tweet.getOrElse( - throw new IllegalArgumentException(s"CandidateTweet.tweet must have a value") - ) - val features = candidate.features.getOrElse( - throw new IllegalArgumentException(s"CandidateTweet.features must have a value") - ) - - CandidateTweet(HydratedTweet(tweet), features) - } -} - -/** - * A candidate Tweet and associated information. - * Model object for CandidateTweet thrift struct. - */ -case class CandidateTweet(hydratedTweet: HydratedTweet, features: ThriftTweetFeatures) { - - def toThrift: thrift.CandidateTweet = { - thrift.CandidateTweet( - tweet = Some(hydratedTweet.tweet), - features = Some(features) - ) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweetsResult.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweetsResult.docx new file mode 100644 index 000000000..44c410c11 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweetsResult.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweetsResult.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweetsResult.scala deleted file mode 100644 index 8ca807711..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/CandidateTweetsResult.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.util.Future - -object CandidateTweetsResult { - val Empty: CandidateTweetsResult = CandidateTweetsResult(Nil, Nil) - val EmptyFuture: Future[CandidateTweetsResult] = Future.value(Empty) - val EmptyCandidateTweet: Seq[CandidateTweet] = Seq.empty[CandidateTweet] - - def fromThrift(response: thrift.GetCandidateTweetsResponse): CandidateTweetsResult = { - val candidates = response.candidates - .map(_.map(CandidateTweet.fromThrift)) - .getOrElse(EmptyCandidateTweet) - val sourceTweets = response.sourceTweets - .map(_.map(CandidateTweet.fromThrift)) - .getOrElse(EmptyCandidateTweet) - if (sourceTweets.nonEmpty) { - require(candidates.nonEmpty, "sourceTweets cannot have a value if candidates list is empty.") - } - CandidateTweetsResult(candidates, sourceTweets) - } -} - -case class CandidateTweetsResult( - candidates: Seq[CandidateTweet], - sourceTweets: Seq[CandidateTweet]) { - - def toThrift: thrift.GetCandidateTweetsResponse = { - val thriftCandidates = candidates.map(_.toThrift) - val thriftSourceTweets = sourceTweets.map(_.toThrift) - thrift.GetCandidateTweetsResponse( - candidates = Some(thriftCandidates), - sourceTweets = Some(thriftSourceTweets) - ) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/HydratedTweetEntry.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/HydratedTweetEntry.docx new file mode 100644 index 000000000..c32c6b723 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/HydratedTweetEntry.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/HydratedTweetEntry.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/HydratedTweetEntry.scala deleted file mode 100644 index 115a0ca52..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/HydratedTweetEntry.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.tweetypie.{thriftscala => tweetypie} - -/** - * Enables HydratedTweet entries to be included in a Timeline. - */ -class HydratedTweetEntry(tweet: tweetypie.Tweet) extends HydratedTweet(tweet) with TimelineEntry { - - def this(hydratedTweet: HydratedTweet) = this(hydratedTweet.tweet) - - override def toTimelineEntryThrift: thrift.TimelineEntry = { - thrift.TimelineEntry.TweetypieTweet(tweet) - } - - override def throwIfInvalid(): Unit = { - // No validation performed. - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Language.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Language.docx new file mode 100644 index 000000000..716734268 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Language.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Language.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Language.scala deleted file mode 100644 index badb4ae70..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Language.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.common.text.language.LocaleUtil -import com.twitter.timelineranker.{thriftscala => thrift} - -object Language { - - def fromThrift(lang: thrift.Language): Language = { - require(lang.language.isDefined, "language can't be None") - require(lang.scope.isDefined, "scope can't be None") - Language(lang.language.get, LanguageScope.fromThrift(lang.scope.get)) - } -} - -/** - * Represents a language and the scope that it relates to. - */ -case class Language(language: String, scope: LanguageScope.Value) { - - throwIfInvalid() - - def toThrift: thrift.Language = { - val scopeOption = Some(LanguageScope.toThrift(scope)) - thrift.Language(Some(language), scopeOption) - } - - def throwIfInvalid(): Unit = { - val result = LocaleUtil.getLocaleOf(language) - require(result != LocaleUtil.UNKNOWN, s"Language ${language} is unsupported") - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/LanguageScope.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/LanguageScope.docx new file mode 100644 index 000000000..e362f3157 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/LanguageScope.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/LanguageScope.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/LanguageScope.scala deleted file mode 100644 index c0b65cbed..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/LanguageScope.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} - -/** - * Represents what this language is associated with. - * For example, "user" is one of the scopes and "event" - * could be another scope. - */ -object LanguageScope extends Enumeration { - - // User scope means that the language is the user's language. - val User: Value = Value(thrift.LanguageScope.User.value) - - // Event scope means that the language is the event's language. - val Event: Value = Value(thrift.LanguageScope.Event.value) - - // list of all LanguageScope values - val All: ValueSet = LanguageScope.ValueSet(User, Event) - - def apply(scope: thrift.LanguageScope): LanguageScope.Value = { - scope match { - case thrift.LanguageScope.User => - User - case thrift.LanguageScope.Event => - Event - case _ => - throw new IllegalArgumentException(s"Unsupported language scope: $scope") - } - } - - def fromThrift(scope: thrift.LanguageScope): LanguageScope.Value = { - apply(scope) - } - - def toThrift(scope: LanguageScope.Value): thrift.LanguageScope = { - scope match { - case LanguageScope.User => - thrift.LanguageScope.User - case LanguageScope.Event => - thrift.LanguageScope.Event - case _ => - throw new IllegalArgumentException(s"Unsupported language scope: $scope") - } - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PartiallyHydratedTweet.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PartiallyHydratedTweet.docx new file mode 100644 index 000000000..a234eae4a Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PartiallyHydratedTweet.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PartiallyHydratedTweet.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PartiallyHydratedTweet.scala deleted file mode 100644 index 29ced28de..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PartiallyHydratedTweet.scala +++ /dev/null @@ -1,184 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelines.util.SnowflakeSortIndexHelper -import com.twitter.tweetypie.{thriftscala => tweetypie} - -object PartiallyHydratedTweet { - private val InvalidValue = "Invalid value" - - /** - * Creates an instance of PartiallyHydratedTweet based on the given search result. - */ - def fromSearchResult(result: ThriftSearchResult): PartiallyHydratedTweet = { - val tweetId = result.id - val metadata = result.metadata.getOrElse( - throw new IllegalArgumentException( - s"cannot initialize PartiallyHydratedTweet $tweetId without ThriftSearchResult metadata." - ) - ) - - val extraMetadataOpt = metadata.extraMetadata - - val userId = metadata.fromUserId - - // The value of referencedTweetAuthorId and sharedStatusId is only considered valid if it is greater than 0. - val referencedTweetAuthorId = - if (metadata.referencedTweetAuthorId > 0) Some(metadata.referencedTweetAuthorId) else None - val sharedStatusId = if (metadata.sharedStatusId > 0) Some(metadata.sharedStatusId) else None - - val isRetweet = metadata.isRetweet.getOrElse(false) - val retweetSourceTweetId = if (isRetweet) sharedStatusId else None - val retweetSourceUserId = if (isRetweet) referencedTweetAuthorId else None - - // The fields sharedStatusId and referencedTweetAuthorId have overloaded meaning when - // this tweet is not a retweet (for retweet, there is only 1 meaning). - // When not a retweet, - // if referencedTweetAuthorId and sharedStatusId are both set, it is considered a reply - // if referencedTweetAuthorId is set and sharedStatusId is not set, it is a directed at tweet. - // References: SEARCH-8561 and SEARCH-13142 - val inReplyToTweetId = if (!isRetweet) sharedStatusId else None - val inReplyToUserId = if (!isRetweet) referencedTweetAuthorId else None - val isReply = metadata.isReply.contains(true) - - val quotedTweetId = extraMetadataOpt.flatMap(_.quotedTweetId) - val quotedUserId = extraMetadataOpt.flatMap(_.quotedUserId) - - val isNullcast = metadata.isNullcast.contains(true) - - val conversationId = extraMetadataOpt.flatMap(_.conversationId) - - // Root author id for the user who posts an exclusive tweet - val exclusiveConversationAuthorId = extraMetadataOpt.flatMap(_.exclusiveConversationAuthorId) - - // Card URI associated with an attached card to this tweet, if it contains one - val cardUri = extraMetadataOpt.flatMap(_.cardUri) - - val tweet = makeTweetyPieTweet( - tweetId, - userId, - inReplyToTweetId, - inReplyToUserId, - retweetSourceTweetId, - retweetSourceUserId, - quotedTweetId, - quotedUserId, - isNullcast, - isReply, - conversationId, - exclusiveConversationAuthorId, - cardUri - ) - new PartiallyHydratedTweet(tweet) - } - - def makeTweetyPieTweet( - tweetId: TweetId, - userId: UserId, - inReplyToTweetId: Option[TweetId], - inReplyToUserId: Option[TweetId], - retweetSourceTweetId: Option[TweetId], - retweetSourceUserId: Option[UserId], - quotedTweetId: Option[TweetId], - quotedUserId: Option[UserId], - isNullcast: Boolean, - isReply: Boolean, - conversationId: Option[Long], - exclusiveConversationAuthorId: Option[Long] = None, - cardUri: Option[String] = None - ): tweetypie.Tweet = { - val isDirectedAt = inReplyToUserId.isDefined - val isRetweet = retweetSourceTweetId.isDefined && retweetSourceUserId.isDefined - - val reply = if (isReply) { - Some( - tweetypie.Reply( - inReplyToStatusId = inReplyToTweetId, - inReplyToUserId = inReplyToUserId.getOrElse(0L) // Required - ) - ) - } else None - - val directedAt = if (isDirectedAt) { - Some( - tweetypie.DirectedAtUser( - userId = inReplyToUserId.get, - screenName = "" // not available from search - ) - ) - } else None - - val share = if (isRetweet) { - Some( - tweetypie.Share( - sourceStatusId = retweetSourceTweetId.get, - sourceUserId = retweetSourceUserId.get, - parentStatusId = - retweetSourceTweetId.get // Not always correct (eg, retweet of a retweet). - ) - ) - } else None - - val quotedTweet = - for { - tweetId <- quotedTweetId - userId <- quotedUserId - } yield tweetypie.QuotedTweet(tweetId = tweetId, userId = userId) - - val coreData = tweetypie.TweetCoreData( - userId = userId, - text = InvalidValue, - createdVia = InvalidValue, - createdAtSecs = SnowflakeSortIndexHelper.idToTimestamp(tweetId).inSeconds, - directedAtUser = directedAt, - reply = reply, - share = share, - nullcast = isNullcast, - conversationId = conversationId - ) - - // Hydrate exclusiveTweetControl which determines whether the user is able to view an exclusive / SuperFollow tweet. - val exclusiveTweetControl = exclusiveConversationAuthorId.map { authorId => - tweetypie.ExclusiveTweetControl(conversationAuthorId = authorId) - } - - val cardReference = cardUri.map { cardUriFromEB => - tweetypie.CardReference(cardUri = cardUriFromEB) - } - - tweetypie.Tweet( - id = tweetId, - quotedTweet = quotedTweet, - coreData = Some(coreData), - exclusiveTweetControl = exclusiveTweetControl, - cardReference = cardReference - ) - } -} - -/** - * Represents an instance of HydratedTweet that is hydrated using search result - * (instead of being hydrated using TweetyPie service). - * - * Not all fields are available using search therefore such fields if accessed - * throw UnsupportedOperationException to ensure that they are not inadvertently - * accessed and relied upon. - */ -class PartiallyHydratedTweet(tweet: tweetypie.Tweet) extends HydratedTweet(tweet) { - override def parentTweetId: Option[TweetId] = throw notSupported("parentTweetId") - override def mentionedUserIds: Seq[UserId] = throw notSupported("mentionedUserIds") - override def takedownCountryCodes: Set[String] = throw notSupported("takedownCountryCodes") - override def hasMedia: Boolean = throw notSupported("hasMedia") - override def isNarrowcast: Boolean = throw notSupported("isNarrowcast") - override def hasTakedown: Boolean = throw notSupported("hasTakedown") - override def isNsfw: Boolean = throw notSupported("isNsfw") - override def isNsfwUser: Boolean = throw notSupported("isNsfwUser") - override def isNsfwAdmin: Boolean = throw notSupported("isNsfwAdmin") - - private def notSupported(name: String): UnsupportedOperationException = { - new UnsupportedOperationException(s"Not supported: $name") - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PriorSeenEntries.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PriorSeenEntries.docx new file mode 100644 index 000000000..58023d8a0 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PriorSeenEntries.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PriorSeenEntries.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PriorSeenEntries.scala deleted file mode 100644 index 1c0b2c2f1..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/PriorSeenEntries.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.model.TweetId - -object PriorSeenEntries { - def fromThrift(entries: thrift.PriorSeenEntries): PriorSeenEntries = { - PriorSeenEntries(seenEntries = entries.seenEntries) - } -} - -case class PriorSeenEntries(seenEntries: Seq[TweetId]) { - - throwIfInvalid() - - def toThrift: thrift.PriorSeenEntries = { - thrift.PriorSeenEntries(seenEntries = seenEntries) - } - - def throwIfInvalid(): Unit = { - // No validation performed. - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQuery.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQuery.docx new file mode 100644 index 000000000..1e5483244 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQuery.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQuery.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQuery.scala deleted file mode 100644 index 4d86ea13b..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQuery.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelineservice.model.TimelineId - -case class RankedTimelineQuery( - override val id: TimelineId, - override val maxCount: Option[Int] = None, - override val range: Option[TimelineRange] = None, - override val options: Option[RankedTimelineQueryOptions] = None) - extends TimelineQuery(thrift.TimelineQueryType.Ranked, id, maxCount, range, options) { - - throwIfInvalid() -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQueryOptions.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQueryOptions.docx new file mode 100644 index 000000000..5eb4bf7da Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQueryOptions.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQueryOptions.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQueryOptions.scala deleted file mode 100644 index 2a9a517d9..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RankedTimelineQueryOptions.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} - -object RankedTimelineQueryOptions { - def fromThrift(options: thrift.RankedTimelineQueryOptions): RankedTimelineQueryOptions = { - RankedTimelineQueryOptions( - seenEntries = options.seenEntries.map(PriorSeenEntries.fromThrift) - ) - } -} - -case class RankedTimelineQueryOptions(seenEntries: Option[PriorSeenEntries]) - extends TimelineQueryOptions { - - throwIfInvalid() - - def toThrift: thrift.RankedTimelineQueryOptions = { - thrift.RankedTimelineQueryOptions(seenEntries = seenEntries.map(_.toThrift)) - } - - def toTimelineQueryOptionsThrift: thrift.TimelineQueryOptions = { - thrift.TimelineQueryOptions.RankedTimelineQueryOptions(toThrift) - } - - def throwIfInvalid(): Unit = { - seenEntries.foreach(_.throwIfInvalid) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RecapQuery.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RecapQuery.docx new file mode 100644 index 000000000..07ec7e5a6 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RecapQuery.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RecapQuery.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RecapQuery.scala deleted file mode 100644 index 6b1279b6e..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/RecapQuery.scala +++ /dev/null @@ -1,278 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.servo.util.Gate -import com.twitter.timelines.model.candidate.CandidateTweetSourceId -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.common.model._ -import com.twitter.timelines.earlybird.common.options.EarlybirdOptions -import com.twitter.timelines.earlybird.common.utils.SearchOperator -import com.twitter.timelines.configapi.{ - DependencyProvider => ConfigApiDependencyProvider, - FutureDependencyProvider => ConfigApiFutureDependencyProvider, - _ -} -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelineservice.DeviceContext - -object RecapQuery { - - val EngagedTweetsSupportedTweetKindOption: TweetKindOption.ValueSet = TweetKindOption( - includeReplies = false, - includeRetweets = false, - includeExtendedReplies = false, - includeOriginalTweetsAndQuotes = true - ) - - val DefaultSearchOperator: SearchOperator.Value = SearchOperator.Exclude - def fromThrift(query: thrift.RecapQuery): RecapQuery = { - - RecapQuery( - userId = query.userId, - maxCount = query.maxCount, - range = query.range.map(TimelineRange.fromThrift), - options = query.options - .map(options => TweetKindOption.fromThrift(options.to[Set])) - .getOrElse(TweetKindOption.None), - searchOperator = query.searchOperator - .map(SearchOperator.fromThrift) - .getOrElse(DefaultSearchOperator), - earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift), - deviceContext = query.deviceContext.map(DeviceContext.fromThrift), - authorIds = query.authorIds, - excludedTweetIds = query.excludedTweetIds, - searchClientSubId = query.searchClientSubId, - candidateTweetSourceId = - query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift), - hydratesContentFeatures = query.hydratesContentFeatures - ) - } - - def fromThrift(query: thrift.RecapHydrationQuery): RecapQuery = { - require(query.tweetIds.nonEmpty, "tweetIds must be non-empty") - - RecapQuery( - userId = query.userId, - tweetIds = Some(query.tweetIds), - searchOperator = DefaultSearchOperator, - earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift), - deviceContext = query.deviceContext.map(DeviceContext.fromThrift), - candidateTweetSourceId = - query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift), - hydratesContentFeatures = query.hydratesContentFeatures - ) - } - - def fromThrift(query: thrift.EngagedTweetsQuery): RecapQuery = { - val options = query.tweetKindOptions - .map(tweetKindOptions => TweetKindOption.fromThrift(tweetKindOptions.to[Set])) - .getOrElse(TweetKindOption.None) - - if (!(options.isEmpty || - (options == EngagedTweetsSupportedTweetKindOption))) { - throw new IllegalArgumentException(s"Unsupported TweetKindOption value: $options") - } - - RecapQuery( - userId = query.userId, - maxCount = query.maxCount, - range = query.range.map(TimelineRange.fromThrift), - options = options, - searchOperator = DefaultSearchOperator, - earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift), - deviceContext = query.deviceContext.map(DeviceContext.fromThrift), - authorIds = query.userIds, - excludedTweetIds = query.excludedTweetIds, - ) - } - - def fromThrift(query: thrift.EntityTweetsQuery): RecapQuery = { - require( - query.semanticCoreIds.isDefined, - "entities(semanticCoreIds) can't be None" - ) - val options = query.tweetKindOptions - .map(tweetKindOptions => TweetKindOption.fromThrift(tweetKindOptions.to[Set])) - .getOrElse(TweetKindOption.None) - - RecapQuery( - userId = query.userId, - maxCount = query.maxCount, - range = query.range.map(TimelineRange.fromThrift), - options = options, - searchOperator = DefaultSearchOperator, - earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift), - deviceContext = query.deviceContext.map(DeviceContext.fromThrift), - excludedTweetIds = query.excludedTweetIds, - semanticCoreIds = query.semanticCoreIds.map(_.map(SemanticCoreAnnotation.fromThrift).toSet), - hashtags = query.hashtags.map(_.toSet), - languages = query.languages.map(_.map(Language.fromThrift).toSet), - candidateTweetSourceId = - query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift), - includeNullcastTweets = query.includeNullcastTweets, - includeTweetsFromArchiveIndex = query.includeTweetsFromArchiveIndex, - authorIds = query.authorIds, - hydratesContentFeatures = query.hydratesContentFeatures - ) - } - - def fromThrift(query: thrift.UtegLikedByTweetsQuery): RecapQuery = { - val options = query.tweetKindOptions - .map(tweetKindOptions => TweetKindOption.fromThrift(tweetKindOptions.to[Set])) - .getOrElse(TweetKindOption.None) - - RecapQuery( - userId = query.userId, - maxCount = query.maxCount, - range = query.range.map(TimelineRange.fromThrift), - options = options, - earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift), - deviceContext = query.deviceContext.map(DeviceContext.fromThrift), - excludedTweetIds = query.excludedTweetIds, - utegLikedByTweetsOptions = for { - utegCount <- query.utegCount - weightedFollowings <- query.weightedFollowings.map(_.toMap) - } yield { - UtegLikedByTweetsOptions( - utegCount = utegCount, - isInNetwork = query.isInNetwork, - weightedFollowings = weightedFollowings - ) - }, - candidateTweetSourceId = - query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift), - hydratesContentFeatures = query.hydratesContentFeatures - ) - } - - val paramGate: (Param[Boolean] => Gate[RecapQuery]) = HasParams.paramGate - - type DependencyProvider[+T] = ConfigApiDependencyProvider[RecapQuery, T] - object DependencyProvider extends DependencyProviderFunctions[RecapQuery] - - type FutureDependencyProvider[+T] = ConfigApiFutureDependencyProvider[RecapQuery, T] - object FutureDependencyProvider extends FutureDependencyProviderFunctions[RecapQuery] -} - -/** - * Model object corresponding to RecapQuery thrift struct. - */ -case class RecapQuery( - userId: UserId, - maxCount: Option[Int] = None, - range: Option[TimelineRange] = None, - options: TweetKindOption.ValueSet = TweetKindOption.None, - searchOperator: SearchOperator.Value = RecapQuery.DefaultSearchOperator, - earlybirdOptions: Option[EarlybirdOptions] = None, - deviceContext: Option[DeviceContext] = None, - authorIds: Option[Seq[UserId]] = None, - tweetIds: Option[Seq[TweetId]] = None, - semanticCoreIds: Option[Set[SemanticCoreAnnotation]] = None, - hashtags: Option[Set[String]] = None, - languages: Option[Set[Language]] = None, - excludedTweetIds: Option[Seq[TweetId]] = None, - // options used only for yml tweets - utegLikedByTweetsOptions: Option[UtegLikedByTweetsOptions] = None, - searchClientSubId: Option[String] = None, - override val params: Params = Params.Empty, - candidateTweetSourceId: Option[CandidateTweetSourceId.Value] = None, - includeNullcastTweets: Option[Boolean] = None, - includeTweetsFromArchiveIndex: Option[Boolean] = None, - hydratesContentFeatures: Option[Boolean] = None) - extends HasParams { - - override def toString: String = { - s"RecapQuery(userId: $userId, maxCount: $maxCount, range: $range, options: $options, searchOperator: $searchOperator, " + - s"earlybirdOptions: $earlybirdOptions, deviceContext: $deviceContext, authorIds: $authorIds, " + - s"tweetIds: $tweetIds, semanticCoreIds: $semanticCoreIds, hashtags: $hashtags, languages: $languages, excludedTweetIds: $excludedTweetIds, " + - s"utegLikedByTweetsOptions: $utegLikedByTweetsOptions, searchClientSubId: $searchClientSubId, " + - s"params: $params, candidateTweetSourceId: $candidateTweetSourceId, includeNullcastTweets: $includeNullcastTweets, " + - s"includeTweetsFromArchiveIndex: $includeTweetsFromArchiveIndex), hydratesContentFeatures: $hydratesContentFeatures" - } - - def throwIfInvalid(): Unit = { - def noDuplicates[T <: Traversable[_]](elements: T) = { - elements.toSet.size == elements.size - } - - maxCount.foreach { max => require(max > 0, "maxCount must be a positive integer") } - range.foreach(_.throwIfInvalid()) - earlybirdOptions.foreach(_.throwIfInvalid()) - tweetIds.foreach { ids => require(ids.nonEmpty, "tweetIds must be nonEmpty if present") } - semanticCoreIds.foreach(_.foreach(_.throwIfInvalid())) - languages.foreach(_.foreach(_.throwIfInvalid())) - languages.foreach { langs => - require(langs.nonEmpty, "languages must be nonEmpty if present") - require(noDuplicates(langs.map(_.language)), "languages must be unique") - } - } - - throwIfInvalid() - - def toThriftRecapQuery: thrift.RecapQuery = { - val thriftOptions = Some(TweetKindOption.toThrift(options)) - thrift.RecapQuery( - userId, - maxCount, - range.map(_.toTimelineRangeThrift), - deprecatedMinCount = None, - thriftOptions, - earlybirdOptions.map(_.toThrift), - deviceContext.map(_.toThrift), - authorIds, - excludedTweetIds, - Some(SearchOperator.toThrift(searchOperator)), - searchClientSubId, - candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift) - ) - } - - def toThriftRecapHydrationQuery: thrift.RecapHydrationQuery = { - require(tweetIds.isDefined && tweetIds.get.nonEmpty, "tweetIds must be present") - thrift.RecapHydrationQuery( - userId, - tweetIds.get, - earlybirdOptions.map(_.toThrift), - deviceContext.map(_.toThrift), - candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift) - ) - } - - def toThriftEntityTweetsQuery: thrift.EntityTweetsQuery = { - val thriftTweetKindOptions = Some(TweetKindOption.toThrift(options)) - thrift.EntityTweetsQuery( - userId = userId, - maxCount = maxCount, - range = range.map(_.toTimelineRangeThrift), - tweetKindOptions = thriftTweetKindOptions, - earlybirdOptions = earlybirdOptions.map(_.toThrift), - deviceContext = deviceContext.map(_.toThrift), - excludedTweetIds = excludedTweetIds, - semanticCoreIds = semanticCoreIds.map(_.map(_.toThrift)), - hashtags = hashtags, - languages = languages.map(_.map(_.toThrift)), - candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift), - includeNullcastTweets = includeNullcastTweets, - includeTweetsFromArchiveIndex = includeTweetsFromArchiveIndex, - authorIds = authorIds - ) - } - - def toThriftUtegLikedByTweetsQuery: thrift.UtegLikedByTweetsQuery = { - - val thriftTweetKindOptions = Some(TweetKindOption.toThrift(options)) - thrift.UtegLikedByTweetsQuery( - userId = userId, - maxCount = maxCount, - utegCount = utegLikedByTweetsOptions.map(_.utegCount), - range = range.map(_.toTimelineRangeThrift), - tweetKindOptions = thriftTweetKindOptions, - earlybirdOptions = earlybirdOptions.map(_.toThrift), - deviceContext = deviceContext.map(_.toThrift), - excludedTweetIds = excludedTweetIds, - isInNetwork = utegLikedByTweetsOptions.map(_.isInNetwork).get, - weightedFollowings = utegLikedByTweetsOptions.map(_.weightedFollowings), - candidateTweetSourceId = candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift) - ) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQuery.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQuery.docx new file mode 100644 index 000000000..cd055da6f Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQuery.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQuery.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQuery.scala deleted file mode 100644 index 2352ed463..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQuery.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelineservice.model.TimelineId - -object ReverseChronTimelineQuery { - def fromTimelineQuery(query: TimelineQuery): ReverseChronTimelineQuery = { - query match { - case q: ReverseChronTimelineQuery => q - case _ => throw new IllegalArgumentException(s"Unsupported query type: $query") - } - } -} - -case class ReverseChronTimelineQuery( - override val id: TimelineId, - override val maxCount: Option[Int] = None, - override val range: Option[TimelineRange] = None, - override val options: Option[ReverseChronTimelineQueryOptions] = None) - extends TimelineQuery(thrift.TimelineQueryType.ReverseChron, id, maxCount, range, options) { - - throwIfInvalid() -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQueryOptions.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQueryOptions.docx new file mode 100644 index 000000000..6a3c24db2 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQueryOptions.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQueryOptions.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQueryOptions.scala deleted file mode 100644 index 4b053863c..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/ReverseChronTimelineQueryOptions.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} - -object ReverseChronTimelineQueryOptions { - val Default: ReverseChronTimelineQueryOptions = ReverseChronTimelineQueryOptions() - - def fromThrift( - options: thrift.ReverseChronTimelineQueryOptions - ): ReverseChronTimelineQueryOptions = { - ReverseChronTimelineQueryOptions( - getTweetsFromArchiveIndex = options.getTweetsFromArchiveIndex - ) - } -} - -case class ReverseChronTimelineQueryOptions(getTweetsFromArchiveIndex: Boolean = true) - extends TimelineQueryOptions { - - throwIfInvalid() - - def toThrift: thrift.ReverseChronTimelineQueryOptions = { - thrift.ReverseChronTimelineQueryOptions(getTweetsFromArchiveIndex = getTweetsFromArchiveIndex) - } - - def toTimelineQueryOptionsThrift: thrift.TimelineQueryOptions = { - thrift.TimelineQueryOptions.ReverseChronTimelineQueryOptions(toThrift) - } - - def throwIfInvalid(): Unit = {} -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimeRange.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimeRange.docx new file mode 100644 index 000000000..5b888394a Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimeRange.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimeRange.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimeRange.scala deleted file mode 100644 index 75e73fb57..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimeRange.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.util.Time - -object TimeRange { - val default: TimeRange = TimeRange(None, None) - - def fromThrift(range: thrift.TimeRange): TimeRange = { - TimeRange( - from = range.fromMs.map(Time.fromMilliseconds), - to = range.toMs.map(Time.fromMilliseconds) - ) - } -} - -case class TimeRange(from: Option[Time], to: Option[Time]) extends TimelineRange { - - throwIfInvalid() - - def throwIfInvalid(): Unit = { - (from, to) match { - case (Some(fromTime), Some(toTime)) => - require(fromTime <= toTime, "from-time must be less than or equal to-time.") - case _ => // valid, do nothing. - } - } - - def toThrift: thrift.TimeRange = { - thrift.TimeRange( - fromMs = from.map(_.inMilliseconds), - toMs = to.map(_.inMilliseconds) - ) - } - - def toTimelineRangeThrift: thrift.TimelineRange = { - thrift.TimelineRange.TimeRange(toThrift) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Timeline.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Timeline.docx new file mode 100644 index 000000000..1c9204fdc Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Timeline.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Timeline.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Timeline.scala deleted file mode 100644 index 3bac0c99d..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Timeline.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.model.UserId -import com.twitter.timelineservice.model.TimelineId -import com.twitter.timelineservice.model.core.TimelineKind - -object Timeline { - def empty(id: TimelineId): Timeline = { - Timeline(id, Nil) - } - - def fromThrift(timeline: thrift.Timeline): Timeline = { - Timeline( - id = TimelineId.fromThrift(timeline.id), - entries = timeline.entries.map(TimelineEntryEnvelope.fromThrift) - ) - } - - def throwIfIdInvalid(id: TimelineId): Unit = { - // Note: if we support timelines other than TimelineKind.home, we need to update - // the implementation of userId method here and in TimelineQuery class. - require(id.kind == TimelineKind.home, s"Expected TimelineKind.home, found: ${id.kind}") - } -} - -case class Timeline(id: TimelineId, entries: Seq[TimelineEntryEnvelope]) { - - throwIfInvalid() - - def userId: UserId = { - id.id - } - - def throwIfInvalid(): Unit = { - Timeline.throwIfIdInvalid(id) - entries.foreach(_.throwIfInvalid()) - } - - def toThrift: thrift.Timeline = { - thrift.Timeline( - id = id.toThrift, - entries = entries.map(_.toThrift) - ) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntry.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntry.docx new file mode 100644 index 000000000..6da38d599 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntry.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntry.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntry.scala deleted file mode 100644 index 6968fc038..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntry.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} - -object TimelineEntry { - def fromThrift(entry: thrift.TimelineEntry): TimelineEntry = { - entry match { - case thrift.TimelineEntry.Tweet(e) => Tweet.fromThrift(e) - case thrift.TimelineEntry.TweetypieTweet(e) => new HydratedTweetEntry(e) - case _ => throw new IllegalArgumentException(s"Unsupported type: $entry") - } - } -} - -trait TimelineEntry { - def toTimelineEntryThrift: thrift.TimelineEntry - def throwIfInvalid(): Unit -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntryEnvelope.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntryEnvelope.docx new file mode 100644 index 000000000..c88eb6b3b Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntryEnvelope.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntryEnvelope.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntryEnvelope.scala deleted file mode 100644 index d016affa4..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineEntryEnvelope.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} - -object TimelineEntryEnvelope { - def fromThrift(entryEnvelope: thrift.TimelineEntryEnvelope): TimelineEntryEnvelope = { - TimelineEntryEnvelope( - entry = TimelineEntry.fromThrift(entryEnvelope.entry) - ) - } -} - -case class TimelineEntryEnvelope(entry: TimelineEntry) { - - throwIfInvalid() - - def toThrift: thrift.TimelineEntryEnvelope = { - thrift.TimelineEntryEnvelope(entry.toTimelineEntryThrift) - } - - def throwIfInvalid(): Unit = { - entry.throwIfInvalid() - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQuery.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQuery.docx new file mode 100644 index 000000000..0a3b3a74e Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQuery.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQuery.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQuery.scala deleted file mode 100644 index de2a8f262..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQuery.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.model.UserId -import com.twitter.timelineservice.model.TimelineId - -object TimelineQuery { - def fromThrift(query: thrift.TimelineQuery): TimelineQuery = { - val queryType = query.queryType - val id = TimelineId.fromThrift(query.timelineId) - val maxCount = query.maxCount - val range = query.range.map(TimelineRange.fromThrift) - val options = query.options.map(TimelineQueryOptions.fromThrift) - - queryType match { - case thrift.TimelineQueryType.Ranked => - val rankedOptions = getRankedOptions(options) - RankedTimelineQuery(id, maxCount, range, rankedOptions) - - case thrift.TimelineQueryType.ReverseChron => - val reverseChronOptions = getReverseChronOptions(options) - ReverseChronTimelineQuery(id, maxCount, range, reverseChronOptions) - - case _ => - throw new IllegalArgumentException(s"Unsupported query type: $queryType") - } - } - - def getRankedOptions( - options: Option[TimelineQueryOptions] - ): Option[RankedTimelineQueryOptions] = { - options.map { - case o: RankedTimelineQueryOptions => o - case _ => - throw new IllegalArgumentException( - "Only RankedTimelineQueryOptions are supported when queryType is TimelineQueryType.Ranked" - ) - } - } - - def getReverseChronOptions( - options: Option[TimelineQueryOptions] - ): Option[ReverseChronTimelineQueryOptions] = { - options.map { - case o: ReverseChronTimelineQueryOptions => o - case _ => - throw new IllegalArgumentException( - "Only ReverseChronTimelineQueryOptions are supported when queryType is TimelineQueryType.ReverseChron" - ) - } - } -} - -abstract class TimelineQuery( - private val queryType: thrift.TimelineQueryType, - val id: TimelineId, - val maxCount: Option[Int], - val range: Option[TimelineRange], - val options: Option[TimelineQueryOptions]) { - - throwIfInvalid() - - def userId: UserId = { - id.id - } - - def throwIfInvalid(): Unit = { - Timeline.throwIfIdInvalid(id) - range.foreach(_.throwIfInvalid()) - options.foreach(_.throwIfInvalid()) - } - - def toThrift: thrift.TimelineQuery = { - thrift.TimelineQuery( - queryType = queryType, - timelineId = id.toThrift, - maxCount = maxCount, - range = range.map(_.toTimelineRangeThrift), - options = options.map(_.toTimelineQueryOptionsThrift) - ) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQueryOptions.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQueryOptions.docx new file mode 100644 index 000000000..cfca6aaf7 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQueryOptions.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQueryOptions.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQueryOptions.scala deleted file mode 100644 index 56ab9873c..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineQueryOptions.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} - -object TimelineQueryOptions { - def fromThrift(options: thrift.TimelineQueryOptions): TimelineQueryOptions = { - options match { - case thrift.TimelineQueryOptions.RankedTimelineQueryOptions(r) => - RankedTimelineQueryOptions.fromThrift(r) - case thrift.TimelineQueryOptions.ReverseChronTimelineQueryOptions(r) => - ReverseChronTimelineQueryOptions.fromThrift(r) - case _ => throw new IllegalArgumentException(s"Unsupported type: $options") - } - } -} - -trait TimelineQueryOptions { - def toTimelineQueryOptionsThrift: thrift.TimelineQueryOptions - def throwIfInvalid(): Unit -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineRange.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineRange.docx new file mode 100644 index 000000000..e4a4cfbb4 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineRange.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineRange.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineRange.scala deleted file mode 100644 index c5bf4abb1..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TimelineRange.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} - -object TimelineRange { - def fromThrift(range: thrift.TimelineRange): TimelineRange = { - range match { - case thrift.TimelineRange.TimeRange(r) => TimeRange.fromThrift(r) - case thrift.TimelineRange.TweetIdRange(r) => TweetIdRange.fromThrift(r) - case _ => throw new IllegalArgumentException(s"Unsupported type: $range") - } - } -} - -trait TimelineRange { - def toTimelineRangeThrift: thrift.TimelineRange - def throwIfInvalid(): Unit -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Tweet.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Tweet.docx new file mode 100644 index 000000000..ede7ebabf Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Tweet.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Tweet.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Tweet.scala deleted file mode 100644 index a5bd45004..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/Tweet.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.search.earlybird.thriftscala._ -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId - -object Tweet { - def fromThrift(tweet: thrift.Tweet): Tweet = { - Tweet(id = tweet.id) - } -} - -case class Tweet( - id: TweetId, - userId: Option[UserId] = None, - sourceTweetId: Option[TweetId] = None, - sourceUserId: Option[UserId] = None) - extends TimelineEntry { - - throwIfInvalid() - - def throwIfInvalid(): Unit = {} - - def toThrift: thrift.Tweet = { - thrift.Tweet( - id = id, - userId = userId, - sourceTweetId = sourceTweetId, - sourceUserId = sourceUserId) - } - - def toTimelineEntryThrift: thrift.TimelineEntry = { - thrift.TimelineEntry.Tweet(toThrift) - } - - def toThriftSearchResult: ThriftSearchResult = { - val metadata = ThriftSearchResultMetadata( - resultType = ThriftSearchResultType.Recency, - fromUserId = userId match { - case Some(id) => id - case None => 0L - }, - isRetweet = - if (sourceUserId.isDefined || sourceUserId.isDefined) Some(true) - else - None, - sharedStatusId = sourceTweetId match { - case Some(id) => id - case None => 0L - }, - referencedTweetAuthorId = sourceUserId match { - case Some(id) => id - case None => 0L - } - ) - ThriftSearchResult( - id = id, - metadata = Some(metadata) - ) - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TweetIdRange.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TweetIdRange.docx new file mode 100644 index 000000000..785868112 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TweetIdRange.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TweetIdRange.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TweetIdRange.scala deleted file mode 100644 index e871e6658..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/TweetIdRange.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.model.TweetId - -object TweetIdRange { - val default: TweetIdRange = TweetIdRange(None, None) - val empty: TweetIdRange = TweetIdRange(Some(0L), Some(0L)) - - def fromThrift(range: thrift.TweetIdRange): TweetIdRange = { - TweetIdRange(fromId = range.fromId, toId = range.toId) - } - - def fromTimelineRange(range: TimelineRange): TweetIdRange = { - range match { - case r: TweetIdRange => r - case _ => - throw new IllegalArgumentException(s"Only Tweet ID range is supported. Found: $range") - } - } -} - -/** - * A range of Tweet IDs with exclusive bounds. - */ -case class TweetIdRange(fromId: Option[TweetId] = None, toId: Option[TweetId] = None) - extends TimelineRange { - - throwIfInvalid() - - def throwIfInvalid(): Unit = { - (fromId, toId) match { - case (Some(fromTweetId), Some(toTweetId)) => - require(fromTweetId <= toTweetId, "fromId must be less than or equal to toId.") - case _ => // valid, do nothing. - } - } - - def toThrift: thrift.TweetIdRange = { - thrift.TweetIdRange(fromId = fromId, toId = toId) - } - - def toTimelineRangeThrift: thrift.TimelineRange = { - thrift.TimelineRange.TweetIdRange(toThrift) - } - - def isEmpty: Boolean = { - (fromId, toId) match { - case (Some(fromId), Some(toId)) if fromId == toId => true - case _ => false - } - } -} diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/UtegLikedByTweetsOptions.docx b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/UtegLikedByTweetsOptions.docx new file mode 100644 index 000000000..800433770 Binary files /dev/null and b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/UtegLikedByTweetsOptions.docx differ diff --git a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/UtegLikedByTweetsOptions.scala b/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/UtegLikedByTweetsOptions.scala deleted file mode 100644 index a102c62b8..000000000 --- a/timelineranker/common/src/main/scala/com/twitter/timelineranker/model/UtegLikedByTweetsOptions.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.timelineranker.model - -import com.twitter.timelines.model.UserId - -case class UtegLikedByTweetsOptions( - utegCount: Int, - isInNetwork: Boolean, - weightedFollowings: Map[UserId, Double]) diff --git a/timelineranker/server/BUILD.bazel b/timelineranker/server/BUILD.bazel deleted file mode 100644 index cb690507f..000000000 --- a/timelineranker/server/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -target( - dependencies = [ - "timelineranker/config", - "timelineranker/server/src/main/scala", - ], -) - -jvm_app( - name = "bundle", - basename = "timelineranker-server-package-dist", - binary = "timelineranker/server/src/main/scala:bin", - bundles = [bundle( - fileset = ["config/**/*"], - owning_target = "timelineranker/server/config:files", - )], - tags = ["bazel-compatible"], -) diff --git a/timelineranker/server/BUILD.docx b/timelineranker/server/BUILD.docx new file mode 100644 index 000000000..0a04a3e9e Binary files /dev/null and b/timelineranker/server/BUILD.docx differ diff --git a/timelineranker/server/config/BUILD b/timelineranker/server/config/BUILD deleted file mode 100644 index 46f6966fe..000000000 --- a/timelineranker/server/config/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -resources( - sources = ["**/*.yml"], -) - -# Created for Bazel compatibility. -# In Bazel, loose files must be part of a target to be included into a bundle. -# See also http://go/bazel-compatibility/bundle_does_not_match_any_files -files( - name = "files", - sources = [ - "!BUILD", - "**/*", - ], -) diff --git a/timelineranker/server/config/BUILD.docx b/timelineranker/server/config/BUILD.docx new file mode 100644 index 000000000..2424645b8 Binary files /dev/null and b/timelineranker/server/config/BUILD.docx differ diff --git a/timelineranker/server/config/decider.docx b/timelineranker/server/config/decider.docx new file mode 100644 index 000000000..488891d05 Binary files /dev/null and b/timelineranker/server/config/decider.docx differ diff --git a/timelineranker/server/config/decider.yml b/timelineranker/server/config/decider.yml deleted file mode 100644 index 90384c052..000000000 --- a/timelineranker/server/config/decider.yml +++ /dev/null @@ -1,153 +0,0 @@ -# Deciders that can be used to control load on TLR or its backends. -enable_max_concurrency_limiting: - comment: "When enabled, limit maxConcurrency filter. Note: Requires system property maxConcurrency to be set." - default_availability: 0 - -# Deciders related to testing / debugging. -enable_routing_to_ranker_dev_proxy: - comment: "Route dark traffic to the TimelineRanker development proxy. 100% means ~100% of requests to a host." - default_availability: 0 - -# Deciders related to authorization. -client_request_authorization: - comment: "Enable client request authorization and rate limiting" - default_availability: 10000 -client_write_whitelist: - comment: "Enable authorization of write protected requests from only whitelisted clients" - default_availability: 0 -allow_timeline_mixer_recap_prod: - comment: "Allow requests from production TimelineMixer/recap" - default_availability: 10000 -allow_timeline_mixer_recycled_prod: - comment: "Allow requests from production TimelineMixer/recycled" - default_availability: 10000 -allow_timeline_mixer_hydrate_prod: - comment: "Allow requests from production TimelineMixer/hydrate" - default_availability: 10000 -allow_timeline_mixer_hydrate_recos_prod: - comment: "Allow requests from production TimelineMixer/hydrate_recos" - default_availability: 10000 -allow_timeline_mixer_seed_authors_prod: - comment: "Allow requests from production TimelineMixer/seed_author_ids" - default_availability: 10000 -allow_timeline_mixer_simcluster_prod: - comment: "Allow requests from production TimelineMixer/simcluster" - default_availability: 10000 -allow_timeline_mixer_entity_tweets_prod: - comment: "Allow requests from production TimelineMixer/entity_tweets" - default_availability: 10000 -allow_timeline_mixer_list_prod: - comment: "Allow requests from production TimelineMixer/list" - default_availability: 10000 -allow_timeline_mixer_list_tweet_prod: - comment: "Allow requests from production TimelineMixer/list_tweet" - default_availability: 10000 -allow_timeline_mixer_uteg_liked_by_tweets_prod: - comment: "Allow requests from production TimelineMixer/uteg_liked_by_tweets" - default_availability: 10000 -allow_timeline_mixer_community_prod: - comment: "Allow requests from production TimelineMixer/community" - default_availability: 10000 -allow_timeline_mixer_community_tweet_prod: - comment: "Allow requests from production TimelineMixer/community_tweet" - default_availability: 10000 -allow_timeline_scorer_recommended_trend_tweet_prod: - comment: "Allow requests from production TimelineMixer/recommended_trend_tweet" - default_availability: 10000 - -allow_timeline_scorer_rec_topic_tweets_prod: - comment: "Allow requests from production TimelineScorer/rec_topic_tweets" - default_availability: 10000 -allow_timeline_scorer_popular_topic_tweets_prod: - comment: "Allow requests from production TimelineScorer/popular_topic_tweets" - default_availability: 10000 - -allow_timelinescorer_hydrate_tweet_scoring_prod: - comment: "Allow requests from production TimelineScorer/hydrate_tweet_scoring" - default_availability: 10000 - -allow_timeline_mixer_staging: - comment: "Allow requests from staging TimelineMixer" - default_availability: 10000 -allow_timeline_ranker_warmup: - comment: "Allow warmup requests from the TLR cluster" - default_availability: 10000 -allow_timeline_ranker_proxy: - comment: "Allow warmup requests from the TimelineRanker proxy" - default_availability: 10000 -allow_timeline_service_prod: - comment: "Allow requests from production TimelineService" - default_availability: 10000 -allow_timeline_service_staging: - comment: "Allow requests from staging TimelineService" - default_availability: 10000 -rate_limit_override_unknown: - comment: "Override the rate limit for unknown clients" - default_availability: 0 - -# Deciders related to reverse-chron home timeline materialization. -multiplier_of_materialization_tweets_fetched: - comment: "Multiplier applied to the number of tweets fetched from search expressed as percentage. 100 means 100%. It can be used to fetch more than the number tweets requested by a caller (to improve similarity) or to fetch less than requested to reduce load." - default_availability: 100 -enable_backfill_filtered_entries: - comment: "Controls whether to back-fill timeline entries that get filtered out by TweetsPostFilter during home timeline materialization." - default_availability: 0 -tweets_filtering_lossage_threshold: - comment: "If back-filling filtered entries is enabled and if percentage of tweets that get filtered out exceeds this value then we will issue a second call to get more tweets. Default value 2000 == 20%" - default_availability: 2000 -tweets_filtering_lossage_limit: - comment: "We need to ensure that the number of tweets requested by the second call are not unbounded (for example, if everything is filtered out in the first call) therefore we limit the actual filtered out percentage to be no greater than the value below. Default value 6000 == 60%. That is, even if the actual lossage is 90% we will consider it to be only 60% for the purpose of back-filling." - default_availability: 6000 -supplement_follows_with_real_graph: - comment: "Whether to fetch additional follows from RealGraph for users with more than the max follows fetched from SGS during home timeline materialization." - default_availability: 0 - -# Deciders related to recap. -recap_enable_content_features_hydration: - comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for recap Tweets. Otherwise those features are not set" - default_availability: 10000 -recap_max_count_multiplier: - comment: "We multiply maxCount (caller supplied value) by this multiplier and fetch those many candidates from search so that we are left with sufficient number of candidates after hydration and filtering. 100 == 1.0" - default_availability: 100 -recap_enable_extra_sorting_in_results: - comment: "If TLR will do extra sorting in search results" - default_availability: 10000 - -# Deciders related to recycled tweets. -recycled_enable_content_features_hydration: - comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for recycled Tweets. Otherwise those features are not set" - default_availability: 0 -recycled_max_count_multiplier: - comment: "We multiply maxCount (caller supplied value) by this multiplier and fetch those many candidates from search so that we are left with sufficient number of candidates after hydration and filtering. 100 == 1.0" - default_availability: 100 - -# Deciders related to entity tweets. -entity_tweets_enable_content_features_hydration: - comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for entity Tweets. Otherwise those features are not set" - default_availability: 10000 - -# Deciders related to both recap and recycled tweets -enable_real_graph_users: - comment: "This is used only if user follows >= 1000. If true, expands user seedset with real graph users and recent followed users. Otherwise, user seedset only includes followed users." - default_availability: 0 -max_real_graph_and_followed_users: - comment: "Maximum number of combined real graph users and recent followed users in the user seedset for recap and recycled tweets if enable_real_graph_users is true and only_real_graph_users is false. This is upper bounded by 2000." - default_availability: 1000 - -# Deciders related to recap author -recap_author_enable_new_pipeline: - comment: "Enable new recap author pipeline" - default_availability: 0 -recap_author_enable_content_features_hydration: - comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for PYLE Tweets. Otherwise those features are not set" - default_availability: 0 - -# Deciders related to recap hydration(rectweet+ranked organic). -recap_hydration_enable_content_features_hydration: - comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for rectweet+ranked organic Tweets. Otherwise those features are not set" - default_availability: 0 - -# Deciders related to uteg liked by tweets -uteg_liked_by_tweets_enable_content_features_hydration: - comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for rectweet+recycled utegLikedBy Tweets. Otherwise those features are not set" - default_availability: 0 diff --git a/timelineranker/server/src/main/resources/BUILD.bazel b/timelineranker/server/src/main/resources/BUILD.bazel deleted file mode 100644 index aa1db34e4..000000000 --- a/timelineranker/server/src/main/resources/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -resources( - sources = [ - "*.xml", - ], -) diff --git a/timelineranker/server/src/main/resources/BUILD.docx b/timelineranker/server/src/main/resources/BUILD.docx new file mode 100644 index 000000000..dc9acb596 Binary files /dev/null and b/timelineranker/server/src/main/resources/BUILD.docx differ diff --git a/timelineranker/server/src/main/resources/logback-timelineranker.docx b/timelineranker/server/src/main/resources/logback-timelineranker.docx new file mode 100644 index 000000000..516b5ce19 Binary files /dev/null and b/timelineranker/server/src/main/resources/logback-timelineranker.docx differ diff --git a/timelineranker/server/src/main/resources/logback-timelineranker.xml b/timelineranker/server/src/main/resources/logback-timelineranker.xml deleted file mode 100644 index 27f266b76..000000000 --- a/timelineranker/server/src/main/resources/logback-timelineranker.xml +++ /dev/null @@ -1,124 +0,0 @@ - - - - - - - - - - true - - - - - ${SERVICE_OUTPUT} - - - ${SERVICE_OUTPUT}.%d.%i.gz - 500MB - - 21 - true - - - ${DEFAULT_SERVICE_PATTERN} - - - - - - ${DEBUG_TRANSCRIPTS_OUTPUT} - - - ${DEBUG_TRANSCRIPTS_OUTPUT}.%d.%i.gz - 500MB - - 21 - true - - - ${DEFAULT_SERVICE_PATTERN} - - - - - - true - loglens - ${log.lens.index:-timelineranker} - ${log.lens.tag} - - %msg%n - - - manhattan-client - .*InvalidRequest.* - - - - - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/timelineranker/server/src/main/scala/BUILD.bazel b/timelineranker/server/src/main/scala/BUILD.bazel deleted file mode 100644 index 1538efd8e..000000000 --- a/timelineranker/server/src/main/scala/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -target( - dependencies = [ - "timelineranker/server/src/main/scala/com/twitter/timelineranker/repository", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/server", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/source", - ], -) - -jvm_binary( - name = "bin", - basename = "timelineranker-server", - main = "com.twitter.timelineranker.server.Main", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - ":scala", - "3rdparty/jvm/org/slf4j:jcl-over-slf4j", # [1] - "3rdparty/jvm/org/slf4j:log4j-over-slf4j", # [1] - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", # [2] - "strato/src/main/scala/com/twitter/strato/logging/logback", # [2] - "timelineranker/server/src/main/resources", # [2] - "twitter-server/logback-classic/src/main/scala", #[2] - ], -) - -# [1] bridge other logging implementations to slf4j-api in addition to JUL -# https://docbird.twitter.biz/core_libraries_guide/logging/twitter_server.html -# without these, c.t.l.Logger become silent/null logger since no proper -# configuration can be found. This can be removed once there are no -# depdency from service to c.t.l.Logger -# -# [2] incur logback implementation diff --git a/timelineranker/server/src/main/scala/BUILD.docx b/timelineranker/server/src/main/scala/BUILD.docx new file mode 100644 index 000000000..cf2a874db Binary files /dev/null and b/timelineranker/server/src/main/scala/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/BUILD deleted file mode 100644 index 1adc23a3e..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "cortex-core/thrift/src/main/thrift:thrift-scala", - "cortex-tweet-annotate/service/src/main/thrift:thrift-scala", - "finagle/finagle-memcached/src/main/scala", - "mediaservices/commons/src/main/thrift:thrift-scala", - "servo/repo", - "servo/util/src/main/scala", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/ml/prediction_service:prediction_service-scala", - "timelines/src/main/scala/com/twitter/timelines/model/types", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "util/util-core:util-core-util", - "util/util-logging/src/main/scala", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/BUILD.docx new file mode 100644 index 000000000..bded76325 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/CortexTweetQueryServiceClient.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/CortexTweetQueryServiceClient.docx new file mode 100644 index 000000000..3c7c092b2 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/CortexTweetQueryServiceClient.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/CortexTweetQueryServiceClient.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/CortexTweetQueryServiceClient.scala deleted file mode 100644 index 60af835bb..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/CortexTweetQueryServiceClient.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.timelineranker.clients - -import com.twitter.cortex_core.thriftscala.ModelName -import com.twitter.cortex_tweet_annotate.thriftscala._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.mediaservices.commons.mediainformation.thriftscala.CalibrationLevel -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.timelines.util.stats.RequestStats -import com.twitter.timelines.util.stats.ScopedFactory -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.util.Future - -object CortexTweetQueryServiceClient { - private[this] val logger = Logger.get(getClass.getSimpleName) - - /** - * A tweet is considered safe if Cortex NSFA model gives it a score that is above the threshold. - * Both the score and the threshold are returned in a response from getTweetSignalByIds endpoint. - */ - private def getSafeTweet( - request: TweetSignalRequest, - response: ModelResponseResult - ): Option[TweetId] = { - val tweetId = request.tweetId - response match { - case ModelResponseResult(ModelResponseState.Success, Some(tid), Some(modelResponse), _) => - val prediction = modelResponse.predictions.flatMap(_.headOption) - val score = prediction.map(_.score.score) - val highRecallBucket = prediction.flatMap(_.calibrationBuckets).flatMap { buckets => - buckets.find(_.description.contains(CalibrationLevel.HighRecall)) - } - val threshold = highRecallBucket.map(_.threshold) - (score, threshold) match { - case (Some(s), Some(t)) if (s > t) => - Some(tid) - case (Some(s), Some(t)) => - logger.ifDebug( - s"Cortex NSFA score for tweet $tweetId is $s (threshold is $t), removing as unsafe." - ) - None - case _ => - logger.ifDebug(s"Unexpected response, removing tweet $tweetId as unsafe.") - None - } - case _ => - logger.ifWarning( - s"Cortex tweet NSFA call was not successful, removing tweet $tweetId as unsafe." - ) - None - } - } -} - -/** - * Enables calling cortex tweet query service to get NSFA scores on the tweet. - */ -class CortexTweetQueryServiceClient( - cortexClient: CortexTweetQueryService.MethodPerEndpoint, - requestScope: RequestScope, - statsReceiver: StatsReceiver) - extends RequestStats { - import CortexTweetQueryServiceClient._ - - private[this] val logger = Logger.get(getClass.getSimpleName) - - private[this] val getTweetSignalByIdsRequestStats = - requestScope.stats("cortex", statsReceiver, suffix = Some("getTweetSignalByIds")) - private[this] val getTweetSignalByIdsRequestScopedStatsReceiver = - getTweetSignalByIdsRequestStats.scopedStatsReceiver - - private[this] val failedCortexTweetQueryServiceScope = - getTweetSignalByIdsRequestScopedStatsReceiver.scope(Failures) - private[this] val failedCortexTweetQueryServiceCallCounter = - failedCortexTweetQueryServiceScope.counter("failOpen") - - private[this] val cortexTweetQueryServiceFailOpenHandler = new FailOpenHandler( - getTweetSignalByIdsRequestScopedStatsReceiver - ) - - def getSafeTweets(tweetIds: Seq[TweetId]): Future[Seq[TweetId]] = { - val requests = tweetIds.map { id => TweetSignalRequest(id, ModelName.TweetToNsfa) } - val results = cortexClient - .getTweetSignalByIds( - GetTweetSignalByIdsRequest(requests) - ) - .map(_.results) - - cortexTweetQueryServiceFailOpenHandler( - results.map { responses => - requests.zip(responses).flatMap { - case (request, response) => - getSafeTweet(request, response) - } - } - ) { _ => - failedCortexTweetQueryServiceCallCounter.incr() - logger.ifWarning(s"Cortex tweet NSFA call failed, considering tweets $tweetIds as unsafe.") - Future.value(Seq()) - } - } -} - -class ScopedCortexTweetQueryServiceClientFactory( - cortexClient: CortexTweetQueryService.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends ScopedFactory[CortexTweetQueryServiceClient] { - - override def scope(scope: RequestScope): CortexTweetQueryServiceClient = { - new CortexTweetQueryServiceClient(cortexClient, scope, statsReceiver) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/MemcacheFactory.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/MemcacheFactory.docx new file mode 100644 index 000000000..e99628cf7 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/MemcacheFactory.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/MemcacheFactory.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/MemcacheFactory.scala deleted file mode 100644 index e69f93ee0..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/MemcacheFactory.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.timelineranker.clients - -import com.twitter.finagle.memcached.{Client => FinagleMemcacheClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.servo.cache.FinagleMemcache -import com.twitter.servo.cache.MemcacheCache -import com.twitter.servo.cache.ObservableMemcache -import com.twitter.servo.cache.Serializer -import com.twitter.servo.cache.StatsReceiverCacheObserver -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.timelines.util.stats.ScopedFactory -import com.twitter.util.Duration - -/** - * Factory to create a servo Memcache-backed Cache object. Clients are required to provide a - * serializer/deserializer for keys and values. - */ -class MemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver) { - private[this] val logger = Logger.get(getClass.getSimpleName) - - def apply[K, V]( - keySerializer: K => String, - valueSerializer: Serializer[V], - ttl: Duration - ): MemcacheCache[K, V] = { - new MemcacheCache[K, V]( - memcache = new ObservableMemcache( - new FinagleMemcache(memcacheClient), - new StatsReceiverCacheObserver(statsReceiver, 1000, logger) - ), - ttl = ttl, - serializer = valueSerializer, - transformKey = keySerializer - ) - } -} - -class ScopedMemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver) - extends ScopedFactory[MemcacheFactory] { - - override def scope(scope: RequestScope): MemcacheFactory = { - new MemcacheFactory( - memcacheClient, - statsReceiver.scope("memcache", scope.scope) - ) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/BUILD deleted file mode 100644 index 32885b372..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/algebird:bijection", - "3rdparty/jvm/com/twitter/bijection:core", - "3rdparty/jvm/com/twitter/bijection:netty", - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/bijection:thrift", - "3rdparty/jvm/com/twitter/bijection:util", - "3rdparty/jvm/com/twitter/storehaus:core", - "finagle/finagle-stats", - "scrooge/scrooge-core/src/main/scala", - "src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits", - "src/thrift/com/twitter/timelines/content_features:thrift-scala", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelines/src/main/scala/com/twitter/timelines/clients/memcache_common", - "timelines/src/main/scala/com/twitter/timelines/model/types", - "util/util-core:util-core-util", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/BUILD.docx new file mode 100644 index 000000000..5be6ebbaa Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/ContentFeaturesMemcacheBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/ContentFeaturesMemcacheBuilder.docx new file mode 100644 index 000000000..cf76e2cbb Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/ContentFeaturesMemcacheBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/ContentFeaturesMemcacheBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/ContentFeaturesMemcacheBuilder.scala deleted file mode 100644 index fe60972a4..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/ContentFeaturesMemcacheBuilder.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.timelineranker.clients.content_features_cache - -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.Store -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelines.clients.memcache_common._ -import com.twitter.timelines.content_features.{thriftscala => thrift} -import com.twitter.timelines.model.TweetId -import com.twitter.util.Duration - -/** - * Content features will be stored by tweetId - */ -class ContentFeaturesMemcacheBuilder( - config: StorehausMemcacheConfig, - ttl: Duration, - statsReceiver: StatsReceiver) { - private[this] val scalaToThriftInjection: Injection[ContentFeatures, thrift.ContentFeatures] = - Injection.build[ContentFeatures, thrift.ContentFeatures](_.toThrift)( - ContentFeatures.tryFromThrift) - - private[this] val thriftToBytesInjection: Injection[thrift.ContentFeatures, Array[Byte]] = - CompactScalaCodec(thrift.ContentFeatures) - - private[this] implicit val valueInjection: Injection[ContentFeatures, Array[Byte]] = - scalaToThriftInjection.andThen(thriftToBytesInjection) - - private[this] val underlyingBuilder = - new MemcacheStoreBuilder[TweetId, ContentFeatures]( - config = config, - scopeName = "contentFeaturesCache", - statsReceiver = statsReceiver, - ttl = ttl - ) - - def build(): Store[TweetId, ContentFeatures] = underlyingBuilder.build() -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/BUILD deleted file mode 100644 index 30c15dbda..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "finagle/finagle-core/src/main", - "servo/util/src/main/scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/search/common:features-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/model/tweet", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/bounds", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelines/src/main/scala/com/twitter/timelines/visibility", - "timelines/src/main/scala/com/twitter/timelines/visibility/model", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-logging/src/main/scala/com/twitter/logging", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/BUILD.docx new file mode 100644 index 000000000..7cef4c8fe Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CandidateGenerationTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CandidateGenerationTransform.docx new file mode 100644 index 000000000..0d1933b85 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CandidateGenerationTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CandidateGenerationTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CandidateGenerationTransform.scala deleted file mode 100644 index 3fed2a7cc..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CandidateGenerationTransform.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.CandidateTweet -import com.twitter.timelineranker.model.CandidateTweetsResult -import com.twitter.util.Future - -class CandidateGenerationTransform(statsReceiver: StatsReceiver) - extends FutureArrow[HydratedCandidatesAndFeaturesEnvelope, CandidateTweetsResult] { - private[this] val numCandidateTweetsStat = statsReceiver.stat("numCandidateTweets") - private[this] val numSourceTweetsStat = statsReceiver.stat("numSourceTweets") - - override def apply( - candidatesAndFeaturesEnvelope: HydratedCandidatesAndFeaturesEnvelope - ): Future[CandidateTweetsResult] = { - val hydratedTweets = candidatesAndFeaturesEnvelope.candidateEnvelope.hydratedTweets.outerTweets - - if (hydratedTweets.nonEmpty) { - val candidates = hydratedTweets.map { hydratedTweet => - CandidateTweet(hydratedTweet, candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId)) - } - numCandidateTweetsStat.add(candidates.size) - - val sourceTweets = - candidatesAndFeaturesEnvelope.candidateEnvelope.sourceHydratedTweets.outerTweets.map { - hydratedTweet => - CandidateTweet( - hydratedTweet, - candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId)) - } - numSourceTweetsStat.add(sourceTweets.size) - - Future.value(CandidateTweetsResult(candidates, sourceTweets)) - } else { - Future.value(CandidateTweetsResult.Empty) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/ContentFeaturesHydrationTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/ContentFeaturesHydrationTransform.docx new file mode 100644 index 000000000..bfd41e724 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/ContentFeaturesHydrationTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/ContentFeaturesHydrationTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/ContentFeaturesHydrationTransform.scala deleted file mode 100644 index 9c548cace..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/ContentFeaturesHydrationTransform.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.servo.util.Gate -import com.twitter.storehaus.Store -import com.twitter.timelineranker.contentfeatures.ContentFeaturesProvider -import com.twitter.timelineranker.core.FutureDependencyTransformer -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.util.SearchResultUtil._ -import com.twitter.timelineranker.util.CachingContentFeaturesProvider -import com.twitter.timelineranker.util.TweetHydrator -import com.twitter.timelineranker.util.TweetypieContentFeaturesProvider -import com.twitter.timelines.clients.tweetypie.TweetyPieClient -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future -import com.twitter.timelines.configapi -import com.twitter.timelines.util.FutureUtils - -class ContentFeaturesHydrationTransformBuilder( - tweetyPieClient: TweetyPieClient, - contentFeaturesCache: Store[TweetId, ContentFeatures], - enableContentFeaturesGate: Gate[RecapQuery], - enableTokensInContentFeaturesGate: Gate[RecapQuery], - enableTweetTextInContentFeaturesGate: Gate[RecapQuery], - enableConversationControlContentFeaturesGate: Gate[RecapQuery], - enableTweetMediaHydrationGate: Gate[RecapQuery], - hydrateInReplyToTweets: Boolean, - statsReceiver: StatsReceiver) { - val scopedStatsReceiver: StatsReceiver = statsReceiver.scope("ContentFeaturesHydrationTransform") - val tweetHydrator: TweetHydrator = new TweetHydrator(tweetyPieClient, scopedStatsReceiver) - val tweetypieContentFeaturesProvider: ContentFeaturesProvider = - new TweetypieContentFeaturesProvider( - tweetHydrator, - enableContentFeaturesGate, - enableTokensInContentFeaturesGate, - enableTweetTextInContentFeaturesGate, - enableConversationControlContentFeaturesGate, - enableTweetMediaHydrationGate, - scopedStatsReceiver - ) - - val cachingContentFeaturesProvider: ContentFeaturesProvider = new CachingContentFeaturesProvider( - underlying = tweetypieContentFeaturesProvider, - contentFeaturesCache = contentFeaturesCache, - statsReceiver = scopedStatsReceiver - ) - - val contentFeaturesProvider: configapi.FutureDependencyTransformer[RecapQuery, Seq[TweetId], Map[ - TweetId, - ContentFeatures - ]] = FutureDependencyTransformer.partition( - gate = enableContentFeaturesGate, - ifTrue = cachingContentFeaturesProvider, - ifFalse = tweetypieContentFeaturesProvider - ) - - lazy val contentFeaturesHydrationTransform: ContentFeaturesHydrationTransform = - new ContentFeaturesHydrationTransform( - contentFeaturesProvider, - enableContentFeaturesGate, - hydrateInReplyToTweets - ) - def build(): ContentFeaturesHydrationTransform = contentFeaturesHydrationTransform -} - -class ContentFeaturesHydrationTransform( - contentFeaturesProvider: ContentFeaturesProvider, - enableContentFeaturesGate: Gate[RecapQuery], - hydrateInReplyToTweets: Boolean) - extends FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ] { - override def apply( - request: HydratedCandidatesAndFeaturesEnvelope - ): Future[HydratedCandidatesAndFeaturesEnvelope] = { - if (enableContentFeaturesGate(request.candidateEnvelope.query)) { - val searchResults = request.candidateEnvelope.searchResults - - val sourceTweetIdMap = searchResults.map { searchResult => - (searchResult.id, getRetweetSourceTweetId(searchResult).getOrElse(searchResult.id)) - }.toMap - - val inReplyToTweetIds = if (hydrateInReplyToTweets) { - searchResults.flatMap(getInReplyToTweetId) - } else { - Seq.empty - } - - val tweetIdsToHydrate = (sourceTweetIdMap.values ++ inReplyToTweetIds).toSeq.distinct - - val contentFeaturesMapFuture = if (tweetIdsToHydrate.nonEmpty) { - contentFeaturesProvider(request.candidateEnvelope.query, tweetIdsToHydrate) - } else { - FutureUtils.EmptyMap[TweetId, ContentFeatures] - } - - Future.value( - request.copy( - contentFeaturesFuture = contentFeaturesMapFuture, - tweetSourceTweetMap = sourceTweetIdMap, - inReplyToTweetIds = inReplyToTweetIds.toSet - ) - ) - } else { - Future.value(request) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CreateCandidateEnvelopeTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CreateCandidateEnvelopeTransform.docx new file mode 100644 index 000000000..4b18487c0 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CreateCandidateEnvelopeTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CreateCandidateEnvelopeTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CreateCandidateEnvelopeTransform.scala deleted file mode 100644 index 7f21cd246..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CreateCandidateEnvelopeTransform.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.util.Future - -/** - * Create a CandidateEnvelope based on the incoming RecapQuery - */ -object CreateCandidateEnvelopeTransform extends FutureArrow[RecapQuery, CandidateEnvelope] { - override def apply(query: RecapQuery): Future[CandidateEnvelope] = { - Future.value(CandidateEnvelope(query)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FeatureHydrationDataTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FeatureHydrationDataTransform.docx new file mode 100644 index 000000000..646f0c28c Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FeatureHydrationDataTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FeatureHydrationDataTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FeatureHydrationDataTransform.scala deleted file mode 100644 index 6cc230137..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FeatureHydrationDataTransform.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.util.Future - -/** - * Fetches all data required for feature hydration and generates the HydratedCandidatesAndFeaturesEnvelope - * @param tweetHydrationAndFilteringPipeline Pipeline which fetches the candidate tweets, hydrates and filters them - * @param languagesService Fetch user languages, required for feature hydration - * @param userProfileInfoService Fetch user profile info, required for feature hydration - */ -class FeatureHydrationDataTransform( - tweetHydrationAndFilteringPipeline: FutureArrow[RecapQuery, CandidateEnvelope], - languagesService: UserLanguagesTransform, - userProfileInfoService: UserProfileInfoTransform) - extends FutureArrow[RecapQuery, HydratedCandidatesAndFeaturesEnvelope] { - override def apply(request: RecapQuery): Future[HydratedCandidatesAndFeaturesEnvelope] = { - Future - .join( - languagesService(request), - userProfileInfoService(request), - tweetHydrationAndFilteringPipeline(request)).map { - case (languages, userProfileInfo, transformedCandidateEnvelope) => - HydratedCandidatesAndFeaturesEnvelope( - transformedCandidateEnvelope, - languages, - userProfileInfo) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowAndRealGraphCombiningTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowAndRealGraphCombiningTransform.docx new file mode 100644 index 000000000..de92867d4 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowAndRealGraphCombiningTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowAndRealGraphCombiningTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowAndRealGraphCombiningTransform.scala deleted file mode 100644 index 3d98a8dae..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowAndRealGraphCombiningTransform.scala +++ /dev/null @@ -1,198 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.parameters.recap.RecapQueryContext -import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams.RecycledMaxFollowedUsersEnableAntiDilutionParam -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.earlybird.common.options.AuthorScoreAdjustments -import com.twitter.util.Future - -/** - * Transform which conditionally augments follow graph data using the real graph, - * derived from the earlybirdOptions passed in the query - * - * @param followGraphDataProvider data provider to be used for fetching updated mutual follow info - * @param maxFollowedUsersProvider max number of users to return - * @param enableRealGraphUsersProvider should we augment using real graph data? - * @param maxRealGraphAndFollowedUsersProvider max combined users to return, overrides maxFollowedUsersProvider above - * @param statsReceiver scoped stats received - */ -class FollowAndRealGraphCombiningTransform( - followGraphDataProvider: FollowGraphDataProvider, - maxFollowedUsersProvider: DependencyProvider[Int], - enableRealGraphUsersProvider: DependencyProvider[Boolean], - maxRealGraphAndFollowedUsersProvider: DependencyProvider[Int], - imputeRealGraphAuthorWeightsProvider: DependencyProvider[Boolean], - imputeRealGraphAuthorWeightsPercentileProvider: DependencyProvider[Int], - statsReceiver: StatsReceiver) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - // Number of authors in the seedset after mixing followed users and real graph users - // Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is true and onlyRealGraphUsers is false - val numFollowAndRealGraphUsersStat: Stat = statsReceiver.stat("numFollowAndRealGraphUsers") - val numFollowAndRealGraphUsersFromFollowGraphStat = - statsReceiver.scope("numFollowAndRealGraphUsers").stat("FollowGraphUsers") - val numFollowAndRealGraphUsersFromRealGraphStat = - statsReceiver.scope("numFollowAndRealGraphUsers").stat("RealGraphUsers") - val numFollowAndRealGraphUsersFromRealGraphCounter = - statsReceiver.scope("numFollowAndRealGraphUsers").counter() - - // Number of authors in the seedset with only followed users - // Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is false - val numFollowedUsersStat: Stat = statsReceiver.stat("numFollowedUsers") - - // Number of authors in the seedset with only followed users - // Only have this stat if user follows < maxFollowedUsers - val numFollowedUsersLessThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersLessThanMax") - val numFollowedUsersLessThanMaxCounter = - statsReceiver.scope("numFollowedUsersLessThanMax").counter() - val numFollowedUsersMoreThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersMoreThanMax") - val numFollowedUsersMoreThanMaxCounter = - statsReceiver.scope("numFollowedUsersMoreThanMax").counter() - - val realGraphAuthorWeightsSumProdStat: Stat = statsReceiver.stat("realGraphAuthorWeightsSumProd") - val realGraphAuthorWeightsSumMinExpStat: Stat = - statsReceiver.stat("realGraphAuthorWeightsSumMinExp") - val realGraphAuthorWeightsSumP50ExpStat: Stat = - statsReceiver.stat("realGraphAuthorWeightsSumP50Exp") - val realGraphAuthorWeightsSumP95ExpStat: Stat = - statsReceiver.stat("realGraphAuthorWeightsSumP95Exp") - val numAuthorsWithoutRealgraphScoreStat: Stat = - statsReceiver.stat("numAuthorsWithoutRealgraphScore") - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val realGraphData = envelope.query.earlybirdOptions - .map(_.authorScoreAdjustments.authorScoreMap) - .getOrElse(Map.empty) - - Future - .join( - envelope.followGraphData.followedUserIdsFuture, - envelope.followGraphData.mutedUserIdsFuture - ).map { - case (followedUserIds, mutedUserIds) => - // Anti-dilution param for DDG-16198 - val recycledMaxFollowedUsersEnableAntiDilutionParamProvider = - DependencyProvider.from(RecycledMaxFollowedUsersEnableAntiDilutionParam) - - val maxFollowedUsers = { - if (followedUserIds.size > RecapQueryContext.MaxFollowedUsers.default && recycledMaxFollowedUsersEnableAntiDilutionParamProvider( - envelope.query)) { - // trigger experiment - maxFollowedUsersProvider(envelope.query) - } else { - maxFollowedUsersProvider(envelope.query) - } - } - - val filteredRealGraphUserIds = realGraphData.keySet - .filterNot(mutedUserIds) - .take(maxFollowedUsers) - .toSeq - - val filteredFollowedUserIds = followedUserIds.filterNot(mutedUserIds) - - if (followedUserIds.size < maxFollowedUsers) { - numFollowedUsersLessThanMaxStat.add(filteredFollowedUserIds.size) - // stats - numFollowedUsersLessThanMaxCounter.incr() - (filteredFollowedUserIds, false) - } else { - numFollowedUsersMoreThanMaxStat.add(filteredFollowedUserIds.size) - numFollowedUsersMoreThanMaxCounter.incr() - if (enableRealGraphUsersProvider(envelope.query)) { - val maxRealGraphAndFollowedUsersNum = - maxRealGraphAndFollowedUsersProvider(envelope.query) - require( - maxRealGraphAndFollowedUsersNum >= maxFollowedUsers, - "maxRealGraphAndFollowedUsers must be greater than or equal to maxFollowedUsers." - ) - val recentFollowedUsersNum = RecapQueryContext.MaxFollowedUsers.bounds - .apply(maxRealGraphAndFollowedUsersNum - filteredRealGraphUserIds.size) - - val recentFollowedUsers = - filteredFollowedUserIds - .filterNot(filteredRealGraphUserIds.contains) - .take(recentFollowedUsersNum) - - val filteredFollowAndRealGraphUserIds = - recentFollowedUsers ++ filteredRealGraphUserIds - - // Track the size of recentFollowedUsers from SGS - numFollowAndRealGraphUsersFromFollowGraphStat.add(recentFollowedUsers.size) - // Track the size of filteredRealGraphUserIds from real graph dataset. - numFollowAndRealGraphUsersFromRealGraphStat.add(filteredRealGraphUserIds.size) - - numFollowAndRealGraphUsersFromRealGraphCounter.incr() - - numFollowAndRealGraphUsersStat.add(filteredFollowAndRealGraphUserIds.size) - - (filteredFollowAndRealGraphUserIds, true) - } else { - numFollowedUsersStat.add(followedUserIds.size) - (filteredFollowedUserIds, false) - } - } - }.map { - case (updatedFollowSeq, shouldUpdateMutualFollows) => - val updatedMutualFollowing = if (shouldUpdateMutualFollows) { - followGraphDataProvider.getMutuallyFollowingUserIds( - envelope.query.userId, - updatedFollowSeq) - } else { - envelope.followGraphData.mutuallyFollowingUserIdsFuture - } - - val followGraphData = envelope.followGraphData.copy( - followedUserIdsFuture = Future.value(updatedFollowSeq), - mutuallyFollowingUserIdsFuture = updatedMutualFollowing - ) - - val authorIdsWithRealgraphScore = realGraphData.keySet - val authorIdsWithoutRealgraphScores = - updatedFollowSeq.filterNot(authorIdsWithRealgraphScore.contains) - - //stat for logging the percentage of users' followings that do not have a realgraph score - if (updatedFollowSeq.nonEmpty) - numAuthorsWithoutRealgraphScoreStat.add( - authorIdsWithoutRealgraphScores.size / updatedFollowSeq.size * 100) - - if (imputeRealGraphAuthorWeightsProvider(envelope.query) && realGraphData.nonEmpty) { - val imputedScorePercentile = - imputeRealGraphAuthorWeightsPercentileProvider(envelope.query) / 100.0 - val existingAuthorIdScores = realGraphData.values.toList.sorted - val imputedScoreIndex = Math.min( - existingAuthorIdScores.length - 1, - (existingAuthorIdScores.length * imputedScorePercentile).toInt) - val imputedScore = existingAuthorIdScores(imputedScoreIndex) - - val updatedAuthorScoreMap = realGraphData ++ authorIdsWithoutRealgraphScores - .map(_ -> imputedScore).toMap - imputedScorePercentile match { - case 0.0 => - realGraphAuthorWeightsSumMinExpStat.add(updatedAuthorScoreMap.values.sum.toFloat) - case 0.5 => - realGraphAuthorWeightsSumP50ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat) - case 0.95 => - realGraphAuthorWeightsSumP95ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat) - case _ => - } - val earlybirdOptionsWithUpdatedAuthorScoreMap = envelope.query.earlybirdOptions - .map(_.copy(authorScoreAdjustments = AuthorScoreAdjustments(updatedAuthorScoreMap))) - val updatedQuery = - envelope.query.copy(earlybirdOptions = earlybirdOptionsWithUpdatedAuthorScoreMap) - envelope.copy(query = updatedQuery, followGraphData = followGraphData) - } else { - envelope.query.earlybirdOptions - .map(_.authorScoreAdjustments.authorScoreMap.values.sum.toFloat).foreach { - realGraphAuthorWeightsSumProdStat.add(_) - } - envelope.copy(followGraphData = followGraphData) - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowGraphDataTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowGraphDataTransform.docx new file mode 100644 index 000000000..02d737d8d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowGraphDataTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowGraphDataTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowGraphDataTransform.scala deleted file mode 100644 index edbc7b6e0..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowGraphDataTransform.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.util.Future - -class FollowGraphDataTransform( - followGraphDataProvider: FollowGraphDataProvider, - maxFollowedUsersProvider: DependencyProvider[Int]) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - - val followGraphData = followGraphDataProvider.getAsync( - envelope.query.userId, - maxFollowedUsersProvider(envelope.query) - ) - - Future.value(envelope.copy(followGraphData = followGraphData)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydrateTweetsAndSourceTweetsInParallelTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydrateTweetsAndSourceTweetsInParallelTransform.docx new file mode 100644 index 000000000..8b6e9fbc4 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydrateTweetsAndSourceTweetsInParallelTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydrateTweetsAndSourceTweetsInParallelTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydrateTweetsAndSourceTweetsInParallelTransform.scala deleted file mode 100644 index e018acf38..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydrateTweetsAndSourceTweetsInParallelTransform.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.util.Future - -/** - * Transform that explicitly hydrates candidate tweets and fetches source tweets in parallel - * and then joins the results back into the original Envelope - * @param candidateTweetHydration Pipeline that hydrates candidate tweets - * @param sourceTweetHydration Pipeline that fetches and hydrates source tweets - */ -class HydrateTweetsAndSourceTweetsInParallelTransform( - candidateTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope], - sourceTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope]) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - Future - .join( - candidateTweetHydration(envelope), - sourceTweetHydration(envelope) - ).map { - case (candidateTweetEnvelope, sourceTweetEnvelope) => - envelope.copy( - hydratedTweets = candidateTweetEnvelope.hydratedTweets, - sourceSearchResults = sourceTweetEnvelope.sourceSearchResults, - sourceHydratedTweets = sourceTweetEnvelope.sourceHydratedTweets - ) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydratedTweetsFilterTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydratedTweetsFilterTransform.docx new file mode 100644 index 000000000..e03ee7694 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydratedTweetsFilterTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydratedTweetsFilterTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydratedTweetsFilterTransform.scala deleted file mode 100644 index 22c3ba161..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydratedTweetsFilterTransform.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.core.HydratedTweets -import com.twitter.timelineranker.util.TweetFilters -import com.twitter.timelineranker.util.TweetsPostFilter -import com.twitter.timelines.model.UserId -import com.twitter.util.Future - -object HydratedTweetsFilterTransform { - val EmptyFollowGraphDataTuple: (Seq[UserId], Seq[UserId], Set[UserId]) = - (Seq.empty[UserId], Seq.empty[UserId], Set.empty[UserId]) - val DefaultNumRetweetsAllowed = 1 - - // Number of duplicate retweets (including the first one) allowed. - // For example, - // If there are 7 retweets of a given tweet, the following value will cause 5 of them - // to be returned after filtering and the additional 2 will be filtered out. - val NumDuplicateRetweetsAllowed = 5 -} - -/** - * Transform which takes TweetFilters ValueSets for inner and outer tweets and uses - * TweetsPostFilter to filter down the HydratedTweets using the supplied filters - * - * @param useFollowGraphData - use follow graph for filtering; otherwise only does filtering - * independent of follow graph data - * @param useSourceTweets - only needed when filtering extended replies - * @param statsReceiver - scoped stats receiver - */ -class HydratedTweetsFilterTransform( - outerFilters: TweetFilters.ValueSet, - innerFilters: TweetFilters.ValueSet, - useFollowGraphData: Boolean, - useSourceTweets: Boolean, - statsReceiver: StatsReceiver, - numRetweetsAllowed: Int = HydratedTweetsFilterTransform.DefaultNumRetweetsAllowed) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - import HydratedTweetsFilterTransform._ - - val logger: Logger = Logger.get(getClass.getSimpleName) - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - if (outerFilters == TweetFilters.None) { - Future.value(envelope) - } else { - val tweetsPostOuterFilter = new TweetsPostFilter(outerFilters, logger, statsReceiver) - val tweetsPostInnerFilter = new TweetsPostFilter(innerFilters, logger, statsReceiver) - - val graphData = if (useFollowGraphData) { - Future.join( - envelope.followGraphData.followedUserIdsFuture, - envelope.followGraphData.inNetworkUserIdsFuture, - envelope.followGraphData.mutedUserIdsFuture - ) - } else { - Future.value(EmptyFollowGraphDataTuple) - } - - val sourceTweets = if (useSourceTweets) { - envelope.sourceHydratedTweets.outerTweets - } else { - Nil - } - - graphData.map { - case (followedUserIds, inNetworkUserIds, mutedUserIds) => - val outerTweets = tweetsPostOuterFilter( - userId = envelope.query.userId, - followedUserIds = followedUserIds, - inNetworkUserIds = inNetworkUserIds, - mutedUserIds = mutedUserIds, - tweets = envelope.hydratedTweets.outerTweets, - numRetweetsAllowed = numRetweetsAllowed, - sourceTweets = sourceTweets - ) - val innerTweets = tweetsPostInnerFilter( - userId = envelope.query.userId, - followedUserIds = followedUserIds, - inNetworkUserIds = inNetworkUserIds, - mutedUserIds = mutedUserIds, - // inner tweets refers to quoted tweets not source tweets, and special rulesets - // in birdherd handle visibility of viewer to inner tweet author for these tweets. - tweets = envelope.hydratedTweets.innerTweets, - numRetweetsAllowed = numRetweetsAllowed, - sourceTweets = sourceTweets - ) - - envelope.copy(hydratedTweets = HydratedTweets(outerTweets, innerTweets)) - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/InNetworkTweetsSearchFeaturesHydrationTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/InNetworkTweetsSearchFeaturesHydrationTransform.docx new file mode 100644 index 000000000..05a4f3754 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/InNetworkTweetsSearchFeaturesHydrationTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/InNetworkTweetsSearchFeaturesHydrationTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/InNetworkTweetsSearchFeaturesHydrationTransform.scala deleted file mode 100644 index 35d90e549..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/InNetworkTweetsSearchFeaturesHydrationTransform.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator -import com.twitter.util.Future - -object InNetworkTweetsSearchFeaturesHydrationTransform - extends FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ] { - override def apply( - request: HydratedCandidatesAndFeaturesEnvelope - ): Future[HydratedCandidatesAndFeaturesEnvelope] = { - Future - .join( - request.candidateEnvelope.followGraphData.followedUserIdsFuture, - request.candidateEnvelope.followGraphData.mutuallyFollowingUserIdsFuture - ).map { - case (followedIds, mutuallyFollowingIds) => - val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate( - searcherUserId = request.candidateEnvelope.query.userId, - searcherProfileInfo = request.userProfileInfo, - followedUserIds = followedIds, - mutuallyFollowingUserIds = mutuallyFollowingIds, - userLanguages = request.userLanguages, - uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode), - searchResults = request.candidateEnvelope.searchResults, - sourceTweetSearchResults = request.candidateEnvelope.sourceSearchResults, - tweets = request.candidateEnvelope.hydratedTweets.outerTweets, - sourceTweets = request.candidateEnvelope.sourceHydratedTweets.outerTweets - ) - - request.copy(features = featuresByTweetId) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/MarkRandomTweetTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/MarkRandomTweetTransform.docx new file mode 100644 index 000000000..b2a48cbc8 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/MarkRandomTweetTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/MarkRandomTweetTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/MarkRandomTweetTransform.scala deleted file mode 100644 index f2c31c36f..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/MarkRandomTweetTransform.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.CandidateTweet -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.util.Future -import com.twitter.util.Time -import scala.util.Random - -/** - * picks up one or more random tweets and sets its tweetFeatures.isRandomTweet field to true. - */ -class MarkRandomTweetTransform( - includeRandomTweetProvider: DependencyProvider[Boolean], - randomGenerator: Random = new Random(Time.now.inMilliseconds), - includeSingleRandomTweetProvider: DependencyProvider[Boolean], - probabilityRandomTweetProvider: DependencyProvider[Double]) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val includeRandomTweet = includeRandomTweetProvider(envelope.query) - val includeSingleRandomTweet = includeSingleRandomTweetProvider(envelope.query) - val probabilityRandomTweet = probabilityRandomTweetProvider(envelope.query) - val searchResults = envelope.searchResults - - if (!includeRandomTweet || searchResults.isEmpty) { // random tweet off - Future.value(envelope) - } else if (includeSingleRandomTweet) { // pick only one - val randomIdx = randomGenerator.nextInt(searchResults.size) - val randomTweet = searchResults(randomIdx) - val randomTweetWithFlag = randomTweet.copy( - tweetFeatures = randomTweet.tweetFeatures - .orElse(Some(CandidateTweet.DefaultFeatures)) - .map(_.copy(isRandomTweet = Some(true))) - ) - val updatedSearchResults = searchResults.updated(randomIdx, randomTweetWithFlag) - - Future.value(envelope.copy(searchResults = updatedSearchResults)) - } else { // pick tweets with perTweetProbability - val updatedSearchResults = searchResults.map { result => - if (randomGenerator.nextDouble() < probabilityRandomTweet) { - result.copy( - tweetFeatures = result.tweetFeatures - .orElse(Some(CandidateTweet.DefaultFeatures)) - .map(_.copy(isRandomTweet = Some(true)))) - - } else - result - } - - Future.value(envelope.copy(searchResults = updatedSearchResults)) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkRepliesToUserIdSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkRepliesToUserIdSearchResultsTransform.docx new file mode 100644 index 000000000..9016cb0ea Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkRepliesToUserIdSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkRepliesToUserIdSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkRepliesToUserIdSearchResultsTransform.scala deleted file mode 100644 index 9d88c2c7d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkRepliesToUserIdSearchResultsTransform.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.util.Future - -object OutOfNetworkRepliesToUserIdSearchResultsTransform { - val DefaultMaxTweetCount = 100 -} - -// Requests search results for out-of-network replies to a user Id -class OutOfNetworkRepliesToUserIdSearchResultsTransform( - searchClient: SearchClient, - statsReceiver: StatsReceiver, - logSearchDebugInfo: Boolean = true) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - private[this] val maxCountStat = statsReceiver.stat("maxCount") - private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch") - private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100") - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val maxCount = envelope.query.maxCount - .getOrElse(OutOfNetworkRepliesToUserIdSearchResultsTransform.DefaultMaxTweetCount) - maxCountStat.add(maxCount) - - envelope.followGraphData.followedUserIdsFuture - .flatMap { - case followedIds => - searchClient - .getOutOfNetworkRepliesToUserId( - userId = envelope.query.userId, - followedUserIds = followedIds.toSet, - maxCount = maxCount, - earlybirdOptions = envelope.query.earlybirdOptions, - logSearchDebugInfo - ).map { results => - numResultsFromSearchStat.add(results.size) - results.foreach { result => - val earlybirdScoreX100 = - result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100 - earlybirdScoreX100Stat.add(earlybirdScoreX100) - } - envelope.copy(searchResults = results) - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkTweetsSearchFeaturesHydrationTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkTweetsSearchFeaturesHydrationTransform.docx new file mode 100644 index 000000000..af4055c11 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkTweetsSearchFeaturesHydrationTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkTweetsSearchFeaturesHydrationTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkTweetsSearchFeaturesHydrationTransform.scala deleted file mode 100644 index 0344fe0d0..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkTweetsSearchFeaturesHydrationTransform.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator -import com.twitter.util.Future - -object OutOfNetworkTweetsSearchFeaturesHydrationTransform - extends FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ] { - override def apply( - request: HydratedCandidatesAndFeaturesEnvelope - ): Future[HydratedCandidatesAndFeaturesEnvelope] = { - val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate( - searcherUserId = request.candidateEnvelope.query.userId, - searcherProfileInfo = request.userProfileInfo, - followedUserIds = Seq.empty, - mutuallyFollowingUserIds = Set.empty, - userLanguages = request.userLanguages, - uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode), - searchResults = request.candidateEnvelope.searchResults, - sourceTweetSearchResults = Seq.empty, - tweets = request.candidateEnvelope.hydratedTweets.outerTweets, - sourceTweets = Seq.empty - ) - - Future.value(request.copy(features = featuresByTweetId)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapHydrationSearchResultsTransformBase.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapHydrationSearchResultsTransformBase.docx new file mode 100644 index 000000000..00285bd41 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapHydrationSearchResultsTransformBase.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapHydrationSearchResultsTransformBase.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapHydrationSearchResultsTransformBase.scala deleted file mode 100644 index bcd8ca1a6..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapHydrationSearchResultsTransformBase.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future - -trait RecapHydrationSearchResultsTransformBase - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - protected def statsReceiver: StatsReceiver - protected def searchClient: SearchClient - private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch") - - def tweetIdsToHydrate(envelope: CandidateEnvelope): Seq[TweetId] - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - searchClient - .getTweetsScoredForRecap( - envelope.query.userId, - tweetIdsToHydrate(envelope), - envelope.query.earlybirdOptions - ).map { results => - numResultsFromSearchStat.add(results.size) - envelope.copy(searchResults = results) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTransform.docx new file mode 100644 index 000000000..9a88b1a77 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTransform.scala deleted file mode 100644 index 613cfc56e..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTransform.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.model.TweetIdRange -import com.twitter.timelineranker.parameters.recap.RecapParams -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes -import com.twitter.util.Future - -/** - * Fetch recap/recycled search results using the search client - * and populate them into the CandidateEnvelope - */ -class RecapSearchResultsTransform( - searchClient: SearchClient, - maxCountProvider: DependencyProvider[Int], - returnAllResultsProvider: DependencyProvider[Boolean], - relevanceOptionsMaxHitsToProcessProvider: DependencyProvider[Int], - enableExcludeSourceTweetIdsProvider: DependencyProvider[Boolean], - enableSettingTweetTypesWithTweetKindOptionProvider: DependencyProvider[Boolean], - perRequestSearchClientIdProvider: DependencyProvider[Option[String]], - relevanceSearchProvider: DependencyProvider[Boolean] = - DependencyProvider.from(RecapParams.EnableRelevanceSearchParam), - statsReceiver: StatsReceiver, - logSearchDebugInfo: Boolean = true) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - private[this] val maxCountStat = statsReceiver.stat("maxCount") - private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch") - private[this] val excludedTweetIdsStat = statsReceiver.stat("excludedTweetIds") - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val maxCount = maxCountProvider(envelope.query) - maxCountStat.add(maxCount) - - val excludedTweetIdsOpt = envelope.query.excludedTweetIds - excludedTweetIdsOpt.foreach { excludedTweetIds => - excludedTweetIdsStat.add(excludedTweetIds.size) - } - - val tweetIdRange = envelope.query.range - .map(TweetIdRange.fromTimelineRange) - .getOrElse(TweetIdRange.default) - - val beforeTweetIdExclusive = tweetIdRange.toId - val afterTweetIdExclusive = tweetIdRange.fromId - - val returnAllResults = returnAllResultsProvider(envelope.query) - val relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcessProvider(envelope.query) - - Future - .join( - envelope.followGraphData.followedUserIdsFuture, - envelope.followGraphData.retweetsMutedUserIdsFuture - ).flatMap { - case (followedIds, retweetsMutedIds) => - val followedIdsIncludingSelf = followedIds.toSet + envelope.query.userId - - searchClient - .getUsersTweetsForRecap( - userId = envelope.query.userId, - followedUserIds = followedIdsIncludingSelf, - retweetsMutedUserIds = retweetsMutedIds, - maxCount = maxCount, - tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options), - searchOperator = envelope.query.searchOperator, - beforeTweetIdExclusive = beforeTweetIdExclusive, - afterTweetIdExclusive = afterTweetIdExclusive, - enableSettingTweetTypesWithTweetKindOption = - enableSettingTweetTypesWithTweetKindOptionProvider(envelope.query), - excludedTweetIds = excludedTweetIdsOpt, - earlybirdOptions = envelope.query.earlybirdOptions, - getOnlyProtectedTweets = false, - logSearchDebugInfo = logSearchDebugInfo, - returnAllResults = returnAllResults, - enableExcludeSourceTweetIdsQuery = - enableExcludeSourceTweetIdsProvider(envelope.query), - relevanceSearch = relevanceSearchProvider(envelope.query), - searchClientId = perRequestSearchClientIdProvider(envelope.query), - relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcess - ).map { results => - numResultsFromSearchStat.add(results.size) - envelope.copy(searchResults = results) - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTruncationTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTruncationTransform.docx new file mode 100644 index 000000000..cf9dfff5a Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTruncationTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTruncationTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTruncationTransform.scala deleted file mode 100644 index ce3649532..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTruncationTransform.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.util.SearchResultUtil -import com.twitter.util.Future - -/** - * Truncate the search results by score. Assumes that the search results are sorted in - * score-descending order unless extraSortBeforeTruncation is set to true. - * - * This transform has two main use cases: - * - * - when returnAllResults is set to true, earlybird returns (numResultsPerShard * number of shards) - * results. this transform is then used to further truncate the result, so that the size will be the - * same as when returnAllResults is set to false. - * - * - we retrieve extra number of results from earlybird, as specified in MaxCountMultiplierParam, - * so that we are left with sufficient number of candidates after hydration and filtering. - * this transform will be used to get rid of extra results we ended up not using. - */ -class RecapSearchResultsTruncationTransform( - extraSortBeforeTruncationGate: DependencyProvider[Boolean], - maxCountProvider: DependencyProvider[Int], - statsReceiver: StatsReceiver) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - private[this] val postTruncationSizeStat = statsReceiver.stat("postTruncationSize") - private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100") - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val sortBeforeTruncation = extraSortBeforeTruncationGate(envelope.query) - val maxCount = maxCountProvider(envelope.query) - val searchResults = envelope.searchResults - - // set aside results that are marked by isRandomTweet field - val (randomTweetSeq, searchResultsExcludingRandom) = searchResults.partition { result => - result.tweetFeatures.flatMap(_.isRandomTweet).getOrElse(false) - } - - // sort and truncate searchResults other than the random tweet - val maxCountExcludingRandom = Math.max(0, maxCount - randomTweetSeq.size) - - val truncatedResultsExcludingRandom = - if (sortBeforeTruncation || searchResultsExcludingRandom.size > maxCountExcludingRandom) { - val sorted = if (sortBeforeTruncation) { - searchResultsExcludingRandom.sortWith( - SearchResultUtil.getScore(_) > SearchResultUtil.getScore(_)) - } else searchResultsExcludingRandom - sorted.take(maxCountExcludingRandom) - } else searchResultsExcludingRandom - - // put back the random tweet set aside previously - val allTruncatedResults = truncatedResultsExcludingRandom ++ randomTweetSeq - - // stats - postTruncationSizeStat.add(allTruncatedResults.size) - allTruncatedResults.foreach { result => - val earlybirdScoreX100 = - result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100 - earlybirdScoreX100Stat.add(earlybirdScoreX100) - } - - Future.value(envelope.copy(searchResults = allTruncatedResults)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SearchResultDedupAndSortingTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SearchResultDedupAndSortingTransform.docx new file mode 100644 index 000000000..5e47ba757 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SearchResultDedupAndSortingTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SearchResultDedupAndSortingTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SearchResultDedupAndSortingTransform.scala deleted file mode 100644 index bf3aa6540..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SearchResultDedupAndSortingTransform.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future -import scala.collection.mutable - -/** - * Remove duplicate search results and order them reverse-chron. - */ -object SearchResultDedupAndSortingTransform - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val seenTweetIds = mutable.Set.empty[TweetId] - val dedupedResults = envelope.searchResults - .filter(result => seenTweetIds.add(result.id)) - .sortBy(_.id)(Ordering[TweetId].reverse) - - val transformedEnvelope = envelope.copy(searchResults = dedupedResults) - Future.value(transformedEnvelope) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SourceTweetsSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SourceTweetsSearchResultsTransform.docx new file mode 100644 index 000000000..c140959dc Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SourceTweetsSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SourceTweetsSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SourceTweetsSearchResultsTransform.scala deleted file mode 100644 index d37bf9653..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SourceTweetsSearchResultsTransform.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.util.SourceTweetsUtil -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.util.Future - -object SourceTweetsSearchResultsTransform { - val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult] - val EmptySearchResultsFuture: Future[Seq[ThriftSearchResult]] = Future.value(EmptySearchResults) -} - -/** - * Fetch source tweets for a given set of search results - * Collects ids of source tweets, including extended reply and reply source tweets if needed, - * fetches those tweets from search and populates them into the envelope - */ -class SourceTweetsSearchResultsTransform( - searchClient: SearchClient, - failOpenHandler: FailOpenHandler, - hydrateReplyRootTweetProvider: DependencyProvider[Boolean], - perRequestSourceSearchClientIdProvider: DependencyProvider[Option[String]], - statsReceiver: StatsReceiver) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - import SourceTweetsSearchResultsTransform._ - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - failOpenHandler { - envelope.followGraphData.followedUserIdsFuture.flatMap { followedUserIds => - // NOTE: tweetIds are pre-computed as a performance optimisation. - val searchResultsTweetIds = envelope.searchResults.map(_.id).toSet - val sourceTweetIds = SourceTweetsUtil.getSourceTweetIds( - searchResults = envelope.searchResults, - searchResultsTweetIds = searchResultsTweetIds, - followedUserIds = followedUserIds, - shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query), - statsReceiver = scopedStatsReceiver - ) - if (sourceTweetIds.isEmpty) { - EmptySearchResultsFuture - } else { - searchClient.getTweetsScoredForRecap( - userId = envelope.query.userId, - tweetIds = sourceTweetIds, - earlybirdOptions = envelope.query.earlybirdOptions, - logSearchDebugInfo = false, - searchClientId = perRequestSourceSearchClientIdProvider(envelope.query) - ) - } - } - } { _: Throwable => EmptySearchResultsFuture }.map { sourceSearchResults => - envelope.copy(sourceSearchResults = sourceSearchResults) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchHydratedTweetsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchHydratedTweetsTransform.docx new file mode 100644 index 000000000..c843c6a95 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchHydratedTweetsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchHydratedTweetsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchHydratedTweetsTransform.scala deleted file mode 100644 index e39b11926..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchHydratedTweetsTransform.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.util.Future - -/** - * trims searchResults to match with hydratedTweets - * (if we previously filtered out hydrated tweets, this transform filters the search result set - * down to match the hydrated tweets.) - */ -object TrimToMatchHydratedTweetsTransform - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val filteredSearchResults = - trimSearchResults(envelope.searchResults, envelope.hydratedTweets.outerTweets) - val filteredSourceSearchResults = - trimSearchResults(envelope.sourceSearchResults, envelope.sourceHydratedTweets.outerTweets) - - Future.value( - envelope.copy( - searchResults = filteredSearchResults, - sourceSearchResults = filteredSourceSearchResults - ) - ) - } - - private def trimSearchResults( - searchResults: Seq[ThriftSearchResult], - hydratedTweets: Seq[HydratedTweet] - ): Seq[ThriftSearchResult] = { - val filteredTweetIds = hydratedTweets.map(_.tweetId).toSet - searchResults.filter(result => filteredTweetIds.contains(result.id)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchSearchResultsTransform.docx new file mode 100644 index 000000000..814462818 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchSearchResultsTransform.scala deleted file mode 100644 index 473daf02f..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchSearchResultsTransform.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.util.SourceTweetsUtil -import com.twitter.util.Future - -/** - * trims elements of the envelope other than the searchResults - * (i.e. sourceSearchResults, hydratedTweets, sourceHydratedTweets) to match with searchResults. - */ -class TrimToMatchSearchResultsTransform( - hydrateReplyRootTweetProvider: DependencyProvider[Boolean], - statsReceiver: StatsReceiver) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val searchResults = envelope.searchResults - val searchResultsIds = searchResults.map(_.id).toSet - - // Trim rest of the seqs to match top search results. - val hydratedTweets = envelope.hydratedTweets.outerTweets - val topHydratedTweets = hydratedTweets.filter(ht => searchResultsIds.contains(ht.tweetId)) - - envelope.followGraphData.followedUserIdsFuture.map { followedUserIds => - val sourceTweetIdsOfTopResults = - SourceTweetsUtil - .getSourceTweetIds( - searchResults = searchResults, - searchResultsTweetIds = searchResultsIds, - followedUserIds = followedUserIds, - shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query), - statsReceiver = scopedStatsReceiver - ).toSet - val sourceTweetSearchResultsForTopN = - envelope.sourceSearchResults.filter(r => sourceTweetIdsOfTopResults.contains(r.id)) - val hydratedSourceTweetsForTopN = - envelope.sourceHydratedTweets.outerTweets.filter(ht => - sourceTweetIdsOfTopResults.contains(ht.tweetId)) - - val hydratedTweetsForEnvelope = envelope.hydratedTweets.copy(outerTweets = topHydratedTweets) - val hydratedSourceTweetsForEnvelope = - envelope.sourceHydratedTweets.copy(outerTweets = hydratedSourceTweetsForTopN) - - envelope.copy( - hydratedTweets = hydratedTweetsForEnvelope, - searchResults = searchResults, - sourceHydratedTweets = hydratedSourceTweetsForEnvelope, - sourceSearchResults = sourceTweetSearchResultsForTopN - ) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetHydrationTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetHydrationTransform.docx new file mode 100644 index 000000000..40a483d7c Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetHydrationTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetHydrationTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetHydrationTransform.scala deleted file mode 100644 index 11a60efb2..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetHydrationTransform.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.IndividualRequestTimeoutException -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.core.HydratedTweets -import com.twitter.timelineranker.model.PartiallyHydratedTweet -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.util.Future - -object TweetHydrationTransform { - val EmptyHydratedTweets: HydratedTweets = - HydratedTweets(Seq.empty[HydratedTweet], Seq.empty[HydratedTweet]) - val EmptyHydratedTweetsFuture: Future[HydratedTweets] = Future.value(EmptyHydratedTweets) -} - -object CandidateTweetHydrationTransform extends TweetHydrationTransform { - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - hydrate( - searchResults = envelope.searchResults, - envelope = envelope - ).map { tweets => envelope.copy(hydratedTweets = tweets) } - } -} - -object SourceTweetHydrationTransform extends TweetHydrationTransform { - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - hydrate( - searchResults = envelope.sourceSearchResults, - envelope = envelope - ).map { tweets => envelope.copy(sourceHydratedTweets = tweets) } - } -} - -// Static IRTE to indicate timeout in tweet hydrator. Placeholder timeout duration of 0 millis is used -// since we are only concerned with the source of the exception. -object TweetHydrationTimeoutException extends IndividualRequestTimeoutException(0.millis) { - serviceName = "tweetHydrator" -} - -/** - * Transform which hydrates tweets in the CandidateEnvelope - **/ -trait TweetHydrationTransform extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - import TweetHydrationTransform._ - - protected def hydrate( - searchResults: Seq[ThriftSearchResult], - envelope: CandidateEnvelope - ): Future[HydratedTweets] = { - if (searchResults.nonEmpty) { - Future.value( - HydratedTweets(searchResults.map(PartiallyHydratedTweet.fromSearchResult)) - ) - } else { - EmptyHydratedTweetsFuture - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetKindOptionHydratedTweetsFilterTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetKindOptionHydratedTweetsFilterTransform.docx new file mode 100644 index 000000000..27aabfe2d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetKindOptionHydratedTweetsFilterTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetKindOptionHydratedTweetsFilterTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetKindOptionHydratedTweetsFilterTransform.scala deleted file mode 100644 index 4d0145063..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetKindOptionHydratedTweetsFilterTransform.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.servo.util.Gate -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelineranker.parameters.recap.RecapParams -import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsParams -import com.twitter.timelineranker.util.TweetFilters -import com.twitter.timelines.common.model.TweetKindOption -import com.twitter.util.Future -import scala.collection.mutable - -object TweetKindOptionHydratedTweetsFilterTransform { - private[common] val enableExpandedExtendedRepliesGate: Gate[RecapQuery] = - RecapQuery.paramGate(RecapParams.EnableExpandedExtendedRepliesFilterParam) - - private[common] val excludeRecommendedRepliesToNonFollowedUsersGate: Gate[RecapQuery] = - RecapQuery.paramGate( - UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeRecommendedRepliesToNonFollowedUsersParam) -} - -/** - * Filter hydrated tweets dynamically based on TweetKindOptions in the query. - */ -class TweetKindOptionHydratedTweetsFilterTransform( - useFollowGraphData: Boolean, - useSourceTweets: Boolean, - statsReceiver: StatsReceiver) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - import TweetKindOptionHydratedTweetsFilterTransform._ - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val filters = convertToFilters(envelope) - - val filterTransform = if (filters == TweetFilters.ValueSet.empty) { - FutureArrow.identity[CandidateEnvelope] - } else { - new HydratedTweetsFilterTransform( - outerFilters = filters, - innerFilters = TweetFilters.None, - useFollowGraphData = useFollowGraphData, - useSourceTweets = useSourceTweets, - statsReceiver = statsReceiver, - numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed - ) - } - - filterTransform.apply(envelope) - } - - /** - * Converts the given query options to equivalent TweetFilter values. - * - * Note: - * -- The semantic of TweetKindOption is opposite of that of TweetFilters. - * TweetKindOption values are of the form IncludeX. That is, they result in X being added. - * TweetFilters values specify what to exclude. - * -- IncludeExtendedReplies requires IncludeReplies to be also specified to be effective. - */ - private[common] def convertToFilters(envelope: CandidateEnvelope): TweetFilters.ValueSet = { - val queryOptions = envelope.query.options - val filters = mutable.Set.empty[TweetFilters.Value] - if (queryOptions.contains(TweetKindOption.IncludeReplies)) { - if (excludeRecommendedRepliesToNonFollowedUsersGate( - envelope.query) && envelope.query.utegLikedByTweetsOptions.isDefined) { - filters += TweetFilters.RecommendedRepliesToNotFollowedUsers - } else if (queryOptions.contains(TweetKindOption.IncludeExtendedReplies)) { - if (enableExpandedExtendedRepliesGate(envelope.query)) { - filters += TweetFilters.NotValidExpandedExtendedReplies - } else { - filters += TweetFilters.NotQualifiedExtendedReplies - } - } else { - filters += TweetFilters.ExtendedReplies - } - } else { - filters += TweetFilters.Replies - } - if (!queryOptions.contains(TweetKindOption.IncludeRetweets)) { - filters += TweetFilters.Retweets - } - TweetFilters.ValueSet.empty ++ filters - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserLanguagesTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserLanguagesTransform.docx new file mode 100644 index 000000000..a77ea0ef3 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserLanguagesTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserLanguagesTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserLanguagesTransform.scala deleted file mode 100644 index 8c86a1a62..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserLanguagesTransform.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.search.common.constants.thriftscala.ThriftLanguage -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelines.clients.manhattan.LanguageUtils -import com.twitter.timelines.clients.manhattan.UserMetadataClient -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.util.Future -import com.twitter.service.metastore.gen.thriftscala.UserLanguages - -object UserLanguagesTransform { - val EmptyUserLanguagesFuture: Future[UserLanguages] = - Future.value(UserMetadataClient.EmptyUserLanguages) -} - -/** - * FutureArrow which fetches user languages - * It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets - */ -class UserLanguagesTransform(handler: FailOpenHandler, userMetadataClient: UserMetadataClient) - extends FutureArrow[RecapQuery, Seq[ThriftLanguage]] { - override def apply(request: RecapQuery): Future[Seq[ThriftLanguage]] = { - import UserLanguagesTransform._ - - handler { - userMetadataClient.getUserLanguages(request.userId) - } { _: Throwable => EmptyUserLanguagesFuture } - }.map(LanguageUtils.computeLanguages(_)) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserProfileInfoTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserProfileInfoTransform.docx new file mode 100644 index 000000000..5256825bd Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserProfileInfoTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserProfileInfoTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserProfileInfoTransform.scala deleted file mode 100644 index 70480efc8..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserProfileInfoTransform.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelines.clients.gizmoduck.GizmoduckClient -import com.twitter.timelines.clients.gizmoduck.UserProfileInfo -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.util.Future - -object UserProfileInfoTransform { - val EmptyUserProfileInfo: UserProfileInfo = UserProfileInfo(None, None, None, None) - val EmptyUserProfileInfoFuture: Future[UserProfileInfo] = Future.value(EmptyUserProfileInfo) -} - -/** - * FutureArrow which fetches user profile info - * It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets - */ -class UserProfileInfoTransform(handler: FailOpenHandler, gizmoduckClient: GizmoduckClient) - extends FutureArrow[RecapQuery, UserProfileInfo] { - import UserProfileInfoTransform._ - override def apply(request: RecapQuery): Future[UserProfileInfo] = { - handler { - gizmoduckClient.getProfileInfo(request.userId).map { profileInfoOpt => - profileInfoOpt.getOrElse(EmptyUserProfileInfo) - } - } { _: Throwable => EmptyUserProfileInfoFuture } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/VisibilityEnforcingTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/VisibilityEnforcingTransform.docx new file mode 100644 index 000000000..43beabbfe Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/VisibilityEnforcingTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/VisibilityEnforcingTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/VisibilityEnforcingTransform.scala deleted file mode 100644 index c1d57af98..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/common/VisibilityEnforcingTransform.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.timelineranker.common - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.core.HydratedTweets -import com.twitter.timelines.visibility.VisibilityEnforcer -import com.twitter.util.Future - -/** - * Transform which uses an instance of a VisiblityEnforcer to filter down HydratedTweets - */ -class VisibilityEnforcingTransform(visibilityEnforcer: VisibilityEnforcer) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - visibilityEnforcer.apply(Some(envelope.query.userId), envelope.hydratedTweets.outerTweets).map { - visibleTweets => - val innerTweets = envelope.hydratedTweets.innerTweets - envelope.copy( - hydratedTweets = HydratedTweets(outerTweets = visibleTweets, innerTweets = innerTweets)) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/BUILD deleted file mode 100644 index 9ab3aa2a2..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/BUILD +++ /dev/null @@ -1,65 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/org/apache/thrift:libthrift", - "abdecider", - "cortex-tweet-annotate/service/src/main/thrift:thrift-scala", - "decider", - "featureswitches/featureswitches-core/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle/finagle-core/src/main", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-stats", - "finagle/finagle-thrift/src/main/java", - "finagle/finagle-thrift/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "loglens/loglens-logging/src/main/scala", - "merlin/util/src/main/scala", - "servo/decider", - "servo/request/src/main/scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/manhattan:v1-scala", - "src/thrift/com/twitter/merlin:thrift-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelineservice:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "strato/src/main/scala/com/twitter/strato/client", - "timelineranker/server/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/clients", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelines:authorization", - "timelines:config", - "timelines:features", - "timelines:util", - "timelines:visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/memcache_common", - "timelines/src/main/scala/com/twitter/timelines/clients/socialgraph", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/realgraph", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/config/configapi", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/logging", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "twitter-server-internal", - "util/util-app", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-stats/src/main/scala", - ], - exports = [ - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/BUILD.docx new file mode 100644 index 000000000..3b95b9036 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/CallInfo.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/CallInfo.docx new file mode 100644 index 000000000..40216caa7 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/CallInfo.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/CallInfo.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/CallInfo.scala deleted file mode 100644 index 7653e4ac1..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/CallInfo.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.conversions.DurationOps._ -import com.twitter.util.Duration -import java.util.concurrent.TimeUnit - -/** - * Information about a single method call. - * - * The purpose of this class is to allow one to express a call graph and latency associated with each (sub)call. - * Once a call graph is defined, calling getOverAllLatency() off the top level call returns total time taken by that call. - * That value can then be compared with the timeout budget allocated to that call to see if the - * value fits within the overall timeout budget of that call. - * - * This is useful in case of a complex call graph where it is hard to mentally estimate the effect on - * overall latency when updating timeout value of one or more sub-calls. - * - * @param methodName name of the called method. - * @param latency P999 Latency incurred in calling a service if the method calls an external service. Otherwise 0. - * @param dependsOn Other calls that this call depends on. - */ -case class Call( - methodName: String, - latency: Duration = 0.milliseconds, - dependsOn: Seq[Call] = Nil) { - - /** - * Latency incurred in this call as well as recursively all calls this call depends on. - */ - def getOverAllLatency: Duration = { - val dependencyLatency = if (dependsOn.isEmpty) { - 0.milliseconds - } else { - dependsOn.map(_.getOverAllLatency).max - } - latency + dependencyLatency - } - - /** - * Call paths starting at this call and recursively traversing all dependencies. - * Typically used for debugging or logging. - */ - def getLatencyPaths: String = { - val sb = new StringBuilder - getLatencyPaths(sb, 1) - sb.toString - } - - def getLatencyPaths(sb: StringBuilder, level: Int): Unit = { - sb.append(s"${getPrefix(level)} ${getLatencyString(getOverAllLatency)} $methodName\n") - if ((latency > 0.milliseconds) && !dependsOn.isEmpty) { - sb.append(s"${getPrefix(level + 1)} ${getLatencyString(latency)} self\n") - } - dependsOn.foreach(_.getLatencyPaths(sb, level + 1)) - } - - private def getLatencyString(latencyValue: Duration): String = { - val latencyMs = latencyValue.inUnit(TimeUnit.MILLISECONDS) - f"[$latencyMs%3d]" - } - - private def getPrefix(level: Int): String = { - " " * (level * 4) + "--" - } -} - -/** - * Information about the getRecapTweetCandidates call. - * - * Acronyms used: - * : call internal to TLR - * EB : Earlybird (search super root) - * GZ : Gizmoduck - * MH : Manhattan - * SGS : Social graph service - * - * The latency values are based on p9999 values observed over 1 week. - */ -object GetRecycledTweetCandidatesCall { - val getUserProfileInfo: Call = Call("GZ.getUserProfileInfo", 200.milliseconds) - val getUserLanguages: Call = Call("MH.getUserLanguages", 300.milliseconds) // p99: 15ms - - val getFollowing: Call = Call("SGS.getFollowing", 250.milliseconds) // p99: 75ms - val getMutuallyFollowing: Call = - Call("SGS.getMutuallyFollowing", 400.milliseconds, Seq(getFollowing)) // p99: 100 - val getVisibilityProfiles: Call = - Call("SGS.getVisibilityProfiles", 400.milliseconds, Seq(getFollowing)) // p99: 100 - val getVisibilityData: Call = Call( - "getVisibilityData", - dependsOn = Seq(getFollowing, getMutuallyFollowing, getVisibilityProfiles) - ) - val getTweetsForRecapRegular: Call = - Call("EB.getTweetsForRecap(regular)", 500.milliseconds, Seq(getVisibilityData)) // p99: 250 - val getTweetsForRecapProtected: Call = - Call("EB.getTweetsForRecap(protected)", 250.milliseconds, Seq(getVisibilityData)) // p99: 150 - val getSearchResults: Call = - Call("getSearchResults", dependsOn = Seq(getTweetsForRecapRegular, getTweetsForRecapProtected)) - val getTweetsScoredForRecap: Call = - Call("EB.getTweetsScoredForRecap", 400.milliseconds, Seq(getSearchResults)) // p99: 100 - - val hydrateSearchResults: Call = Call("hydrateSearchResults") - val getSourceTweetSearchResults: Call = - Call("getSourceTweetSearchResults", dependsOn = Seq(getSearchResults)) - val hydrateTweets: Call = - Call("hydrateTweets", dependsOn = Seq(getSearchResults, hydrateSearchResults)) - val hydrateSourceTweets: Call = - Call("hydrateSourceTweets", dependsOn = Seq(getSourceTweetSearchResults, hydrateSearchResults)) - val topLevel: Call = Call( - "getRecapTweetCandidates", - dependsOn = Seq( - getUserProfileInfo, - getUserLanguages, - getVisibilityData, - getSearchResults, - hydrateSearchResults, - hydrateSourceTweets - ) - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientAccessPermissions.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientAccessPermissions.docx new file mode 100644 index 000000000..daad55ba5 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientAccessPermissions.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientAccessPermissions.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientAccessPermissions.scala deleted file mode 100644 index 4a9b717eb..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientAccessPermissions.scala +++ /dev/null @@ -1,287 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.timelineranker.decider.DeciderKey._ -import com.twitter.timelines.authorization.TrustedPermission -import com.twitter.timelines.authorization.RateLimitingTrustedPermission -import com.twitter.timelines.authorization.RateLimitingUntrustedPermission -import com.twitter.timelines.authorization.ClientDetails - -object ClientAccessPermissions { - // We want timelineranker locked down for requests outside of what's defined here. - val DefaultRateLimit = 0d - - def unknown(name: String): ClientDetails = { - ClientDetails(name, RateLimitingUntrustedPermission(RateLimitOverrideUnknown, DefaultRateLimit)) - } - - val All: Seq[ClientDetails] = Seq( - /** - * Production clients for timelinemixer. - */ - new ClientDetails( - "timelinemixer.recap.prod", - RateLimitingTrustedPermission(AllowTimelineMixerRecapProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.recycled.prod", - RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.hydrate.prod", - RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.hydrate_recos.prod", - RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.seed_author_ids.prod", - RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.simcluster.prod", - RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.entity_tweets.prod", - RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd), - protectedWriteAccess = TrustedPermission - ), - /** - * This client is whitelisted for timelinemixer only as it used by - * List injection service which will not be migrated to timelinescorer. - */ - new ClientDetails( - "timelinemixer.list.prod", - RateLimitingTrustedPermission(AllowTimelineMixerListProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.list_tweet.prod", - RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.community.prod", - RateLimitingTrustedPermission(AllowTimelineMixerCommunityProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.community_tweet.prod", - RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.uteg_liked_by_tweets.prod", - RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd), - protectedWriteAccess = TrustedPermission - ), - /** - * Production clients for timelinescorer. Most of these clients have their - * equivalents under the timelinemixer scope (with exception of list injection - * client). - */ - new ClientDetails( - "timelinescorer.recap.prod", - RateLimitingTrustedPermission(AllowTimelineMixerRecapProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.recycled.prod", - RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.hydrate.prod", - RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.hydrate_recos.prod", - RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.seed_author_ids.prod", - RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.simcluster.prod", - RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.entity_tweets.prod", - RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.list_tweet.prod", - RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.uteg_liked_by_tweets.prod", - RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelineservice.prod", - RateLimitingTrustedPermission(AllowTimelineServiceProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.hydrate_tweet_scoring.prod", - RateLimitingTrustedPermission(AllowTimelineScorerHydrateTweetScoringProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.community_tweet.prod", - RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.recommended_trend_tweet.prod", - RateLimitingTrustedPermission(AllowTimelineScorerRecommendedTrendTweetProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.rec_topic_tweets.prod", - RateLimitingTrustedPermission(AllowTimelineScorerRecTopicTweetsProd), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.popular_topic_tweets.prod", - RateLimitingTrustedPermission(AllowTimelineScorerPopularTopicTweetsProd), - protectedWriteAccess = TrustedPermission - ), - /** - * TimelineRanker utilities. Traffic proxy, warmups, and console. - */ - new ClientDetails( - "timelineranker.proxy", - RateLimitingTrustedPermission(AllowTimelineRankerProxy), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - TimelineRankerConstants.WarmupClientName, - RateLimitingTrustedPermission(AllowTimelineRankerWarmup), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - TimelineRankerConstants.ForwardedClientName, - RateLimitingTrustedPermission(AllowTimelineRankerWarmup), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelineranker.console", - RateLimitingUntrustedPermission(RateLimitOverrideUnknown, 1d), - protectedWriteAccess = TrustedPermission - ), - /** - * Staging clients. - */ - new ClientDetails( - "timelinemixer.recap.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.recycled.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.hydrate.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.hydrate_recos.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.seed_author_ids.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.simcluster.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.entity_tweets.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.list.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.list_tweet.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.community.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.community_tweet.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.community_tweet.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.recommended_trend_tweet.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.uteg_liked_by_tweets.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinemixer.entity_tweets.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.hydrate_tweet_scoring.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.rec_topic_tweets.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelinescorer.popular_topic_tweets.staging", - RateLimitingTrustedPermission(AllowTimelineMixerStaging), - protectedWriteAccess = TrustedPermission - ), - new ClientDetails( - "timelineservice.staging", - RateLimitingTrustedPermission(AllowTimelineServiceStaging), - protectedWriteAccess = TrustedPermission - ) - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrapperFactories.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrapperFactories.docx new file mode 100644 index 000000000..3644da2dd Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrapperFactories.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrapperFactories.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrapperFactories.scala deleted file mode 100644 index 048d22cc1..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrapperFactories.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.servo.util.Gate -import com.twitter.timelineranker.clients.ScopedCortexTweetQueryServiceClientFactory -import com.twitter.timelines.clients.gizmoduck.ScopedGizmoduckClientFactory -import com.twitter.timelines.clients.manhattan.ScopedUserMetadataClientFactory -import com.twitter.timelines.clients.socialgraph.ScopedSocialGraphClientFactory -import com.twitter.timelines.clients.strato.realgraph.ScopedRealGraphClientFactory -import com.twitter.timelines.clients.tweetypie.AdditionalFieldConfig -import com.twitter.timelines.clients.tweetypie.ScopedTweetyPieClientFactory -import com.twitter.timelines.visibility.VisibilityEnforcerFactory -import com.twitter.timelines.visibility.VisibilityProfileHydratorFactory -import com.twitter.tweetypie.thriftscala.{Tweet => TTweet} - -class ClientWrapperFactories(config: RuntimeConfiguration) { - private[this] val statsReceiver = config.statsReceiver - - val cortexTweetQueryServiceClientFactory: ScopedCortexTweetQueryServiceClientFactory = - new ScopedCortexTweetQueryServiceClientFactory( - config.underlyingClients.cortexTweetQueryServiceClient, - statsReceiver = statsReceiver - ) - - val gizmoduckClientFactory: ScopedGizmoduckClientFactory = new ScopedGizmoduckClientFactory( - config.underlyingClients.gizmoduckClient, - statsReceiver = statsReceiver - ) - - val socialGraphClientFactory: ScopedSocialGraphClientFactory = new ScopedSocialGraphClientFactory( - config.underlyingClients.sgsClient, - statsReceiver - ) - - val visibilityEnforcerFactory: VisibilityEnforcerFactory = new VisibilityEnforcerFactory( - gizmoduckClientFactory, - socialGraphClientFactory, - statsReceiver - ) - - val tweetyPieAdditionalFieldsToDisable: Seq[Short] = Seq( - TTweet.MediaTagsField.id, - TTweet.SchedulingInfoField.id, - TTweet.EscherbirdEntityAnnotationsField.id, - TTweet.CardReferenceField.id, - TTweet.SelfPermalinkField.id, - TTweet.ExtendedTweetMetadataField.id, - TTweet.CommunitiesField.id, - TTweet.VisibleTextRangeField.id - ) - - val tweetyPieHighQoSClientFactory: ScopedTweetyPieClientFactory = - new ScopedTweetyPieClientFactory( - tweetyPieClient = config.underlyingClients.tweetyPieHighQoSClient, - additionalFieldConfig = AdditionalFieldConfig( - fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap - ), - includePartialResults = Gate.False, - statsReceiver = statsReceiver - ) - - val tweetyPieLowQoSClientFactory: ScopedTweetyPieClientFactory = new ScopedTweetyPieClientFactory( - tweetyPieClient = config.underlyingClients.tweetyPieLowQoSClient, - additionalFieldConfig = AdditionalFieldConfig( - fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap - ), - includePartialResults = Gate.False, - statsReceiver = statsReceiver - ) - - val userMetadataClientFactory: ScopedUserMetadataClientFactory = - new ScopedUserMetadataClientFactory( - config.underlyingClients.manhattanStarbuckClient, - TimelineRankerConstants.ManhattanStarbuckAppId, - statsReceiver - ) - - val visibilityProfileHydratorFactory: VisibilityProfileHydratorFactory = - new VisibilityProfileHydratorFactory( - gizmoduckClientFactory, - socialGraphClientFactory, - statsReceiver - ) - - val realGraphClientFactory = - new ScopedRealGraphClientFactory(config.underlyingClients.stratoClient, statsReceiver) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrappers.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrappers.docx new file mode 100644 index 000000000..449f5edc7 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrappers.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrappers.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrappers.scala deleted file mode 100644 index 3a93a4074..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrappers.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.storehaus.Store -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelines.model.TweetId -class ClientWrappers(config: RuntimeConfiguration) { - private[this] val backendClientConfiguration = config.underlyingClients - - val contentFeaturesCache: Store[TweetId, ContentFeatures] = - backendClientConfiguration.contentFeaturesCache -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/DefaultUnderlyingClientConfiguration.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/DefaultUnderlyingClientConfiguration.docx new file mode 100644 index 000000000..c6fe68e95 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/DefaultUnderlyingClientConfiguration.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/DefaultUnderlyingClientConfiguration.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/DefaultUnderlyingClientConfiguration.scala deleted file mode 100644 index 537181f24..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/DefaultUnderlyingClientConfiguration.scala +++ /dev/null @@ -1,158 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.PercentOps._ -import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.util.DefaultTimer -import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck} -import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1} -import com.twitter.merlin.thriftscala.UserRolesService -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.storehaus.Store -import com.twitter.strato.client.Strato -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.timelineranker.clients.content_features_cache.ContentFeaturesMemcacheBuilder -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.thriftscala.TimelineRanker -import com.twitter.timelines.clients.memcache_common.StorehausMemcacheConfig -import com.twitter.timelines.model.TweetId -import com.twitter.timelineservice.thriftscala.TimelineService -import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie} -import com.twitter.util.Timer -import org.apache.thrift.protocol.TCompactProtocol - -class DefaultUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver) - extends UnderlyingClientConfiguration(flags, statsReceiver) { top => - - val timer: Timer = DefaultTimer - val twCachePrefix = "/srv#/prod/local/cache" - - override val cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint = { - methodPerEndpointClient[ - CortexTweetQueryService.ServicePerEndpoint, - CortexTweetQueryService.MethodPerEndpoint]( - thriftMuxClientBuilder("cortex-tweet-query", requireMutualTls = true) - .dest("/s/cortex-tweet-annotate/cortex-tweet-query") - .requestTimeout(200.milliseconds) - .timeout(500.milliseconds) - ) - } - - override val gizmoduckClient: Gizmoduck.MethodPerEndpoint = { - methodPerEndpointClient[Gizmoduck.ServicePerEndpoint, Gizmoduck.MethodPerEndpoint]( - thriftMuxClientBuilder("gizmoduck", requireMutualTls = true) - .dest("/s/gizmoduck/gizmoduck") - .requestTimeout(400.milliseconds) - .timeout(900.milliseconds) - ) - } - - override lazy val manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint = { - methodPerEndpointClient[ManhattanV1.ServicePerEndpoint, ManhattanV1.MethodPerEndpoint]( - thriftMuxClientBuilder("manhattan.starbuck", requireMutualTls = true) - .dest("/s/manhattan/starbuck.native-thrift") - .requestTimeout(600.millis) - ) - } - - override val sgsClient: SocialGraphService.MethodPerEndpoint = { - methodPerEndpointClient[ - SocialGraphService.ServicePerEndpoint, - SocialGraphService.MethodPerEndpoint]( - thriftMuxClientBuilder("socialgraph", requireMutualTls = true) - .dest("/s/socialgraph/socialgraph") - .requestTimeout(350.milliseconds) - .timeout(700.milliseconds) - ) - } - - override lazy val timelineRankerForwardingClient: TimelineRanker.FinagledClient = - new TimelineRanker.FinagledClient( - thriftMuxClientBuilder( - TimelineRankerConstants.ForwardedClientName, - ClientId(TimelineRankerConstants.ForwardedClientName), - protocolFactoryOption = Some(new TCompactProtocol.Factory()), - requireMutualTls = true - ).dest("/s/timelineranker/timelineranker:compactthrift").build(), - protocolFactory = new TCompactProtocol.Factory() - ) - - override val timelineServiceClient: TimelineService.MethodPerEndpoint = { - methodPerEndpointClient[TimelineService.ServicePerEndpoint, TimelineService.MethodPerEndpoint]( - thriftMuxClientBuilder("timelineservice", requireMutualTls = true) - .dest("/s/timelineservice/timelineservice") - .requestTimeout(600.milliseconds) - .timeout(800.milliseconds) - ) - } - - override val tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint = { - methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint]( - thriftMuxClientBuilder("tweetypieHighQoS", requireMutualTls = true) - .dest("/s/tweetypie/tweetypie") - .requestTimeout(450.milliseconds) - .timeout(800.milliseconds), - maxExtraLoadPercent = Some(1.percent) - ) - } - - /** - * Provide less costly TweetPie client with the trade-off of reduced quality. Intended for use cases - * which are not essential for successful completion of timeline requests. Currently this client differs - * from the highQoS endpoint by having retries count set to 1 instead of 2. - */ - override val tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint = { - methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint]( - thriftMuxClientBuilder("tweetypieLowQoS", requireMutualTls = true) - .dest("/s/tweetypie/tweetypie") - .retryPolicy(mkRetryPolicy(1)) // override default value - .requestTimeout(450.milliseconds) - .timeout(800.milliseconds), - maxExtraLoadPercent = Some(1.percent) - ) - } - - override val userRolesServiceClient: UserRolesService.MethodPerEndpoint = { - methodPerEndpointClient[ - UserRolesService.ServicePerEndpoint, - UserRolesService.MethodPerEndpoint]( - thriftMuxClientBuilder("merlin", requireMutualTls = true) - .dest("/s/merlin/merlin") - .requestTimeout(1.second) - ) - } - - lazy val contentFeaturesCache: Store[TweetId, ContentFeatures] = - new ContentFeaturesMemcacheBuilder( - config = new StorehausMemcacheConfig( - destName = s"$twCachePrefix/timelines_content_features:twemcaches", - keyPrefix = "", - requestTimeout = 50.milliseconds, - numTries = 1, - globalTimeout = 60.milliseconds, - tcpConnectTimeout = 50.milliseconds, - connectionAcquisitionTimeout = 25.milliseconds, - numPendingRequests = 100, - isReadOnly = false, - serviceIdentifier = serviceIdentifier - ), - ttl = 48.hours, - statsReceiver - ).build - - override val userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient = - new UserTweetEntityGraph.FinagledClient( - thriftMuxClientBuilder("user_tweet_entity_graph", requireMutualTls = true) - .dest("/s/cassowary/user_tweet_entity_graph") - .retryPolicy(mkRetryPolicy(2)) - .requestTimeout(300.milliseconds) - .build() - ) - - override val stratoClient: StratoClient = - Strato.client.withMutualTls(serviceIdentifier, OpportunisticTls.Required).build() -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RequestScopes.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RequestScopes.docx new file mode 100644 index 000000000..35699447c Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RequestScopes.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RequestScopes.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RequestScopes.scala deleted file mode 100644 index b2804c5a3..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RequestScopes.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.timelines.util.stats.RequestScope - -object RequestScopes { - val HomeTimelineMaterialization: RequestScope = RequestScope("homeMaterialization") - val InNetworkTweetSource: RequestScope = RequestScope("inNetworkTweet") - val RecapHydrationSource: RequestScope = RequestScope("recapHydration") - val RecapAuthorSource: RequestScope = RequestScope("recapAuthor") - val ReverseChronHomeTimelineSource: RequestScope = RequestScope("reverseChronHomeTimelineSource") - val EntityTweetsSource: RequestScope = RequestScope("entityTweets") - val UtegLikedByTweetsSource: RequestScope = RequestScope("utegLikedByTweets") -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RuntimeConfiguration.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RuntimeConfiguration.docx new file mode 100644 index 000000000..a39f62822 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RuntimeConfiguration.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RuntimeConfiguration.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RuntimeConfiguration.scala deleted file mode 100644 index ddc0c6467..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RuntimeConfiguration.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.abdecider.ABDeciderFactory -import com.twitter.abdecider.LoggingABDecider -import com.twitter.decider.Decider -import com.twitter.featureswitches.Value -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.util.Effect -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelines.authorization.TimelinesClientRequestAuthorizer -import com.twitter.timelines.config._ -import com.twitter.timelines.config.configapi._ -import com.twitter.timelines.features._ -import com.twitter.timelines.util.ImpressionCountingABDecider -import com.twitter.timelines.util.logging.Scribe -import com.twitter.util.Await -import com.twitter.servo.util.Gate -import com.twitter.timelines.model.UserId - -trait ClientProvider { - def clientWrappers: ClientWrappers - def underlyingClients: UnderlyingClientConfiguration -} - -trait UtilityProvider { - def abdecider: LoggingABDecider - def clientRequestAuthorizer: TimelinesClientRequestAuthorizer - def configStore: ConfigStore - def decider: Decider - def deciderGateBuilder: DeciderGateBuilder - def employeeGate: UserRolesGate.EmployeeGate - def configApiConfiguration: ConfigApiConfiguration - def statsReceiver: StatsReceiver - def whitelist: UserList -} - -trait RuntimeConfiguration extends ClientProvider with UtilityProvider with ConfigUtils { - def isProd: Boolean - def maxConcurrency: Int - def clientEventScribe: Effect[String] - def clientWrapperFactories: ClientWrapperFactories -} - -class RuntimeConfigurationImpl( - flags: TimelineRankerFlags, - configStoreFactory: DynamicConfigStoreFactory, - val decider: Decider, - val forcedFeatureValues: Map[String, Value] = Map.empty[String, Value], - val statsReceiver: StatsReceiver) - extends RuntimeConfiguration { - - // Creates and initialize config store as early as possible so other parts could have a dependency on it for settings. - override val configStore: DynamicConfigStore = - configStoreFactory.createDcEnvAwareFileBasedConfigStore( - relativeConfigFilePath = "timelines/timelineranker/service_settings.yml", - dc = flags.getDatacenter, - env = flags.getEnv, - configBusConfig = ConfigBusProdConfig, - onUpdate = ConfigStore.NullOnUpdateCallback, - statsReceiver = statsReceiver - ) - Await.result(configStore.init) - - val environment: Env.Value = flags.getEnv - override val isProd: Boolean = isProdEnv(environment) - val datacenter: Datacenter.Value = flags.getDatacenter - val abDeciderPath = "/usr/local/config/abdecider/abdecider.yml" - override val maxConcurrency: Int = flags.maxConcurrency() - - val deciderGateBuilder: DeciderGateBuilder = new DeciderGateBuilder(decider) - - val clientRequestAuthorizer: TimelinesClientRequestAuthorizer = - new TimelinesClientRequestAuthorizer( - deciderGateBuilder = deciderGateBuilder, - clientDetails = ClientAccessPermissions.All, - unknownClientDetails = ClientAccessPermissions.unknown, - clientAuthorizationGate = - deciderGateBuilder.linearGate(DeciderKey.ClientRequestAuthorization), - clientWriteWhitelistGate = deciderGateBuilder.linearGate(DeciderKey.ClientWriteWhitelist), - globalCapacityQPS = flags.requestRateLimit(), - statsReceiver = statsReceiver - ) - override val clientEventScribe = Scribe.clientEvent(isProd, statsReceiver) - val abdecider: LoggingABDecider = new ImpressionCountingABDecider( - abdecider = ABDeciderFactory.withScribeEffect( - abDeciderYmlPath = abDeciderPath, - scribeEffect = clientEventScribe, - decider = None, - environment = Some("production"), - ).buildWithLogging(), - statsReceiver = statsReceiver - ) - - val underlyingClients: UnderlyingClientConfiguration = buildUnderlyingClientConfiguration - - val clientWrappers: ClientWrappers = new ClientWrappers(this) - override val clientWrapperFactories: ClientWrapperFactories = new ClientWrapperFactories(this) - - private[this] val userRolesCacheFactory = new UserRolesCacheFactory( - userRolesService = underlyingClients.userRolesServiceClient, - statsReceiver = statsReceiver - ) - override val whitelist: Whitelist = Whitelist( - configStoreFactory = configStoreFactory, - userRolesCacheFactory = userRolesCacheFactory, - statsReceiver = statsReceiver - ) - - override val employeeGate: Gate[UserId] = UserRolesGate( - userRolesCacheFactory.create(UserRoles.EmployeesRoleName) - ) - - private[this] val featureRecipientFactory = - new UserRolesCachingFeatureRecipientFactory(userRolesCacheFactory, statsReceiver) - - override val configApiConfiguration: FeatureSwitchesV2ConfigApiConfiguration = - FeatureSwitchesV2ConfigApiConfiguration( - datacenter = flags.getDatacenter, - serviceName = ServiceName.TimelineRanker, - abdecider = abdecider, - featureRecipientFactory = featureRecipientFactory, - forcedValues = forcedFeatureValues, - statsReceiver = statsReceiver - ) - - private[this] def buildUnderlyingClientConfiguration: UnderlyingClientConfiguration = { - environment match { - case Env.prod => new DefaultUnderlyingClientConfiguration(flags, statsReceiver) - case _ => new StagingUnderlyingClientConfiguration(flags, statsReceiver) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/StagingUnderlyingConfiguration.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/StagingUnderlyingConfiguration.docx new file mode 100644 index 000000000..54427ca6f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/StagingUnderlyingConfiguration.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/StagingUnderlyingConfiguration.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/StagingUnderlyingConfiguration.scala deleted file mode 100644 index 56bc7cb92..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/StagingUnderlyingConfiguration.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.finagle.stats.StatsReceiver - -class StagingUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver) - extends DefaultUnderlyingClientConfiguration(flags, statsReceiver) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerConstants.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerConstants.docx new file mode 100644 index 000000000..16d82950b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerConstants.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerConstants.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerConstants.scala deleted file mode 100644 index bcbfc74ea..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerConstants.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.timelineranker.config - -object TimelineRankerConstants { - val ClientPrefix = "timelineranker." - val ManhattanStarbuckAppId = "timelineranker" - val WarmupClientName = "timelineranker.warmup" - val ForwardedClientName = "timelineranker.forwarded" -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerFlags.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerFlags.docx new file mode 100644 index 000000000..4a7bee1aa Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerFlags.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerFlags.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerFlags.scala deleted file mode 100644 index 19e423c10..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerFlags.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.app.Flags -import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.timelines.config.CommonFlags -import com.twitter.timelines.config.ConfigUtils -import com.twitter.timelines.config.Datacenter -import com.twitter.timelines.config.Env -import com.twitter.timelines.config.ProvidesServiceIdentifier -import java.net.InetSocketAddress -import com.twitter.app.Flag - -class TimelineRankerFlags(flag: Flags) - extends CommonFlags(flag) - with ConfigUtils - with ProvidesServiceIdentifier { - val dc: Flag[String] = flag( - "dc", - "smf1", - "Name of data center in which this instance will execute" - ) - val environment: Flag[String] = flag( - "environment", - "devel", - "The mesos environment in which this instance will be running" - ) - val maxConcurrency: Flag[Int] = flag( - "maxConcurrency", - 200, - "Maximum concurrent requests" - ) - val servicePort: Flag[InetSocketAddress] = flag( - "service.port", - new InetSocketAddress(8287), - "Port number that this thrift service will listen on" - ) - val serviceCompactPort: Flag[InetSocketAddress] = flag( - "service.compact.port", - new InetSocketAddress(8288), - "Port number that the TCompactProtocol-based thrift service will listen on" - ) - - val serviceIdentifier: Flag[ServiceIdentifier] = flag[ServiceIdentifier]( - "service.identifier", - EmptyServiceIdentifier, - "service identifier for this service for use with mutual TLS, " + - "format is expected to be -service.identifier=\"role:service:environment:zone\"" - ) - - val opportunisticTlsLevel = flag[String]( - "opportunistic.tls.level", - "desired", - "The server's OpportunisticTls level." - ) - - val requestRateLimit: Flag[Double] = flag[Double]( - "requestRateLimit", - 1000.0, - "Request rate limit to be used by the client request authorizer" - ) - - val enableThriftmuxCompression = flag( - "enableThriftmuxServerCompression", - true, - "build server with thriftmux compression enabled" - ) - - def getDatacenter: Datacenter.Value = getDC(dc()) - def getEnv: Env.Value = getEnv(environment()) - override def getServiceIdentifier: ServiceIdentifier = serviceIdentifier() -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/UnderlyingClientConfiguration.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/UnderlyingClientConfiguration.docx new file mode 100644 index 000000000..bd09d7210 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/UnderlyingClientConfiguration.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/UnderlyingClientConfiguration.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/UnderlyingClientConfiguration.scala deleted file mode 100644 index 1bfa23fd8..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/config/UnderlyingClientConfiguration.scala +++ /dev/null @@ -1,107 +0,0 @@ -package com.twitter.timelineranker.config - -import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService -import com.twitter.finagle.Service -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.ThriftClientRequest -import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck} -import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1} -import com.twitter.merlin.thriftscala.UserRolesService -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.storehaus.Store -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.thriftscala.TimelineRanker -import com.twitter.timelines.config.ConfigUtils -import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration -import com.twitter.timelines.model.TweetId -import com.twitter.timelineservice.thriftscala.TimelineService -import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie} -import com.twitter.util.Duration -import com.twitter.util.Try -import org.apache.thrift.protocol.TCompactProtocol - -abstract class UnderlyingClientConfiguration( - flags: TimelineRankerFlags, - val statsReceiver: StatsReceiver) - extends TimelinesUnderlyingClientConfiguration - with ConfigUtils { - - lazy val thriftClientId: ClientId = timelineRankerClientId() - override lazy val serviceIdentifier: ServiceIdentifier = flags.getServiceIdentifier - - def timelineRankerClientId(scope: Option[String] = None): ClientId = { - clientIdWithScopeOpt("timelineranker", flags.getEnv, scope) - } - - def createEarlybirdClient( - scope: String, - requestTimeout: Duration, - timeout: Duration, - retryPolicy: RetryPolicy[Try[Nothing]] - ): EarlybirdService.MethodPerEndpoint = { - val scopedName = s"earlybird/$scope" - - methodPerEndpointClient[ - EarlybirdService.ServicePerEndpoint, - EarlybirdService.MethodPerEndpoint]( - thriftMuxClientBuilder( - scopedName, - protocolFactoryOption = Some(new TCompactProtocol.Factory), - requireMutualTls = true) - .dest("/s/earlybird-root-superroot/root-superroot") - .timeout(timeout) - .requestTimeout(requestTimeout) - .retryPolicy(retryPolicy) - ) - } - - def createEarlybirdRealtimeCgClient( - scope: String, - requestTimeout: Duration, - timeout: Duration, - retryPolicy: RetryPolicy[Try[Nothing]] - ): EarlybirdService.MethodPerEndpoint = { - val scopedName = s"earlybird/$scope" - - methodPerEndpointClient[ - EarlybirdService.ServicePerEndpoint, - EarlybirdService.MethodPerEndpoint]( - thriftMuxClientBuilder( - scopedName, - protocolFactoryOption = Some(new TCompactProtocol.Factory), - requireMutualTls = true) - .dest("/s/earlybird-rootrealtimecg/root-realtime_cg") - .timeout(timeout) - .requestTimeout(requestTimeout) - .retryPolicy(retryPolicy) - ) - } - - def cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint - def gizmoduckClient: Gizmoduck.MethodPerEndpoint - def manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint - def sgsClient: SocialGraphService.MethodPerEndpoint - def timelineRankerForwardingClient: TimelineRanker.FinagledClient - def timelineServiceClient: TimelineService.MethodPerEndpoint - def tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint - def tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint - def userRolesServiceClient: UserRolesService.MethodPerEndpoint - def contentFeaturesCache: Store[TweetId, ContentFeatures] - def userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient - def stratoClient: StratoClient - - def darkTrafficProxy: Option[Service[ThriftClientRequest, Array[Byte]]] = { - if (flags.darkTrafficDestination.trim.nonEmpty) { - Some(darkTrafficClient(flags.darkTrafficDestination)) - } else { - None - } - } - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/BUILD deleted file mode 100644 index 1536f3f08..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelines/src/main/scala/com/twitter/timelines/model/types", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/BUILD.docx new file mode 100644 index 000000000..13bcdfd94 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/package.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/package.docx new file mode 100644 index 000000000..05a83e6e6 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/package.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/package.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/package.scala deleted file mode 100644 index bcebb2ed5..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/package.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.timelineranker - -import com.twitter.timelineranker.core.FutureDependencyTransformer -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelines.model.TweetId - -package object contentfeatures { - type ContentFeaturesProvider = - FutureDependencyTransformer[Seq[TweetId], Map[TweetId, ContentFeatures]] -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/BUILD deleted file mode 100644 index 68e4a6e01..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-scala", - "src/thrift/com/twitter/search/common:features-scala", - "timelineranker/common:model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/model/tweet", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/visibility/model", - "util/util-core:util-core-util", - ], - exports = [ - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/BUILD.docx new file mode 100644 index 000000000..f856f1169 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/CandidateEnvelope.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/CandidateEnvelope.docx new file mode 100644 index 000000000..611ee669a Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/CandidateEnvelope.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/CandidateEnvelope.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/CandidateEnvelope.scala deleted file mode 100644 index 1b876b563..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/CandidateEnvelope.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.timelineranker.core - -import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelines.model.TweetId - -object CandidateEnvelope { - val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult] - val EmptyHydratedTweets: HydratedTweets = HydratedTweets(Seq.empty, Seq.empty) - val EmptyUtegResults: Map[TweetId, TweetRecommendation] = Map.empty[TweetId, TweetRecommendation] -} - -case class CandidateEnvelope( - query: RecapQuery, - searchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults, - utegResults: Map[TweetId, TweetRecommendation] = CandidateEnvelope.EmptyUtegResults, - hydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets, - followGraphData: FollowGraphDataFuture = FollowGraphDataFuture.EmptyFollowGraphDataFuture, - // The source tweets are - // - the retweeted tweet, for retweets - // - the inReplyTo tweet, for extended replies - sourceSearchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults, - sourceHydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphData.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphData.docx new file mode 100644 index 000000000..00cb79f9e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphData.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphData.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphData.scala deleted file mode 100644 index b5d1e890d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphData.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.timelineranker.core - -import com.twitter.timelines.model.UserId - -/** - * Follow graph details of a given user. Includes users followed, but also followed users in various - * states of mute. - * - * @param userId ID of a given user. - * @param followedUserIds IDs of users who the given user follows. - * @param mutuallyFollowingUserIds A subset of followedUserIds where followed users follow back the given user. - * @param mutedUserIds A subset of followedUserIds that the given user has muted. - * @param retweetsMutedUserIds A subset of followedUserIds whose retweets are muted by the given user. - */ -case class FollowGraphData( - userId: UserId, - followedUserIds: Seq[UserId], - mutuallyFollowingUserIds: Set[UserId], - mutedUserIds: Set[UserId], - retweetsMutedUserIds: Set[UserId]) { - val filteredFollowedUserIds: Seq[UserId] = followedUserIds.filterNot(mutedUserIds) - val allUserIds: Seq[UserId] = filteredFollowedUserIds :+ userId - val inNetworkUserIds: Seq[UserId] = followedUserIds :+ userId -} - -object FollowGraphData { - val Empty: FollowGraphData = FollowGraphData( - 0L, - Seq.empty[UserId], - Set.empty[UserId], - Set.empty[UserId], - Set.empty[UserId] - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphDataFuture.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphDataFuture.docx new file mode 100644 index 000000000..e11576990 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphDataFuture.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphDataFuture.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphDataFuture.scala deleted file mode 100644 index eef6f7a09..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphDataFuture.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.timelineranker.core - -import com.twitter.timelines.model.UserId -import com.twitter.util.Future - -/** - * Similar to FollowGraphData but makes available its fields as separate futures. - */ -case class FollowGraphDataFuture( - userId: UserId, - followedUserIdsFuture: Future[Seq[UserId]], - mutuallyFollowingUserIdsFuture: Future[Set[UserId]], - mutedUserIdsFuture: Future[Set[UserId]], - retweetsMutedUserIdsFuture: Future[Set[UserId]]) { - - def inNetworkUserIdsFuture: Future[Seq[UserId]] = followedUserIdsFuture.map(_ :+ userId) - - def get(): Future[FollowGraphData] = { - Future - .join( - followedUserIdsFuture, - mutuallyFollowingUserIdsFuture, - mutedUserIdsFuture, - retweetsMutedUserIdsFuture - ) - .map { - case (followedUserIds, mutuallyFollowingUserIds, mutedUserIds, retweetsMutedUserIds) => - FollowGraphData( - userId, - followedUserIds, - mutuallyFollowingUserIds, - mutedUserIds, - retweetsMutedUserIds - ) - } - } -} - -object FollowGraphDataFuture { - private def mkEmptyFuture(field: String) = { - Future.exception( - new IllegalStateException(s"FollowGraphDataFuture field $field accessed without being set") - ) - } - - val EmptyFollowGraphDataFuture: FollowGraphDataFuture = FollowGraphDataFuture( - userId = 0L, - followedUserIdsFuture = mkEmptyFuture("followedUserIdsFuture"), - mutuallyFollowingUserIdsFuture = mkEmptyFuture("mutuallyFollowingUserIdsFuture"), - mutedUserIdsFuture = mkEmptyFuture("mutedUserIdsFuture"), - retweetsMutedUserIdsFuture = mkEmptyFuture("retweetsMutedUserIdsFuture") - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedCandidatesAndFeaturesEnvelope.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedCandidatesAndFeaturesEnvelope.docx new file mode 100644 index 000000000..111ae607f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedCandidatesAndFeaturesEnvelope.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedCandidatesAndFeaturesEnvelope.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedCandidatesAndFeaturesEnvelope.scala deleted file mode 100644 index 64f4cd416..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedCandidatesAndFeaturesEnvelope.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.timelineranker.core - -import com.twitter.search.common.constants.thriftscala.ThriftLanguage -import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelines.clients.gizmoduck.UserProfileInfo -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.util.FutureUtils -import com.twitter.util.Future - -case class HydratedCandidatesAndFeaturesEnvelope( - candidateEnvelope: CandidateEnvelope, - userLanguages: Seq[ThriftLanguage], - userProfileInfo: UserProfileInfo, - features: Map[TweetId, ThriftTweetFeatures] = Map.empty, - contentFeaturesFuture: Future[Map[TweetId, ContentFeatures]] = FutureUtils.EmptyMap, - tweetSourceTweetMap: Map[TweetId, TweetId] = Map.empty, - inReplyToTweetIds: Set[TweetId] = Set.empty) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedTweets.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedTweets.docx new file mode 100644 index 000000000..0f081cfad Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedTweets.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedTweets.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedTweets.scala deleted file mode 100644 index cf3fdb52f..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedTweets.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.timelineranker.core - -import com.twitter.timelines.model.tweet.HydratedTweet - -case class HydratedTweets( - outerTweets: Seq[HydratedTweet], - innerTweets: Seq[HydratedTweet] = Seq.empty) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/package.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/package.docx new file mode 100644 index 000000000..338726031 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/package.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/package.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/package.scala deleted file mode 100644 index 1b641c822..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/core/package.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.timelineranker - -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelines.configapi - -package object core { - type FutureDependencyTransformer[-U, +V] = configapi.FutureDependencyTransformer[RecapQuery, U, V] - object FutureDependencyTransformer - extends configapi.FutureDependencyTransformerFunctions[RecapQuery] - - type DependencyTransformer[-U, +V] = configapi.DependencyTransformer[RecapQuery, U, V] - object DependencyTransformer extends configapi.DependencyTransformerFunctions[RecapQuery] -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/BUILD deleted file mode 100644 index a4e6b28a9..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - sources = ["*.scala"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "servo/decider", - "timelineranker/server/config", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/BUILD.docx new file mode 100644 index 000000000..44a50c1b4 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/DeciderKey.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/DeciderKey.docx new file mode 100644 index 000000000..371c90574 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/DeciderKey.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/DeciderKey.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/DeciderKey.scala deleted file mode 100644 index aa89aadcb..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/DeciderKey.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.timelineranker.decider - -import com.twitter.servo.decider.DeciderKeyEnum - -object DeciderKey extends DeciderKeyEnum { - // Deciders that can be used to control load on TLR or its backends. - val EnableMaxConcurrencyLimiting: Value = Value("enable_max_concurrency_limiting") - - // Deciders related to testing / debugging. - val EnableRoutingToRankerDevProxy: Value = Value("enable_routing_to_ranker_dev_proxy") - - // Deciders related to authorization. - val ClientRequestAuthorization: Value = Value("client_request_authorization") - val ClientWriteWhitelist: Value = Value("client_write_whitelist") - val AllowTimelineMixerRecapProd: Value = Value("allow_timeline_mixer_recap_prod") - val AllowTimelineMixerRecycledProd: Value = Value("allow_timeline_mixer_recycled_prod") - val AllowTimelineMixerHydrateProd: Value = Value("allow_timeline_mixer_hydrate_prod") - val AllowTimelineMixerHydrateRecosProd: Value = Value("allow_timeline_mixer_hydrate_recos_prod") - val AllowTimelineMixerSeedAuthorsProd: Value = Value("allow_timeline_mixer_seed_authors_prod") - val AllowTimelineMixerSimclusterProd: Value = Value("allow_timeline_mixer_simcluster_prod") - val AllowTimelineMixerEntityTweetsProd: Value = Value("allow_timeline_mixer_entity_tweets_prod") - val AllowTimelineMixerListProd: Value = Value("allow_timeline_mixer_list_prod") - val AllowTimelineMixerListTweetProd: Value = Value("allow_timeline_mixer_list_tweet_prod") - val AllowTimelineMixerCommunityProd: Value = Value("allow_timeline_mixer_community_prod") - val AllowTimelineMixerCommunityTweetProd: Value = Value( - "allow_timeline_mixer_community_tweet_prod") - val AllowTimelineScorerRecommendedTrendTweetProd: Value = Value( - "allow_timeline_scorer_recommended_trend_tweet_prod") - val AllowTimelineMixerUtegLikedByTweetsProd: Value = Value( - "allow_timeline_mixer_uteg_liked_by_tweets_prod") - val AllowTimelineMixerStaging: Value = Value("allow_timeline_mixer_staging") - val AllowTimelineRankerProxy: Value = Value("allow_timeline_ranker_proxy") - val AllowTimelineRankerWarmup: Value = Value("allow_timeline_ranker_warmup") - val AllowTimelineScorerRecTopicTweetsProd: Value = - Value("allow_timeline_scorer_rec_topic_tweets_prod") - val AllowTimelineScorerPopularTopicTweetsProd: Value = - Value("allow_timeline_scorer_popular_topic_tweets_prod") - val AllowTimelineScorerHydrateTweetScoringProd: Value = Value( - "allow_timelinescorer_hydrate_tweet_scoring_prod") - val AllowTimelineServiceProd: Value = Value("allow_timeline_service_prod") - val AllowTimelineServiceStaging: Value = Value("allow_timeline_service_staging") - val RateLimitOverrideUnknown: Value = Value("rate_limit_override_unknown") - - // Deciders related to reverse-chron home timeline materialization. - val MultiplierOfMaterializationTweetsFetched: Value = Value( - "multiplier_of_materialization_tweets_fetched" - ) - val BackfillFilteredEntries: Value = Value("enable_backfill_filtered_entries") - val TweetsFilteringLossageThreshold: Value = Value("tweets_filtering_lossage_threshold") - val TweetsFilteringLossageLimit: Value = Value("tweets_filtering_lossage_limit") - val SupplementFollowsWithRealGraph: Value = Value("supplement_follows_with_real_graph") - - // Deciders related to recap. - val RecapEnableContentFeaturesHydration: Value = Value("recap_enable_content_features_hydration") - val RecapMaxCountMultiplier: Value = Value("recap_max_count_multiplier") - val RecapEnableExtraSortingInResults: Value = Value("recap_enable_extra_sorting_in_results") - - // Deciders related to recycled tweets. - val RecycledMaxCountMultiplier: Value = Value("recycled_max_count_multiplier") - val RecycledEnableContentFeaturesHydration: Value = Value( - "recycled_enable_content_features_hydration") - - // Deciders related to entity tweets. - val EntityTweetsEnableContentFeaturesHydration: Value = Value( - "entity_tweets_enable_content_features_hydration") - - // Deciders related to both recap and recycled tweets - val EnableRealGraphUsers: Value = Value("enable_real_graph_users") - val MaxRealGraphAndFollowedUsers: Value = Value("max_real_graph_and_followed_users") - - // Deciders related to recap author - val RecapAuthorEnableNewPipeline: Value = Value("recap_author_enable_new_pipeline") - val RecapAuthorEnableContentFeaturesHydration: Value = Value( - "recap_author_enable_content_features_hydration") - - // Deciders related to recap hydration (rectweet and ranked organic). - val RecapHydrationEnableContentFeaturesHydration: Value = Value( - "recap_hydration_enable_content_features_hydration") - - // Deciders related to uteg liked by tweets - val UtegLikedByTweetsEnableContentFeaturesHydration: Value = Value( - "uteg_liked_by_tweets_enable_content_features_hydration") -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/BUILD.bazel b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/BUILD.bazel deleted file mode 100644 index 9e745e7a0..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/BUILD.bazel +++ /dev/null @@ -1,39 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", - "finagle/finagle-core/src/main", - "servo/util/src/main/scala", - "src/thrift/com/twitter/search:earlybird-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/common", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/repository", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelines/src/main/scala/com/twitter/timelines/visibility", - "timelines/src/main/scala/com/twitter/timelines/visibility/model", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/BUILD.docx new file mode 100644 index 000000000..8f201b5bf Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepository.docx new file mode 100644 index 000000000..ed728aab9 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepository.scala deleted file mode 100644 index 1e20bd689..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepository.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.timelineranker.entity_tweets - -import com.twitter.timelineranker.model.CandidateTweetsResult -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.util.Future - -/** - * A repository of entity tweets candidates. - * - * For now, it does not cache any results therefore forwards all calls to the underlying source. - */ -class EntityTweetsRepository(source: EntityTweetsSource) { - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - source.get(query) - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - source.get(queries) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepositoryBuilder.docx new file mode 100644 index 000000000..52896f442 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepositoryBuilder.scala deleted file mode 100644 index f8e1197a4..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepositoryBuilder.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.timelineranker.entity_tweets - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelineranker.config.RequestScopes -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.parameters.ConfigBuilder -import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder -import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.util.Duration - -class EntityTweetsRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder) - extends CandidatesRepositoryBuilder(config) { - - // Default client id for this repository if the upstream requests doesn't indicate one. - override val clientSubId = "community" - override val requestScope: RequestScope = RequestScopes.EntityTweetsSource - override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet = - SgsFollowGraphDataFields.ValueSet( - SgsFollowGraphDataFields.FollowedUserIds, - SgsFollowGraphDataFields.MutuallyFollowingUserIds, - SgsFollowGraphDataFields.MutedUserIds - ) - - /** - * [1] timeout is derived from p9999 TLR <-> Earlybird latency and shall be less than - * request timeout of timelineranker client within downstream timelinemixer, which is - * 1s now - * - * [2] processing timeout is less than request timeout [1] with 100ms space for networking and - * other times such as gc - */ - override val searchProcessingTimeout: Duration = 550.milliseconds // [2] - override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdClient( - scope = scope, - requestTimeout = 650.milliseconds, // [1] - timeout = 900.milliseconds, // [1] - retryPolicy = config.underlyingClients.DefaultRetryPolicy - ) - - def apply(): EntityTweetsRepository = { - val entityTweetsSource = new EntityTweetsSource( - gizmoduckClient, - searchClient, - tweetyPieHighQoSClient, - userMetadataClient, - followGraphDataProvider, - clientFactories.visibilityEnforcerFactory.apply( - VisibilityRules, - RequestScopes.EntityTweetsSource - ), - config.underlyingClients.contentFeaturesCache, - config.statsReceiver - ) - - new EntityTweetsRepository(entityTweetsSource) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSearchResultsTransform.docx new file mode 100644 index 000000000..585810581 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSearchResultsTransform.scala deleted file mode 100644 index 0158b833a..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSearchResultsTransform.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.timelineranker.entity_tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.TweetIdRange -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future - -object EntityTweetsSearchResultsTransform { - // If EntityTweetsQuery.maxCount is not specified, the following count is used. - val DefaultEntityTweetsMaxTweetCount = 200 -} - -/** - * Fetch entity tweets search results using the search client - * and populate them into the CandidateEnvelope - */ -class EntityTweetsSearchResultsTransform( - searchClient: SearchClient, - statsReceiver: StatsReceiver, - logSearchDebugInfo: Boolean = false) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - import EntityTweetsSearchResultsTransform._ - - private[this] val maxCountStat = statsReceiver.stat("maxCount") - private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch") - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val maxCount = envelope.query.maxCount.getOrElse(DefaultEntityTweetsMaxTweetCount) - maxCountStat.add(maxCount) - - val tweetIdRange = envelope.query.range - .map(TweetIdRange.fromTimelineRange) - .getOrElse(TweetIdRange.default) - - val beforeTweetIdExclusive = tweetIdRange.toId - val afterTweetIdExclusive = tweetIdRange.fromId - - val excludedTweetIds = envelope.query.excludedTweetIds.getOrElse(Seq.empty[TweetId]).toSet - val languages = envelope.query.languages.map(_.map(_.language)) - - envelope.followGraphData.inNetworkUserIdsFuture.flatMap { inNetworkUserIds => - searchClient - .getEntityTweets( - userId = Some(envelope.query.userId), - followedUserIds = inNetworkUserIds.toSet, - maxCount = maxCount, - beforeTweetIdExclusive = beforeTweetIdExclusive, - afterTweetIdExclusive = afterTweetIdExclusive, - earlybirdOptions = envelope.query.earlybirdOptions, - semanticCoreIds = envelope.query.semanticCoreIds, - hashtags = envelope.query.hashtags, - languages = languages, - tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options), - searchOperator = envelope.query.searchOperator, - excludedTweetIds = excludedTweetIds, - logSearchDebugInfo = logSearchDebugInfo, - includeNullcastTweets = envelope.query.includeNullcastTweets.getOrElse(false), - includeTweetsFromArchiveIndex = - envelope.query.includeTweetsFromArchiveIndex.getOrElse(false), - authorIds = envelope.query.authorIds.map(_.toSet) - ).map { results => - numResultsFromSearchStat.add(results.size) - envelope.copy(searchResults = results) - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSource.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSource.docx new file mode 100644 index 000000000..ac68a3bd1 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSource.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSource.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSource.scala deleted file mode 100644 index 722429d38..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSource.scala +++ /dev/null @@ -1,146 +0,0 @@ -package com.twitter.timelineranker.entity_tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.storehaus.Store -import com.twitter.timelineranker.common._ -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.model._ -import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsParams._ -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform -import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform -import com.twitter.timelineranker.util.TweetFilters -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.clients.gizmoduck.GizmoduckClient -import com.twitter.timelines.clients.manhattan.UserMetadataClient -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.tweetypie.TweetyPieClient -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.timelines.util.stats.RequestStatsReceiver -import com.twitter.timelines.visibility.VisibilityEnforcer -import com.twitter.util.Future - -class EntityTweetsSource( - gizmoduckClient: GizmoduckClient, - searchClient: SearchClient, - tweetyPieClient: TweetyPieClient, - userMetadataClient: UserMetadataClient, - followGraphDataProvider: FollowGraphDataProvider, - visibilityEnforcer: VisibilityEnforcer, - contentFeaturesCache: Store[TweetId, ContentFeatures], - statsReceiver: StatsReceiver) { - - private[this] val baseScope = statsReceiver.scope("entityTweetsSource") - private[this] val requestStats = RequestStatsReceiver(baseScope) - - private[this] val failOpenScope = baseScope.scope("failOpen") - private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo") - private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages") - - private[this] val followGraphDataTransform = new FollowGraphDataTransform( - followGraphDataProvider = followGraphDataProvider, - maxFollowedUsersProvider = DependencyProvider.from(MaxFollowedUsersParam) - ) - private[this] val fetchSearchResultsTransform = new EntityTweetsSearchResultsTransform( - searchClient = searchClient, - statsReceiver = baseScope - ) - private[this] val userProfileInfoTransform = - new UserProfileInfoTransform(userProfileHandler, gizmoduckClient) - private[this] val languagesTransform = - new UserLanguagesTransform(userLanguagesHandler, userMetadataClient) - - private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform( - visibilityEnforcer - ) - - private[this] val filters = TweetFilters.ValueSet( - TweetFilters.DuplicateTweets, - TweetFilters.DuplicateRetweets - ) - - private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform( - outerFilters = filters, - innerFilters = TweetFilters.None, - useFollowGraphData = false, - useSourceTweets = false, - statsReceiver = baseScope, - numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed - ) - - private[this] val contentFeaturesHydrationTransform = - new ContentFeaturesHydrationTransformBuilder( - tweetyPieClient = tweetyPieClient, - contentFeaturesCache = contentFeaturesCache, - enableContentFeaturesGate = RecapQuery.paramGate(EnableContentFeaturesHydrationParam), - enableTokensInContentFeaturesGate = - RecapQuery.paramGate(EnableTokensInContentFeaturesHydrationParam), - enableTweetTextInContentFeaturesGate = - RecapQuery.paramGate(EnableTweetTextInContentFeaturesHydrationParam), - enableConversationControlContentFeaturesGate = - RecapQuery.paramGate(EnableConversationControlInContentFeaturesHydrationParam), - enableTweetMediaHydrationGate = RecapQuery.paramGate(EnableTweetMediaHydrationParam), - hydrateInReplyToTweets = false, - statsReceiver = baseScope - ).build() - - private[this] def hydratesContentFeatures( - hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope - ): Boolean = - hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true) - - private[this] val contentFeaturesTransformer = FutureArrow.choose( - predicate = hydratesContentFeatures, - ifTrue = contentFeaturesHydrationTransform - .andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform) - .andThen(CopyContentFeaturesIntoHydratedTweetsTransform), - ifFalse = FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ](Future.value) // empty transformer - ) - - private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope) - - private[this] val hydrationAndFilteringPipeline = - CreateCandidateEnvelopeTransform - .andThen(followGraphDataTransform) // Fetch follow graph data - .andThen(fetchSearchResultsTransform) // fetch search results - .andThen(SearchResultDedupAndSortingTransform) // dedup and order search results - .andThen(CandidateTweetHydrationTransform) // hydrate search results - .andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones - .andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter - .andThen( - TrimToMatchHydratedTweetsTransform - ) // trim search result set to match filtered hydrated tweets (this needs to be accurate for feature hydration) - - // runs the main pipeline in parallel with fetching user profile info and user languages - private[this] val featureHydrationDataTransform = - new FeatureHydrationDataTransform( - hydrationAndFilteringPipeline, - languagesTransform, - userProfileInfoTransform - ) - - private[this] val tweetFeaturesHydrationTransform = - OutOfNetworkTweetsSearchFeaturesHydrationTransform - .andThen(contentFeaturesTransformer) - - private[this] val featureHydrationPipeline = - featureHydrationDataTransform - .andThen(tweetFeaturesHydrationTransform) - .andThen(candidateGenerationTransform) - - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - requestStats.addEventStats { - featureHydrationPipeline(query) - } - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - Future.collect(queries.map(get)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/BUILD deleted file mode 100644 index 0277c6aaa..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/BUILD +++ /dev/null @@ -1,41 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", - "finagle/finagle-core/src/main", - "servo/util/src/main/scala", - "src/thrift/com/twitter/search:earlybird-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/common", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/repository", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/tweet", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelines/src/main/scala/com/twitter/timelines/visibility", - "timelines/src/main/scala/com/twitter/timelines/visibility/model", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/BUILD.docx new file mode 100644 index 000000000..d2d434163 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepository.docx new file mode 100644 index 000000000..b012631b6 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepository.scala deleted file mode 100644 index 34884a700..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepository.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.timelineranker.in_network_tweets - -import com.twitter.timelineranker.model.CandidateTweetsResult -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams -import com.twitter.util.Future - -/** - * A repository of in-network tweet candidates. - * For now, it does not cache any results therefore forwards all calls to the underlying source. - */ -class InNetworkTweetRepository( - source: InNetworkTweetSource, - realtimeCGSource: InNetworkTweetSource) { - - private[this] val enableRealtimeCGProvider = - DependencyProvider.from(InNetworkTweetParams.EnableEarlybirdRealtimeCgMigrationParam) - - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - if (enableRealtimeCGProvider(query)) { - realtimeCGSource.get(query) - } else { - source.get(query) - } - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - Future.collect(queries.map(query => get(query))) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepositoryBuilder.docx new file mode 100644 index 000000000..df69f6e7f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepositoryBuilder.scala deleted file mode 100644 index ff61c5bd0..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepositoryBuilder.scala +++ /dev/null @@ -1,109 +0,0 @@ -package com.twitter.timelineranker.in_network_tweets - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.service.RetryPolicy -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.timelineranker.config.RequestScopes -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.parameters.ConfigBuilder -import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder -import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.timelines.visibility.model.CheckedUserActorType -import com.twitter.timelines.visibility.model.ExclusionReason -import com.twitter.timelines.visibility.model.VisibilityCheckStatus -import com.twitter.timelines.visibility.model.VisibilityCheckUser -import com.twitter.util.Duration - -object InNetworkTweetRepositoryBuilder { - val VisibilityRuleExclusions: Set[ExclusionReason] = Set[ExclusionReason]( - ExclusionReason( - CheckedUserActorType(Some(false), VisibilityCheckUser.SourceUser), - Set(VisibilityCheckStatus.Blocked) - ) - ) - - private val EarlybirdTimeout = 600.milliseconds - private val EarlybirdRequestTimeout = 600.milliseconds - - /** - * The timeouts below are only used for the Earlybird Cluster Migration - */ - private val EarlybirdRealtimeCGTimeout = 600.milliseconds - private val EarlybirdRealtimeCGRequestTimeout = 600.milliseconds -} - -class InNetworkTweetRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder) - extends CandidatesRepositoryBuilder(config) { - import InNetworkTweetRepositoryBuilder._ - - override val clientSubId = "recycled_tweets" - override val requestScope: RequestScope = RequestScopes.InNetworkTweetSource - override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet = - SgsFollowGraphDataFields.ValueSet( - SgsFollowGraphDataFields.FollowedUserIds, - SgsFollowGraphDataFields.MutuallyFollowingUserIds, - SgsFollowGraphDataFields.MutedUserIds, - SgsFollowGraphDataFields.RetweetsMutedUserIds - ) - override val searchProcessingTimeout: Duration = 200.milliseconds - - override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdClient( - scope = scope, - requestTimeout = EarlybirdRequestTimeout, - timeout = EarlybirdTimeout, - retryPolicy = RetryPolicy.Never - ) - - private lazy val searchClientForSourceTweets = - newSearchClient(clientId = clientSubId + "_source_tweets") - - /** The RealtimeCG clients below are only used for the Earlybird Cluster Migration */ - private def earlybirdRealtimeCGClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdRealtimeCgClient( - scope = scope, - requestTimeout = EarlybirdRealtimeCGRequestTimeout, - timeout = EarlybirdRealtimeCGTimeout, - retryPolicy = RetryPolicy.Never - ) - private val realtimeCGClientSubId = "realtime_cg_recycled_tweets" - private lazy val searchRealtimeCGClient = - newSearchClient(earlybirdRealtimeCGClient, clientId = realtimeCGClientSubId) - - def apply(): InNetworkTweetRepository = { - val inNetworkTweetSource = new InNetworkTweetSource( - gizmoduckClient, - searchClient, - searchClientForSourceTweets, - tweetyPieHighQoSClient, - userMetadataClient, - followGraphDataProvider, - config.underlyingClients.contentFeaturesCache, - clientFactories.visibilityEnforcerFactory.apply( - VisibilityRules, - RequestScopes.InNetworkTweetSource, - reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions - ), - config.statsReceiver - ) - - val inNetworkTweetRealtimeCGSource = new InNetworkTweetSource( - gizmoduckClient, - searchRealtimeCGClient, - searchClientForSourceTweets, // do not migrate source_tweets as they are sharded by TweetID - tweetyPieHighQoSClient, - userMetadataClient, - followGraphDataProvider, - config.underlyingClients.contentFeaturesCache, - clientFactories.visibilityEnforcerFactory.apply( - VisibilityRules, - RequestScopes.InNetworkTweetSource, - reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions - ), - config.statsReceiver.scope("replacementRealtimeCG") - ) - - new InNetworkTweetRepository(inNetworkTweetSource, inNetworkTweetRealtimeCGSource) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetSource.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetSource.docx new file mode 100644 index 000000000..ca8931359 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetSource.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetSource.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetSource.scala deleted file mode 100644 index 9c015123b..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetSource.scala +++ /dev/null @@ -1,271 +0,0 @@ -package com.twitter.timelineranker.in_network_tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.storehaus.Store -import com.twitter.timelineranker.common._ -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.model._ -import com.twitter.timelineranker.monitoring.UsersSearchResultMonitoringTransform -import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams -import com.twitter.timelineranker.parameters.monitoring.MonitoringParams -import com.twitter.timelineranker.parameters.recap.RecapParams -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform -import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform -import com.twitter.timelineranker.util.TweetFilters -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.clients.gizmoduck.GizmoduckClient -import com.twitter.timelines.clients.manhattan.UserMetadataClient -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.tweetypie.TweetyPieClient -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.timelines.util.stats.RequestStatsReceiver -import com.twitter.timelines.visibility.VisibilityEnforcer -import com.twitter.util.Future - -class InNetworkTweetSource( - gizmoduckClient: GizmoduckClient, - searchClient: SearchClient, - searchClientForSourceTweets: SearchClient, - tweetyPieClient: TweetyPieClient, - userMetadataClient: UserMetadataClient, - followGraphDataProvider: FollowGraphDataProvider, - contentFeaturesCache: Store[TweetId, ContentFeatures], - visibilityEnforcer: VisibilityEnforcer, - statsReceiver: StatsReceiver) { - private[this] val baseScope = statsReceiver.scope("recycledTweetSource") - private[this] val requestStats = RequestStatsReceiver(baseScope) - - private[this] val failOpenScope = baseScope.scope("failOpen") - private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo") - private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages") - private[this] val sourceTweetSearchHandler = - new FailOpenHandler(failOpenScope, "sourceTweetSearch") - - private[this] val filters = TweetFilters.ValueSet( - TweetFilters.DuplicateTweets, - TweetFilters.DuplicateRetweets, - TweetFilters.TweetsFromNotFollowedUsers, - TweetFilters.NonReplyDirectedAtNotFollowedUsers - ) - - private[this] val hydrateReplyRootTweetProvider = - DependencyProvider.from(InNetworkTweetParams.EnableReplyRootTweetHydrationParam) - - private[this] val sourceTweetsSearchResultsTransform = new SourceTweetsSearchResultsTransform( - searchClientForSourceTweets, - sourceTweetSearchHandler, - hydrateReplyRootTweetProvider = hydrateReplyRootTweetProvider, - perRequestSourceSearchClientIdProvider = DependencyProvider.None, - baseScope - ) - - private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform( - visibilityEnforcer - ) - - private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform( - outerFilters = filters, - innerFilters = TweetFilters.None, - useFollowGraphData = true, - useSourceTweets = true, - statsReceiver = baseScope, - numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed - ) - - private[this] val dynamicHydratedTweetsFilter = new TweetKindOptionHydratedTweetsFilterTransform( - useFollowGraphData = true, - useSourceTweets = true, - statsReceiver = baseScope - ) - - private[this] val userProfileInfoTransform = - new UserProfileInfoTransform(userProfileHandler, gizmoduckClient) - private[this] val languagesTransform = - new UserLanguagesTransform(userLanguagesHandler, userMetadataClient) - - private[this] def hydratesContentFeatures( - hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope - ): Boolean = - hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true) - - private[this] val contentFeaturesTransformer = FutureArrow.choose( - predicate = hydratesContentFeatures, - ifTrue = contentFeaturesHydrationTransform - .andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform) - .andThen(CopyContentFeaturesIntoHydratedTweetsTransform), - ifFalse = FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ](Future.value) // empty transformer - ) - - private[this] val contentFeaturesHydrationTransform = - new ContentFeaturesHydrationTransformBuilder( - tweetyPieClient = tweetyPieClient, - contentFeaturesCache = contentFeaturesCache, - enableContentFeaturesGate = - RecapQuery.paramGate(InNetworkTweetParams.EnableContentFeaturesHydrationParam), - enableTokensInContentFeaturesGate = - RecapQuery.paramGate(InNetworkTweetParams.EnableTokensInContentFeaturesHydrationParam), - enableTweetTextInContentFeaturesGate = - RecapQuery.paramGate(InNetworkTweetParams.EnableTweetTextInContentFeaturesHydrationParam), - enableConversationControlContentFeaturesGate = RecapQuery.paramGate( - InNetworkTweetParams.EnableConversationControlInContentFeaturesHydrationParam), - enableTweetMediaHydrationGate = RecapQuery.paramGate( - InNetworkTweetParams.EnableTweetMediaHydrationParam - ), - hydrateInReplyToTweets = true, - statsReceiver = baseScope - ).build() - - private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope) - - private[this] val maxFollowedUsersProvider = - DependencyProvider.from(InNetworkTweetParams.MaxFollowedUsersParam) - private[this] val earlybirdReturnAllResultsProvider = - DependencyProvider.from(InNetworkTweetParams.EnableEarlybirdReturnAllResultsParam) - private[this] val relevanceOptionsMaxHitsToProcessProvider = - DependencyProvider.from(InNetworkTweetParams.RelevanceOptionsMaxHitsToProcessParam) - - private[this] val followGraphDataTransform = - new FollowGraphDataTransform(followGraphDataProvider, maxFollowedUsersProvider) - - private[this] val enableRealGraphUsersProvider = - DependencyProvider.from(RecapParams.EnableRealGraphUsersParam) - private[this] val maxRealGraphAndFollowedUsersProvider = - DependencyProvider.from(RecapParams.MaxRealGraphAndFollowedUsersParam) - private[this] val maxRealGraphAndFollowedUsersFSOverrideProvider = - DependencyProvider.from(RecapParams.MaxRealGraphAndFollowedUsersFSOverrideParam) - private[this] val imputeRealGraphAuthorWeightsProvider = - DependencyProvider.from(RecapParams.ImputeRealGraphAuthorWeightsParam) - private[this] val imputeRealGraphAuthorWeightsPercentileProvider = - DependencyProvider.from(RecapParams.ImputeRealGraphAuthorWeightsPercentileParam) - private[this] val maxRealGraphAndFollowedUsersFromDeciderAndFS = DependencyProvider { envelope => - maxRealGraphAndFollowedUsersFSOverrideProvider(envelope).getOrElse( - maxRealGraphAndFollowedUsersProvider(envelope)) - } - private[this] val followAndRealGraphCombiningTransform = new FollowAndRealGraphCombiningTransform( - followGraphDataProvider = followGraphDataProvider, - maxFollowedUsersProvider = maxFollowedUsersProvider, - enableRealGraphUsersProvider = enableRealGraphUsersProvider, - maxRealGraphAndFollowedUsersProvider = maxRealGraphAndFollowedUsersFromDeciderAndFS, - imputeRealGraphAuthorWeightsProvider = imputeRealGraphAuthorWeightsProvider, - imputeRealGraphAuthorWeightsPercentileProvider = imputeRealGraphAuthorWeightsPercentileProvider, - statsReceiver = baseScope - ) - - private[this] val maxCountProvider = DependencyProvider { query => - query.maxCount.getOrElse(query.params(InNetworkTweetParams.DefaultMaxTweetCount)) - } - - private[this] val maxCountWithMarginProvider = DependencyProvider { query => - val maxCount = query.maxCount.getOrElse(query.params(InNetworkTweetParams.DefaultMaxTweetCount)) - val multiplier = query.params(InNetworkTweetParams.MaxCountMultiplierParam) - (maxCount * multiplier).toInt - } - - private[this] val debugAuthorsMonitoringProvider = - DependencyProvider.from(MonitoringParams.DebugAuthorsAllowListParam) - - private[this] val retrieveSearchResultsTransform = new RecapSearchResultsTransform( - searchClient = searchClient, - maxCountProvider = maxCountWithMarginProvider, - returnAllResultsProvider = earlybirdReturnAllResultsProvider, - relevanceOptionsMaxHitsToProcessProvider = relevanceOptionsMaxHitsToProcessProvider, - enableExcludeSourceTweetIdsProvider = DependencyProvider.True, - enableSettingTweetTypesWithTweetKindOptionProvider = - DependencyProvider.from(RecapParams.EnableSettingTweetTypesWithTweetKindOption), - perRequestSearchClientIdProvider = DependencyProvider.None, - statsReceiver = baseScope, - logSearchDebugInfo = false - ) - - private[this] val preTruncateSearchResultsTransform = - new UsersSearchResultMonitoringTransform( - name = "RecapSearchResultsTruncationTransform", - new RecapSearchResultsTruncationTransform( - extraSortBeforeTruncationGate = DependencyProvider.True, - maxCountProvider = maxCountWithMarginProvider, - statsReceiver = baseScope.scope("afterSearchResultsTransform") - ), - baseScope.scope("afterSearchResultsTransform"), - debugAuthorsMonitoringProvider - ) - - private[this] val finalTruncationTransform = new UsersSearchResultMonitoringTransform( - name = "RecapSearchResultsTruncationTransform", - new RecapSearchResultsTruncationTransform( - extraSortBeforeTruncationGate = DependencyProvider.True, - maxCountProvider = maxCountProvider, - statsReceiver = baseScope.scope("finalTruncation") - ), - baseScope.scope("finalTruncation"), - debugAuthorsMonitoringProvider - ) - - // Fetch source tweets based on search results present in the envelope - // and hydrate them. - private[this] val fetchAndHydrateSourceTweets = - sourceTweetsSearchResultsTransform - .andThen(SourceTweetHydrationTransform) - - // Hydrate candidate tweets and fetch source tweets in parallel - private[this] val hydrateTweetsAndSourceTweetsInParallel = - new HydrateTweetsAndSourceTweetsInParallelTransform( - candidateTweetHydration = CandidateTweetHydrationTransform, - sourceTweetHydration = fetchAndHydrateSourceTweets - ) - - private[this] val trimToMatchSearchResultsTransform = new TrimToMatchSearchResultsTransform( - hydrateReplyRootTweetProvider = hydrateReplyRootTweetProvider, - statsReceiver = baseScope - ) - - private[this] val hydrationAndFilteringPipeline = - CreateCandidateEnvelopeTransform // Create empty CandidateEnvelope - .andThen(followGraphDataTransform) // Fetch follow graph data - .andThen(followAndRealGraphCombiningTransform) // Experiment: expand seed author set - .andThen(retrieveSearchResultsTransform) // Fetch search results - .andThen( - preTruncateSearchResultsTransform - ) // truncate the search result up to maxCount + some margin, preserving the random tweet - .andThen(SearchResultDedupAndSortingTransform) // dedups, and sorts reverse-chron - .andThen(hydrateTweetsAndSourceTweetsInParallel) // candidates + source tweets in parallel - .andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones - .andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter - .andThen(dynamicHydratedTweetsFilter) // filter hydrated tweets based on query TweetKindOption - .andThen(TrimToMatchHydratedTweetsTransform) // trim searchResult to match with hydratedTweets - .andThen( - finalTruncationTransform - ) // truncate the searchResult to exactly up to maxCount, preserving the random tweet - .andThen( - trimToMatchSearchResultsTransform - ) // trim other fields to match with the final searchResult - - // runs the main pipeline in parallel with fetching user profile info and user languages - private[this] val featureHydrationDataTransform = new FeatureHydrationDataTransform( - hydrationAndFilteringPipeline, - languagesTransform, - userProfileInfoTransform - ) - - private[this] val featureHydrationPipeline = - featureHydrationDataTransform - .andThen(InNetworkTweetsSearchFeaturesHydrationTransform) - .andThen(contentFeaturesTransformer) - .andThen(candidateGenerationTransform) - - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - requestStats.addEventStats { - featureHydrationPipeline(query) - } - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - Future.collect(queries.map(get)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/BUILD deleted file mode 100644 index 90defcc0c..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/thrift:libthrift", - "scrooge/scrooge-core/src/main/scala", - "servo/util", - "src/thrift/com/twitter/search:earlybird-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/BUILD.docx new file mode 100644 index 000000000..3af51086a Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/UsersSearchResultMonitoringTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/UsersSearchResultMonitoringTransform.docx new file mode 100644 index 000000000..72e70c34e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/UsersSearchResultMonitoringTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/UsersSearchResultMonitoringTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/UsersSearchResultMonitoringTransform.scala deleted file mode 100644 index d0968a9b5..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/UsersSearchResultMonitoringTransform.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.timelineranker.monitoring - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.util.Future - -/** - * Captures tweet counts pre and post transformation for a list of users - */ -class UsersSearchResultMonitoringTransform( - name: String, - underlyingTransformer: FutureArrow[CandidateEnvelope, CandidateEnvelope], - statsReceiver: StatsReceiver, - debugAuthorListDependencyProvider: DependencyProvider[Seq[Long]]) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - private val scopedStatsReceiver = statsReceiver.scope(name) - private val preTransformCounter = (userId: Long) => - scopedStatsReceiver - .scope("pre_transform").scope(userId.toString).counter("debug_author_list") - private val postTransformCounter = (userId: Long) => - scopedStatsReceiver - .scope("post_transform").scope(userId.toString).counter("debug_author_list") - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val debugAuthorList = debugAuthorListDependencyProvider.apply(envelope.query) - envelope.searchResults - .filter(isTweetFromDebugAuthorList(_, debugAuthorList)) - .flatMap(_.metadata) - .foreach(metadata => preTransformCounter(metadata.fromUserId).incr()) - - underlyingTransformer - .apply(envelope) - .map { result => - envelope.searchResults - .filter(isTweetFromDebugAuthorList(_, debugAuthorList)) - .flatMap(_.metadata) - .foreach(metadata => postTransformCounter(metadata.fromUserId).incr()) - result - } - } - - private def isTweetFromDebugAuthorList( - searchResult: ThriftSearchResult, - debugAuthorList: Seq[Long] - ): Boolean = - searchResult.metadata.exists(metadata => debugAuthorList.contains(metadata.fromUserId)) - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/BUILD.bazel b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/BUILD.bazel deleted file mode 100644 index 66b19aefc..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/thrift/com/twitter/timelineranker/server/model:thrift-scala", - "timelineranker/common:model", - "timelines:observe", - "timelines/src/main/scala/com/twitter/timelines/authorization", - "timelines/src/main/scala/com/twitter/timelines/features", - "timelines/src/main/scala/com/twitter/timelines/model/types", - "util/util-core:util-core-util", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/BUILD.docx new file mode 100644 index 000000000..9f438697f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/DebugObserverBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/DebugObserverBuilder.docx new file mode 100644 index 000000000..6e3013161 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/DebugObserverBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/DebugObserverBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/DebugObserverBuilder.scala deleted file mode 100644 index 43598d7fb..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/DebugObserverBuilder.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.timelineranker.observe - -import com.twitter.servo.util.Gate -import com.twitter.timelineranker.model.TimelineQuery -import com.twitter.timelines.features.Features -import com.twitter.timelines.features.UserList -import com.twitter.timelines.observe.DebugObserver -import com.twitter.timelineranker.{thriftscala => thrift} - -/** - * Builds the DebugObserver that is attached to thrift requests. - * This class exists to centralize the gates that determine whether or not - * to enable debug transcripts for a particular request. - */ -class DebugObserverBuilder(whitelist: UserList) { - - lazy val observer: DebugObserver = build() - - private[this] def build(): DebugObserver = { - new DebugObserver(queryGate) - } - - private[observe] def queryGate: Gate[Any] = { - val shouldEnableDebug = whitelist.userIdGate(Features.DebugTranscript) - - Gate { a: Any => - a match { - case q: thrift.EngagedTweetsQuery => shouldEnableDebug(q.userId) - case q: thrift.RecapHydrationQuery => shouldEnableDebug(q.userId) - case q: thrift.RecapQuery => shouldEnableDebug(q.userId) - case q: TimelineQuery => shouldEnableDebug(q.userId) - case _ => false - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/ObservedRequests.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/ObservedRequests.docx new file mode 100644 index 000000000..a31754f97 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/ObservedRequests.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/ObservedRequests.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/ObservedRequests.scala deleted file mode 100644 index c73600845..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/ObservedRequests.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.timelineranker.observe - -import com.twitter.timelines.authorization.ReadRequest -import com.twitter.timelines.model.UserId -import com.twitter.timelines.observe.ObservedAndValidatedRequests -import com.twitter.timelines.observe.ServiceObserver -import com.twitter.timelines.observe.ServiceTracer -import com.twitter.util.Future - -trait ObservedRequests extends ObservedAndValidatedRequests { - - def observeAndValidate[R, Q]( - request: Q, - viewerIds: Seq[UserId], - stats: ServiceObserver.Stats[Q], - exceptionHandler: PartialFunction[Throwable, Future[R]] - )( - f: Q => Future[R] - ): Future[R] = { - super.observeAndValidate[Q, R]( - request, - viewerIds, - ReadRequest, - validateRequest, - exceptionHandler, - stats, - ServiceTracer.identity[Q] - )(f) - } - - def validateRequest[Q](request: Q): Unit = { - // TimelineQuery and its derived classes do not permit invalid instances to be constructed. - // Therefore no additional validation is required. - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/BUILD deleted file mode 100644 index b60f8c7ce..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "servo/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/BUILD.docx new file mode 100644 index 000000000..5b99c9a9e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/ConfigBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/ConfigBuilder.docx new file mode 100644 index 000000000..7dd2d54bc Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/ConfigBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/ConfigBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/ConfigBuilder.scala deleted file mode 100644 index 2698fe818..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/ConfigBuilder.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.timelineranker.parameters - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsProduction -import com.twitter.timelineranker.parameters.recap.RecapProduction -import com.twitter.timelineranker.parameters.recap_author.RecapAuthorProduction -import com.twitter.timelineranker.parameters.recap_hydration.RecapHydrationProduction -import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetProduction -import com.twitter.timelineranker.parameters.revchron.ReverseChronProduction -import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsProduction -import com.twitter.timelineranker.parameters.monitoring.MonitoringProduction -import com.twitter.timelines.configapi.CompositeConfig -import com.twitter.timelines.configapi.Config - -/** - * Builds global composite config containing prioritized "layers" of parameter overrides - * based on whitelists, experiments, and deciders. Generated config can be used in tests with - * mocked decider and whitelist. - */ -class ConfigBuilder(deciderGateBuilder: DeciderGateBuilder, statsReceiver: StatsReceiver) { - - /** - * Production config which includes all configs which contribute to production behavior. At - * minimum, it should include all configs containing decider-based param overrides. - * - * It is important that the production config include all production param overrides as it is - * used to build holdback experiment configs; If the production config doesn't include all param - * overrides supporting production behavior then holdback experiment "production" buckets will - * not reflect production behavior. - */ - val prodConfig: Config = new CompositeConfig( - Seq( - new RecapProduction(deciderGateBuilder, statsReceiver).config, - new InNetworkTweetProduction(deciderGateBuilder).config, - new ReverseChronProduction(deciderGateBuilder).config, - new EntityTweetsProduction(deciderGateBuilder).config, - new RecapAuthorProduction(deciderGateBuilder).config, - new RecapHydrationProduction(deciderGateBuilder).config, - new UtegLikedByTweetsProduction(deciderGateBuilder).config, - MonitoringProduction.config - ), - "prodConfig" - ) - - val whitelistConfig: Config = new CompositeConfig( - Seq( - // No whitelists configured at present. - ), - "whitelistConfig" - ) - - val rootConfig: Config = new CompositeConfig( - Seq( - whitelistConfig, - prodConfig - ), - "rootConfig" - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/BUILD deleted file mode 100644 index 74953405d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", - "servo/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/BUILD.docx new file mode 100644 index 000000000..305404fb0 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsParams.docx new file mode 100644 index 000000000..87c743a6d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsParams.scala deleted file mode 100644 index c269093f3..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsParams.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.timelineranker.parameters.entity_tweets - -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelines.configapi.decider.DeciderParam -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object EntityTweetsParams { - - /** - * Controls limit on the number of followed users fetched from SGS. - */ - object MaxFollowedUsersParam - extends FSBoundedParam[Int]( - name = "entity_tweets_max_followed_users", - default = 1000, - min = 0, - max = 5000 - ) - - /** - * Enables semantic core, penguin, and tweetypie content features in entity tweets source. - */ - object EnableContentFeaturesHydrationParam - extends DeciderParam[Boolean]( - decider = DeciderKey.EntityTweetsEnableContentFeaturesHydration, - default = false - ) - - /** - * additionally enables tokens when hydrating content features. - */ - object EnableTokensInContentFeaturesHydrationParam - extends FSParam( - name = "entity_tweets_enable_tokens_in_content_features_hydration", - default = false - ) - - /** - * additionally enables tweet text when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableTweetTextInContentFeaturesHydrationParam - extends FSParam( - name = "entity_tweets_enable_tweet_text_in_content_features_hydration", - default = false - ) - - /** - * additionally enables conversationControl when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableConversationControlInContentFeaturesHydrationParam - extends FSParam( - name = "conversation_control_in_content_features_hydration_entity_enable", - default = false - ) - - object EnableTweetMediaHydrationParam - extends FSParam( - name = "tweet_media_hydration_entity_tweets_enable", - default = false - ) - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsProduction.docx new file mode 100644 index 000000000..45872bab2 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsProduction.scala deleted file mode 100644 index 9f0a44b5a..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsProduction.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.timelineranker.parameters.entity_tweets - -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsParams._ -import com.twitter.timelines.configapi.decider.DeciderUtils -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object EntityTweetsProduction { - val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName]( - EnableContentFeaturesHydrationParam -> DeciderKey.EntityTweetsEnableContentFeaturesHydration - ) -} - -case class EntityTweetsProduction(deciderGateBuilder: DeciderGateBuilder) { - - val booleanDeciderOverrides = DeciderUtils.getBooleanDeciderOverrides( - deciderGateBuilder, - EnableContentFeaturesHydrationParam - ) - - val booleanFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableTokensInContentFeaturesHydrationParam, - EnableTweetTextInContentFeaturesHydrationParam, - EnableConversationControlInContentFeaturesHydrationParam, - EnableTweetMediaHydrationParam - ) - - val intFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxFollowedUsersParam - ) - - val config: BaseConfig = new BaseConfigBuilder() - .set(booleanDeciderOverrides: _*) - .set(booleanFeatureSwitchOverrides: _*) - .set(intFeatureSwitchOverrides: _*) - .build() -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/BUILD deleted file mode 100644 index 0d3eee95d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", - "servo/decider", - "servo/util/src/main/scala", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/BUILD.docx new file mode 100644 index 000000000..2318a5bc0 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetParams.docx new file mode 100644 index 000000000..2f649c6e6 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetParams.scala deleted file mode 100644 index c869a4f44..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetParams.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.timelineranker.parameters.in_network_tweets - -import com.twitter.timelineranker.parameters.recap.RecapQueryContext -import com.twitter.timelines.configapi.decider._ -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object InNetworkTweetParams { - import RecapQueryContext._ - - /** - * Controls limit on the number of followed users fetched from SGS. - * - * The specific default value below is for blender-timelines parity. - */ - object MaxFollowedUsersParam - extends FSBoundedParam[Int]( - name = "recycled_max_followed_users", - default = MaxFollowedUsers.default, - min = MaxFollowedUsers.bounds.minInclusive, - max = MaxFollowedUsers.bounds.maxInclusive - ) - - /** - * Controls limit on the number of hits for Earlybird. - * - */ - object RelevanceOptionsMaxHitsToProcessParam - extends FSBoundedParam[Int]( - name = "recycled_relevance_options_max_hits_to_process", - default = 500, - min = 100, - max = 20000 - ) - - /** - * Fallback value for maximum number of search results, if not specified by query.maxCount - */ - object DefaultMaxTweetCount extends Param(200) - - /** - * We multiply maxCount (caller supplied value) by this multiplier and fetch those many - * candidates from search so that we are left with sufficient number of candidates after - * hydration and filtering. - */ - object MaxCountMultiplierParam - extends Param(MaxCountMultiplier.default) - with DeciderValueConverter[Double] { - override def convert: IntConverter[Double] = - OutputBoundIntConverter[Double](divideDeciderBy100 _, MaxCountMultiplier.bounds) - } - - /** - * Enable [[SearchQueryBuilder.createExcludedSourceTweetIdsQuery]] - */ - object EnableExcludeSourceTweetIdsQueryParam - extends FSParam[Boolean]( - name = "recycled_exclude_source_tweet_ids_query_enable", - default = false - ) - - object EnableEarlybirdReturnAllResultsParam - extends FSParam[Boolean]( - name = "recycled_enable_earlybird_return_all_results", - default = true - ) - - /** - * FS-controlled param to enable anti-dilution transform for DDG-16198 - */ - object RecycledMaxFollowedUsersEnableAntiDilutionParam - extends FSParam[Boolean]( - name = "recycled_max_followed_users_enable_anti_dilution", - default = false - ) - - /** - * Enables semantic core, penguin, and tweetypie content features in recycled source. - */ - object EnableContentFeaturesHydrationParam extends Param(default = true) - - /** - * additionally enables tokens when hydrating content features. - */ - object EnableTokensInContentFeaturesHydrationParam - extends FSParam( - name = "recycled_enable_tokens_in_content_features_hydration", - default = false - ) - - /** - * additionally enables tweet text when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableTweetTextInContentFeaturesHydrationParam - extends FSParam( - name = "recycled_enable_tweet_text_in_content_features_hydration", - default = false - ) - - /** - * Enables hydrating root tweet of in-network replies and extended replies - */ - object EnableReplyRootTweetHydrationParam - extends FSParam( - name = "recycled_enable_reply_root_tweet_hydration", - default = true - ) - - /** - * additionally enables conversationControl when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableConversationControlInContentFeaturesHydrationParam - extends FSParam( - name = "conversation_control_in_content_features_hydration_recycled_enable", - default = false - ) - - object EnableTweetMediaHydrationParam - extends FSParam( - name = "tweet_media_hydration_recycled_enable", - default = false - ) - - object EnableEarlybirdRealtimeCgMigrationParam - extends FSParam( - name = "recycled_enable_earlybird_realtime_cg_migration", - default = false - ) - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetProduction.docx new file mode 100644 index 000000000..b9b091f18 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetProduction.scala deleted file mode 100644 index bf48c1701..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetProduction.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.timelineranker.parameters.in_network_tweets - -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams._ -import com.twitter.timelineranker.parameters.util.ConfigHelper -import com.twitter.timelines.configapi._ -import com.twitter.servo.decider.DeciderKeyEnum - -object InNetworkTweetProduction { - val deciderByParam: Map[Param[_], DeciderKeyEnum#Value] = Map[Param[_], DeciderKeyName]( - EnableContentFeaturesHydrationParam -> DeciderKey.RecycledEnableContentFeaturesHydration, - MaxCountMultiplierParam -> DeciderKey.RecycledMaxCountMultiplier - ) - - val doubleParams: Seq[MaxCountMultiplierParam.type] = Seq( - MaxCountMultiplierParam - ) - - val booleanDeciderParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq( - EnableContentFeaturesHydrationParam - ) - - val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq( - EnableExcludeSourceTweetIdsQueryParam, - EnableTokensInContentFeaturesHydrationParam, - EnableReplyRootTweetHydrationParam, - EnableTweetTextInContentFeaturesHydrationParam, - EnableConversationControlInContentFeaturesHydrationParam, - EnableTweetMediaHydrationParam, - EnableEarlybirdReturnAllResultsParam, - EnableEarlybirdRealtimeCgMigrationParam, - RecycledMaxFollowedUsersEnableAntiDilutionParam - ) - - val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq( - MaxFollowedUsersParam, - RelevanceOptionsMaxHitsToProcessParam - ) -} - -class InNetworkTweetProduction(deciderGateBuilder: DeciderGateBuilder) { - val configHelper: ConfigHelper = - new ConfigHelper(InNetworkTweetProduction.deciderByParam, deciderGateBuilder) - val doubleDeciderOverrides: Seq[OptionalOverride[Double]] = - configHelper.createDeciderBasedOverrides(InNetworkTweetProduction.doubleParams) - val booleanDeciderOverrides: Seq[OptionalOverride[Boolean]] = - configHelper.createDeciderBasedBooleanOverrides(InNetworkTweetProduction.booleanDeciderParams) - val boundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Int]] = - FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - InNetworkTweetProduction.boundedIntFeatureSwitchParams: _*) - val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] = - FeatureSwitchOverrideUtil.getBooleanFSOverrides( - InNetworkTweetProduction.booleanFeatureSwitchParams: _*) - - val config: BaseConfig = new BaseConfigBuilder() - .set( - booleanDeciderOverrides: _* - ) - .set( - doubleDeciderOverrides: _* - ) - .set( - boundedIntFeatureSwitchOverrides: _* - ) - .set( - booleanFeatureSwitchOverrides: _* - ) - .build(InNetworkTweetProduction.getClass.getSimpleName) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/BUILD deleted file mode 100644 index 8bf1d2ecc..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "servo/decider", - "servo/util/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/BUILD.docx new file mode 100644 index 000000000..69353df34 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringParams.docx new file mode 100644 index 000000000..108f172fd Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringParams.scala deleted file mode 100644 index 62d9bb64e..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringParams.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.timelineranker.parameters.monitoring - -import com.twitter.timelines.configapi.FSParam - -object MonitoringParams { - - object DebugAuthorsAllowListParam - extends FSParam[Seq[Long]]( - name = "monitoring_debug_authors_allow_list", - default = Seq.empty[Long] - ) - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringProduction.docx new file mode 100644 index 000000000..edba1555f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringProduction.scala deleted file mode 100644 index 7a6417cad..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringProduction.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.timelineranker.parameters.monitoring - -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelineranker.parameters.monitoring.MonitoringParams.DebugAuthorsAllowListParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil - -object MonitoringProduction { - private val longSeqOverrides = - FeatureSwitchOverrideUtil.getLongSeqFSOverrides(DebugAuthorsAllowListParam) - - val config = BaseConfigBuilder() - .set(longSeqOverrides: _*) - .build() -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/BUILD deleted file mode 100644 index a457e03b6..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider", - "servo/decider", - "servo/util/src/main/scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - "timelines/src/main/scala/com/twitter/timelines/experiment", - "util/util-logging", - "util/util-stats", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/BUILD.docx new file mode 100644 index 000000000..0e20d449f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapParams.docx new file mode 100644 index 000000000..5bc78352d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapParams.scala deleted file mode 100644 index 050ee22d1..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapParams.scala +++ /dev/null @@ -1,231 +0,0 @@ -package com.twitter.timelineranker.parameters.recap - -import com.twitter.timelines.configapi.decider._ -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.util.bounds.BoundsWithDefault - -object RecapParams { - val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 3000, 1000) - val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.1, 2.0, 2.0) - val MaxRealGraphAndFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, 2000, 1000) - val ProbabilityRandomTweet: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.0, 1.0, 0.0) - - /** - * Controls limit on the number of followed users fetched from SGS. - * - * The specific default value below is for blender-timelines parity. - */ - object MaxFollowedUsersParam - extends FSBoundedParam[Int]( - name = "recap_max_followed_users", - default = MaxFollowedUsers.default, - min = MaxFollowedUsers.bounds.minInclusive, - max = MaxFollowedUsers.bounds.maxInclusive - ) - - /** - * Controls limit on the number of hits for Earlybird. - * We added it solely for backward compatibility, to align with recycled. - * RecapSource is deprecated, but, this param is used by RecapAuthor source - */ - object RelevanceOptionsMaxHitsToProcessParam - extends FSBoundedParam[Int]( - name = "recap_relevance_options_max_hits_to_process", - default = 500, - min = 100, - max = 20000 - ) - - /** - * Enables fetching author seedset from real graph users. Only used if user follows >= 1000. - * If true, expands author seedset with real graph users and recent followed users. - * Otherwise, user seedset only includes followed users. - */ - object EnableRealGraphUsersParam extends Param(false) - - /** - * Only used if EnableRealGraphUsersParam is true and OnlyRealGraphUsersParam is false. - * Maximum number of real graph users and recent followed users when mixing recent/real-graph users. - */ - object MaxRealGraphAndFollowedUsersParam - extends Param(MaxRealGraphAndFollowedUsers.default) - with DeciderValueConverter[Int] { - override def convert: IntConverter[Int] = - OutputBoundIntConverter(MaxRealGraphAndFollowedUsers.bounds) - } - - /** - * FS-controlled param to override the MaxRealGraphAndFollowedUsersParam decider value for experiments - */ - object MaxRealGraphAndFollowedUsersFSOverrideParam - extends FSBoundedParam[Option[Int]]( - name = "max_real_graph_and_followers_users_fs_override_param", - default = None, - min = Some(100), - max = Some(10000) - ) - - /** - * Experimental params for leveling the playing field between user folowees received from - * real-graph and follow-graph stores. - * Author relevance scores returned by real-graph are currently being used for light-ranking - * in-network tweet candidates. - * Follow-graph store returns the most recent followees without any relevance scores - * We are trying to impute the missing scores by using aggregated statistics (min, avg, p50, etc.) - * of real-graph scores. - */ - object ImputeRealGraphAuthorWeightsParam - extends FSParam(name = "impute_real_graph_author_weights", default = false) - - object ImputeRealGraphAuthorWeightsPercentileParam - extends FSBoundedParam[Int]( - name = "impute_real_graph_author_weights_percentile", - default = 50, - min = 0, - max = 99) - - /** - * Enable running the new pipeline for recap author source - */ - object EnableNewRecapAuthorPipeline extends Param(false) - - /** - * Fallback value for maximum number of search results, if not specified by query.maxCount - */ - object DefaultMaxTweetCount extends Param(200) - - /** - * We multiply maxCount (caller supplied value) by this multiplier and fetch those many - * candidates from search so that we are left with sufficient number of candidates after - * hydration and filtering. - */ - object MaxCountMultiplierParam - extends Param(MaxCountMultiplier.default) - with DeciderValueConverter[Double] { - override def convert: IntConverter[Double] = - OutputBoundIntConverter[Double](divideDeciderBy100 _, MaxCountMultiplier.bounds) - } - - /** - * Enables return all results from search index. - */ - object EnableReturnAllResultsParam - extends FSParam(name = "recap_enable_return_all_results", default = false) - - /** - * Includes one or multiple random tweets in the response. - */ - object IncludeRandomTweetParam - extends FSParam(name = "recap_include_random_tweet", default = false) - - /** - * One single random tweet (true) or tag tweet as random with given probability (false). - */ - object IncludeSingleRandomTweetParam - extends FSParam(name = "recap_include_random_tweet_single", default = true) - - /** - * Probability to tag a tweet as random (will not be ranked). - */ - object ProbabilityRandomTweetParam - extends FSBoundedParam( - name = "recap_include_random_tweet_probability", - default = ProbabilityRandomTweet.default, - min = ProbabilityRandomTweet.bounds.minInclusive, - max = ProbabilityRandomTweet.bounds.maxInclusive) - - /** - * Enable extra sorting by score for search results. - */ - object EnableExtraSortingInSearchResultParam extends Param(true) - - /** - * Enables semantic core, penguin, and tweetypie content features in recap source. - */ - object EnableContentFeaturesHydrationParam extends Param(true) - - /** - * additionally enables tokens when hydrating content features. - */ - object EnableTokensInContentFeaturesHydrationParam - extends FSParam( - name = "recap_enable_tokens_in_content_features_hydration", - default = false - ) - - /** - * additionally enables tweet text when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableTweetTextInContentFeaturesHydrationParam - extends FSParam( - name = "recap_enable_tweet_text_in_content_features_hydration", - default = false - ) - - /** - * Enables hydrating in-network inReplyToTweet features - */ - object EnableInNetworkInReplyToTweetFeaturesHydrationParam - extends FSParam( - name = "recap_enable_in_network_in_reply_to_tweet_features_hydration", - default = false - ) - - /** - * Enables hydrating root tweet of in-network replies and extended replies - */ - object EnableReplyRootTweetHydrationParam - extends FSParam( - name = "recap_enable_reply_root_tweet_hydration", - default = false - ) - - /** - * Enable setting tweetTypes in search queries with TweetKindOption in RecapQuery - */ - object EnableSettingTweetTypesWithTweetKindOption - extends FSParam( - name = "recap_enable_setting_tweet_types_with_tweet_kind_option", - default = false - ) - - /** - * Enable relevance search, otherwise recency search from earlybird. - */ - object EnableRelevanceSearchParam - extends FSParam( - name = "recap_enable_relevance_search", - default = true - ) - - object EnableExpandedExtendedRepliesFilterParam - extends FSParam( - name = "recap_enable_expanded_extended_replies_filter", - default = false - ) - - /** - * additionally enables conversationControl when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableConversationControlInContentFeaturesHydrationParam - extends FSParam( - name = "conversation_control_in_content_features_hydration_recap_enable", - default = false - ) - - object EnableTweetMediaHydrationParam - extends FSParam( - name = "tweet_media_hydration_recap_enable", - default = false - ) - - object EnableExcludeSourceTweetIdsQueryParam - extends FSParam[Boolean]( - name = "recap_exclude_source_tweet_ids_query_enable", - default = false - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapProduction.docx new file mode 100644 index 000000000..3cad8064f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapProduction.scala deleted file mode 100644 index 0d5625181..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapProduction.scala +++ /dev/null @@ -1,115 +0,0 @@ -package com.twitter.timelineranker.parameters.recap - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.parameters.recap.RecapParams._ -import com.twitter.timelineranker.parameters.util.ConfigHelper -import com.twitter.timelines.configapi._ -import com.twitter.servo.decider.DeciderKeyEnum - -object RecapProduction { - val deciderByParam: Map[Param[_], DeciderKeyEnum#Value] = Map[Param[_], DeciderKeyName]( - EnableRealGraphUsersParam -> DeciderKey.EnableRealGraphUsers, - MaxRealGraphAndFollowedUsersParam -> DeciderKey.MaxRealGraphAndFollowedUsers, - EnableContentFeaturesHydrationParam -> DeciderKey.RecapEnableContentFeaturesHydration, - MaxCountMultiplierParam -> DeciderKey.RecapMaxCountMultiplier, - EnableNewRecapAuthorPipeline -> DeciderKey.RecapAuthorEnableNewPipeline, - RecapParams.EnableExtraSortingInSearchResultParam -> DeciderKey.RecapEnableExtraSortingInResults - ) - - val intParams: Seq[MaxRealGraphAndFollowedUsersParam.type] = Seq( - MaxRealGraphAndFollowedUsersParam - ) - - val doubleParams: Seq[MaxCountMultiplierParam.type] = Seq( - MaxCountMultiplierParam - ) - - val boundedDoubleFeatureSwitchParams: Seq[FSBoundedParam[Double]] = Seq( - RecapParams.ProbabilityRandomTweetParam - ) - - val booleanParams: Seq[Param[Boolean]] = Seq( - EnableRealGraphUsersParam, - EnableContentFeaturesHydrationParam, - EnableNewRecapAuthorPipeline, - RecapParams.EnableExtraSortingInSearchResultParam - ) - - val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq( - RecapParams.EnableReturnAllResultsParam, - RecapParams.IncludeRandomTweetParam, - RecapParams.IncludeSingleRandomTweetParam, - RecapParams.EnableInNetworkInReplyToTweetFeaturesHydrationParam, - RecapParams.EnableReplyRootTweetHydrationParam, - RecapParams.EnableSettingTweetTypesWithTweetKindOption, - RecapParams.EnableRelevanceSearchParam, - EnableTokensInContentFeaturesHydrationParam, - EnableTweetTextInContentFeaturesHydrationParam, - EnableExpandedExtendedRepliesFilterParam, - EnableConversationControlInContentFeaturesHydrationParam, - EnableTweetMediaHydrationParam, - ImputeRealGraphAuthorWeightsParam, - EnableExcludeSourceTweetIdsQueryParam - ) - - val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq( - RecapParams.MaxFollowedUsersParam, - ImputeRealGraphAuthorWeightsPercentileParam, - RecapParams.RelevanceOptionsMaxHitsToProcessParam - ) -} - -class RecapProduction(deciderGateBuilder: DeciderGateBuilder, statsReceiver: StatsReceiver) { - - val configHelper: ConfigHelper = - new ConfigHelper(RecapProduction.deciderByParam, deciderGateBuilder) - val intOverrides: Seq[OptionalOverride[Int]] = - configHelper.createDeciderBasedOverrides(RecapProduction.intParams) - val optionalBoundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Option[Int]]] = - FeatureSwitchOverrideUtil.getBoundedOptionalIntOverrides( - ( - MaxRealGraphAndFollowedUsersFSOverrideParam, - "max_real_graph_and_followers_users_fs_override_defined", - "max_real_graph_and_followers_users_fs_override_value" - ) - ) - val doubleOverrides: Seq[OptionalOverride[Double]] = - configHelper.createDeciderBasedOverrides(RecapProduction.doubleParams) - val booleanOverrides: Seq[OptionalOverride[Boolean]] = - configHelper.createDeciderBasedBooleanOverrides(RecapProduction.booleanParams) - val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] = - FeatureSwitchOverrideUtil.getBooleanFSOverrides(RecapProduction.booleanFeatureSwitchParams: _*) - val boundedDoubleFeatureSwitchOverrides: Seq[OptionalOverride[Double]] = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - RecapProduction.boundedDoubleFeatureSwitchParams: _*) - val boundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Int]] = - FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - RecapProduction.boundedIntFeatureSwitchParams: _*) - - val config: BaseConfig = new BaseConfigBuilder() - .set( - intOverrides: _* - ) - .set( - booleanOverrides: _* - ) - .set( - doubleOverrides: _* - ) - .set( - booleanFeatureSwitchOverrides: _* - ) - .set( - boundedIntFeatureSwitchOverrides: _* - ) - .set( - optionalBoundedIntFeatureSwitchOverrides: _* - ) - .set( - boundedDoubleFeatureSwitchOverrides: _* - ) - .build(RecapProduction.getClass.getSimpleName) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapQueryContext.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapQueryContext.docx new file mode 100644 index 000000000..1b1866cc7 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapQueryContext.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapQueryContext.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapQueryContext.scala deleted file mode 100644 index a69fb8033..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapQueryContext.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.timelineranker.parameters.recap - -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelines.util.bounds.BoundsWithDefault - -object RecapQueryContext { - val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 3000, 1000) - val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.1, 2.0, 2.0) - val MaxRealGraphAndFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, 2000, 1000) - - def getDefaultContext(query: RecapQuery): RecapQueryContext = { - new RecapQueryContextImpl( - query, - getEnableHydrationUsingTweetyPie = () => false, - getMaxFollowedUsers = () => MaxFollowedUsers.default, - getMaxCountMultiplier = () => MaxCountMultiplier.default, - getEnableRealGraphUsers = () => false, - getOnlyRealGraphUsers = () => false, - getMaxRealGraphAndFollowedUsers = () => MaxRealGraphAndFollowedUsers.default, - getEnableTextFeatureHydration = () => false - ) - } -} - -// Note that methods that return parameter value always use () to indicate that -// side effects may be involved in their invocation. -trait RecapQueryContext { - def query: RecapQuery - - // If true, tweet hydration are performed by calling TweetyPie. - // Otherwise, tweets are partially hydrated based on information in ThriftSearchResult. - def enableHydrationUsingTweetyPie(): Boolean - - // Maximum number of followed user accounts to use when fetching recap tweets. - def maxFollowedUsers(): Int - - // We multiply maxCount (caller supplied value) by this multiplier and fetch those many - // candidates from search so that we are left with sufficient number of candidates after - // hydration and filtering. - def maxCountMultiplier(): Double - - // Only used if user follows >= 1000. - // If true, fetches recap/recycled tweets using author seedset mixing with real graph users and followed users. - // Otherwise, fetches recap/recycled tweets only using followed users - def enableRealGraphUsers(): Boolean - - // Only used if enableRealGraphUsers is true. - // If true, user seedset only contains real graph users. - // Otherwise, user seedset contains real graph users and recent followed users. - def onlyRealGraphUsers(): Boolean - - // Only used if enableRealGraphUsers is true and onlyRealGraphUsers is false. - // Maximum number of real graph users and recent followed users when mixing recent/real-graph users. - def maxRealGraphAndFollowedUsers(): Int - - // If true, text features are hydrated for prediction. - // Otherwise those feature values are not set at all. - def enableTextFeatureHydration(): Boolean -} - -class RecapQueryContextImpl( - override val query: RecapQuery, - getEnableHydrationUsingTweetyPie: () => Boolean, - getMaxFollowedUsers: () => Int, - getMaxCountMultiplier: () => Double, - getEnableRealGraphUsers: () => Boolean, - getOnlyRealGraphUsers: () => Boolean, - getMaxRealGraphAndFollowedUsers: () => Int, - getEnableTextFeatureHydration: () => Boolean) - extends RecapQueryContext { - - override def enableHydrationUsingTweetyPie(): Boolean = { getEnableHydrationUsingTweetyPie() } - override def maxFollowedUsers(): Int = { getMaxFollowedUsers() } - override def maxCountMultiplier(): Double = { getMaxCountMultiplier() } - override def enableRealGraphUsers(): Boolean = { getEnableRealGraphUsers() } - override def onlyRealGraphUsers(): Boolean = { getOnlyRealGraphUsers() } - override def maxRealGraphAndFollowedUsers(): Int = { getMaxRealGraphAndFollowedUsers() } - override def enableTextFeatureHydration(): Boolean = { getEnableTextFeatureHydration() } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/BUILD deleted file mode 100644 index 256daa9c7..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "servo/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/BUILD.docx new file mode 100644 index 000000000..ed3cc6128 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorParams.docx new file mode 100644 index 000000000..3f6f0c535 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorParams.scala deleted file mode 100644 index e3f336ca9..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorParams.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.timelineranker.parameters.recap_author - -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object RecapAuthorParams { - - /** - * Enables semantic core, penguin, and tweetypie content features in recap author source. - */ - object EnableContentFeaturesHydrationParam extends Param(false) - - /** - * additionally enables tokens when hydrating content features. - */ - object EnableTokensInContentFeaturesHydrationParam - extends FSParam( - name = "recap_author_enable_tokens_in_content_features_hydration", - default = false - ) - - /** - * additionally enables tweet text when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableTweetTextInContentFeaturesHydrationParam - extends FSParam( - name = "recap_author_enable_tweet_text_in_content_features_hydration", - default = false - ) - - object EnableEarlybirdRealtimeCgMigrationParam - extends FSParam( - name = "recap_author_enable_earlybird_realtime_cg_migration", - default = false - ) - /** - * additionally enables conversationControl when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableConversationControlInContentFeaturesHydrationParam - extends FSParam( - name = "conversation_control_in_content_features_hydration_recap_author_enable", - default = false - ) - - object EnableTweetMediaHydrationParam - extends FSParam( - name = "tweet_media_hydration_recap_author_enable", - default = false - ) - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorProduction.docx new file mode 100644 index 000000000..0c066e9ec Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorProduction.scala deleted file mode 100644 index 89908c28a..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorProduction.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.timelineranker.parameters.recap_author - -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.parameters.recap_author.RecapAuthorParams._ -import com.twitter.timelineranker.parameters.util.ConfigHelper -import com.twitter.timelines.configapi._ - -object RecapAuthorProduction { - val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName]( - EnableContentFeaturesHydrationParam -> DeciderKey.RecapAuthorEnableContentFeaturesHydration - ) - - val booleanParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq( - EnableContentFeaturesHydrationParam - ) - - val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq( - EnableTokensInContentFeaturesHydrationParam, - EnableTweetTextInContentFeaturesHydrationParam, - EnableConversationControlInContentFeaturesHydrationParam, - EnableTweetMediaHydrationParam, - EnableEarlybirdRealtimeCgMigrationParam - ) -} - -class RecapAuthorProduction(deciderGateBuilder: DeciderGateBuilder) { - val configHelper: ConfigHelper = - new ConfigHelper(RecapAuthorProduction.deciderByParam, deciderGateBuilder) - val booleanOverrides: Seq[OptionalOverride[Boolean]] = - configHelper.createDeciderBasedBooleanOverrides(RecapAuthorProduction.booleanParams) - - val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] = - FeatureSwitchOverrideUtil.getBooleanFSOverrides( - RecapAuthorProduction.booleanFeatureSwitchParams: _* - ) - - val config: BaseConfig = new BaseConfigBuilder() - .set( - booleanOverrides: _* - ).set( - booleanFeatureSwitchOverrides: _* - ) - .build(RecapAuthorProduction.getClass.getSimpleName) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/BUILD deleted file mode 100644 index 256daa9c7..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "servo/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/BUILD.docx new file mode 100644 index 000000000..ed3cc6128 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationParams.docx new file mode 100644 index 000000000..d5b41dbef Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationParams.scala deleted file mode 100644 index 454fbb5af..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationParams.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.timelineranker.parameters.recap_hydration - -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object RecapHydrationParams { - - /** - * Enables semantic core, penguin, and tweetypie content features in recap hydration source. - */ - object EnableContentFeaturesHydrationParam extends Param(false) - - /** - * additionally enables tokens when hydrating content features. - */ - object EnableTokensInContentFeaturesHydrationParam - extends FSParam( - name = "recap_hydration_enable_tokens_in_content_features_hydration", - default = false - ) - - /** - * additionally enables tweet text when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableTweetTextInContentFeaturesHydrationParam - extends FSParam( - name = "recap_hydration_enable_tweet_text_in_content_features_hydration", - default = false - ) - - /** - * additionally enables conversationControl when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableConversationControlInContentFeaturesHydrationParam - extends FSParam( - name = "conversation_control_in_content_features_hydration_recap_hydration_enable", - default = false - ) - - object EnableTweetMediaHydrationParam - extends FSParam( - name = "tweet_media_hydration_recap_hydration_enable", - default = false - ) - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationProduction.docx new file mode 100644 index 000000000..0f37110c9 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationProduction.scala deleted file mode 100644 index 9d3d596d6..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationProduction.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.timelineranker.parameters.recap_hydration - -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.parameters.recap_hydration.RecapHydrationParams._ -import com.twitter.timelineranker.parameters.util.ConfigHelper -import com.twitter.timelines.configapi._ - -object RecapHydrationProduction { - val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName]( - EnableContentFeaturesHydrationParam -> DeciderKey.RecapHydrationEnableContentFeaturesHydration - ) - - val booleanParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq( - EnableContentFeaturesHydrationParam - ) - - val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq( - EnableTokensInContentFeaturesHydrationParam, - EnableTweetTextInContentFeaturesHydrationParam, - EnableConversationControlInContentFeaturesHydrationParam, - EnableTweetMediaHydrationParam - ) -} - -class RecapHydrationProduction(deciderGateBuilder: DeciderGateBuilder) { - val configHelper: ConfigHelper = - new ConfigHelper(RecapHydrationProduction.deciderByParam, deciderGateBuilder) - val booleanOverrides: Seq[OptionalOverride[Boolean]] = - configHelper.createDeciderBasedBooleanOverrides(RecapHydrationProduction.booleanParams) - - val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] = - FeatureSwitchOverrideUtil.getBooleanFSOverrides( - RecapHydrationProduction.booleanFeatureSwitchParams: _* - ) - - val config: BaseConfig = new BaseConfigBuilder() - .set( - booleanOverrides: _* - ).set( - booleanFeatureSwitchOverrides: _* - ) - .build(RecapHydrationProduction.getClass.getSimpleName) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/BUILD deleted file mode 100644 index 662fcc7c3..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "servo/decider", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/decider", - "timelines/src/main/scala/com/twitter/timelines/util/bounds", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/BUILD.docx new file mode 100644 index 000000000..a0e171c9d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronParams.docx new file mode 100644 index 000000000..2a8ea614b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronParams.scala deleted file mode 100644 index f6fb36ea3..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronParams.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.timelineranker.parameters.revchron - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object ReverseChronParams { - import ReverseChronTimelineQueryContext._ - - /** - * Controls limit on the number of followed users fetched from SGS when materializing home timelines. - */ - object MaxFollowedUsersParam - extends FSBoundedParam( - "reverse_chron_max_followed_users", - default = MaxFollowedUsers.default, - min = MaxFollowedUsers.bounds.minInclusive, - max = MaxFollowedUsers.bounds.maxInclusive - ) - - object ReturnEmptyWhenOverMaxFollowsParam - extends FSParam( - name = "reverse_chron_return_empty_when_over_max_follows", - default = true - ) - - /** - * When true, search requests for the reverse chron timeline will include an additional operator - * so that search will not return tweets that are directed at non-followed users. - */ - object DirectedAtNarrowcastingViaSearchParam - extends FSParam( - name = "reverse_chron_directed_at_narrowcasting_via_search", - default = false - ) - - /** - * When true, search requests for the reverse chron timeline will request additional metadata - * from search and use this metadata for post filtering. - */ - object PostFilteringBasedOnSearchMetadataEnabledParam - extends FSParam( - name = "reverse_chron_post_filtering_based_on_search_metadata_enabled", - default = true - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronProduction.docx new file mode 100644 index 000000000..ccf967e75 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronProduction.scala deleted file mode 100644 index 2c6114371..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronProduction.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.timelineranker.parameters.revchron - -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi._ - -object ReverseChronProduction { - val intFeatureSwitchParams = Seq(ReverseChronParams.MaxFollowedUsersParam) - val booleanFeatureSwitchParams = Seq( - ReverseChronParams.ReturnEmptyWhenOverMaxFollowsParam, - ReverseChronParams.DirectedAtNarrowcastingViaSearchParam, - ReverseChronParams.PostFilteringBasedOnSearchMetadataEnabledParam - ) -} - -class ReverseChronProduction(deciderGateBuilder: DeciderGateBuilder) { - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - ReverseChronProduction.intFeatureSwitchParams: _* - ) - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - ReverseChronProduction.booleanFeatureSwitchParams: _* - ) - - val config: BaseConfig = new BaseConfigBuilder() - .set(intOverrides: _*) - .set(booleanOverrides: _*) - .build(ReverseChronProduction.getClass.getSimpleName) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContext.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContext.docx new file mode 100644 index 000000000..7905a53cf Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContext.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContext.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContext.scala deleted file mode 100644 index 13e4c0c46..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContext.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.timelineranker.parameters.revchron - -import com.twitter.timelineranker.model.ReverseChronTimelineQuery -import com.twitter.timelines.util.bounds.BoundsWithDefault -import com.twitter.timelineservice.model.core.TimelineKind -import com.twitter.timelineservice.model.core.TimelineLimits - -object ReverseChronTimelineQueryContext { - val MaxCountLimit: Int = TimelineLimits.default.lengthLimit(TimelineKind.home) - val MaxCount: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, MaxCountLimit, MaxCountLimit) - val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.5, 2.0, 1.0) - val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 15000, 5000) - val TweetsFilteringLossageThresholdPercent: BoundsWithDefault[Int] = - BoundsWithDefault[Int](10, 100, 20) - val TweetsFilteringLossageLimitPercent: BoundsWithDefault[Int] = - BoundsWithDefault[Int](40, 65, 60) - - def getDefaultContext(query: ReverseChronTimelineQuery): ReverseChronTimelineQueryContext = { - new ReverseChronTimelineQueryContextImpl( - query, - getMaxCount = () => MaxCount.default, - getMaxCountMultiplier = () => MaxCountMultiplier.default, - getMaxFollowedUsers = () => MaxFollowedUsers.default, - getReturnEmptyWhenOverMaxFollows = () => true, - getDirectedAtNarrowastingViaSearch = () => false, - getPostFilteringBasedOnSearchMetadataEnabled = () => true, - getBackfillFilteredEntries = () => false, - getTweetsFilteringLossageThresholdPercent = () => - TweetsFilteringLossageThresholdPercent.default, - getTweetsFilteringLossageLimitPercent = () => TweetsFilteringLossageLimitPercent.default - ) - } -} - -// Note that methods that return parameter value always use () to indicate that -// side effects may be involved in their invocation. -// for example, A likely side effect is to cause experiment impression. -trait ReverseChronTimelineQueryContext { - def query: ReverseChronTimelineQuery - - // Maximum number of tweets to be returned to caller. - def maxCount(): Int - - // Multiplier applied to the number of tweets fetched from search expressed as percentage. - // It can be used to fetch more than the number tweets requested by a caller (to improve similarity) - // or to fetch less than requested to reduce load. - def maxCountMultiplier(): Double - - // Maximum number of followed user accounts to use when materializing home timelines. - def maxFollowedUsers(): Int - - // When true, if the user follows more than maxFollowedUsers, return an empty timeline. - def returnEmptyWhenOverMaxFollows(): Boolean - - // When true, appends an operator for directed-at narrowcasting to the home materialization - // search request - def directedAtNarrowcastingViaSearch(): Boolean - - // When true, requests additional metadata from search and use this metadata for post filtering. - def postFilteringBasedOnSearchMetadataEnabled(): Boolean - - // Controls whether to back-fill timeline entries that get filtered out by TweetsPostFilter - // during home timeline materialization. - def backfillFilteredEntries(): Boolean - - // If back-filling filtered entries is enabled and if number of tweets that get filtered out - // exceed this percentage then we will issue a second call to get more tweets. - def tweetsFilteringLossageThresholdPercent(): Int - - // We need to ensure that the number of tweets requested by the second call - // are not unbounded (for example, if everything is filtered out in the first call) - // therefore we adjust the actual filtered out percentage to be no greater than - // the value below. - def tweetsFilteringLossageLimitPercent(): Int - - // We need to indicate to search if we should use the archive cluster - // this option will come from ReverseChronTimelineQueryOptions and - // will be `true` by default if the options are not present. - def getTweetsFromArchiveIndex(): Boolean = - query.options.map(_.getTweetsFromArchiveIndex).getOrElse(true) -} - -class ReverseChronTimelineQueryContextImpl( - override val query: ReverseChronTimelineQuery, - getMaxCount: () => Int, - getMaxCountMultiplier: () => Double, - getMaxFollowedUsers: () => Int, - getReturnEmptyWhenOverMaxFollows: () => Boolean, - getDirectedAtNarrowastingViaSearch: () => Boolean, - getPostFilteringBasedOnSearchMetadataEnabled: () => Boolean, - getBackfillFilteredEntries: () => Boolean, - getTweetsFilteringLossageThresholdPercent: () => Int, - getTweetsFilteringLossageLimitPercent: () => Int) - extends ReverseChronTimelineQueryContext { - override def maxCount(): Int = { getMaxCount() } - override def maxCountMultiplier(): Double = { getMaxCountMultiplier() } - override def maxFollowedUsers(): Int = { getMaxFollowedUsers() } - override def backfillFilteredEntries(): Boolean = { getBackfillFilteredEntries() } - override def tweetsFilteringLossageThresholdPercent(): Int = { - getTweetsFilteringLossageThresholdPercent() - } - override def tweetsFilteringLossageLimitPercent(): Int = { - getTweetsFilteringLossageLimitPercent() - } - override def returnEmptyWhenOverMaxFollows(): Boolean = { - getReturnEmptyWhenOverMaxFollows() - } - override def directedAtNarrowcastingViaSearch(): Boolean = { - getDirectedAtNarrowastingViaSearch() - } - override def postFilteringBasedOnSearchMetadataEnabled(): Boolean = { - getPostFilteringBasedOnSearchMetadataEnabled() - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContextBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContextBuilder.docx new file mode 100644 index 000000000..af58600c6 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContextBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContextBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContextBuilder.scala deleted file mode 100644 index 714a8e4f9..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContextBuilder.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.timelineranker.parameters.revchron - -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.model._ -import com.twitter.timelineranker.parameters.util.RequestContextBuilder -import com.twitter.timelines.configapi.Config -import com.twitter.timelines.decider.FeatureValue -import com.twitter.util.Future - -object ReverseChronTimelineQueryContextBuilder { - val MaxCountLimitKey: Seq[String] = Seq("search_request_max_count_limit") -} - -class ReverseChronTimelineQueryContextBuilder( - config: Config, - runtimeConfig: RuntimeConfiguration, - requestContextBuilder: RequestContextBuilder) { - - import ReverseChronTimelineQueryContext._ - import ReverseChronTimelineQueryContextBuilder._ - - private val maxCountMultiplier = FeatureValue( - runtimeConfig.deciderGateBuilder, - DeciderKey.MultiplierOfMaterializationTweetsFetched, - MaxCountMultiplier, - value => (value / 100.0) - ) - - private val backfillFilteredEntriesGate = - runtimeConfig.deciderGateBuilder.linearGate(DeciderKey.BackfillFilteredEntries) - - private val tweetsFilteringLossageThresholdPercent = FeatureValue( - runtimeConfig.deciderGateBuilder, - DeciderKey.TweetsFilteringLossageThreshold, - TweetsFilteringLossageThresholdPercent, - value => (value / 100) - ) - - private val tweetsFilteringLossageLimitPercent = FeatureValue( - runtimeConfig.deciderGateBuilder, - DeciderKey.TweetsFilteringLossageLimit, - TweetsFilteringLossageLimitPercent, - value => (value / 100) - ) - - private def getMaxCountFromConfigStore(): Int = { - runtimeConfig.configStore.getAsInt(MaxCountLimitKey).getOrElse(MaxCount.default) - } - - def apply(query: ReverseChronTimelineQuery): Future[ReverseChronTimelineQueryContext] = { - requestContextBuilder(Some(query.userId), deviceContext = None).map { baseContext => - val params = config(baseContext, runtimeConfig.statsReceiver) - - new ReverseChronTimelineQueryContextImpl( - query, - getMaxCount = () => getMaxCountFromConfigStore(), - getMaxCountMultiplier = () => maxCountMultiplier(), - getMaxFollowedUsers = () => params(ReverseChronParams.MaxFollowedUsersParam), - getReturnEmptyWhenOverMaxFollows = - () => params(ReverseChronParams.ReturnEmptyWhenOverMaxFollowsParam), - getDirectedAtNarrowastingViaSearch = - () => params(ReverseChronParams.DirectedAtNarrowcastingViaSearchParam), - getPostFilteringBasedOnSearchMetadataEnabled = - () => params(ReverseChronParams.PostFilteringBasedOnSearchMetadataEnabledParam), - getBackfillFilteredEntries = () => backfillFilteredEntriesGate(), - getTweetsFilteringLossageThresholdPercent = () => tweetsFilteringLossageThresholdPercent(), - getTweetsFilteringLossageLimitPercent = () => tweetsFilteringLossageLimitPercent() - ) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/BUILD deleted file mode 100644 index 9b4f0f80d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "servo/decider/src/main/scala", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - "timelines/src/main/scala/com/twitter/timelines/util/bounds", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/BUILD.docx new file mode 100644 index 000000000..9332566c0 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsParams.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsParams.docx new file mode 100644 index 000000000..1978efcb9 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsParams.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsParams.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsParams.scala deleted file mode 100644 index a9baada3a..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsParams.scala +++ /dev/null @@ -1,174 +0,0 @@ -package com.twitter.timelineranker.parameters.uteg_liked_by_tweets - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.util.bounds.BoundsWithDefault - -object UtegLikedByTweetsParams { - - val ProbabilityRandomTweet: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.0, 1.0, 0.0) - - object DefaultUTEGInNetworkCount extends Param(200) - - object DefaultUTEGOutOfNetworkCount extends Param(800) - - object DefaultMaxTweetCount extends Param(200) - - /** - * Enables semantic core, penguin, and tweetypie content features in uteg liked by tweets source. - */ - object EnableContentFeaturesHydrationParam extends Param(false) - - /** - * additionally enables tokens when hydrating content features. - */ - object EnableTokensInContentFeaturesHydrationParam - extends FSParam( - name = "uteg_liked_by_tweets_enable_tokens_in_content_features_hydration", - default = false - ) - - /** - * additionally enables tweet text when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - object EnableTweetTextInContentFeaturesHydrationParam - extends FSParam( - name = "uteg_liked_by_tweets_enable_tweet_text_in_content_features_hydration", - default = false - ) - - /** - * A multiplier for earlybird score when combining earlybird score and real graph score for ranking. - * Note multiplier for realgraph score := 1.0, and only change earlybird score multiplier. - */ - object EarlybirdScoreMultiplierParam - extends FSBoundedParam( - "uteg_liked_by_tweets_earlybird_score_multiplier_param", - 1.0, - 0, - 20.0 - ) - - object UTEGRecommendationsFilter { - - /** - * enable filtering of UTEG recommendations based on social proof type - */ - object EnableParam - extends FSParam( - "uteg_liked_by_tweets_uteg_recommendations_filter_enable", - false - ) - - /** - * filters out UTEG recommendations that have been tweeted by someone the user follows - */ - object ExcludeTweetParam - extends FSParam( - "uteg_liked_by_tweets_uteg_recommendations_filter_exclude_tweet", - false - ) - - /** - * filters out UTEG recommendations that have been retweeted by someone the user follows - */ - object ExcludeRetweetParam - extends FSParam( - "uteg_liked_by_tweets_uteg_recommendations_filter_exclude_retweet", - false - ) - - /** - * filters out UTEG recommendations that have been replied to by someone the user follows - * not filtering out the replies - */ - object ExcludeReplyParam - extends FSParam( - "uteg_liked_by_tweets_uteg_recommendations_filter_exclude_reply", - false - ) - - /** - * filters out UTEG recommendations that have been quoted by someone the user follows - */ - object ExcludeQuoteTweetParam - extends FSParam( - "uteg_liked_by_tweets_uteg_recommendations_filter_exclude_quote", - false - ) - - /** - * filters out recommended replies that have been directed at out of network users. - */ - object ExcludeRecommendedRepliesToNonFollowedUsersParam - extends FSParam( - name = - "uteg_liked_by_tweets_uteg_recommendations_filter_exclude_recommended_replies_to_non_followed_users", - default = false - ) - } - - /** - * Minimum number of favorited-by users required for recommended tweets. - */ - object MinNumFavoritedByUserIdsParam extends Param(1) - - /** - * Includes one or multiple random tweets in the response. - */ - object IncludeRandomTweetParam - extends FSParam(name = "uteg_liked_by_tweets_include_random_tweet", default = false) - - /** - * One single random tweet (true) or tag tweet as random with given probability (false). - */ - object IncludeSingleRandomTweetParam - extends FSParam(name = "uteg_liked_by_tweets_include_random_tweet_single", default = false) - - /** - * Probability to tag a tweet as random (will not be ranked). - */ - object ProbabilityRandomTweetParam - extends FSBoundedParam( - name = "uteg_liked_by_tweets_include_random_tweet_probability", - default = ProbabilityRandomTweet.default, - min = ProbabilityRandomTweet.bounds.minInclusive, - max = ProbabilityRandomTweet.bounds.maxInclusive) - - /** - * additionally enables conversationControl when hydrating content features. - * This only works if EnableContentFeaturesHydrationParam is set to true - */ - - object EnableConversationControlInContentFeaturesHydrationParam - extends FSParam( - name = "conversation_control_in_content_features_hydration_uteg_liked_by_tweets_enable", - default = false - ) - - object EnableTweetMediaHydrationParam - extends FSParam( - name = "tweet_media_hydration_uteg_liked_by_tweets_enable", - default = false - ) - - object NumAdditionalRepliesParam - extends FSBoundedParam( - name = "uteg_liked_by_tweets_num_additional_replies", - default = 0, - min = 0, - max = 1000 - ) - - /** - * Enable relevance search, otherwise recency search from earlybird. - */ - object EnableRelevanceSearchParam - extends FSParam( - name = "uteg_liked_by_tweets_enable_relevance_search", - default = true - ) - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsProduction.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsProduction.docx new file mode 100644 index 000000000..d43eea70b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsProduction.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsProduction.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsProduction.scala deleted file mode 100644 index 9b1d8a638..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsProduction.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.timelineranker.parameters.uteg_liked_by_tweets - -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsParams._ -import com.twitter.timelineranker.parameters.util.ConfigHelper -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.OptionalOverride -import com.twitter.timelines.configapi.Param - -object UtegLikedByTweetsProduction { - val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName]( - EnableContentFeaturesHydrationParam -> DeciderKey.UtegLikedByTweetsEnableContentFeaturesHydration - ) - - val booleanDeciderParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq( - EnableContentFeaturesHydrationParam - ) - - val intParams: Seq[Param[Int]] = Seq( - DefaultUTEGInNetworkCount, - DefaultMaxTweetCount, - DefaultUTEGOutOfNetworkCount, - MinNumFavoritedByUserIdsParam - ) - - val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq( - UTEGRecommendationsFilter.EnableParam, - UTEGRecommendationsFilter.ExcludeQuoteTweetParam, - UTEGRecommendationsFilter.ExcludeReplyParam, - UTEGRecommendationsFilter.ExcludeRetweetParam, - UTEGRecommendationsFilter.ExcludeTweetParam, - EnableTokensInContentFeaturesHydrationParam, - EnableConversationControlInContentFeaturesHydrationParam, - UTEGRecommendationsFilter.ExcludeRecommendedRepliesToNonFollowedUsersParam, - EnableTweetTextInContentFeaturesHydrationParam, - EnableTweetMediaHydrationParam, - UtegLikedByTweetsParams.IncludeRandomTweetParam, - UtegLikedByTweetsParams.IncludeSingleRandomTweetParam, - UtegLikedByTweetsParams.EnableRelevanceSearchParam - ) - val boundedDoubleFeatureSwitchParams: Seq[FSBoundedParam[Double]] = Seq( - EarlybirdScoreMultiplierParam, - UtegLikedByTweetsParams.ProbabilityRandomTweetParam - ) - val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq( - UtegLikedByTweetsParams.NumAdditionalRepliesParam - ) - -} - -class UtegLikedByTweetsProduction(deciderGateBuilder: DeciderGateBuilder) { - val configHelper: ConfigHelper = - new ConfigHelper(UtegLikedByTweetsProduction.deciderByParam, deciderGateBuilder) - val booleanDeciderOverrides: Seq[OptionalOverride[Boolean]] = - configHelper.createDeciderBasedBooleanOverrides( - UtegLikedByTweetsProduction.booleanDeciderParams) - val boundedDoubleFeatureSwitchOverrides: Seq[OptionalOverride[Double]] = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - UtegLikedByTweetsProduction.boundedDoubleFeatureSwitchParams: _*) - val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] = - FeatureSwitchOverrideUtil.getBooleanFSOverrides( - UtegLikedByTweetsProduction.booleanFeatureSwitchParams: _*) - val boundedIntFeaturesSwitchOverrides: Seq[OptionalOverride[Int]] = - FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - UtegLikedByTweetsProduction.boundedIntFeatureSwitchParams: _*) - - val config: BaseConfig = new BaseConfigBuilder() - .set( - booleanDeciderOverrides: _* - ) - .set( - boundedDoubleFeatureSwitchOverrides: _* - ) - .set( - booleanFeatureSwitchOverrides: _* - ) - .set( - boundedIntFeaturesSwitchOverrides: _* - ) - .build(UtegLikedByTweetsProduction.getClass.getSimpleName) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/BUILD deleted file mode 100644 index cf1a99870..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "configapi/configapi-decider", - "decider/src/main/scala", - "servo/decider", - "servo/util/src/main/scala", - "timelineranker/common:model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/config/configapi", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/BUILD.docx new file mode 100644 index 000000000..02cff5278 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/CommonRequestContext.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/CommonRequestContext.docx new file mode 100644 index 000000000..1be132c8b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/CommonRequestContext.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/CommonRequestContext.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/CommonRequestContext.scala deleted file mode 100644 index b16b9af2b..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/CommonRequestContext.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.timelineranker.parameters.util - -import com.twitter.servo.util.Gate -import com.twitter.timelines.configapi.BaseRequestContext -import com.twitter.timelines.configapi.WithExperimentContext -import com.twitter.timelines.configapi.WithFeatureContext -import com.twitter.timelines.configapi.WithUserId -import com.twitter.timelines.model.UserId -import com.twitter.timelineservice.DeviceContext -import com.twitter.timelineservice.model.RequestContextFactory -import com.twitter.util.Future - -trait CommonRequestContext - extends BaseRequestContext - with WithExperimentContext - with WithUserId - with WithFeatureContext - -trait RequestContextBuilder { - def apply( - recipientUserId: Option[UserId], - deviceContext: Option[DeviceContext] - ): Future[CommonRequestContext] -} - -class RequestContextBuilderImpl(requestContextFactory: RequestContextFactory) - extends RequestContextBuilder { - override def apply( - recipientUserId: Option[UserId], - deviceContextOpt: Option[DeviceContext] - ): Future[CommonRequestContext] = { - val requestContextFut = requestContextFactory( - contextualUserIdOpt = recipientUserId, - deviceContext = deviceContextOpt.getOrElse(DeviceContext.empty), - experimentConfigurationOpt = None, - requestLogOpt = None, - contextualUserContext = None, - useRolesCache = Gate.True, - timelineId = None - ) - - requestContextFut.map { requestContext => - new CommonRequestContext { - override val userId = recipientUserId - override val experimentContext = requestContext.experimentContext - override val featureContext = requestContext.featureContext - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/ConfigHelper.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/ConfigHelper.docx new file mode 100644 index 000000000..13f6bc194 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/ConfigHelper.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/ConfigHelper.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/ConfigHelper.scala deleted file mode 100644 index b1c600b15..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/ConfigHelper.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.timelineranker.parameters.util - -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.servo.decider.DeciderKeyName -import com.twitter.timelines.configapi._ -import com.twitter.timelines.configapi.decider.DeciderIntSpaceOverrideValue -import com.twitter.timelines.configapi.decider.DeciderSwitchOverrideValue -import com.twitter.timelines.configapi.decider.DeciderValueConverter -import com.twitter.timelines.configapi.decider.RecipientBuilder - -class ConfigHelper( - deciderByParam: Map[Param[_], DeciderKeyName], - deciderGateBuilder: DeciderGateBuilder) { - def createDeciderBasedBooleanOverrides( - parameters: Seq[Param[Boolean]] - ): Seq[OptionalOverride[Boolean]] = { - parameters.map { parameter => - parameter.optionalOverrideValue( - DeciderSwitchOverrideValue( - feature = deciderGateBuilder.keyToFeature(deciderByParam(parameter)), - recipientBuilder = RecipientBuilder.User, - enabledValue = true, - disabledValueOption = Some(false) - ) - ) - } - } - - def createDeciderBasedOverrides[T]( - parameters: Seq[Param[T] with DeciderValueConverter[T]] - ): Seq[OptionalOverride[T]] = { - parameters.map { parameter => - parameter.optionalOverrideValue( - DeciderIntSpaceOverrideValue( - feature = deciderGateBuilder.keyToFeature(deciderByParam(parameter)), - conversion = parameter.convert - ) - ) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/RecapQueryParamInitializer.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/RecapQueryParamInitializer.docx new file mode 100644 index 000000000..c07b4df1b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/RecapQueryParamInitializer.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/RecapQueryParamInitializer.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/RecapQueryParamInitializer.scala deleted file mode 100644 index ba96366f4..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/RecapQueryParamInitializer.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.timelineranker.parameters.util - -import com.twitter.servo.util.FunctionArrow -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelines.configapi.Config -import com.twitter.util.Future - -class RecapQueryParamInitializer(config: Config, runtimeConfig: RuntimeConfiguration) - extends FunctionArrow[RecapQuery, Future[RecapQuery]] { - private[this] val requestContextBuilder = - new RequestContextBuilderImpl(runtimeConfig.configApiConfiguration.requestContextFactory) - - def apply(query: RecapQuery): Future[RecapQuery] = { - requestContextBuilder(Some(query.userId), query.deviceContext).map { baseContext => - val params = config(baseContext, runtimeConfig.statsReceiver) - query.copy(params = params) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/BUILD deleted file mode 100644 index 54e356a77..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/BUILD.docx new file mode 100644 index 000000000..1247002c2 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/BUILD deleted file mode 100644 index 572408d68..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/thrift/com/twitter/escherbird:tweet-annotation-scala", - "src/thrift/com/twitter/timelines/content_features:thrift-scala", - "src/thrift/com/twitter/tweetypie:media-entity-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - ], - exports = [ - "src/thrift/com/twitter/timelines/content_features:thrift-scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/BUILD.docx new file mode 100644 index 000000000..ca84fb26e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/ContentFeatures.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/ContentFeatures.docx new file mode 100644 index 000000000..051e1eb9e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/ContentFeatures.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/ContentFeatures.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/ContentFeatures.scala deleted file mode 100644 index ab3489c41..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model/ContentFeatures.scala +++ /dev/null @@ -1,222 +0,0 @@ -package com.twitter.timelineranker.recap.model - -import com.twitter.escherbird.thriftscala.TweetEntityAnnotation -import com.twitter.timelines.content_features.v1.thriftscala.{ContentFeatures => ContentFeaturesV1} -import com.twitter.timelines.content_features.{thriftscala => thrift} -import com.twitter.tweetypie.thriftscala.ConversationControl -import com.twitter.tweetypie.thriftscala.MediaEntity -import com.twitter.tweetypie.thriftscala.SelfThreadMetadata -import scala.util.Failure -import scala.util.Success -import scala.util.{Try => ScalaTry} - -case class ContentFeatures( - length: Short, - hasQuestion: Boolean, - numCaps: Short, - numWhiteSpaces: Short, - numNewlines: Option[Short], - videoDurationMs: Option[Int], - bitRate: Option[Int], - aspectRatioNum: Option[Short], - aspectRatioDen: Option[Short], - widths: Option[Seq[Short]], - heights: Option[Seq[Short]], - resizeMethods: Option[Seq[Short]], - numMediaTags: Option[Short], - mediaTagScreenNames: Option[Seq[String]], - emojiTokens: Option[Set[String]], - emoticonTokens: Option[Set[String]], - phrases: Option[Set[String]], - faceAreas: Option[Seq[Int]], - dominantColorRed: Option[Short], - dominantColorBlue: Option[Short], - dominantColorGreen: Option[Short], - numColors: Option[Short], - stickerIds: Option[Seq[Long]], - mediaOriginProviders: Option[Seq[String]], - isManaged: Option[Boolean], - is360: Option[Boolean], - viewCount: Option[Long], - isMonetizable: Option[Boolean], - isEmbeddable: Option[Boolean], - hasSelectedPreviewImage: Option[Boolean], - hasTitle: Option[Boolean], - hasDescription: Option[Boolean], - hasVisitSiteCallToAction: Option[Boolean], - hasAppInstallCallToAction: Option[Boolean], - hasWatchNowCallToAction: Option[Boolean], - media: Option[Seq[MediaEntity]], - dominantColorPercentage: Option[Double], - posUnigrams: Option[Set[String]], - posBigrams: Option[Set[String]], - semanticCoreAnnotations: Option[Seq[TweetEntityAnnotation]], - selfThreadMetadata: Option[SelfThreadMetadata], - tokens: Option[Seq[String]], - tweetText: Option[String], - conversationControl: Option[ConversationControl]) { - def toThrift: thrift.ContentFeatures = - thrift.ContentFeatures.V1(toThriftV1) - - def toThriftV1: ContentFeaturesV1 = - ContentFeaturesV1( - length = length, - hasQuestion = hasQuestion, - numCaps = numCaps, - numWhiteSpaces = numWhiteSpaces, - numNewlines = numNewlines, - videoDurationMs = videoDurationMs, - bitRate = bitRate, - aspectRatioNum = aspectRatioNum, - aspectRatioDen = aspectRatioDen, - widths = widths, - heights = heights, - resizeMethods = resizeMethods, - numMediaTags = numMediaTags, - mediaTagScreenNames = mediaTagScreenNames, - emojiTokens = emojiTokens, - emoticonTokens = emoticonTokens, - phrases = phrases, - faceAreas = faceAreas, - dominantColorRed = dominantColorRed, - dominantColorBlue = dominantColorBlue, - dominantColorGreen = dominantColorGreen, - numColors = numColors, - stickerIds = stickerIds, - mediaOriginProviders = mediaOriginProviders, - isManaged = isManaged, - is360 = is360, - viewCount = viewCount, - isMonetizable = isMonetizable, - isEmbeddable = isEmbeddable, - hasSelectedPreviewImage = hasSelectedPreviewImage, - hasTitle = hasTitle, - hasDescription = hasDescription, - hasVisitSiteCallToAction = hasVisitSiteCallToAction, - hasAppInstallCallToAction = hasAppInstallCallToAction, - hasWatchNowCallToAction = hasWatchNowCallToAction, - dominantColorPercentage = dominantColorPercentage, - posUnigrams = posUnigrams, - posBigrams = posBigrams, - semanticCoreAnnotations = semanticCoreAnnotations, - selfThreadMetadata = selfThreadMetadata, - tokens = tokens, - tweetText = tweetText, - conversationControl = conversationControl, - media = media - ) -} - -object ContentFeatures { - val Empty: ContentFeatures = ContentFeatures( - 0.toShort, - false, - 0.toShort, - 0.toShort, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None - ) - - def fromThrift(contentFeatures: thrift.ContentFeatures): Option[ContentFeatures] = - contentFeatures match { - case thrift.ContentFeatures.V1(contentFeaturesV1) => - Some(fromThriftV1(contentFeaturesV1)) - case _ => - None - } - - private val failure = - Failure[ContentFeatures](new Exception("Failure to convert content features from thrift")) - - def tryFromThrift(contentFeaturesThrift: thrift.ContentFeatures): ScalaTry[ContentFeatures] = - fromThrift(contentFeaturesThrift) match { - case Some(contentFeatures) => Success[ContentFeatures](contentFeatures) - case None => failure - } - - def fromThriftV1(contentFeaturesV1: ContentFeaturesV1): ContentFeatures = - ContentFeatures( - length = contentFeaturesV1.length, - hasQuestion = contentFeaturesV1.hasQuestion, - numCaps = contentFeaturesV1.numCaps, - numWhiteSpaces = contentFeaturesV1.numWhiteSpaces, - numNewlines = contentFeaturesV1.numNewlines, - videoDurationMs = contentFeaturesV1.videoDurationMs, - bitRate = contentFeaturesV1.bitRate, - aspectRatioNum = contentFeaturesV1.aspectRatioNum, - aspectRatioDen = contentFeaturesV1.aspectRatioDen, - widths = contentFeaturesV1.widths, - heights = contentFeaturesV1.heights, - resizeMethods = contentFeaturesV1.resizeMethods, - numMediaTags = contentFeaturesV1.numMediaTags, - mediaTagScreenNames = contentFeaturesV1.mediaTagScreenNames, - emojiTokens = contentFeaturesV1.emojiTokens.map(_.toSet), - emoticonTokens = contentFeaturesV1.emoticonTokens.map(_.toSet), - phrases = contentFeaturesV1.phrases.map(_.toSet), - faceAreas = contentFeaturesV1.faceAreas, - dominantColorRed = contentFeaturesV1.dominantColorRed, - dominantColorBlue = contentFeaturesV1.dominantColorBlue, - dominantColorGreen = contentFeaturesV1.dominantColorGreen, - numColors = contentFeaturesV1.numColors, - stickerIds = contentFeaturesV1.stickerIds, - mediaOriginProviders = contentFeaturesV1.mediaOriginProviders, - isManaged = contentFeaturesV1.isManaged, - is360 = contentFeaturesV1.is360, - viewCount = contentFeaturesV1.viewCount, - isMonetizable = contentFeaturesV1.isMonetizable, - isEmbeddable = contentFeaturesV1.isEmbeddable, - hasSelectedPreviewImage = contentFeaturesV1.hasSelectedPreviewImage, - hasTitle = contentFeaturesV1.hasTitle, - hasDescription = contentFeaturesV1.hasDescription, - hasVisitSiteCallToAction = contentFeaturesV1.hasVisitSiteCallToAction, - hasAppInstallCallToAction = contentFeaturesV1.hasAppInstallCallToAction, - hasWatchNowCallToAction = contentFeaturesV1.hasWatchNowCallToAction, - dominantColorPercentage = contentFeaturesV1.dominantColorPercentage, - posUnigrams = contentFeaturesV1.posUnigrams.map(_.toSet), - posBigrams = contentFeaturesV1.posBigrams.map(_.toSet), - semanticCoreAnnotations = contentFeaturesV1.semanticCoreAnnotations, - selfThreadMetadata = contentFeaturesV1.selfThreadMetadata, - tokens = contentFeaturesV1.tokens.map(_.toSeq), - tweetText = contentFeaturesV1.tweetText, - conversationControl = contentFeaturesV1.conversationControl, - media = contentFeaturesV1.media - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/BUILD.bazel b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/BUILD.bazel deleted file mode 100644 index b96717315..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/BUILD.bazel +++ /dev/null @@ -1,41 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "finagle/finagle-core/src/main", - "servo/util/src/main/scala", - "src/thrift/com/twitter/search:earlybird-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/common", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/repository", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/bounds", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelines/src/main/scala/com/twitter/timelines/visibility", - "timelines/src/main/scala/com/twitter/timelines/visibility/model", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/BUILD.docx new file mode 100644 index 000000000..9081ce5f3 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepository.docx new file mode 100644 index 000000000..19357d445 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepository.scala deleted file mode 100644 index 1d31f32d7..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepository.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.timelineranker.recap_author - -import com.twitter.timelineranker.model.CandidateTweetsResult -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.util.Future -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.parameters.recap_author.RecapAuthorParams - -/** - * A repository of recap author results. - * - * For now, it does not cache any results therefore forwards all calls to the underlying source. - */ -class RecapAuthorRepository(source: RecapAuthorSource, realtimeCGSource: RecapAuthorSource) { - private[this] val enableRealtimeCGProvider = - DependencyProvider.from(RecapAuthorParams.EnableEarlybirdRealtimeCgMigrationParam) - - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - if (enableRealtimeCGProvider(query)) { - realtimeCGSource.get(query) - } else { - source.get(query) - } - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - Future.collect(queries.map(query => get(query))) - } -} \ No newline at end of file diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepositoryBuilder.docx new file mode 100644 index 000000000..dd894da00 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepositoryBuilder.scala deleted file mode 100644 index 3508fb6c0..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorRepositoryBuilder.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.timelineranker.recap_author - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.service.RetryPolicy -import com.twitter.timelineranker.config.RequestScopes -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder -import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.util.Duration - -class RecapAuthorRepositoryBuilder(config: RuntimeConfiguration) - extends CandidatesRepositoryBuilder(config) { - override val clientSubId = "recap_by_author" - override val requestScope: RequestScope = RequestScopes.RecapAuthorSource - override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet = - SgsFollowGraphDataFields.ValueSet( - SgsFollowGraphDataFields.FollowedUserIds, - SgsFollowGraphDataFields.MutuallyFollowingUserIds, - SgsFollowGraphDataFields.MutedUserIds - ) - - /** - * Budget for processing within the search root cluster for the recap_by_author query. - */ - override val searchProcessingTimeout: Duration = 250.milliseconds - private val EarlybirdTimeout = 650.milliseconds - private val EarlybirdRequestTimeout = 600.milliseconds - - private val EarlybirdRealtimeCGTimeout = 650.milliseconds - private val EarlybirdRealtimeCGRequestTimeout = 600.milliseconds - - /** - * TLM -> TLR timeout is 1s for candidate retrieval, so make the finagle TLR -> EB timeout - * a bit shorter than 1s. - */ - override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdClient( - scope = scope, - requestTimeout = EarlybirdRequestTimeout, - // timeout is slight less than timelineranker client timeout in timelinemixer - timeout = EarlybirdTimeout, - retryPolicy = RetryPolicy.Never - ) - - /** The RealtimeCG clients below are only used for the Earlybird Cluster Migration */ - private def earlybirdRealtimeCGClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdRealtimeCgClient( - scope = scope, - requestTimeout = EarlybirdRealtimeCGRequestTimeout, - timeout = EarlybirdRealtimeCGTimeout, - retryPolicy = RetryPolicy.Never - ) - - private val realtimeCGClientSubId = "realtime_cg_recap_by_author" - private lazy val searchRealtimeCGClient = - newSearchClient(earlybirdRealtimeCGClient, clientId = realtimeCGClientSubId) - - def apply(): RecapAuthorRepository = { - val recapAuthorSource = new RecapAuthorSource( - gizmoduckClient, - searchClient, - tweetyPieLowQoSClient, - userMetadataClient, - followGraphDataProvider, // Used to early-enforce visibility filtering, even though authorIds is part of query - config.underlyingClients.contentFeaturesCache, - clientFactories.visibilityEnforcerFactory.apply( - VisibilityRules, - RequestScopes.RecapAuthorSource - ), - config.statsReceiver - ) - val recapAuthorRealtimeCGSource = new RecapAuthorSource( - gizmoduckClient, - searchRealtimeCGClient, - tweetyPieLowQoSClient, - userMetadataClient, - followGraphDataProvider, // Used to early-enforce visibility filtering, even though authorIds is part of query - config.underlyingClients.contentFeaturesCache, - clientFactories.visibilityEnforcerFactory.apply( - VisibilityRules, - RequestScopes.RecapAuthorSource - ), - config.statsReceiver.scope("replacementRealtimeCG") - ) - - new RecapAuthorRepository(recapAuthorSource, recapAuthorRealtimeCGSource) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSearchResultsTransform.docx new file mode 100644 index 000000000..bc3b57334 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSearchResultsTransform.scala deleted file mode 100644 index 288230b2d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSearchResultsTransform.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.timelineranker.recap_author - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.model.TweetIdRange -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes -import com.twitter.timelines.model.UserId -import com.twitter.util.Future - -/** - * Fetch recap results based on an author id set passed into the query. - * Calls into the same search method as Recap, but uses the authorIds instead of the SGS-provided followedIds. - */ -class RecapAuthorSearchResultsTransform( - searchClient: SearchClient, - maxCountProvider: DependencyProvider[Int], - relevanceOptionsMaxHitsToProcessProvider: DependencyProvider[Int], - enableSettingTweetTypesWithTweetKindOptionProvider: DependencyProvider[Boolean], - statsReceiver: StatsReceiver, - logSearchDebugInfo: Boolean = false) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - private[this] val maxCountStat = statsReceiver.stat("maxCount") - private[this] val numInputAuthorsStat = statsReceiver.stat("numInputAuthors") - private[this] val excludedTweetIdsStat = statsReceiver.stat("excludedTweetIds") - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val maxCount = maxCountProvider(envelope.query) - maxCountStat.add(maxCount) - - val authorIds = envelope.query.authorIds.getOrElse(Seq.empty[UserId]) - numInputAuthorsStat.add(authorIds.size) - - val excludedTweetIdsOpt = envelope.query.excludedTweetIds - excludedTweetIdsOpt.map { excludedTweetIds => excludedTweetIdsStat.add(excludedTweetIds.size) } - - val tweetIdRange = envelope.query.range - .map(TweetIdRange.fromTimelineRange) - .getOrElse(TweetIdRange.default) - - val beforeTweetIdExclusive = tweetIdRange.toId - val afterTweetIdExclusive = tweetIdRange.fromId - - val relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcessProvider(envelope.query) - - searchClient - .getUsersTweetsForRecap( - userId = envelope.query.userId, - followedUserIds = authorIds.toSet, // user authorIds as the set of followed users - retweetsMutedUserIds = Set.empty, - maxCount = maxCount, - tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options), - searchOperator = envelope.query.searchOperator, - beforeTweetIdExclusive = beforeTweetIdExclusive, - afterTweetIdExclusive = afterTweetIdExclusive, - enableSettingTweetTypesWithTweetKindOption = - enableSettingTweetTypesWithTweetKindOptionProvider(envelope.query), - excludedTweetIds = excludedTweetIdsOpt, - earlybirdOptions = envelope.query.earlybirdOptions, - getOnlyProtectedTweets = false, - logSearchDebugInfo = logSearchDebugInfo, - returnAllResults = true, - enableExcludeSourceTweetIdsQuery = false, - relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcess - ).map { results => envelope.copy(searchResults = results) } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSource.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSource.docx new file mode 100644 index 000000000..3c6b54211 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSource.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSource.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSource.scala deleted file mode 100644 index 4f9e5b9d7..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author/RecapAuthorSource.scala +++ /dev/null @@ -1,212 +0,0 @@ -package com.twitter.timelineranker.recap_author - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.storehaus.Store -import com.twitter.timelineranker.common._ -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.model._ -import com.twitter.timelineranker.monitoring.UsersSearchResultMonitoringTransform -import com.twitter.timelineranker.parameters.monitoring.MonitoringParams -import com.twitter.timelineranker.parameters.recap.RecapParams -import com.twitter.timelineranker.parameters.recap_author.RecapAuthorParams -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform -import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform -import com.twitter.timelineranker.util.TweetFilters -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.clients.gizmoduck.GizmoduckClient -import com.twitter.timelines.clients.manhattan.UserMetadataClient -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.tweetypie.TweetyPieClient -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.timelines.util.stats.RequestStatsReceiver -import com.twitter.timelines.visibility.VisibilityEnforcer -import com.twitter.util.Future - -/** - * This source controls what tweets are fetched from earlybird given a - * list of authors to fetch tweets from. The controls available are: - * 1. The ''filters'' val, which is also overridden - * by the query options in TweetKindOptions (see Recap.scala, the - * parent class, for details on how this override works). For example, one - * can choose to retrieve replies, retweets and/or extended replies - * by changing the options passed in, which get added to ''filters''. - * 2. The visiblityEnforcer passed in, which controls what visibility rules - * are applied to the tweets returned from earlybird (e.g. mutes, blocks). - */ -class RecapAuthorSource( - gizmoduckClient: GizmoduckClient, - searchClient: SearchClient, - tweetyPieClient: TweetyPieClient, - userMetadataClient: UserMetadataClient, - followGraphDataProvider: FollowGraphDataProvider, - contentFeaturesCache: Store[TweetId, ContentFeatures], - visibilityEnforcer: VisibilityEnforcer, - statsReceiver: StatsReceiver) { - - private[this] val baseScope = statsReceiver.scope("recapAuthor") - private[this] val requestStats = RequestStatsReceiver(baseScope) - - private[this] val failOpenScope = baseScope.scope("failOpen") - private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo") - private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages") - - /* - * Similar to RecapSource, we filter out tweets directed at non-followed users that - * are not "replies" i.e. those that begin with the @-handle. - * For tweets to non-followed users that are replies, these are "extended replies" - * and are handled separately by the dynamic filters (see Recap.scala for details). - * Reply and retweet filtering is also handled by dynamic filters, overriden by - * TweetKindOptions passed in with the query (again, details in Recap.scala) - * We however do not filter out tweets from non-followed users, unlike RecapSource, - * because one of the main use cases of this endpoint is to retrieve tweets from out - * of network authors. - */ - val filters: TweetFilters.ValueSet = TweetFilters.ValueSet( - TweetFilters.DuplicateTweets, - TweetFilters.DuplicateRetweets, - TweetFilters.DirectedAtNotFollowedUsers, - TweetFilters.NonReplyDirectedAtNotFollowedUsers - ) - - private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform( - visibilityEnforcer - ) - - private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform( - outerFilters = filters, - innerFilters = TweetFilters.None, - useFollowGraphData = true, - useSourceTweets = false, - statsReceiver = baseScope, - numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed - ) - - private[this] val dynamicHydratedTweetsFilter = new TweetKindOptionHydratedTweetsFilterTransform( - useFollowGraphData = false, - useSourceTweets = false, - statsReceiver = baseScope - ) - - private[this] val maxFollowedUsersProvider = - DependencyProvider.value(RecapParams.MaxFollowedUsers.default) - private[this] val followGraphDataTransform = - new FollowGraphDataTransform(followGraphDataProvider, maxFollowedUsersProvider) - private[this] val maxSearchResultCountProvider = DependencyProvider { query => - query.maxCount.getOrElse(query.params(RecapParams.DefaultMaxTweetCount)) - } - private[this] val relevanceOptionsMaxHitsToProcessProvider = - DependencyProvider.from(RecapParams.RelevanceOptionsMaxHitsToProcessParam) - - private[this] val retrieveSearchResultsTransform = new RecapAuthorSearchResultsTransform( - searchClient = searchClient, - maxCountProvider = maxSearchResultCountProvider, - relevanceOptionsMaxHitsToProcessProvider = relevanceOptionsMaxHitsToProcessProvider, - enableSettingTweetTypesWithTweetKindOptionProvider = - DependencyProvider.from(RecapParams.EnableSettingTweetTypesWithTweetKindOption), - statsReceiver = baseScope, - logSearchDebugInfo = false) - - private[this] val debugAuthorsMonitoringProvider = - DependencyProvider.from(MonitoringParams.DebugAuthorsAllowListParam) - private[this] val preTruncateSearchResultsTransform = - new UsersSearchResultMonitoringTransform( - name = "RecapSearchResultsTruncationTransform", - new RecapSearchResultsTruncationTransform( - extraSortBeforeTruncationGate = DependencyProvider.True, - maxCountProvider = maxSearchResultCountProvider, - statsReceiver = baseScope.scope("afterSearchResultsTransform") - ), - baseScope.scope("afterSearchResultsTransform"), - debugAuthorsMonitoringProvider - ) - - private[this] val searchResultsTransform = retrieveSearchResultsTransform - .andThen(preTruncateSearchResultsTransform) - - private[this] val userProfileInfoTransform = - new UserProfileInfoTransform(userProfileHandler, gizmoduckClient) - private[this] val languagesTransform = - new UserLanguagesTransform(userLanguagesHandler, userMetadataClient) - - private[this] val contentFeaturesHydrationTransform = - new ContentFeaturesHydrationTransformBuilder( - tweetyPieClient = tweetyPieClient, - contentFeaturesCache = contentFeaturesCache, - enableContentFeaturesGate = - RecapQuery.paramGate(RecapAuthorParams.EnableContentFeaturesHydrationParam), - enableTokensInContentFeaturesGate = - RecapQuery.paramGate(RecapAuthorParams.EnableTokensInContentFeaturesHydrationParam), - enableTweetTextInContentFeaturesGate = - RecapQuery.paramGate(RecapAuthorParams.EnableTweetTextInContentFeaturesHydrationParam), - enableConversationControlContentFeaturesGate = RecapQuery.paramGate( - RecapAuthorParams.EnableConversationControlInContentFeaturesHydrationParam), - enableTweetMediaHydrationGate = - RecapQuery.paramGate(RecapAuthorParams.EnableTweetMediaHydrationParam), - hydrateInReplyToTweets = false, - statsReceiver = baseScope - ).build() - - private[this] def hydratesContentFeatures( - hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope - ): Boolean = - hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true) - - private[this] val contentFeaturesTransformer = FutureArrow.choose( - predicate = hydratesContentFeatures, - ifTrue = contentFeaturesHydrationTransform - .andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform) - .andThen(CopyContentFeaturesIntoHydratedTweetsTransform), - ifFalse = FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ](Future.value) // empty transformer - ) - - private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope) - - val hydrationAndFilteringPipeline: FutureArrow[RecapQuery, CandidateEnvelope] = - CreateCandidateEnvelopeTransform // Create empty CandidateEnvelope - .andThen(followGraphDataTransform) // Fetch follow graph data - .andThen(searchResultsTransform) // Fetch search results - .andThen(SearchResultDedupAndSortingTransform) - .andThen(CandidateTweetHydrationTransform) // candidate hydration - .andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones - .andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter - .andThen(dynamicHydratedTweetsFilter) // filter hydrated tweets based on query TweetKindOption - .andThen( - TrimToMatchHydratedTweetsTransform - ) // trim search result set to match filtered hydrated tweets (this needs to be accurate for feature hydration) - - // runs the main pipeline in parallel with fetching user profile info and user languages - val featureHydrationDataTransform: FeatureHydrationDataTransform = - new FeatureHydrationDataTransform( - hydrationAndFilteringPipeline, - languagesTransform, - userProfileInfoTransform - ) - - // Copy transforms must go after the search features transform, as the search transform - // overwrites the ThriftTweetFeatures. - val featureHydrationPipeline: FutureArrow[RecapQuery, CandidateTweetsResult] = - featureHydrationDataTransform - .andThen( - InNetworkTweetsSearchFeaturesHydrationTransform - ) // RecapAuthorSource uses InNetworkTweetsSearchFeaturesHydrationTransform because PYLE uses the in-network model and features. - .andThen(contentFeaturesTransformer) - .andThen(candidateGenerationTransform) - - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - requestStats.addEventStats { - featureHydrationPipeline(query) - } - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - Future.collect(queries.map(get)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/BUILD.bazel b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/BUILD.bazel deleted file mode 100644 index 56effdae1..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "finagle/finagle-core/src/main", - "servo/util/src/main/scala", - "src/thrift/com/twitter/search:earlybird-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/common", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/repository", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/bounds", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/BUILD.docx new file mode 100644 index 000000000..43846f91b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepository.docx new file mode 100644 index 000000000..2b6ddc05f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepository.scala deleted file mode 100644 index e8bcf84df..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepository.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.timelineranker.recap_hydration - -import com.twitter.timelineranker.model.CandidateTweetsResult -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.util.Future - -/** - * A repository of recap hydration results. - * - * For now, it does not cache any results therefore forwards all calls to the underlying source. - */ -class RecapHydrationRepository(source: RecapHydrationSource) { - def hydrate(query: RecapQuery): Future[CandidateTweetsResult] = { - source.hydrate(query) - } - - def hydrate(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - source.hydrate(queries) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepositoryBuilder.docx new file mode 100644 index 000000000..d766fb67a Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepositoryBuilder.scala deleted file mode 100644 index b9917a320..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationRepositoryBuilder.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.timelineranker.recap_hydration - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.service.RetryPolicy -import com.twitter.timelineranker.config.RequestScopes -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.parameters.ConfigBuilder -import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder -import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.util.Duration - -class RecapHydrationRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder) - extends CandidatesRepositoryBuilder(config) { - - override val clientSubId = "feature_hydration" - override val requestScope: RequestScope = RequestScopes.RecapHydrationSource - override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet = - SgsFollowGraphDataFields.ValueSet( - SgsFollowGraphDataFields.FollowedUserIds, - SgsFollowGraphDataFields.MutuallyFollowingUserIds - ) - override val searchProcessingTimeout: Duration = 200.milliseconds //[2] - - override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdClient( - scope = scope, - requestTimeout = 500.milliseconds, // [1] - timeout = 500.milliseconds, // [1] - retryPolicy = RetryPolicy.Never - ) - - def apply(): RecapHydrationRepository = { - val recapHydrationSource = new RecapHydrationSource( - gizmoduckClient, - searchClient, - tweetyPieLowQoSClient, - userMetadataClient, - followGraphDataProvider, - config.underlyingClients.contentFeaturesCache, - config.statsReceiver - ) - - new RecapHydrationRepository(recapHydrationSource) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSearchResultsTransform.docx new file mode 100644 index 000000000..961778308 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSearchResultsTransform.scala deleted file mode 100644 index 59a00ecf9..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSearchResultsTransform.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.timelineranker.recap_hydration - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.timelineranker.common.RecapHydrationSearchResultsTransformBase -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.model.TweetId - -class RecapHydrationSearchResultsTransform( - override protected val searchClient: SearchClient, - override protected val statsReceiver: StatsReceiver) - extends RecapHydrationSearchResultsTransformBase { - override def tweetIdsToHydrate(envelope: CandidateEnvelope): Seq[TweetId] = - envelope.query.tweetIds.get -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSource.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSource.docx new file mode 100644 index 000000000..c61cafe6f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSource.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSource.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSource.scala deleted file mode 100644 index e089ba363..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration/RecapHydrationSource.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.timelineranker.recap_hydration - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FutureArrow -import com.twitter.storehaus.Store -import com.twitter.timelineranker.common._ -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.model._ -import com.twitter.timelineranker.parameters.recap.RecapParams -import com.twitter.timelineranker.parameters.recap_hydration.RecapHydrationParams -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform -import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.clients.gizmoduck.GizmoduckClient -import com.twitter.timelines.clients.manhattan.UserMetadataClient -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.tweetypie.TweetyPieClient -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.timelines.util.stats.RequestStatsReceiver -import com.twitter.util.Future - -class RecapHydrationSource( - gizmoduckClient: GizmoduckClient, - searchClient: SearchClient, - tweetyPieClient: TweetyPieClient, - userMetadataClient: UserMetadataClient, - followGraphDataProvider: FollowGraphDataProvider, - contentFeaturesCache: Store[TweetId, ContentFeatures], - statsReceiver: StatsReceiver) { - - private[this] val baseScope = statsReceiver.scope("recapHydration") - private[this] val requestStats = RequestStatsReceiver(baseScope) - private[this] val numInputTweetsStat = baseScope.stat("numInputTweets") - - private[this] val failOpenScope = baseScope.scope("failOpen") - private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo") - private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages") - - private[this] val maxFollowedUsersProvider = - DependencyProvider.value(RecapParams.MaxFollowedUsers.default) - private[this] val followGraphDataTransform = - new FollowGraphDataTransform(followGraphDataProvider, maxFollowedUsersProvider) - - private[this] val searchResultsTransform = - new RecapHydrationSearchResultsTransform(searchClient, baseScope) - - private[this] val userProfileInfoTransform = - new UserProfileInfoTransform(userProfileHandler, gizmoduckClient) - private[this] val languagesTransform = - new UserLanguagesTransform(userLanguagesHandler, userMetadataClient) - - private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope) - - private[this] val hydrationAndFilteringPipeline = - CreateCandidateEnvelopeTransform - .andThen(followGraphDataTransform) - .andThen(searchResultsTransform) - .andThen(CandidateTweetHydrationTransform) - - // runs the main pipeline in parallel with fetching user profile info and user languages - private[this] val featureHydrationDataTransform = new FeatureHydrationDataTransform( - hydrationAndFilteringPipeline, - languagesTransform, - userProfileInfoTransform - ) - - private[this] val contentFeaturesHydrationTransform = - new ContentFeaturesHydrationTransformBuilder( - tweetyPieClient = tweetyPieClient, - contentFeaturesCache = contentFeaturesCache, - enableContentFeaturesGate = - RecapQuery.paramGate(RecapHydrationParams.EnableContentFeaturesHydrationParam), - enableTokensInContentFeaturesGate = - RecapQuery.paramGate(RecapHydrationParams.EnableTokensInContentFeaturesHydrationParam), - enableTweetTextInContentFeaturesGate = - RecapQuery.paramGate(RecapHydrationParams.EnableTweetTextInContentFeaturesHydrationParam), - enableConversationControlContentFeaturesGate = RecapQuery.paramGate( - RecapHydrationParams.EnableConversationControlInContentFeaturesHydrationParam), - enableTweetMediaHydrationGate = RecapQuery.paramGate( - RecapHydrationParams.EnableTweetMediaHydrationParam - ), - hydrateInReplyToTweets = true, - statsReceiver = baseScope - ).build() - - private[this] def hydratesContentFeatures( - hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope - ): Boolean = - hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true) - - private[this] val contentFeaturesTransformer = FutureArrow.choose( - predicate = hydratesContentFeatures, - ifTrue = contentFeaturesHydrationTransform - .andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform) - .andThen(CopyContentFeaturesIntoHydratedTweetsTransform), - ifFalse = FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ](Future.value) // empty transformer - ) - - private[this] val featureHydrationPipeline = - featureHydrationDataTransform - .andThen(InNetworkTweetsSearchFeaturesHydrationTransform) - .andThen(contentFeaturesTransformer) - .andThen(candidateGenerationTransform) - - def hydrate(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - Future.collect(queries.map(hydrate)) - } - - def hydrate(query: RecapQuery): Future[CandidateTweetsResult] = { - require(query.tweetIds.isDefined && query.tweetIds.get.nonEmpty, "tweetIds must be present") - query.tweetIds.foreach(ids => numInputTweetsStat.add(ids.size)) - - requestStats.addEventStats { - featureHydrationPipeline(query) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/BUILD deleted file mode 100644 index 74119dd8b..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "finagle/finagle-core/src/main", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "timelineranker/common:model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/source", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/clients/socialgraph", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph", - "timelines/src/main/scala/com/twitter/timelines/config/configapi", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelines/src/main/scala/com/twitter/timelines/visibility", - "timelines/src/main/scala/com/twitter/timelines/visibility/model", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/BUILD.docx new file mode 100644 index 000000000..c8bf75ab1 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/CandidatesRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/CandidatesRepositoryBuilder.docx new file mode 100644 index 000000000..b9ab0a45d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/CandidatesRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/CandidatesRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/CandidatesRepositoryBuilder.scala deleted file mode 100644 index c6769c626..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/CandidatesRepositoryBuilder.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.servo.util.Gate -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields -import com.twitter.timelineranker.visibility.ScopedSgsFollowGraphDataProviderFactory -import com.twitter.timelines.clients.relevance_search.ScopedSearchClientFactory -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.user_tweet_entity_graph.UserTweetEntityGraphClient -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.util.Duration -import com.twitter.timelineranker.config.ClientWrapperFactories -import com.twitter.timelineranker.config.UnderlyingClientConfiguration -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.clients.gizmoduck.GizmoduckClient -import com.twitter.timelines.clients.manhattan.ManhattanUserMetadataClient -import com.twitter.timelines.clients.tweetypie.TweetyPieClient - -abstract class CandidatesRepositoryBuilder(config: RuntimeConfiguration) extends RepositoryBuilder { - - def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint - def searchProcessingTimeout: Duration - def clientSubId: String - def requestScope: RequestScope - def followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet - - protected lazy val clientConfig: UnderlyingClientConfiguration = config.underlyingClients - - protected lazy val clientFactories: ClientWrapperFactories = config.clientWrapperFactories - protected lazy val gizmoduckClient: GizmoduckClient = - clientFactories.gizmoduckClientFactory.scope(requestScope) - protected lazy val searchClient: SearchClient = newSearchClient(clientId = clientSubId) - protected lazy val tweetyPieHighQoSClient: TweetyPieClient = - clientFactories.tweetyPieHighQoSClientFactory.scope(requestScope) - protected lazy val tweetyPieLowQoSClient: TweetyPieClient = - clientFactories.tweetyPieLowQoSClientFactory.scope(requestScope) - protected lazy val followGraphDataProvider: FollowGraphDataProvider = - new ScopedSgsFollowGraphDataProviderFactory( - clientFactories.socialGraphClientFactory, - clientFactories.visibilityProfileHydratorFactory, - followGraphDataFieldsToFetch, - config.statsReceiver - ).scope(requestScope) - protected lazy val userMetadataClient: ManhattanUserMetadataClient = - clientFactories.userMetadataClientFactory.scope(requestScope) - protected lazy val userTweetEntityGraphClient: UserTweetEntityGraphClient = - new UserTweetEntityGraphClient( - config.underlyingClients.userTweetEntityGraphClient, - config.statsReceiver - ) - - protected lazy val perRequestSearchClientIdProvider: DependencyProvider[Option[String]] = - DependencyProvider { recapQuery: RecapQuery => - recapQuery.searchClientSubId.map { subId => - clientConfig.timelineRankerClientId(Some(s"$subId.$clientSubId")).name - } - } - - protected lazy val perRequestSourceSearchClientIdProvider: DependencyProvider[Option[String]] = - DependencyProvider { recapQuery: RecapQuery => - recapQuery.searchClientSubId.map { subId => - clientConfig.timelineRankerClientId(Some(s"$subId.${clientSubId}_source_tweets")).name - } - } - - protected def newSearchClient(clientId: String): SearchClient = - new ScopedSearchClientFactory( - searchServiceClient = earlybirdClient(clientId), - clientId = clientConfig.timelineRankerClientId(Some(clientId)).name, - processingTimeout = Some(searchProcessingTimeout), - collectConversationIdGate = Gate.True, - statsReceiver = config.statsReceiver - ).scope(requestScope) - - protected def newSearchClient( - earlybirdReplacementClient: String => EarlybirdService.MethodPerEndpoint, - clientId: String - ): SearchClient = - new ScopedSearchClientFactory( - searchServiceClient = earlybirdReplacementClient(clientId), - clientId = clientConfig.timelineRankerClientId(Some(clientId)).name, - processingTimeout = Some(searchProcessingTimeout), - collectConversationIdGate = Gate.True, - statsReceiver = config.statsReceiver - ).scope(requestScope) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RankedHomeTimelineRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RankedHomeTimelineRepository.docx new file mode 100644 index 000000000..69d8d3de7 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RankedHomeTimelineRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RankedHomeTimelineRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RankedHomeTimelineRepository.scala deleted file mode 100644 index 4ce7ccdec..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RankedHomeTimelineRepository.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.timelineranker.model.Timeline -import com.twitter.timelineranker.model.TimelineQuery -import com.twitter.util.Future - -/** - * A repository of ranked home timelines. - */ -class RankedHomeTimelineRepository extends TimelineRepository { - def get(queries: Seq[TimelineQuery]): Seq[Future[Timeline]] = { - queries.map { _ => - Future.exception(new UnsupportedOperationException("ranked timelines are not yet supported.")) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RepositoryBuilder.docx new file mode 100644 index 000000000..4c0558e06 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RepositoryBuilder.scala deleted file mode 100644 index 6ad842e57..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RepositoryBuilder.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.timelines.visibility.model.VisibilityRule - -object RepositoryBuilder { - val VisibilityRules: Set[VisibilityRule.Value] = Set( - VisibilityRule.Blocked, - VisibilityRule.BlockedBy, - VisibilityRule.Muted, - VisibilityRule.Protected, - VisibilityRule.AccountStatus - ) -} - -trait RepositoryBuilder { - val VisibilityRules: Set[VisibilityRule.Value] = RepositoryBuilder.VisibilityRules -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepository.docx new file mode 100644 index 000000000..16e9739d5 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepository.scala deleted file mode 100644 index d7da864d7..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepository.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.timelineranker.model.ReverseChronTimelineQuery -import com.twitter.timelineranker.model.Timeline -import com.twitter.timelineranker.parameters.revchron.ReverseChronTimelineQueryContextBuilder -import com.twitter.timelineranker.source.ReverseChronHomeTimelineSource -import com.twitter.util.Future - -/** - * A repository of reverse-chron home timelines. - * - * It does not cache any results therefore forwards all calls to the underlying source. - */ -class ReverseChronHomeTimelineRepository( - source: ReverseChronHomeTimelineSource, - contextBuilder: ReverseChronTimelineQueryContextBuilder) { - def get(query: ReverseChronTimelineQuery): Future[Timeline] = { - contextBuilder(query).flatMap(source.get) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepositoryBuilder.docx new file mode 100644 index 000000000..70cad140c Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepositoryBuilder.scala deleted file mode 100644 index b651c2e98..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/ReverseChronHomeTimelineRepositoryBuilder.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelineranker.config.RequestScopes -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.parameters.ConfigBuilder -import com.twitter.timelineranker.parameters.revchron.ReverseChronTimelineQueryContextBuilder -import com.twitter.timelineranker.parameters.util.RequestContextBuilderImpl -import com.twitter.timelineranker.source.ReverseChronHomeTimelineSource -import com.twitter.timelineranker.visibility.RealGraphFollowGraphDataProvider -import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.util.Duration - -class ReverseChronHomeTimelineRepositoryBuilder( - config: RuntimeConfiguration, - configBuilder: ConfigBuilder) - extends CandidatesRepositoryBuilder(config) { - - override val clientSubId = "home_materialization" - override val requestScope: RequestScope = RequestScopes.HomeTimelineMaterialization - override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet = - SgsFollowGraphDataFields.ValueSet( - SgsFollowGraphDataFields.FollowedUserIds, - SgsFollowGraphDataFields.MutedUserIds, - SgsFollowGraphDataFields.RetweetsMutedUserIds - ) - override val searchProcessingTimeout: Duration = 800.milliseconds // [3] - - override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdClient( - scope = scope, - requestTimeout = 1.second, // [1] - timeout = 1900.milliseconds, // [2] - retryPolicy = config.underlyingClients.DefaultRetryPolicy - ) - - val realGraphFollowGraphDataProvider = new RealGraphFollowGraphDataProvider( - followGraphDataProvider, - config.clientWrapperFactories.realGraphClientFactory - .scope(RequestScopes.ReverseChronHomeTimelineSource), - config.clientWrapperFactories.socialGraphClientFactory - .scope(RequestScopes.ReverseChronHomeTimelineSource), - config.deciderGateBuilder.idGate(DeciderKey.SupplementFollowsWithRealGraph), - config.statsReceiver.scope(RequestScopes.ReverseChronHomeTimelineSource.scope) - ) - - def apply(): ReverseChronHomeTimelineRepository = { - val reverseChronTimelineSource = new ReverseChronHomeTimelineSource( - searchClient, - realGraphFollowGraphDataProvider, - clientFactories.visibilityEnforcerFactory.apply( - VisibilityRules, - RequestScopes.ReverseChronHomeTimelineSource - ), - config.statsReceiver - ) - - val contextBuilder = new ReverseChronTimelineQueryContextBuilder( - configBuilder.rootConfig, - config, - new RequestContextBuilderImpl(config.configApiConfiguration.requestContextFactory) - ) - - new ReverseChronHomeTimelineRepository( - reverseChronTimelineSource, - contextBuilder - ) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepository.docx new file mode 100644 index 000000000..784c28578 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepository.scala deleted file mode 100644 index 418c85b80..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepository.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.timelineranker.model._ -import com.twitter.util.Future - -class RoutingTimelineRepository( - reverseChronTimelineRepository: ReverseChronHomeTimelineRepository, - rankedTimelineRepository: RankedHomeTimelineRepository) - extends TimelineRepository { - - override def get(query: TimelineQuery): Future[Timeline] = { - query match { - case q: ReverseChronTimelineQuery => reverseChronTimelineRepository.get(q) - case q: RankedTimelineQuery => rankedTimelineRepository.get(q) - case _ => - throw new IllegalArgumentException( - s"Query types other than RankedTimelineQuery and ReverseChronTimelineQuery are not supported. Found: $query" - ) - } - } - - override def get(queries: Seq[TimelineQuery]): Seq[Future[Timeline]] = { - queries.map(get) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepositoryBuilder.docx new file mode 100644 index 000000000..0ac3cb3be Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepositoryBuilder.scala deleted file mode 100644 index f35d5001d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/RoutingTimelineRepositoryBuilder.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.parameters.ConfigBuilder - -object RoutingTimelineRepositoryBuilder { - def apply( - config: RuntimeConfiguration, - configBuilder: ConfigBuilder - ): RoutingTimelineRepository = { - - val reverseChronTimelineRepository = - new ReverseChronHomeTimelineRepositoryBuilder(config, configBuilder).apply - val rankedTimelineRepository = new RankedHomeTimelineRepository - - new RoutingTimelineRepository(reverseChronTimelineRepository, rankedTimelineRepository) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/TimelineRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/TimelineRepository.docx new file mode 100644 index 000000000..2f7c0fad3 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/TimelineRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/TimelineRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/TimelineRepository.scala deleted file mode 100644 index f553ff835..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/repository/TimelineRepository.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.timelineranker.repository - -import com.twitter.timelineranker.model.Timeline -import com.twitter.timelineranker.model.TimelineQuery -import com.twitter.util.Future - -trait TimelineRepository { - def get(queries: Seq[TimelineQuery]): Seq[Future[Timeline]] - def get(query: TimelineQuery): Future[Timeline] = get(Seq(query)).head -} - -class EmptyTimelineRepository extends TimelineRepository { - def get(queries: Seq[TimelineQuery]): Seq[Future[Timeline]] = { - queries.map(q => Future.value(Timeline.empty(q.id))) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/BUILD.bazel b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/BUILD.bazel deleted file mode 100644 index 564d7c0c8..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/BUILD.bazel +++ /dev/null @@ -1,65 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/thrift:libthrift", - "abdecider/src/main/scala", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "decider/src/main/scala", - "featureswitches/featureswitches-core/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authorization/server", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/server", - "finagle/finagle-core/src/main", - "finagle/finagle-thrift/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", - "servo/decider", - "servo/request/src/main/scala", - "servo/util", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelineranker/server/model:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", - "thrift-web-forms", - "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/model", - "thrift-web-forms/src/main/scala/com/twitter/thriftwebforms/view", - "timelineranker/common:model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/decider", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/observe", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_author", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap_hydration", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/repository", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets", - "timelines:observe", - "timelines:warmup", - "timelines/src/main/scala/com/twitter/timelines/authorization", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/features", - "timelines/src/main/scala/com/twitter/timelines/features/app", - "timelines/src/main/scala/com/twitter/timelines/filter", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/server", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/logging", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "twitter-server-internal", - "twitter-server/server/src/main/scala", - "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/logging", - "util/util-app/src/main/scala", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/concurrent", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-logging", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/BUILD.docx new file mode 100644 index 000000000..ceaa7b361 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Main.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Main.docx new file mode 100644 index 000000000..d54e21165 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Main.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Main.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Main.scala deleted file mode 100644 index f7cd3d4be..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Main.scala +++ /dev/null @@ -1,182 +0,0 @@ -package com.twitter.timelineranker.server - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.server.MtlsStackServer._ -import com.twitter.finagle.mux -import com.twitter.finagle.param.Reporter -import com.twitter.finagle.stats.DefaultStatsReceiver -import com.twitter.finagle.util.NullReporterFactory -import com.twitter.finagle.ListeningServer -import com.twitter.finagle.ServiceFactory -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authorization.server.MtlsServerSessionTrackerFilter -import com.twitter.finagle.ssl.OpportunisticTls -import com.twitter.finatra.thrift.filters.LoggingMDCFilter -import com.twitter.finatra.thrift.filters.ThriftMDCFilter -import com.twitter.finatra.thrift.filters.TraceIdMDCFilter -import com.twitter.logging.Logger -import com.twitter.server.TwitterServer -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.thriftwebforms.MethodOptions -import com.twitter.thriftwebforms.TwitterServerThriftWebForms -import com.twitter.timelineranker.config.RuntimeConfigurationImpl -import com.twitter.timelineranker.config.TimelineRankerFlags -import com.twitter.timelineranker.thriftscala -import com.twitter.timelines.config.DefaultDynamicConfigStoreFactory -import com.twitter.timelines.config.EmptyDynamicConfigStoreFactory -import com.twitter.timelines.config.Env -import com.twitter.timelines.features.app.ForcibleFeatureValues -import com.twitter.timelines.server.AdminMutableDeciders -import com.twitter.timelines.warmup.NoWarmup -import com.twitter.timelines.warmup.WarmupFlag -import com.twitter.util.Await -import java.net.SocketAddress -import org.apache.thrift.protocol.TBinaryProtocol -import org.apache.thrift.protocol.TCompactProtocol -import org.apache.thrift.protocol.TProtocolFactory - -object Main extends TimelineRankerServer - -class TimelineRankerServer extends { - override val statsReceiver: MemoizingStatsReceiver = new MemoizingStatsReceiver( - DefaultStatsReceiver) -} with TwitterServer with AdminMutableDeciders with ForcibleFeatureValues with WarmupFlag { - - val timelineRankerFlags: TimelineRankerFlags = new TimelineRankerFlags(flag) - lazy val mainLogger: Logger = Logger.get("Main") - - private[this] lazy val thriftWebFormsAccess = if (timelineRankerFlags.getEnv == Env.local) { - MethodOptions.Access.Default - } else { - MethodOptions.Access.ByLdapGroup(Seq("timeline-team", "timelineranker-twf-read")) - } - - private[this] def mkThriftWebFormsRoutes() = - TwitterServerThriftWebForms[thriftscala.TimelineRanker.MethodPerEndpoint]( - thriftServicePort = timelineRankerFlags.servicePort().getPort, - defaultMethodAccess = thriftWebFormsAccess, - methodOptions = TimelineRankerThriftWebForms.methodOptions, - serviceIdentifier = timelineRankerFlags.serviceIdentifier(), - opportunisticTlsLevel = OpportunisticTls.Required, - ) - - override protected def failfastOnFlagsNotParsed = true - override val defaultCloseGracePeriod = 10.seconds - - private[this] def mkServer( - labelSuffix: String, - socketAddress: SocketAddress, - protocolFactory: TProtocolFactory, - serviceFactory: ServiceFactory[Array[Byte], Array[Byte]], - opportunisticTlsLevel: OpportunisticTls.Level, - ): ListeningServer = { - val compressor = Seq(mux.transport.Compression.lz4Compressor(highCompression = false)) - val decompressor = Seq(mux.transport.Compression.lz4Decompressor()) - val compressionLevel = - if (timelineRankerFlags.enableThriftmuxCompression()) { - mux.transport.CompressionLevel.Desired - } else { - mux.transport.CompressionLevel.Off - } - - val mtlsSessionTrackerFilter = - new MtlsServerSessionTrackerFilter[mux.Request, mux.Response](statsReceiver) - val loggingMDCFilter = { new LoggingMDCFilter }.toFilter[mux.Request, mux.Response] - val traceIdMDCFilter = { new TraceIdMDCFilter }.toFilter[mux.Request, mux.Response] - val thriftMDCFilter = { new ThriftMDCFilter }.toFilter[mux.Request, mux.Response] - val filters = mtlsSessionTrackerFilter - .andThen(loggingMDCFilter) - .andThen(traceIdMDCFilter) - .andThen(thriftMDCFilter) - - ThriftMux.server - // By default, finagle logs exceptions to chickadee, which is deprecated and - // basically unused. To avoid wasted overhead, we explicitly disable the reporter. - .configured(Reporter(NullReporterFactory)) - .withLabel("timelineranker." + labelSuffix) - .withMutualTls(timelineRankerFlags.getServiceIdentifier) - .withOpportunisticTls(opportunisticTlsLevel) - .withProtocolFactory(protocolFactory) - .withCompressionPreferences.compression(compressionLevel, compressor) - .withCompressionPreferences.decompression(compressionLevel, decompressor) - .filtered(filters) - .serve(socketAddress, serviceFactory) - } - - def main(): Unit = { - try { - val parsedOpportunisticTlsLevel = OpportunisticTls.Values - .find( - _.value.toLowerCase == timelineRankerFlags.opportunisticTlsLevel().toLowerCase).getOrElse( - OpportunisticTls.Desired) - - TwitterServerThriftWebForms.addAdminRoutes(this, mkThriftWebFormsRoutes()) - addAdminMutableDeciderRoutes(timelineRankerFlags.getEnv) - - val configStoreFactory = if (timelineRankerFlags.getEnv == Env.local) { - EmptyDynamicConfigStoreFactory - } else { - new DefaultDynamicConfigStoreFactory - } - - val runtimeConfiguration = new RuntimeConfigurationImpl( - timelineRankerFlags, - configStoreFactory, - decider, - forcedFeatureValues = getFeatureSwitchOverrides, - statsReceiver - ) - - val timelineRankerBuilder = new TimelineRankerBuilder(runtimeConfiguration) - - val warmup = if (shouldWarmup) { - new Warmup( - timelineRankerBuilder.timelineRanker, - runtimeConfiguration.underlyingClients.timelineRankerForwardingClient, - mainLogger - ) - } else { - new NoWarmup() - } - - warmup.prebindWarmup() - - // Create Thrift services that use the binary Thrift protocol, and the compact one. - val server = - mkServer( - "binary", - timelineRankerFlags.servicePort(), - new TBinaryProtocol.Factory(), - timelineRankerBuilder.serviceFactory, - parsedOpportunisticTlsLevel, - ) - - val compactServer = - mkServer( - "compact", - timelineRankerFlags.serviceCompactPort(), - new TCompactProtocol.Factory(), - timelineRankerBuilder.compactProtocolServiceFactory, - parsedOpportunisticTlsLevel, - ) - - mainLogger.info( - s"Thrift binary server bound to service port [${timelineRankerFlags.servicePort()}]") - closeOnExit(server) - mainLogger.info( - s"Thrift compact server bound to service port [${timelineRankerFlags.serviceCompactPort()}]") - closeOnExit(compactServer) - - warmup.warmupComplete() - - mainLogger.info("ready: server") - Await.ready(server) - Await.ready(compactServer) - } catch { - case e: Throwable => - e.printStackTrace() - mainLogger.error(e, s"failure in main") - System.exit(1) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRanker.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRanker.docx new file mode 100644 index 000000000..a109d46c3 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRanker.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRanker.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRanker.scala deleted file mode 100644 index 6e608786d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRanker.scala +++ /dev/null @@ -1,255 +0,0 @@ -package com.twitter.timelineranker.server - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.servo.util.FunctionArrow -import com.twitter.timelineranker.entity_tweets.EntityTweetsRepository -import com.twitter.timelineranker.in_network_tweets.InNetworkTweetRepository -import com.twitter.timelineranker.model._ -import com.twitter.timelineranker.observe.ObservedRequests -import com.twitter.timelineranker.recap_author.RecapAuthorRepository -import com.twitter.timelineranker.recap_hydration.RecapHydrationRepository -import com.twitter.timelineranker.repository._ -import com.twitter.timelineranker.uteg_liked_by_tweets.UtegLikedByTweetsRepository -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.timelines.authorization.TimelinesClientRequestAuthorizer -import com.twitter.timelines.observe.DebugObserver -import com.twitter.timelines.observe.ObservedAndValidatedRequests -import com.twitter.timelines.observe.QueryWidth -import com.twitter.timelines.observe.ServiceObserver -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try - -object TimelineRanker { - def toTimelineErrorThriftResponse( - ex: Throwable, - reason: Option[thrift.ErrorReason] = None - ): thrift.GetTimelineResponse = { - thrift.GetTimelineResponse( - error = Some(thrift.TimelineError(message = ex.toString, reason)) - ) - } - - def getTimelinesExceptionHandler: PartialFunction[ - Throwable, - Future[thrift.GetTimelineResponse] - ] = { - case e: TimeoutException => - Future.value(toTimelineErrorThriftResponse(e, Some(thrift.ErrorReason.UpstreamTimeout))) - case e: Throwable if ObservedAndValidatedRequests.isOverCapacityException(e) => - Future.value(toTimelineErrorThriftResponse(e, Some(thrift.ErrorReason.OverCapacity))) - case e => Future.value(toTimelineErrorThriftResponse(e)) - } - - def toErrorThriftResponse( - ex: Throwable, - reason: Option[thrift.ErrorReason] = None - ): thrift.GetCandidateTweetsResponse = { - thrift.GetCandidateTweetsResponse( - error = Some(thrift.TimelineError(message = ex.toString, reason)) - ) - } - - def exceptionHandler: PartialFunction[Throwable, Future[thrift.GetCandidateTweetsResponse]] = { - case e: TimeoutException => - Future.value(toErrorThriftResponse(e, Some(thrift.ErrorReason.UpstreamTimeout))) - case e: Throwable if ObservedAndValidatedRequests.isOverCapacityException(e) => - Future.value(toErrorThriftResponse(e, Some(thrift.ErrorReason.OverCapacity))) - case e => Future.value(toErrorThriftResponse(e)) - } -} - -class TimelineRanker( - routingRepository: RoutingTimelineRepository, - inNetworkTweetRepository: InNetworkTweetRepository, - recapHydrationRepository: RecapHydrationRepository, - recapAuthorRepository: RecapAuthorRepository, - entityTweetsRepository: EntityTweetsRepository, - utegLikedByTweetsRepository: UtegLikedByTweetsRepository, - serviceObserver: ServiceObserver, - val abdecider: Option[LoggingABDecider], - override val clientRequestAuthorizer: TimelinesClientRequestAuthorizer, - override val debugObserver: DebugObserver, - queryParamInitializer: FunctionArrow[RecapQuery, Future[RecapQuery]], - statsReceiver: StatsReceiver) - extends thrift.TimelineRanker.MethodPerEndpoint - with ObservedRequests { - - override val requestWidthStats: Stat = statsReceiver.stat("TimelineRanker/requestWidth") - - private[this] val getTimelinesStats = serviceObserver.readMethodStats( - "getTimelines", - QueryWidth.one[TimelineQuery] - ) - - private[this] val getInNetworkTweetCandidatesStats = serviceObserver.readMethodStats( - "getInNetworkTweetCandidates", - QueryWidth.one[RecapQuery] - ) - - private[this] val hydrateTweetCandidatesStats = serviceObserver.readMethodStats( - "hydrateTweetCandidates", - QueryWidth.one[RecapQuery] - ) - - private[this] val getRecapCandidatesFromAuthorsStats = serviceObserver.readMethodStats( - "getRecapCandidatesFromAuthors", - QueryWidth.one[RecapQuery] - ) - - private[this] val getEntityTweetCandidatesStats = serviceObserver.readMethodStats( - "getEntityTweetCandidates", - QueryWidth.one[RecapQuery] - ) - - private[this] val getUtegLikedByTweetCandidatesStats = serviceObserver.readMethodStats( - "getUtegLikedByTweetCandidates", - QueryWidth.one[RecapQuery] - ) - - def getTimelines( - thriftQueries: Seq[thrift.TimelineQuery] - ): Future[Seq[thrift.GetTimelineResponse]] = { - Future.collect( - thriftQueries.map { thriftQuery => - Try(TimelineQuery.fromThrift(thriftQuery)) match { - case Return(query) => - observeAndValidate( - query, - Seq(query.userId), - getTimelinesStats, - TimelineRanker.getTimelinesExceptionHandler) { validatedQuery => - routingRepository.get(validatedQuery).map { timeline => - thrift.GetTimelineResponse(Some(timeline.toThrift)) - } - } - case Throw(e) => Future.value(TimelineRanker.toTimelineErrorThriftResponse(e)) - } - } - ) - } - - def getRecycledTweetCandidates( - thriftQueries: Seq[thrift.RecapQuery] - ): Future[Seq[thrift.GetCandidateTweetsResponse]] = { - Future.collect( - thriftQueries.map { thriftQuery => - Try(RecapQuery.fromThrift(thriftQuery)) match { - case Return(query) => - observeAndValidate( - query, - Seq(query.userId), - getInNetworkTweetCandidatesStats, - TimelineRanker.exceptionHandler - ) { validatedQuery => - Future(queryParamInitializer(validatedQuery)).flatten.liftToTry.flatMap { - case Return(q) => inNetworkTweetRepository.get(q).map(_.toThrift) - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - ) - } - - def hydrateTweetCandidates( - thriftQueries: Seq[thrift.RecapHydrationQuery] - ): Future[Seq[thrift.GetCandidateTweetsResponse]] = { - Future.collect( - thriftQueries.map { thriftQuery => - Try(RecapQuery.fromThrift(thriftQuery)) match { - case Return(query) => - observeAndValidate( - query, - Seq(query.userId), - hydrateTweetCandidatesStats, - TimelineRanker.exceptionHandler - ) { validatedQuery => - Future(queryParamInitializer(validatedQuery)).flatten.liftToTry.flatMap { - case Return(q) => recapHydrationRepository.hydrate(q).map(_.toThrift) - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - ) - } - - def getRecapCandidatesFromAuthors( - thriftQueries: Seq[thrift.RecapQuery] - ): Future[Seq[thrift.GetCandidateTweetsResponse]] = { - Future.collect( - thriftQueries.map { thriftQuery => - Try(RecapQuery.fromThrift(thriftQuery)) match { - case Return(query) => - observeAndValidate( - query, - Seq(query.userId), - getRecapCandidatesFromAuthorsStats, - TimelineRanker.exceptionHandler - ) { validatedQuery => - Future(queryParamInitializer(validatedQuery)).flatten.liftToTry.flatMap { - case Return(q) => recapAuthorRepository.get(q).map(_.toThrift) - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - ) - } - - def getEntityTweetCandidates( - thriftQueries: Seq[thrift.EntityTweetsQuery] - ): Future[Seq[thrift.GetCandidateTweetsResponse]] = { - Future.collect( - thriftQueries.map { thriftQuery => - Try(RecapQuery.fromThrift(thriftQuery)) match { - case Return(query) => - observeAndValidate( - query, - Seq(query.userId), - getEntityTweetCandidatesStats, - TimelineRanker.exceptionHandler - ) { validatedQuery => - Future(queryParamInitializer(validatedQuery)).flatten.liftToTry.flatMap { - case Return(q) => entityTweetsRepository.get(q).map(_.toThrift) - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - ) - } - - def getUtegLikedByTweetCandidates( - thriftQueries: Seq[thrift.UtegLikedByTweetsQuery] - ): Future[Seq[thrift.GetCandidateTweetsResponse]] = { - Future.collect( - thriftQueries.map { thriftQuery => - Try(RecapQuery.fromThrift(thriftQuery)) match { - case Return(query) => - observeAndValidate( - query, - Seq(query.userId), - getUtegLikedByTweetCandidatesStats, - TimelineRanker.exceptionHandler - ) { validatedQuery => - Future(queryParamInitializer(validatedQuery)).flatten.liftToTry.flatMap { - case Return(q) => utegLikedByTweetsRepository.get(q).map(_.toThrift) - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - case Throw(e) => Future.value(TimelineRanker.toErrorThriftResponse(e)) - } - } - ) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerBuilder.docx new file mode 100644 index 000000000..d4c81cee1 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerBuilder.scala deleted file mode 100644 index 3c1ffc793..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerBuilder.scala +++ /dev/null @@ -1,127 +0,0 @@ -package com.twitter.timelineranker.server - -import com.twitter.concurrent.AsyncSemaphore -import com.twitter.finagle.Filter -import com.twitter.finagle.ServiceFactory -import com.twitter.finagle.thrift.filter.ThriftForwardingWarmUpFilter -import com.twitter.finagle.thrift.ClientIdRequiredFilter -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.config.TimelineRankerConstants -import com.twitter.timelineranker.decider.DeciderKey -import com.twitter.timelineranker.entity_tweets.EntityTweetsRepositoryBuilder -import com.twitter.timelineranker.observe.DebugObserverBuilder -import com.twitter.timelineranker.parameters.ConfigBuilder -import com.twitter.timelineranker.parameters.util.RecapQueryParamInitializer -import com.twitter.timelineranker.recap_author.RecapAuthorRepositoryBuilder -import com.twitter.timelineranker.recap_hydration.RecapHydrationRepositoryBuilder -import com.twitter.timelineranker.in_network_tweets.InNetworkTweetRepositoryBuilder -import com.twitter.timelineranker.repository._ -import com.twitter.timelineranker.thriftscala.TimelineRanker$FinagleService -import com.twitter.timelineranker.uteg_liked_by_tweets.UtegLikedByTweetsRepositoryBuilder -import com.twitter.timelines.filter.DarkTrafficFilterBuilder -import com.twitter.timelines.observe.ServiceObserver -import com.twitter.timelines.util.DeciderableRequestSemaphoreFilter -import org.apache.thrift.protocol.TBinaryProtocol -import org.apache.thrift.protocol.TCompactProtocol -import org.apache.thrift.protocol.TProtocolFactory - -class TimelineRankerBuilder(config: RuntimeConfiguration) { - - private[this] val underlyingClients = config.underlyingClients - - private[this] val configBuilder = - new ConfigBuilder(config.deciderGateBuilder, config.statsReceiver) - private[this] val debugObserverBuilder = new DebugObserverBuilder(config.whitelist) - private[this] val serviceObserver = new ServiceObserver(config.statsReceiver) - private[this] val routingRepository = RoutingTimelineRepositoryBuilder(config, configBuilder) - private[this] val inNetworkTweetRepository = - new InNetworkTweetRepositoryBuilder(config, configBuilder).apply() - private[this] val recapHydrationRepository = - new RecapHydrationRepositoryBuilder(config, configBuilder).apply() - private[this] val recapAuthorRepository = new RecapAuthorRepositoryBuilder(config).apply() - private[this] val entityTweetsRepository = - new EntityTweetsRepositoryBuilder(config, configBuilder).apply() - private[this] val utegLikedByTweetsRepository = - new UtegLikedByTweetsRepositoryBuilder(config, configBuilder).apply() - - private[this] val queryParamInitializer = new RecapQueryParamInitializer( - config = configBuilder.rootConfig, - runtimeConfig = config - ) - - val timelineRanker: TimelineRanker = new TimelineRanker( - routingRepository = routingRepository, - inNetworkTweetRepository = inNetworkTweetRepository, - recapHydrationRepository = recapHydrationRepository, - recapAuthorRepository = recapAuthorRepository, - entityTweetsRepository = entityTweetsRepository, - utegLikedByTweetsRepository = utegLikedByTweetsRepository, - serviceObserver = serviceObserver, - abdecider = Some(config.abdecider), - clientRequestAuthorizer = config.clientRequestAuthorizer, - debugObserver = debugObserverBuilder.observer, - queryParamInitializer = queryParamInitializer, - statsReceiver = config.statsReceiver - ) - - private[this] def mkServiceFactory( - protocolFactory: TProtocolFactory - ): ServiceFactory[Array[Byte], Array[Byte]] = { - val clientIdFilter = new ClientIdRequiredFilter[Array[Byte], Array[Byte]]( - config.statsReceiver.scope("service").scope("filter") - ) - - // Limits the total number of concurrent requests handled by the TimelineRanker - val maxConcurrencyFilter = { - val asyncSemaphore = new AsyncSemaphore( - initialPermits = config.maxConcurrency, - maxWaiters = 0 - ) - val enableLimiting = config.deciderGateBuilder.linearGate( - DeciderKey.EnableMaxConcurrencyLimiting - ) - - new DeciderableRequestSemaphoreFilter( - enableFilter = enableLimiting, - semaphore = asyncSemaphore, - statsReceiver = config.statsReceiver - ) - } - - // Forwards a percentage of traffic via the DarkTrafficFilter to the TimelineRanker proxy, which in turn can be - // used to forward dark traffic to staged instances - val darkTrafficFilter = DarkTrafficFilterBuilder( - config.deciderGateBuilder, - DeciderKey.EnableRoutingToRankerDevProxy, - TimelineRankerConstants.ClientPrefix, - underlyingClients.darkTrafficProxy, - config.statsReceiver - ) - - val warmupForwardingFilter = if (config.isProd) { - new ThriftForwardingWarmUpFilter( - Warmup.WarmupForwardingTime, - underlyingClients.timelineRankerForwardingClient.service, - config.statsReceiver.scope("warmupForwardingFilter"), - isBypassClient = { _.name.startsWith("timelineranker.") } - ) - } else Filter.identity[Array[Byte], Array[Byte]] - - val serviceFilterChain = clientIdFilter - .andThen(maxConcurrencyFilter) - .andThen(warmupForwardingFilter) - .andThen(darkTrafficFilter) - .andThen(serviceObserver.thriftExceptionFilter) - - val finagleService = - new TimelineRanker$FinagleService(timelineRanker, protocolFactory) - - ServiceFactory.const(serviceFilterChain andThen finagleService) - } - - val serviceFactory: ServiceFactory[Array[Byte], Array[Byte]] = - mkServiceFactory(new TBinaryProtocol.Factory()) - - val compactProtocolServiceFactory: ServiceFactory[Array[Byte], Array[Byte]] = - mkServiceFactory(new TCompactProtocol.Factory()) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerThriftWebForms.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerThriftWebForms.docx new file mode 100644 index 000000000..4f076e0a0 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerThriftWebForms.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerThriftWebForms.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerThriftWebForms.scala deleted file mode 100644 index d767f10e5..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/TimelineRankerThriftWebForms.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.timelineranker.server - -import com.twitter.thriftwebforms.MethodOptions -import com.twitter.thriftwebforms.view.ServiceResponseView -import com.twitter.timelineranker.{thriftscala => thrift} -import com.twitter.util.Future - -object TimelineRankerThriftWebForms { - - private def renderTweetIds(tweetIDs: Seq[Long]): Future[ServiceResponseView] = { - val html = tweetIDs.map { tweetID => - s"""""" - }.mkString - Future.value( - ServiceResponseView( - "Tweets", - html, - Seq("//platform.twitter.com/widgets.js") - ) - ) - } - - private def renderGetCandidateTweetsResponse(r: AnyRef): Future[ServiceResponseView] = { - val responses = r.asInstanceOf[Seq[thrift.GetCandidateTweetsResponse]] - val tweetIds = responses.flatMap( - _.candidates.map(_.flatMap(_.tweet.map(_.id))).getOrElse(Nil) - ) - renderTweetIds(tweetIds) - } - - def methodOptions: Map[String, MethodOptions] = - Map( - thrift.TimelineRanker.GetRecycledTweetCandidates.name -> MethodOptions( - responseRenderers = Seq(renderGetCandidateTweetsResponse) - ), - thrift.TimelineRanker.HydrateTweetCandidates.name -> MethodOptions( - responseRenderers = Seq(renderGetCandidateTweetsResponse) - ), - thrift.TimelineRanker.GetRecapCandidatesFromAuthors.name -> MethodOptions( - responseRenderers = Seq(renderGetCandidateTweetsResponse) - ), - thrift.TimelineRanker.GetEntityTweetCandidates.name -> MethodOptions( - responseRenderers = Seq(renderGetCandidateTweetsResponse) - ) - ) -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Warmup.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Warmup.docx new file mode 100644 index 000000000..fafea2c76 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Warmup.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Warmup.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Warmup.scala deleted file mode 100644 index e60ba9c10..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/server/Warmup.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.timelineranker.server - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thrift.ClientId -import com.twitter.logging.Logger -import com.twitter.timelineranker.model._ -import com.twitter.timelines.warmup.TwitterServerWarmup -import com.twitter.timelineservice.model.TimelineId -import com.twitter.timelineservice.model.core.TimelineKind -import com.twitter.timelineranker.config.TimelineRankerConstants -import com.twitter.timelineranker.thriftscala.{TimelineRanker => ThriftTimelineRanker} -import com.twitter.util.Future -import com.twitter.util.Duration - -object Warmup { - val WarmupForwardingTime: Duration = 25.seconds -} - -class Warmup( - localInstance: TimelineRanker, - forwardingClient: ThriftTimelineRanker.MethodPerEndpoint, - override val logger: Logger) - extends TwitterServerWarmup { - override val WarmupClientId: ClientId = ClientId(TimelineRankerConstants.WarmupClientName) - override val NumWarmupRequests = 20 - override val MinSuccessfulRequests = 10 - - private[this] val warmupUserId = Math.abs(scala.util.Random.nextLong()) - private[server] val reverseChronQuery = ReverseChronTimelineQuery( - id = new TimelineId(warmupUserId, TimelineKind.home), - maxCount = Some(20), - range = Some(TweetIdRange.default) - ).toThrift - private[server] val recapQuery = RecapQuery( - userId = warmupUserId, - maxCount = Some(20), - range = Some(TweetIdRange.default) - ).toThriftRecapQuery - - override def sendSingleWarmupRequest(): Future[Unit] = { - val localWarmups = Seq( - localInstance.getTimelines(Seq(reverseChronQuery)), - localInstance.getRecycledTweetCandidates(Seq(recapQuery)) - ) - - // send forwarding requests but ignore failures - forwardingClient.getTimelines(Seq(reverseChronQuery)).unit.handle { - case e => logger.warning(e, "fowarding request failed") - } - - Future.join(localWarmups).unit - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/BUILD deleted file mode 100644 index 469252a4e..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/org/apache/hadoop:hadoop-client-default", - "timelineranker/common:model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelines/src/main/scala/com/twitter/timelines/visibility", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-core:util-core-util", - "util/util-logging/src/main/scala", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/BUILD.docx new file mode 100644 index 000000000..2c62f6fd5 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/ReverseChronHomeTimelineSource.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/ReverseChronHomeTimelineSource.docx new file mode 100644 index 000000000..1da9ffd5b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/ReverseChronHomeTimelineSource.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/ReverseChronHomeTimelineSource.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/ReverseChronHomeTimelineSource.scala deleted file mode 100644 index c9c3f5f8f..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/ReverseChronHomeTimelineSource.scala +++ /dev/null @@ -1,327 +0,0 @@ -package com.twitter.timelineranker.source - -import com.google.common.annotations.VisibleForTesting -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.timelineranker.core.FollowGraphData -import com.twitter.timelineranker.model._ -import com.twitter.timelineranker.parameters.revchron.ReverseChronTimelineQueryContext -import com.twitter.timelineranker.util.TweetFiltersBasedOnSearchMetadata -import com.twitter.timelineranker.util.TweetsPostFilterBasedOnSearchMetadata -import com.twitter.timelineranker.util.SearchResultWithVisibilityActors -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelines.util.stats.RequestStats -import com.twitter.timelines.util.stats.RequestStatsReceiver -import com.twitter.timelines.visibility.VisibilityEnforcer -import com.twitter.timelineservice.model.TimelineId -import com.twitter.timelineservice.model.core.TimelineKind -import com.twitter.util.Future - -object ReverseChronHomeTimelineSource { - - // Post search filters applied to tweets using metadata included in search results. - val FiltersBasedOnSearchMetadata: TweetFiltersBasedOnSearchMetadata.ValueSet = - TweetFiltersBasedOnSearchMetadata.ValueSet( - TweetFiltersBasedOnSearchMetadata.DuplicateRetweets, - TweetFiltersBasedOnSearchMetadata.DuplicateTweets - ) - - object GetTweetsResult { - val Empty: GetTweetsResult = GetTweetsResult(0, 0L, Nil) - val EmptyFuture: Future[GetTweetsResult] = Future.value(Empty) - } - - case class GetTweetsResult( - // numSearchResults is the result count before filtering so may not match tweets.size - numSearchResults: Int, - minTweetIdFromSearch: TweetId, - tweets: Seq[Tweet]) -} - -/** - * Timeline source that enables materializing reverse chron timelines - * using search infrastructure. - */ -class ReverseChronHomeTimelineSource( - searchClient: SearchClient, - followGraphDataProvider: FollowGraphDataProvider, - visibilityEnforcer: VisibilityEnforcer, - statsReceiver: StatsReceiver) - extends RequestStats { - - import ReverseChronHomeTimelineSource._ - - private[this] val logger = Logger.get("ReverseChronHomeTimelineSource") - private[this] val scope = statsReceiver.scope("reverseChronSource") - private[this] val stats = RequestStatsReceiver(scope) - private[this] val emptyTimelineReturnedCounter = - scope.counter("emptyTimelineReturnedDueToMaxFollows") - private[this] val maxCountStat = scope.stat("maxCount") - private[this] val numTweetsStat = scope.stat("numTweets") - private[this] val requestedAdditionalTweetsAfterFilter = - scope.counter("requestedAdditionalTweetsAfterFilter") - private[this] val emptyTimelines = scope.counter("emptyTimelines") - private[this] val emptyTimelinesWithSignificantFollowing = - scope.counter("emptyTimelinesWithSignificantFollowing") - - // Threshold to use to determine if a user has a significant followings list size - private[this] val SignificantFollowingThreshold = 20 - - def get(contexts: Seq[ReverseChronTimelineQueryContext]): Seq[Future[Timeline]] = { - contexts.map(get) - } - - def get(context: ReverseChronTimelineQueryContext): Future[Timeline] = { - stats.addEventStats { - val query: ReverseChronTimelineQuery = context.query - - // We only support Tweet ID range at present. - val tweetIdRange = - query.range.map(TweetIdRange.fromTimelineRange).getOrElse(TweetIdRange.default) - - val userId = query.userId - val timelineId = TimelineId(userId, TimelineKind.home) - val maxFollowingCount = context.maxFollowedUsers() - - followGraphDataProvider - .get( - userId, - maxFollowingCount - ) - .flatMap { followGraphData => - // We return an empty timeline if a given user follows more than the limit - // on the number of users. This is because, such a user's timeline will quickly - // fill up displacing materialized tweets wasting the materialation work. - // This behavior can be disabled via featureswitches to support non-materialization - // use cases when we should always return a timeline. - if (followGraphData.filteredFollowedUserIds.isEmpty || - (followGraphData.followedUserIds.size >= maxFollowingCount && context - .returnEmptyWhenOverMaxFollows())) { - if (followGraphData.followedUserIds.size >= maxFollowingCount) { - emptyTimelineReturnedCounter.incr() - } - Future.value(Timeline.empty(timelineId)) - } else { - val maxCount = getMaxCount(context) - val numEntriesToRequest = (maxCount * context.maxCountMultiplier()).toInt - maxCountStat.add(numEntriesToRequest) - - val allUserIds = followGraphData.followedUserIds :+ userId - getTweets( - userId, - allUserIds, - followGraphData, - numEntriesToRequest, - tweetIdRange, - context - ).map { tweets => - if (tweets.isEmpty) { - emptyTimelines.incr() - if (followGraphData.followedUserIds.size >= SignificantFollowingThreshold) { - emptyTimelinesWithSignificantFollowing.incr() - logger.debug( - "Search returned empty home timeline for user %s (follow count %s), query: %s", - userId, - followGraphData.followedUserIds.size, - query) - } - } - // If we had requested more entries than maxCount (due to multiplier being > 1.0) - // then we need to trim it back to maxCount. - val truncatedTweets = tweets.take(maxCount) - numTweetsStat.add(truncatedTweets.size) - Timeline( - timelineId, - truncatedTweets.map(tweet => TimelineEntryEnvelope(tweet)) - ) - } - } - } - } - } - - /** - * Gets tweets from search and performs post-filtering. - * - * If we do not end up with sufficient tweets after post-filtering, - * we issue a second call to search to get more tweets if: - * -- such behavior is enabled by setting backfillFilteredEntries to true. - * -- the original call to search returned requested number of tweets. - * -- after post-filtering, the percentage of filtered out tweets - * exceeds the value of tweetsFilteringLossageThresholdPercent. - */ - private def getTweets( - userId: UserId, - allUserIds: Seq[UserId], - followGraphData: FollowGraphData, - numEntriesToRequest: Int, - tweetIdRange: TweetIdRange, - context: ReverseChronTimelineQueryContext - ): Future[Seq[Tweet]] = { - getTweetsHelper( - userId, - allUserIds, - followGraphData, - numEntriesToRequest, - tweetIdRange, - context.directedAtNarrowcastingViaSearch(), - context.postFilteringBasedOnSearchMetadataEnabled(), - context.getTweetsFromArchiveIndex() - ).flatMap { result => - val numAdditionalTweetsToRequest = getNumAdditionalTweetsToRequest( - numEntriesToRequest, - result.numSearchResults, - result.numSearchResults - result.tweets.size, - context - ) - - if (numAdditionalTweetsToRequest > 0) { - requestedAdditionalTweetsAfterFilter.incr() - val updatedRange = tweetIdRange.copy(toId = Some(result.minTweetIdFromSearch)) - getTweetsHelper( - userId, - allUserIds, - followGraphData, - numAdditionalTweetsToRequest, - updatedRange, - context.directedAtNarrowcastingViaSearch(), - context.postFilteringBasedOnSearchMetadataEnabled(), - context.getTweetsFromArchiveIndex() - ).map { result2 => result.tweets ++ result2.tweets } - } else { - Future.value(result.tweets) - } - } - } - - private[source] def getNumAdditionalTweetsToRequest( - numTweetsRequested: Int, - numTweetsFoundBySearch: Int, - numTweetsFilteredOut: Int, - context: ReverseChronTimelineQueryContext - ): Int = { - require(numTweetsFoundBySearch <= numTweetsRequested) - - if (!context.backfillFilteredEntries() || (numTweetsFoundBySearch < numTweetsRequested)) { - // If multiple calls are not enabled or if search did not find enough tweets, - // there is no point in making another call to get more. - 0 - } else { - val numTweetsFilteredOutPercent = numTweetsFilteredOut * 100.0 / numTweetsFoundBySearch - if (numTweetsFilteredOutPercent > context.tweetsFilteringLossageThresholdPercent()) { - - // We assume that the next call will also have lossage percentage similar to the first call. - // Therefore, we proactively request proportionately more tweets so that we do not - // end up needing a third call. - // In any case, regardless of what we get in the second call, we do not make any subsequent calls. - val adjustedFilteredOutPercent = - math.min(numTweetsFilteredOutPercent, context.tweetsFilteringLossageLimitPercent()) - val numTweetsToRequestMultiplier = 100 / (100 - adjustedFilteredOutPercent) - val numTweetsToRequest = (numTweetsFilteredOut * numTweetsToRequestMultiplier).toInt - - numTweetsToRequest - } else { - // Did not have sufficient lossage to warrant an extra call. - 0 - } - } - } - - private def getClientId(subClientId: String): String = { - // Hacky: Extract the environment from the existing clientId set by TimelineRepositoryBuilder - val env = searchClient.clientId.split('.').last - - s"timelineranker.$subClientId.$env" - } - - private def getTweetsHelper( - userId: UserId, - allUserIds: Seq[UserId], - followGraphData: FollowGraphData, - maxCount: Int, - tweetIdRange: TweetIdRange, - withDirectedAtNarrowcasting: Boolean, - postFilteringBasedOnSearchMetadataEnabled: Boolean, - getTweetsFromArchiveIndex: Boolean - ): Future[GetTweetsResult] = { - val beforeTweetIdExclusive = tweetIdRange.toId - val afterTweetIdExclusive = tweetIdRange.fromId - val searchClientId: Option[String] = if (!getTweetsFromArchiveIndex) { - // Set a custom clientId which has different QPS quota and access. - // Used for notify we are fetching from realtime only. - // see: SEARCH-42651 - Some(getClientId("home_materialization_realtime_only")) - } else { - // Let the searchClient derive its clientId for the regular case of fetching from archive - None - } - - searchClient - .getUsersTweetsReverseChron( - userId = userId, - followedUserIds = allUserIds.toSet, - retweetsMutedUserIds = followGraphData.retweetsMutedUserIds, - maxCount = maxCount, - beforeTweetIdExclusive = beforeTweetIdExclusive, - afterTweetIdExclusive = afterTweetIdExclusive, - withDirectedAtNarrowcasting = withDirectedAtNarrowcasting, - postFilteringBasedOnSearchMetadataEnabled = postFilteringBasedOnSearchMetadataEnabled, - getTweetsFromArchiveIndex = getTweetsFromArchiveIndex, - searchClientId = searchClientId - ) - .flatMap { searchResults => - if (searchResults.nonEmpty) { - val minTweetId = searchResults.last.id - val filteredTweetsFuture = filterTweets( - userId, - followGraphData.inNetworkUserIds, - searchResults, - FiltersBasedOnSearchMetadata, - postFilteringBasedOnSearchMetadataEnabled = postFilteringBasedOnSearchMetadataEnabled, - visibilityEnforcer - ) - filteredTweetsFuture.map(tweets => - GetTweetsResult(searchResults.size, minTweetId, tweets)) - } else { - GetTweetsResult.EmptyFuture - } - } - } - - def filterTweets( - userId: UserId, - inNetworkUserIds: Seq[UserId], - searchResults: Seq[ThriftSearchResult], - filtersBasedOnSearchMetadata: TweetFiltersBasedOnSearchMetadata.ValueSet, - postFilteringBasedOnSearchMetadataEnabled: Boolean = true, - visibilityEnforcer: VisibilityEnforcer - ): Future[Seq[Tweet]] = { - val filteredTweets = if (postFilteringBasedOnSearchMetadataEnabled) { - val tweetsPostFilterBasedOnSearchMetadata = - new TweetsPostFilterBasedOnSearchMetadata(filtersBasedOnSearchMetadata, logger, scope) - tweetsPostFilterBasedOnSearchMetadata.apply(userId, inNetworkUserIds, searchResults) - } else { - searchResults - } - visibilityEnforcer - .apply(Some(userId), filteredTweets.map(SearchResultWithVisibilityActors(_, scope))) - .map(_.map { searchResult => - new Tweet( - id = searchResult.tweetId, - userId = Some(searchResult.userId), - sourceTweetId = searchResult.sourceTweetId, - sourceUserId = searchResult.sourceUserId) - }) - } - - @VisibleForTesting - private[source] def getMaxCount(context: ReverseChronTimelineQueryContext): Int = { - val maxCountFromQuery = ReverseChronTimelineQueryContext.MaxCount(context.query.maxCount) - val maxCountFromContext = context.maxCount() - math.min(maxCountFromQuery, maxCountFromContext) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/TimelineSource.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/TimelineSource.docx new file mode 100644 index 000000000..5c5511d4d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/TimelineSource.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/TimelineSource.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/TimelineSource.scala deleted file mode 100644 index 15bb5b126..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/source/TimelineSource.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.timelineranker.source - -import com.twitter.timelineranker.model.Timeline -import com.twitter.timelineranker.model.TimelineQuery -import com.twitter.util.Future - -trait TimelineSource { - def get(queries: Seq[TimelineQuery]): Seq[Future[Timeline]] - def get(query: TimelineQuery): Future[Timeline] = get(Seq(query)).head -} - -class EmptyTimelineSource extends TimelineSource { - def get(queries: Seq[TimelineQuery]): Seq[Future[Timeline]] = { - queries.map(q => Future.value(Timeline.empty(q.id))) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/BUILD.bazel b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/BUILD.bazel deleted file mode 100644 index 8e3564aad..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/BUILD.bazel +++ /dev/null @@ -1,51 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "finagle/finagle-core/src/main", - "servo/util/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/thrift/com/twitter/recos:recos-common-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:features-scala", - "timelineranker/common/src/main/scala/com/twitter/timelineranker/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/common", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/config", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/repository", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/util", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/manhattan", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph", - "timelines/src/main/scala/com/twitter/timelines/config", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/options", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/tweet", - "timelines/src/main/scala/com/twitter/timelines/uteg_utils", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/bounds", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "timelines/src/main/scala/com/twitter/timelines/visibility/model", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-stats/src/main/scala", - ], - exports = [ - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/BUILD.docx new file mode 100644 index 000000000..6498d4dca Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/CombinedScoreAndTruncateTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/CombinedScoreAndTruncateTransform.docx new file mode 100644 index 000000000..0468f9725 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/CombinedScoreAndTruncateTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/CombinedScoreAndTruncateTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/CombinedScoreAndTruncateTransform.scala deleted file mode 100644 index 0bfaa9677..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/CombinedScoreAndTruncateTransform.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.search.earlybird.thriftscala.ThriftSearchResultMetadata -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future - -object CombinedScoreAndTruncateTransform { - val DefaultRealGraphWeight = 1.0 - val DefaultEmptyScore = 0.0 -} - -/** - * Rank and truncate search results according to - * DefaultRealGraphWeight * real_graph_score + earlybird_score_multiplier * earlybird_score - * Note: scoring and truncation only applies to out of network candidates - */ -class CombinedScoreAndTruncateTransform( - maxTweetCountProvider: DependencyProvider[Int], - earlybirdScoreMultiplierProvider: DependencyProvider[Double], - numAdditionalRepliesProvider: DependencyProvider[Int], - statsReceiver: StatsReceiver) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - import CombinedScoreAndTruncateTransform._ - - private[this] val scopedStatsReceiver = statsReceiver.scope("CombinedScoreAndTruncateTransform") - private[this] val earlybirdScoreX100Stat = scopedStatsReceiver.stat("earlybirdScoreX100") - private[this] val realGraphScoreX100Stat = scopedStatsReceiver.stat("realGraphScoreX100") - private[this] val additionalReplyCounter = scopedStatsReceiver.counter("additionalReplies") - private[this] val resultCounter = scopedStatsReceiver.counter("results") - - private[this] def getRealGraphScore( - searchResult: ThriftSearchResult, - utegResults: Map[TweetId, TweetRecommendation] - ): Double = { - utegResults.get(searchResult.id).map(_.score).getOrElse(DefaultEmptyScore) - } - - private[this] def getEarlybirdScore(metadataOpt: Option[ThriftSearchResultMetadata]): Double = { - metadataOpt - .flatMap(metadata => metadata.score) - .getOrElse(DefaultEmptyScore) - } - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val maxCount = maxTweetCountProvider(envelope.query) - val earlybirdScoreMultiplier = earlybirdScoreMultiplierProvider(envelope.query) - val realGraphScoreMultiplier = DefaultRealGraphWeight - - val searchResultsAndScore = envelope.searchResults.map { searchResult => - val realGraphScore = getRealGraphScore(searchResult, envelope.utegResults) - val earlybirdScore = getEarlybirdScore(searchResult.metadata) - earlybirdScoreX100Stat.add(earlybirdScore.toFloat * 100) - realGraphScoreX100Stat.add(realGraphScore.toFloat * 100) - val combinedScore = - realGraphScoreMultiplier * realGraphScore + earlybirdScoreMultiplier * earlybirdScore - (searchResult, combinedScore) - } - - // set aside results that are marked by isRandomTweet field - val (randomSearchResults, otherSearchResults) = searchResultsAndScore.partition { - resultAndScore => - resultAndScore._1.tweetFeatures.flatMap(_.isRandomTweet).getOrElse(false) - } - - val (topResults, remainingResults) = otherSearchResults - .sortBy(_._2)(Ordering[Double].reverse).map(_._1).splitAt( - maxCount - randomSearchResults.length) - - val numAdditionalReplies = numAdditionalRepliesProvider(envelope.query) - val additionalReplies = { - if (numAdditionalReplies > 0) { - val replyTweetIdSet = - envelope.hydratedTweets.outerTweets.filter(_.hasReply).map(_.tweetId).toSet - remainingResults.filter(result => replyTweetIdSet(result.id)).take(numAdditionalReplies) - } else { - Seq.empty - } - } - - val transformedSearchResults = - topResults ++ additionalReplies ++ randomSearchResults - .map(_._1) - - resultCounter.incr(transformedSearchResults.size) - additionalReplyCounter.incr(additionalReplies.size) - - Future.value(envelope.copy(searchResults = transformedSearchResults)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/MinNumNonAuthorFavoritedByUserIdsFilterTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/MinNumNonAuthorFavoritedByUserIdsFilterTransform.docx new file mode 100644 index 000000000..1cba47d1e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/MinNumNonAuthorFavoritedByUserIdsFilterTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/MinNumNonAuthorFavoritedByUserIdsFilterTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/MinNumNonAuthorFavoritedByUserIdsFilterTransform.scala deleted file mode 100644 index dc5925dcb..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/MinNumNonAuthorFavoritedByUserIdsFilterTransform.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.recos.recos_common.thriftscala.SocialProofType -import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future - -class MinNumNonAuthorFavoritedByUserIdsFilterTransform( - minNumFavoritedByUserIdsProvider: DependencyProvider[Int]) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val filteredSearchResults = envelope.searchResults.filter { searchResult => - numNonAuthorFavs( - searchResult = searchResult, - utegResultsMap = envelope.utegResults - ).exists(_ >= minNumFavoritedByUserIdsProvider(envelope.query)) - } - Future.value(envelope.copy(searchResults = filteredSearchResults)) - } - - // return number of non-author users that faved the tweet in a searchResult - // return None if author is None or if the tweet is not found in utegResultsMap - protected def numNonAuthorFavs( - searchResult: ThriftSearchResult, - utegResultsMap: Map[TweetId, TweetRecommendation] - ): Option[Int] = { - for { - metadata <- searchResult.metadata - authorId = metadata.fromUserId - tweetRecommendation <- utegResultsMap.get(searchResult.id) - favedByUserIds <- tweetRecommendation.socialProofByType.get(SocialProofType.Favorite) - } yield favedByUserIds.filterNot(_ == authorId).size - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/RemoveCandidatesAuthoredByWeightedFollowingsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/RemoveCandidatesAuthoredByWeightedFollowingsTransform.docx new file mode 100644 index 000000000..286c28716 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/RemoveCandidatesAuthoredByWeightedFollowingsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/RemoveCandidatesAuthoredByWeightedFollowingsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/RemoveCandidatesAuthoredByWeightedFollowingsTransform.scala deleted file mode 100644 index afafb5f6b..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/RemoveCandidatesAuthoredByWeightedFollowingsTransform.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelines.model.UserId -import com.twitter.util.Future - -object RemoveCandidatesAuthoredByWeightedFollowingsTransform - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val filteredSearchResults = envelope.query.utegLikedByTweetsOptions match { - case Some(opts) => - envelope.searchResults.filterNot(isAuthorInWeightedFollowings(_, opts.weightedFollowings)) - case None => envelope.searchResults - } - Future.value(envelope.copy(searchResults = filteredSearchResults)) - } - - private def isAuthorInWeightedFollowings( - searchResult: ThriftSearchResult, - weightedFollowings: Map[UserId, Double] - ): Boolean = { - searchResult.metadata match { - case Some(metadata) => weightedFollowings.contains(metadata.fromUserId) - case None => false - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/SocialProofAndUTEGScoreHydrationTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/SocialProofAndUTEGScoreHydrationTransform.docx new file mode 100644 index 000000000..f1125f602 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/SocialProofAndUTEGScoreHydrationTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/SocialProofAndUTEGScoreHydrationTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/SocialProofAndUTEGScoreHydrationTransform.scala deleted file mode 100644 index 286411c70..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/SocialProofAndUTEGScoreHydrationTransform.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.util.Future - -object SocialProofAndUTEGScoreHydrationTransform - extends FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ] { - override def apply( - request: HydratedCandidatesAndFeaturesEnvelope - ): Future[HydratedCandidatesAndFeaturesEnvelope] = { - - val updatedFeatures = request.features.map { - case (tweetId, features) => - tweetId -> - features.copy( - utegSocialProofByType = - request.candidateEnvelope.utegResults.get(tweetId).map(_.socialProofByType), - utegScore = request.candidateEnvelope.utegResults.get(tweetId).map(_.score) - ) - } - - Future.value(request.copy(features = updatedFeatures)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UTEGResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UTEGResultsTransform.docx new file mode 100644 index 000000000..a51e14c0e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UTEGResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UTEGResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UTEGResultsTransform.scala deleted file mode 100644 index 6a65fb970..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UTEGResultsTransform.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.recos.recos_common.thriftscala.SocialProofType -import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetEntityDisplayLocation -import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation -import com.twitter.servo.util.FutureArrow -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.core.DependencyTransformer -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelineranker.model.TimeRange -import com.twitter.timelineranker.model.TweetIdRange -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelines.clients.user_tweet_entity_graph.RecommendTweetEntityQuery -import com.twitter.timelines.clients.user_tweet_entity_graph.UserTweetEntityGraphClient -import com.twitter.util.Future - -object UTEGResultsTransform { - val MaxUserSocialProofSize = 10 - val MaxTweetSocialProofSize = 10 - val MinUserSocialProofSize = 1 - - def requiredTweetAuthors(query: RecapQuery): Option[Set[Long]] = { - query.utegLikedByTweetsOptions - .filter(_.isInNetwork) - .map(_.weightedFollowings.keySet) - } - - def makeUTEGQuery( - query: RecapQuery, - socialProofTypes: Seq[SocialProofType], - utegCountProvider: DependencyProvider[Int] - ): RecommendTweetEntityQuery = { - val utegLikedByTweetsOpt = query.utegLikedByTweetsOptions - RecommendTweetEntityQuery( - userId = query.userId, - displayLocation = TweetEntityDisplayLocation.HomeTimeline, - seedUserIdsWithWeights = utegLikedByTweetsOpt.map(_.weightedFollowings).getOrElse(Map.empty), - maxTweetResults = utegCountProvider(query), - maxTweetAgeInMillis = // the "to" in the Range field is not supported by this new endpoint - query.range match { - case Some(TimeRange(from, _)) => from.map(_.untilNow.inMillis) - case Some(TweetIdRange(from, _)) => from.map(SnowflakeId.timeFromId(_).untilNow.inMillis) - case _ => None - }, - excludedTweetIds = query.excludedTweetIds, - maxUserSocialProofSize = Some(MaxUserSocialProofSize), - maxTweetSocialProofSize = Some(MaxTweetSocialProofSize), - minUserSocialProofSize = Some(MinUserSocialProofSize), - socialProofTypes = socialProofTypes, - tweetAuthors = requiredTweetAuthors(query) - ) - } -} - -class UTEGResultsTransform( - userTweetEntityGraphClient: UserTweetEntityGraphClient, - utegCountProvider: DependencyProvider[Int], - recommendationsFilter: DependencyTransformer[Seq[TweetRecommendation], Seq[TweetRecommendation]], - socialProofTypes: Seq[SocialProofType]) - extends FutureArrow[CandidateEnvelope, CandidateEnvelope] { - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - val utegQuery = - UTEGResultsTransform.makeUTEGQuery(envelope.query, socialProofTypes, utegCountProvider) - userTweetEntityGraphClient.findTweetRecommendations(utegQuery).map { recommendations => - val filteredRecommendations = recommendationsFilter(envelope.query, recommendations) - val utegResultsMap = filteredRecommendations.map { recommendation => - recommendation.tweetId -> recommendation - }.toMap - envelope.copy(utegResults = utegResultsMap) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepository.docx new file mode 100644 index 000000000..e4a3f8c20 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepository.scala deleted file mode 100644 index 4db0e105c..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepository.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.timelineranker.model.CandidateTweetsResult -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.util.Future - -/** - * A repository of YML tweets candidiates - */ -class UtegLikedByTweetsRepository(source: UtegLikedByTweetsSource) { - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - source.get(query) - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - source.get(queries) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepositoryBuilder.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepositoryBuilder.docx new file mode 100644 index 000000000..b5890eb63 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepositoryBuilder.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepositoryBuilder.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepositoryBuilder.scala deleted file mode 100644 index 0f5098cae..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsRepositoryBuilder.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelineranker.config.RequestScopes -import com.twitter.timelineranker.config.RuntimeConfiguration -import com.twitter.timelineranker.parameters.ConfigBuilder -import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder -import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.timelines.util.stats.RequestScope -import com.twitter.util.Duration - -class UtegLikedByTweetsRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder) - extends CandidatesRepositoryBuilder(config) { - override val clientSubId = "uteg_liked_by_tweets" - override val requestScope: RequestScope = RequestScopes.UtegLikedByTweetsSource - override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet = - SgsFollowGraphDataFields.ValueSet( - SgsFollowGraphDataFields.FollowedUserIds, - SgsFollowGraphDataFields.MutuallyFollowingUserIds, - SgsFollowGraphDataFields.MutedUserIds - ) - override val searchProcessingTimeout: Duration = 400.milliseconds - override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint = - config.underlyingClients.createEarlybirdClient( - scope = scope, - requestTimeout = 500.milliseconds, - timeout = 900.milliseconds, - retryPolicy = config.underlyingClients.DefaultRetryPolicy - ) - - def apply(): UtegLikedByTweetsRepository = { - val utegLikedByTweetsSource = new UtegLikedByTweetsSource( - userTweetEntityGraphClient = userTweetEntityGraphClient, - gizmoduckClient = gizmoduckClient, - searchClient = searchClient, - tweetyPieClient = tweetyPieHighQoSClient, - userMetadataClient = userMetadataClient, - followGraphDataProvider = followGraphDataProvider, - contentFeaturesCache = config.underlyingClients.contentFeaturesCache, - statsReceiver = config.statsReceiver - ) - - new UtegLikedByTweetsRepository(utegLikedByTweetsSource) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSearchResultsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSearchResultsTransform.docx new file mode 100644 index 000000000..e77b75876 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSearchResultsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSearchResultsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSearchResultsTransform.scala deleted file mode 100644 index 5d6dac1f6..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSearchResultsTransform.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.timelineranker.common.RecapHydrationSearchResultsTransformBase -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future - -class UtegLikedByTweetsSearchResultsTransform( - override protected val searchClient: SearchClient, - override protected val statsReceiver: StatsReceiver, - relevanceSearchProvider: DependencyProvider[Boolean]) - extends RecapHydrationSearchResultsTransformBase { - - private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch") - - override def tweetIdsToHydrate(envelope: CandidateEnvelope): Seq[TweetId] = - envelope.utegResults.keys.toSeq - - override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = { - searchClient - .getTweetsScoredForRecap( - userId = envelope.query.userId, - tweetIds = tweetIdsToHydrate(envelope), - earlybirdOptions = envelope.query.earlybirdOptions, - logSearchDebugInfo = false, - searchClientId = None, - relevanceSearch = relevanceSearchProvider(envelope.query) - ).map { results => - numResultsFromSearchStat.add(results.size) - envelope.copy(searchResults = results) - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSource.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSource.docx new file mode 100644 index 000000000..da9217554 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSource.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSource.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSource.scala deleted file mode 100644 index 8cb126f78..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/uteg_liked_by_tweets/UtegLikedByTweetsSource.scala +++ /dev/null @@ -1,306 +0,0 @@ -package com.twitter.timelineranker.uteg_liked_by_tweets - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.recos_common.thriftscala.SocialProofType -import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation -import com.twitter.servo.util.FutureArrow -import com.twitter.servo.util.Gate -import com.twitter.storehaus.Store -import com.twitter.timelineranker.common._ -import com.twitter.timelineranker.core.CandidateEnvelope -import com.twitter.timelineranker.core.DependencyTransformer -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.model.CandidateTweetsResult -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelineranker.model.RecapQuery.DependencyProvider -import com.twitter.timelineranker.monitoring.UsersSearchResultMonitoringTransform -import com.twitter.timelineranker.parameters.recap.RecapParams -import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsParams -import com.twitter.timelineranker.parameters.monitoring.MonitoringParams -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform -import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform -import com.twitter.timelineranker.visibility.FollowGraphDataProvider -import com.twitter.timelines.clients.gizmoduck.GizmoduckClient -import com.twitter.timelines.clients.manhattan.UserMetadataClient -import com.twitter.timelines.clients.relevance_search.SearchClient -import com.twitter.timelines.clients.tweetypie.TweetyPieClient -import com.twitter.timelines.clients.user_tweet_entity_graph.UserTweetEntityGraphClient -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.uteg_utils.UTEGRecommendationsFilterBuilder -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.timelines.util.stats.RequestStatsReceiver -import com.twitter.util.Future - -class UtegLikedByTweetsSource( - userTweetEntityGraphClient: UserTweetEntityGraphClient, - gizmoduckClient: GizmoduckClient, - searchClient: SearchClient, - tweetyPieClient: TweetyPieClient, - userMetadataClient: UserMetadataClient, - followGraphDataProvider: FollowGraphDataProvider, - contentFeaturesCache: Store[TweetId, ContentFeatures], - statsReceiver: StatsReceiver) { - - private[this] val socialProofTypes = Seq(SocialProofType.Favorite) - - private[this] val baseScope = statsReceiver.scope("utegLikedByTweetsSource") - private[this] val requestStats = RequestStatsReceiver(baseScope) - - private[this] val failOpenScope = baseScope.scope("failOpen") - private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo") - private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages") - - private[this] val debugAuthorsMonitoringProvider = - DependencyProvider.from(MonitoringParams.DebugAuthorsAllowListParam) - - private[this] val maxFollowedUsersProvider = - DependencyProvider.value(RecapParams.MaxFollowedUsers.default) - private[this] val followGraphDataTransform = - new FollowGraphDataTransform(followGraphDataProvider, maxFollowedUsersProvider) - - private[this] val searchResultsTransform = - new UtegLikedByTweetsSearchResultsTransform( - searchClient = searchClient, - statsReceiver = baseScope, - relevanceSearchProvider = - DependencyProvider.from(UtegLikedByTweetsParams.EnableRelevanceSearchParam) - ) - - private[this] val userProfileInfoTransform = - new UserProfileInfoTransform(userProfileHandler, gizmoduckClient) - private[this] val languagesTransform = - new UserLanguagesTransform(userLanguagesHandler, userMetadataClient) - - private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope) - - private[this] val maxCandidatesToFetchFromUtegProvider = DependencyProvider { query => - query.utegLikedByTweetsOptions - .map(_.utegCount).getOrElse( - query.utegLikedByTweetsOptions match { - case Some(opts) => - if (opts.isInNetwork) query.params(UtegLikedByTweetsParams.DefaultUTEGInNetworkCount) - else query.params(UtegLikedByTweetsParams.DefaultUTEGOutOfNetworkCount) - case None => 0 - } - ) - } - - private[this] def isInNetwork(envelope: CandidateEnvelope): Boolean = - isInNetwork(envelope.query) - - private[this] def isInNetwork(query: RecapQuery): Boolean = - query.utegLikedByTweetsOptions.exists(_.isInNetwork) - - private[this] def isInNetwork(hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope): Boolean = - isInNetwork(hydratedEnvelope.candidateEnvelope) - - private[this] val recommendationsFilter = - DependencyTransformer.partition[Seq[TweetRecommendation], Seq[TweetRecommendation]]( - gate = Gate[RecapQuery](f = (query: RecapQuery) => isInNetwork(query)), - ifTrue = DependencyTransformer.identity, - ifFalse = new UTEGRecommendationsFilterBuilder[RecapQuery]( - enablingGate = - RecapQuery.paramGate(UtegLikedByTweetsParams.UTEGRecommendationsFilter.EnableParam), - excludeTweetGate = - RecapQuery.paramGate(UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeTweetParam), - excludeRetweetGate = RecapQuery.paramGate( - UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeRetweetParam), - excludeReplyGate = - RecapQuery.paramGate(UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeReplyParam), - excludeQuoteGate = RecapQuery.paramGate( - UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeQuoteTweetParam - ), - statsReceiver = baseScope - ).build - ) - - private[this] val utegResultsTransform = new UTEGResultsTransform( - userTweetEntityGraphClient, - maxCandidatesToFetchFromUtegProvider, - recommendationsFilter, - socialProofTypes - ) - - private[this] val earlybirdScoreMultiplierProvider = - DependencyProvider.from(UtegLikedByTweetsParams.EarlybirdScoreMultiplierParam) - private[this] val maxCandidatesToReturnToCallerProvider = DependencyProvider { query => - query.maxCount.getOrElse(query.params(UtegLikedByTweetsParams.DefaultMaxTweetCount)) - } - - private[this] val minNumFavedByUserIdsProvider = DependencyProvider { query => - query.params(UtegLikedByTweetsParams.MinNumFavoritedByUserIdsParam) - } - - private[this] val removeTweetsAuthoredBySeedSetForOutOfNetworkPipeline = - FutureArrow.choose[CandidateEnvelope, CandidateEnvelope]( - predicate = isInNetwork, - ifTrue = FutureArrow.identity, - ifFalse = new UsersSearchResultMonitoringTransform( - name = "RemoveCandidatesAuthoredByWeightedFollowingsTransform", - RemoveCandidatesAuthoredByWeightedFollowingsTransform, - baseScope, - debugAuthorsMonitoringProvider - ) - ) - - private[this] val minNumFavoritedByUserIdsFilterTransform = - FutureArrow.choose[CandidateEnvelope, CandidateEnvelope]( - predicate = isInNetwork, - ifTrue = FutureArrow.identity, - ifFalse = new UsersSearchResultMonitoringTransform( - name = "MinNumNonAuthorFavoritedByUserIdsFilterTransform", - new MinNumNonAuthorFavoritedByUserIdsFilterTransform( - minNumFavoritedByUserIdsProvider = minNumFavedByUserIdsProvider - ), - baseScope, - debugAuthorsMonitoringProvider - ) - ) - - private[this] val includeRandomTweetProvider = - DependencyProvider.from(UtegLikedByTweetsParams.IncludeRandomTweetParam) - private[this] val includeSingleRandomTweetProvider = - DependencyProvider.from(UtegLikedByTweetsParams.IncludeSingleRandomTweetParam) - private[this] val probabilityRandomTweetProvider = - DependencyProvider.from(UtegLikedByTweetsParams.ProbabilityRandomTweetParam) - - private[this] val markRandomTweetTransform = new MarkRandomTweetTransform( - includeRandomTweetProvider = includeRandomTweetProvider, - includeSingleRandomTweetProvider = includeSingleRandomTweetProvider, - probabilityRandomTweetProvider = probabilityRandomTweetProvider, - ) - - private[this] val combinedScoreTruncateTransform = - FutureArrow.choose[CandidateEnvelope, CandidateEnvelope]( - predicate = isInNetwork, - ifTrue = FutureArrow.identity, - ifFalse = new CombinedScoreAndTruncateTransform( - maxTweetCountProvider = maxCandidatesToReturnToCallerProvider, - earlybirdScoreMultiplierProvider = earlybirdScoreMultiplierProvider, - numAdditionalRepliesProvider = - DependencyProvider.from(UtegLikedByTweetsParams.NumAdditionalRepliesParam), - statsReceiver = baseScope - ) - ) - - private[this] val excludeRecommendedRepliesToNonFollowedUsersGate: Gate[RecapQuery] = - RecapQuery.paramGate( - UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeRecommendedRepliesToNonFollowedUsersParam) - - private[this] def enableUseFollowGraphDataForRecommendedReplies( - envelope: CandidateEnvelope - ): Boolean = - excludeRecommendedRepliesToNonFollowedUsersGate(envelope.query) - - val dynamicHydratedTweetsFilter: FutureArrow[CandidateEnvelope, CandidateEnvelope] = - FutureArrow.choose[CandidateEnvelope, CandidateEnvelope]( - predicate = enableUseFollowGraphDataForRecommendedReplies, - ifTrue = new TweetKindOptionHydratedTweetsFilterTransform( - useFollowGraphData = true, - useSourceTweets = false, - statsReceiver = baseScope - ), - ifFalse = new TweetKindOptionHydratedTweetsFilterTransform( - useFollowGraphData = false, - useSourceTweets = false, - statsReceiver = baseScope - ) - ) - - private[this] val trimToMatchSearchResultsTransform = - new UsersSearchResultMonitoringTransform( - name = "TrimToMatchSearchResultsTransform", - new TrimToMatchSearchResultsTransform( - hydrateReplyRootTweetProvider = DependencyProvider.False, - statsReceiver = baseScope - ), - baseScope, - debugAuthorsMonitoringProvider - ) - - // combine score and truncate tweet candidates immediately after - private[this] val hydrationAndFilteringPipeline = - CreateCandidateEnvelopeTransform - .andThen(followGraphDataTransform) - .andThen(utegResultsTransform) - .andThen(searchResultsTransform) - // For out of network tweets, remove tweets whose author is contained in the weighted following seed set passed into TLR - .andThen(removeTweetsAuthoredBySeedSetForOutOfNetworkPipeline) - .andThen(minNumFavoritedByUserIdsFilterTransform) - .andThen(CandidateTweetHydrationTransform) - .andThen(markRandomTweetTransform) - .andThen(dynamicHydratedTweetsFilter) - .andThen(TrimToMatchHydratedTweetsTransform) - .andThen(combinedScoreTruncateTransform) - .andThen(trimToMatchSearchResultsTransform) - - // runs the main pipeline in parallel with fetching user profile info and user languages - private[this] val featureHydrationDataTransform = new FeatureHydrationDataTransform( - hydrationAndFilteringPipeline, - languagesTransform, - userProfileInfoTransform - ) - - private[this] val contentFeaturesHydrationTransform = - new ContentFeaturesHydrationTransformBuilder( - tweetyPieClient, - contentFeaturesCache, - enableContentFeaturesGate = - RecapQuery.paramGate(UtegLikedByTweetsParams.EnableContentFeaturesHydrationParam), - enableTokensInContentFeaturesGate = - RecapQuery.paramGate(UtegLikedByTweetsParams.EnableTokensInContentFeaturesHydrationParam), - enableTweetTextInContentFeaturesGate = RecapQuery.paramGate( - UtegLikedByTweetsParams.EnableTweetTextInContentFeaturesHydrationParam), - enableConversationControlContentFeaturesGate = RecapQuery.paramGate( - UtegLikedByTweetsParams.EnableConversationControlInContentFeaturesHydrationParam), - enableTweetMediaHydrationGate = RecapQuery.paramGate( - UtegLikedByTweetsParams.EnableTweetMediaHydrationParam - ), - hydrateInReplyToTweets = true, - statsReceiver = baseScope - ).build() - - // use OutOfNetworkTweetsSearchFeaturesHydrationTransform for rectweets - private[this] val tweetsSearchFeaturesHydrationTransform = - FutureArrow - .choose[HydratedCandidatesAndFeaturesEnvelope, HydratedCandidatesAndFeaturesEnvelope]( - predicate = isInNetwork, - ifTrue = InNetworkTweetsSearchFeaturesHydrationTransform, - ifFalse = OutOfNetworkTweetsSearchFeaturesHydrationTransform - ) - - private[this] def hydratesContentFeatures( - hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope - ): Boolean = - hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true) - - private[this] val contentFeaturesTransformer = FutureArrow.choose( - predicate = hydratesContentFeatures, - ifTrue = contentFeaturesHydrationTransform - .andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform) - .andThen(CopyContentFeaturesIntoHydratedTweetsTransform), - ifFalse = FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ](Future.value) // empty transformer - ) - - private[this] val featureHydrationPipeline = - featureHydrationDataTransform - .andThen(tweetsSearchFeaturesHydrationTransform) - .andThen(SocialProofAndUTEGScoreHydrationTransform) - .andThen(contentFeaturesTransformer) - .andThen(candidateGenerationTransform) - - def get(query: RecapQuery): Future[CandidateTweetsResult] = { - requestStats.addEventStats { - featureHydrationPipeline(query) - } - } - - def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = { - Future.collect(queries.map(get)) - } - -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/BUILD b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/BUILD deleted file mode 100644 index 9594393e4..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/BUILD +++ /dev/null @@ -1,39 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/thrift:libthrift", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "finagle/finagle-core/src/main", - "mediaservices/commons/src/main/thrift:thrift-scala", - "servo/util/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/common/text/tagger", - "src/java/com/twitter/common/text/token", - "src/java/com/twitter/common_internal/text", - "src/java/com/twitter/common_internal/text/version", - "src/java/com/twitter/search/common/util/text", - "src/resources/com/twitter/ml/feature/generator", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:features-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/tweetypie:media-entity-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "timelineranker/common:model", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/core", - "timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model", - "timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck", - "timelines/src/main/scala/com/twitter/timelines/clients/tweetypie", - "timelines/src/main/scala/com/twitter/timelines/model/tweet", - "timelines/src/main/scala/com/twitter/timelines/util", - "timelines/src/main/scala/com/twitter/timelines/util/stats", - "util/util-core:util-core-util", - "util/util-logging/src/main/scala", - "util/util-stats/src/main/scala", - ], -) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/BUILD.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/BUILD.docx new file mode 100644 index 000000000..c16b4c043 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/BUILD.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CachingContentFeaturesProvider.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CachingContentFeaturesProvider.docx new file mode 100644 index 000000000..54a0be965 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CachingContentFeaturesProvider.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CachingContentFeaturesProvider.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CachingContentFeaturesProvider.scala deleted file mode 100644 index c904fe3b6..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CachingContentFeaturesProvider.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.Store -import com.twitter.timelineranker.contentfeatures.ContentFeaturesProvider -import com.twitter.timelineranker.model.RecapQuery -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.util.FailOpenHandler -import com.twitter.timelines.util.FutureUtils -import com.twitter.timelines.util.stats.FutureObserver -import com.twitter.util.Future - -object CachingContentFeaturesProvider { - private sealed trait CacheResult - private object CacheFailure extends CacheResult - private object CacheMiss extends CacheResult - private case class CacheHit(t: ContentFeatures) extends CacheResult - def isHit(result: CacheResult): Boolean = result != CacheMiss && result != CacheFailure - def isMiss(result: CacheResult): Boolean = result == CacheMiss -} - -class CachingContentFeaturesProvider( - underlying: ContentFeaturesProvider, - contentFeaturesCache: Store[TweetId, ContentFeatures], - statsReceiver: StatsReceiver) - extends ContentFeaturesProvider { - import CachingContentFeaturesProvider._ - - private val scopedStatsReceiver = statsReceiver.scope("CachingContentFeaturesProvider") - private val cacheScope = scopedStatsReceiver.scope("cache") - private val cacheReadsCounter = cacheScope.counter("reads") - private val cacheReadFailOpenHandler = new FailOpenHandler(cacheScope.scope("reads")) - private val cacheHitsCounter = cacheScope.counter("hits") - private val cacheMissesCounter = cacheScope.counter("misses") - private val cacheFailuresCounter = cacheScope.counter("failures") - private val cacheWritesCounter = cacheScope.counter("writes") - private val cacheWriteObserver = FutureObserver(cacheScope.scope("writes")) - private val underlyingScope = scopedStatsReceiver.scope("underlying") - private val underlyingReadsCounter = underlyingScope.counter("reads") - - override def apply( - query: RecapQuery, - tweetIds: Seq[TweetId] - ): Future[Map[TweetId, ContentFeatures]] = { - if (tweetIds.nonEmpty) { - val distinctTweetIds = tweetIds.toSet - readFromCache(distinctTweetIds).flatMap { cacheResultsFuture => - val (resultsFromCache, missedTweetIds) = partitionHitsMisses(cacheResultsFuture) - - if (missedTweetIds.nonEmpty) { - underlyingReadsCounter.incr(missedTweetIds.size) - val resultsFromUnderlyingFu = underlying(query, missedTweetIds) - resultsFromUnderlyingFu.onSuccess(writeToCache) - resultsFromUnderlyingFu - .map(resultsFromUnderlying => resultsFromCache ++ resultsFromUnderlying) - } else { - Future.value(resultsFromCache) - } - } - } else { - FutureUtils.EmptyMap - } - } - - private def readFromCache(tweetIds: Set[TweetId]): Future[Seq[(TweetId, CacheResult)]] = { - cacheReadsCounter.incr(tweetIds.size) - Future.collect( - contentFeaturesCache - .multiGet(tweetIds) - .toSeq - .map { - case (tweetId, cacheResultOptionFuture) => - cacheReadFailOpenHandler( - cacheResultOptionFuture.map { - case Some(t: ContentFeatures) => tweetId -> CacheHit(t) - case None => tweetId -> CacheMiss - } - ) { _: Throwable => Future.value(tweetId -> CacheFailure) } - } - ) - } - - private def partitionHitsMisses( - cacheResults: Seq[(TweetId, CacheResult)] - ): (Map[TweetId, ContentFeatures], Seq[TweetId]) = { - val (hits, missesAndFailures) = cacheResults.partition { - case (_, cacheResult) => isHit(cacheResult) - } - - val (misses, cacheFailures) = missesAndFailures.partition { - case (_, cacheResult) => isMiss(cacheResult) - } - - val cacheHits = hits.collect { case (tweetId, CacheHit(t)) => (tweetId, t) }.toMap - val cacheMisses = misses.collect { case (tweetId, _) => tweetId } - - cacheHitsCounter.incr(cacheHits.size) - cacheMissesCounter.incr(cacheMisses.size) - cacheFailuresCounter.incr(cacheFailures.size) - - (cacheHits, cacheMisses) - } - - private def writeToCache(results: Map[TweetId, ContentFeatures]): Unit = { - if (results.nonEmpty) { - cacheWritesCounter.incr(results.size) - val indexedResults = results.map { - case (tweetId, contentFeatures) => - (tweetId, Some(contentFeatures)) - } - contentFeaturesCache - .multiPut(indexedResults) - .map { - case (_, statusFu) => - cacheWriteObserver(statusFu) - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoHydratedTweetsTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoHydratedTweetsTransform.docx new file mode 100644 index 000000000..dfcd63729 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoHydratedTweetsTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoHydratedTweetsTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoHydratedTweetsTransform.scala deleted file mode 100644 index b7bb87ece..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoHydratedTweetsTransform.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.util.Future - -object CopyContentFeaturesIntoHydratedTweetsTransform - extends FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ] { - - override def apply( - request: HydratedCandidatesAndFeaturesEnvelope - ): Future[HydratedCandidatesAndFeaturesEnvelope] = { - - request.contentFeaturesFuture.map { sourceTweetContentFeaturesMap => - val updatedHyratedTweets = request.candidateEnvelope.hydratedTweets.outerTweets.map { - hydratedTweet => - val contentFeaturesOpt = request.tweetSourceTweetMap - .get(hydratedTweet.tweetId) - .flatMap(sourceTweetContentFeaturesMap.get) - - val updatedHyratedTweet = contentFeaturesOpt match { - case Some(contentFeatures: ContentFeatures) => - copyContentFeaturesIntoHydratedTweets( - contentFeatures, - hydratedTweet - ) - case _ => hydratedTweet - } - - updatedHyratedTweet - } - - request.copy( - candidateEnvelope = request.candidateEnvelope.copy( - hydratedTweets = request.candidateEnvelope.hydratedTweets.copy( - outerTweets = updatedHyratedTweets - ) - ) - ) - } - } - - def copyContentFeaturesIntoHydratedTweets( - contentFeatures: ContentFeatures, - hydratedTweet: HydratedTweet - ): HydratedTweet = { - HydratedTweet( - hydratedTweet.tweet.copy( - selfThreadMetadata = contentFeatures.selfThreadMetadata, - media = contentFeatures.media - ) - ) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoThriftTweetFeaturesTransform.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoThriftTweetFeaturesTransform.docx new file mode 100644 index 000000000..8336ab1e0 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoThriftTweetFeaturesTransform.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoThriftTweetFeaturesTransform.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoThriftTweetFeaturesTransform.scala deleted file mode 100644 index e75ec0eb7..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/CopyContentFeaturesIntoThriftTweetFeaturesTransform.scala +++ /dev/null @@ -1,106 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures -import com.twitter.servo.util.FutureArrow -import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.timelines.model.TweetId -import com.twitter.util.Future - -/** - * Populates features with tweetId -> thriftTweetFeatures pairs. - * - * If a tweetId from contentFeatures is from searchResults, its content features are copied to - * thriftTweetFeatures. If the tweet is a retweet, the original tweet's content features are copied. - * - * If the tweetId is not found in searchResults, but is an inReplyToTweet of a searchResult, the - * tweetId -> thriftTweetFeatures pair is added to features. This is because in TLM, reply tweets - * have features that are their inReplyToTweets' content features. This also allows scoring - * inReplyToTweet with content features populated when scoring replies. - */ -object CopyContentFeaturesIntoThriftTweetFeaturesTransform - extends FutureArrow[ - HydratedCandidatesAndFeaturesEnvelope, - HydratedCandidatesAndFeaturesEnvelope - ] { - - override def apply( - request: HydratedCandidatesAndFeaturesEnvelope - ): Future[HydratedCandidatesAndFeaturesEnvelope] = { - - // Content Features Request Failures are handled in [[TweetypieContentFeaturesProvider]] - request.contentFeaturesFuture.map { contentFeaturesMap => - val features = request.features.map { - case (tweetId: TweetId, thriftTweetFeatures: ThriftTweetFeatures) => - val contentFeaturesOpt = request.tweetSourceTweetMap - .get(tweetId) - .orElse( - request.inReplyToTweetIds.contains(tweetId) match { - case true => Some(tweetId) - case false => None - } - ) - .flatMap(contentFeaturesMap.get) - - val thriftTweetFeaturesWithContentFeatures = contentFeaturesOpt match { - case Some(contentFeatures: ContentFeatures) => - copyContentFeaturesIntoThriftTweetFeatures(contentFeatures, thriftTweetFeatures) - case _ => thriftTweetFeatures - } - - (tweetId, thriftTweetFeaturesWithContentFeatures) - } - - request.copy(features = features) - } - } - - def copyContentFeaturesIntoThriftTweetFeatures( - contentFeatures: ContentFeatures, - thriftTweetFeatures: ThriftTweetFeatures - ): ThriftTweetFeatures = { - thriftTweetFeatures.copy( - tweetLength = Some(contentFeatures.length.toInt), - hasQuestion = Some(contentFeatures.hasQuestion), - numCaps = Some(contentFeatures.numCaps.toInt), - numWhitespaces = Some(contentFeatures.numWhiteSpaces.toInt), - numNewlines = contentFeatures.numNewlines, - videoDurationMs = contentFeatures.videoDurationMs, - bitRate = contentFeatures.bitRate, - aspectRatioNum = contentFeatures.aspectRatioNum, - aspectRatioDen = contentFeatures.aspectRatioDen, - widths = contentFeatures.widths.map(_.map(_.toInt)), - heights = contentFeatures.heights.map(_.map(_.toInt)), - resizeMethods = contentFeatures.resizeMethods.map(_.map(_.toInt)), - numMediaTags = contentFeatures.numMediaTags.map(_.toInt), - mediaTagScreenNames = contentFeatures.mediaTagScreenNames, - emojiTokens = contentFeatures.emojiTokens, - emoticonTokens = contentFeatures.emoticonTokens, - phrases = contentFeatures.phrases, - textTokens = contentFeatures.tokens, - faceAreas = contentFeatures.faceAreas, - dominantColorRed = contentFeatures.dominantColorRed, - dominantColorBlue = contentFeatures.dominantColorBlue, - dominantColorGreen = contentFeatures.dominantColorGreen, - numColors = contentFeatures.numColors.map(_.toInt), - stickerIds = contentFeatures.stickerIds, - mediaOriginProviders = contentFeatures.mediaOriginProviders, - isManaged = contentFeatures.isManaged, - is360 = contentFeatures.is360, - viewCount = contentFeatures.viewCount, - isMonetizable = contentFeatures.isMonetizable, - isEmbeddable = contentFeatures.isEmbeddable, - hasSelectedPreviewImage = contentFeatures.hasSelectedPreviewImage, - hasTitle = contentFeatures.hasTitle, - hasDescription = contentFeatures.hasDescription, - hasVisitSiteCallToAction = contentFeatures.hasVisitSiteCallToAction, - hasAppInstallCallToAction = contentFeatures.hasAppInstallCallToAction, - hasWatchNowCallToAction = contentFeatures.hasWatchNowCallToAction, - dominantColorPercentage = contentFeatures.dominantColorPercentage, - posUnigrams = contentFeatures.posUnigrams, - posBigrams = contentFeatures.posBigrams, - semanticCoreAnnotations = contentFeatures.semanticCoreAnnotations, - conversationControl = contentFeatures.conversationControl - ) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ExtendedRepliesFilter.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ExtendedRepliesFilter.docx new file mode 100644 index 000000000..43df6c83b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ExtendedRepliesFilter.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ExtendedRepliesFilter.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ExtendedRepliesFilter.scala deleted file mode 100644 index 567915955..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ExtendedRepliesFilter.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelines.model.tweet.HydratedTweet - -object ExtendedRepliesFilter { - private[util] def isExtendedReply(tweet: HydratedTweet, followedUserIds: Seq[UserId]): Boolean = { - tweet.hasReply && - tweet.directedAtUser.exists(!followedUserIds.contains(_)) && - followedUserIds.contains(tweet.userId) - } - - private[util] def isNotQualifiedExtendedReply( - tweet: HydratedTweet, - userId: UserId, - followedUserIds: Seq[UserId], - mutedUserIds: Set[UserId], - sourceTweetsById: Map[TweetId, HydratedTweet] - ): Boolean = { - val currentUserId = userId - isExtendedReply(tweet, followedUserIds) && - !( - !tweet.isRetweet && - // and the extended reply must be directed at someone other than the current user - tweet.directedAtUser.exists(_ != currentUserId) && - // There must be a source tweet - tweet.inReplyToTweetId - .flatMap(sourceTweetsById.get) - .filter { c => - // and the author of the source tweet must be non zero - (c.userId != 0) && - (c.userId != currentUserId) && // and not by the current user - (!c.hasReply) && // and a root tweet, i.e. not a reply - (!c.isRetweet) && // and not a retweet - (c.userId != tweet.userId) // and not a by the same user - } - // and not by a muted user - .exists(sourceTweet => !mutedUserIds.contains(sourceTweet.userId)) - ) - } - - private[util] def isNotValidExpandedExtendedReply( - tweet: HydratedTweet, - viewingUserId: UserId, - followedUserIds: Seq[UserId], - mutedUserIds: Set[UserId], - sourceTweetsById: Map[TweetId, HydratedTweet] - ): Boolean = { - // An extended reply is valid if we hydrated the in-reply to tweet - val isValidExtendedReply = - !tweet.isRetweet && // extended replies must be source tweets - tweet.directedAtUser.exists( - _ != viewingUserId) && // the extended reply must be directed at someone other than the viewing user - tweet.inReplyToTweetId - .flatMap( - sourceTweetsById.get - ) // there must be an in-reply-to tweet matching the following properities - .exists { inReplyToTweet => - (inReplyToTweet.userId > 0) && // and the in-reply to author is valid - (inReplyToTweet.userId != viewingUserId) && // the reply can not be in reply to the viewing user's tweet - !inReplyToTweet.isRetweet && // and the in-reply-to tweet is not a retweet (this should always be true?) - !mutedUserIds.contains( - inReplyToTweet.userId) && // and the in-reply-to user is not muted - inReplyToTweet.inReplyToUserId.forall(r => - !mutedUserIds - .contains(r)) // if there is an in-reply-to-in-reply-to user they are not muted - } - // filter any invalid extended reply - isExtendedReply(tweet, followedUserIds) && !isValidExtendedReply - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/LatentRepository.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/LatentRepository.docx new file mode 100644 index 000000000..28690ea97 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/LatentRepository.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/LatentRepository.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/LatentRepository.scala deleted file mode 100644 index 49842972d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/LatentRepository.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.finagle.util.DefaultTimer -import com.twitter.servo.repository.Repository -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Timer -import scala.util.Random - -// Inject an artificial delay into an underlying repository's response to match the provided p50 -// and max latencies. -class LatentRepository[Q, R]( - underlying: Repository[Q, R], - p50: Duration, - max: Duration, - random: Random = new Random, - timer: Timer = DefaultTimer) - extends Repository[Q, R] { - import scala.math.ceil - import scala.math.pow - - val p50Millis: Long = p50.inMilliseconds - val maxMillis: Long = max.inMilliseconds - require(p50Millis > 0 && maxMillis > 0 && maxMillis > p50Millis) - - override def apply(query: Q): Future[R] = { - val x = random.nextDouble() - val sleepTime = ceil(pow(p50Millis, 2 * (1 - x)) / pow(maxMillis, 1 - 2 * x)).toInt - Future.sleep(Duration.fromMilliseconds(sleepTime))(timer).flatMap { _ => underlying(query) } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/RecommendedRepliesFilter.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/RecommendedRepliesFilter.docx new file mode 100644 index 000000000..b7103817e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/RecommendedRepliesFilter.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/RecommendedRepliesFilter.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/RecommendedRepliesFilter.scala deleted file mode 100644 index fecb3a815..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/RecommendedRepliesFilter.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.timelines.model.UserId -import com.twitter.timelines.model.tweet.HydratedTweet - -object RecommendedRepliesFilter { - private[util] def isRecommendedReply( - tweet: HydratedTweet, - followedUserIds: Seq[UserId] - ): Boolean = { - tweet.hasReply && tweet.inReplyToTweetId.nonEmpty && - (!followedUserIds.contains(tweet.userId)) - } - - private[util] def isRecommendedReplyToNotFollowedUser( - tweet: HydratedTweet, - viewingUserId: UserId, - followedUserIds: Seq[UserId], - mutedUserIds: Set[UserId] - ): Boolean = { - val isValidRecommendedReply = - !tweet.isRetweet && - tweet.inReplyToUserId.exists(followedUserIds.contains(_)) && - !mutedUserIds.contains(tweet.userId) - - isRecommendedReply(tweet, followedUserIds) && !isValidRecommendedReply - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ReverseExtendedRepliesFilter.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ReverseExtendedRepliesFilter.docx new file mode 100644 index 000000000..cb94fff48 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ReverseExtendedRepliesFilter.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ReverseExtendedRepliesFilter.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ReverseExtendedRepliesFilter.scala deleted file mode 100644 index b8cd39fbe..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/ReverseExtendedRepliesFilter.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelines.model.tweet.HydratedTweet - -object ReverseExtendedRepliesFilter { - private[util] def isQualifiedReverseExtendedReply( - tweet: HydratedTweet, - currentUserId: UserId, - followedUserIds: Seq[UserId], - mutedUserIds: Set[UserId], - sourceTweetsById: Map[TweetId, HydratedTweet] - ): Boolean = { - // tweet author is out of the current user's network - !followedUserIds.contains(tweet.userId) && - // tweet author is not muted - !mutedUserIds.contains(tweet.userId) && - // tweet is not a retweet - !tweet.isRetweet && - // there must be a source tweet - tweet.inReplyToTweetId - .flatMap(sourceTweetsById.get) - .filter { sourceTweet => - (!sourceTweet.isRetweet) && // and it's not a retweet - (!sourceTweet.hasReply) && // and it's not a reply - (sourceTweet.userId != 0) && // and the author's id must be non zero - followedUserIds.contains(sourceTweet.userId) // and the author is followed - } // and the author has not been muted - .exists(sourceTweet => !mutedUserIds.contains(sourceTweet.userId)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultUtil.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultUtil.docx new file mode 100644 index 000000000..7f4661d2d Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultUtil.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultUtil.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultUtil.scala deleted file mode 100644 index e80e80c76..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultUtil.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId - -object SearchResultUtil { - val DefaultScore = 0.0 - def getScore(result: ThriftSearchResult): Double = { - result.metadata.flatMap(_.score).filterNot(_.isNaN).getOrElse(DefaultScore) - } - - def isRetweet(result: ThriftSearchResult): Boolean = { - result.metadata.flatMap(_.isRetweet).getOrElse(false) - } - - def isReply(result: ThriftSearchResult): Boolean = { - result.metadata.flatMap(_.isReply).getOrElse(false) - } - - def isEligibleReply(result: ThriftSearchResult): Boolean = { - isReply(result) && !isRetweet(result) - } - - def authorId(result: ThriftSearchResult): Option[UserId] = { - // fromUserId defaults to 0L if unset. None is cleaner - result.metadata.map(_.fromUserId).filter(_ != 0L) - } - - def referencedTweetAuthorId(result: ThriftSearchResult): Option[UserId] = { - // referencedTweetAuthorId defaults to 0L by default. None is cleaner - result.metadata.map(_.referencedTweetAuthorId).filter(_ != 0L) - } - - /** - * Extended replies are replies, that are not retweets (see below), from a followed userId - * towards a non-followed userId. - * - * In Thrift SearchResult it is possible to have both isRetweet and isReply set to true, - * in the case of the retweeted reply. This is confusing edge case as the retweet object - * is not itself a reply, but the original tweet is reply. - */ - def isExtendedReply(followedUserIds: Seq[UserId])(result: ThriftSearchResult): Boolean = { - isEligibleReply(result) && - authorId(result).exists(followedUserIds.contains(_)) && // author is followed - referencedTweetAuthorId(result).exists(!followedUserIds.contains(_)) // referenced author is not - } - - /** - * If a tweet is a reply that is not a retweet, and both the user follows both the reply author - * and the reply parent's author - */ - def isInNetworkReply(followedUserIds: Seq[UserId])(result: ThriftSearchResult): Boolean = { - isEligibleReply(result) && - authorId(result).exists(followedUserIds.contains(_)) && // author is followed - referencedTweetAuthorId(result).exists(followedUserIds.contains(_)) // referenced author is - } - - /** - * If a tweet is a retweet, and user follows author of outside tweet but not following author of - * source/inner tweet. This tweet is also called oon-retweet - */ - def isOutOfNetworkRetweet(followedUserIds: Seq[UserId])(result: ThriftSearchResult): Boolean = { - isRetweet(result) && - authorId(result).exists(followedUserIds.contains(_)) && // author is followed - referencedTweetAuthorId(result).exists(!followedUserIds.contains(_)) // referenced author is not - } - - /** - * From official documentation in thrift on sharedStatusId: - * When isRetweet (or packed features equivalent) is true, this is the status id of the - * original tweet. When isReply and getReplySource are true, this is the status id of the - * original tweet. In all other circumstances this is 0. - * - * If a tweet is a retweet of a reply, this is the status id of the reply (the original tweet - * of the retweet), not the reply's in-reply-to tweet status id. - */ - def getSourceTweetId(result: ThriftSearchResult): Option[TweetId] = { - result.metadata.map(_.sharedStatusId).filter(_ != 0L) - } - - def getRetweetSourceTweetId(result: ThriftSearchResult): Option[TweetId] = { - if (isRetweet(result)) { - getSourceTweetId(result) - } else { - None - } - } - - def getInReplyToTweetId(result: ThriftSearchResult): Option[TweetId] = { - if (isReply(result)) { - getSourceTweetId(result) - } else { - None - } - } - - def getReplyRootTweetId(result: ThriftSearchResult): Option[TweetId] = { - if (isEligibleReply(result)) { - for { - meta <- result.metadata - extraMeta <- meta.extraMetadata - conversationId <- extraMeta.conversationId - } yield { - conversationId - } - } else { - None - } - } - - /** - * For retweet: selfTweetId + sourceTweetId, (however selfTweetId is redundant here, since Health - * score retweet by tweetId == sourceTweetId) - * For replies: selfTweetId + immediate ancestor tweetId + root ancestor tweetId. - * Use set to de-duplicate the case when source tweet == root tweet. (like A->B, B is root and source). - */ - def getOriginalTweetIdAndAncestorTweetIds(searchResult: ThriftSearchResult): Set[TweetId] = { - Set(searchResult.id) ++ - SearchResultUtil.getSourceTweetId(searchResult).toSet ++ - SearchResultUtil.getReplyRootTweetId(searchResult).toSet - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultWithVisibilityActors.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultWithVisibilityActors.docx new file mode 100644 index 000000000..83291364e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultWithVisibilityActors.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultWithVisibilityActors.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultWithVisibilityActors.scala deleted file mode 100644 index c1b5ebcbf..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SearchResultWithVisibilityActors.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelines.visibility.model.CheckedUserActor -import com.twitter.timelines.visibility.model.HasVisibilityActors -import com.twitter.timelines.visibility.model.VisibilityCheckUser - -case class SearchResultWithVisibilityActors( - searchResult: ThriftSearchResult, - statsReceiver: StatsReceiver) - extends HasVisibilityActors { - - private[this] val searchResultWithoutMetadata = - statsReceiver.counter("searchResultWithoutMetadata") - - val tweetId: TweetId = searchResult.id - val metadata = searchResult.metadata - val (userId, isRetweet, sourceUserId, sourceTweetId) = metadata match { - case Some(md) => { - ( - md.fromUserId, - md.isRetweet, - md.isRetweet.getOrElse(false) match { - case true => Some(md.referencedTweetAuthorId) - case false => None - }, - // metadata.sharedStatusId is defaulting to 0 for tweets that don't have one - // 0 is not a valid tweet id so converting to None. Also disregarding sharedStatusId - // for non-retweets. - if (md.isRetweet.isDefined && md.isRetweet.get) - md.sharedStatusId match { - case 0 => None - case id => Some(id) - } - else None - ) - } - case None => { - searchResultWithoutMetadata.incr() - throw new IllegalArgumentException( - "searchResult is missing metadata: " + searchResult.toString) - } - } - - /** - * Returns the set of users (or 'actors') relevant for Tweet visibility filtering. Usually the - * Tweet author, but if this is a Retweet, then the source Tweet author is also relevant. - */ - def getVisibilityActors(viewerIdOpt: Option[UserId]): Seq[CheckedUserActor] = { - val isSelf = isViewerAlsoTweetAuthor(viewerIdOpt, Some(userId)) - Seq( - Some(CheckedUserActor(isSelf, VisibilityCheckUser.Tweeter, userId)), - sourceUserId.map { - CheckedUserActor(isSelf, VisibilityCheckUser.SourceUser, _) - } - ).flatten - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SnowflakeUtils.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SnowflakeUtils.docx new file mode 100644 index 000000000..5f4fb998e Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SnowflakeUtils.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SnowflakeUtils.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SnowflakeUtils.scala deleted file mode 100644 index b288d519c..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SnowflakeUtils.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.util.Duration -import com.twitter.util.Time - -object SnowflakeUtils { - def mutateIdTime(id: Long, timeOp: Time => Time): Long = { - SnowflakeId.firstIdFor(timeOp(SnowflakeId(id).time)) - } - - def quantizeDown(id: Long, step: Duration): Long = { - mutateIdTime(id, _.floor(step)) - } - - def quantizeUp(id: Long, step: Duration): Long = { - mutateIdTime(id, _.ceil(step)) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SourceTweetsUtil.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SourceTweetsUtil.docx new file mode 100644 index 000000000..871995420 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SourceTweetsUtil.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SourceTweetsUtil.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SourceTweetsUtil.scala deleted file mode 100644 index 59488a9ac..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/SourceTweetsUtil.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId - -object SourceTweetsUtil { - def getSourceTweetIds( - searchResults: Seq[ThriftSearchResult], - searchResultsTweetIds: Set[TweetId], - followedUserIds: Seq[TweetId], - shouldIncludeReplyRootTweets: Boolean, - statsReceiver: StatsReceiver - ): Seq[TweetId] = { - val replyRootTweetCounter = statsReceiver.counter("replyRootTweet") - - val retweetSourceTweetIds = getRetweetSourceTweetIds(searchResults, searchResultsTweetIds) - - val inNetworkReplyInReplyToTweetIds = getInNetworkInReplyToTweetIds( - searchResults, - searchResultsTweetIds, - followedUserIds - ) - - val extendedRepliesSourceTweetIds = getExtendedReplySourceTweetIds( - searchResults, - searchResultsTweetIds, - followedUserIds - ) - - val replyRootTweetIds = if (shouldIncludeReplyRootTweets) { - val rootTweetIds = getReplyRootTweetIds( - searchResults, - searchResultsTweetIds - ) - replyRootTweetCounter.incr(rootTweetIds.size) - - rootTweetIds - } else { - Seq.empty - } - - (retweetSourceTweetIds ++ extendedRepliesSourceTweetIds ++ - inNetworkReplyInReplyToTweetIds ++ replyRootTweetIds).distinct - } - - def getInNetworkInReplyToTweetIds( - searchResults: Seq[ThriftSearchResult], - searchResultsTweetIds: Set[TweetId], - followedUserIds: Seq[UserId] - ): Seq[TweetId] = { - searchResults - .filter(SearchResultUtil.isInNetworkReply(followedUserIds)) - .flatMap(SearchResultUtil.getSourceTweetId) - .filterNot(searchResultsTweetIds.contains) - } - - def getReplyRootTweetIds( - searchResults: Seq[ThriftSearchResult], - searchResultsTweetIds: Set[TweetId] - ): Seq[TweetId] = { - searchResults - .flatMap(SearchResultUtil.getReplyRootTweetId) - .filterNot(searchResultsTweetIds.contains) - } - - def getRetweetSourceTweetIds( - searchResults: Seq[ThriftSearchResult], - searchResultsTweetIds: Set[TweetId] - ): Seq[TweetId] = { - searchResults - .filter(SearchResultUtil.isRetweet) - .flatMap(SearchResultUtil.getSourceTweetId) - .filterNot(searchResultsTweetIds.contains) - } - - def getExtendedReplySourceTweetIds( - searchResults: Seq[ThriftSearchResult], - searchResultsTweetIds: Set[TweetId], - followedUserIds: Seq[UserId] - ): Seq[TweetId] = { - searchResults - .filter(SearchResultUtil.isExtendedReply(followedUserIds)) - .flatMap(SearchResultUtil.getSourceTweetId) - .filterNot(searchResultsTweetIds.contains) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetAnnotationFeaturesExtractor.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetAnnotationFeaturesExtractor.docx new file mode 100644 index 000000000..fdf5e521a Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetAnnotationFeaturesExtractor.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetAnnotationFeaturesExtractor.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetAnnotationFeaturesExtractor.scala deleted file mode 100644 index 7e35727eb..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetAnnotationFeaturesExtractor.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.tweetypie.{thriftscala => tweetypie} -import com.twitter.timelineranker.recap.model.ContentFeatures - -object TweetAnnotationFeaturesExtractor { - def addAnnotationFeaturesFromTweet( - inputFeatures: ContentFeatures, - tweet: tweetypie.Tweet, - hydrateSemanticCoreFeatures: Boolean - ): ContentFeatures = { - if (hydrateSemanticCoreFeatures) { - val annotations = tweet.escherbirdEntityAnnotations.map(_.entityAnnotations) - inputFeatures.copy(semanticCoreAnnotations = annotations) - } else { - inputFeatures - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetHydrator.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetHydrator.docx new file mode 100644 index 000000000..5ba627115 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetHydrator.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetHydrator.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetHydrator.scala deleted file mode 100644 index 45d99bb3e..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetHydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.timelineranker.core.HydratedTweets -import com.twitter.timelines.clients.tweetypie.TweetyPieClient -import com.twitter.timelines.model._ -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.timelines.model.tweet.HydratedTweetUtils -import com.twitter.timelines.util.stats.RequestStats -import com.twitter.tweetypie.thriftscala.TweetInclude -import com.twitter.util.Future - -object TweetHydrator { - val FieldsToHydrate: Set[TweetInclude] = TweetyPieClient.CoreTweetFields - val EmptyHydratedTweets: HydratedTweets = - HydratedTweets(Seq.empty[HydratedTweet], Seq.empty[HydratedTweet]) - val EmptyHydratedTweetsFuture: Future[HydratedTweets] = Future.value(EmptyHydratedTweets) -} - -class TweetHydrator(tweetyPieClient: TweetyPieClient, statsReceiver: StatsReceiver) - extends RequestStats { - - private[this] val hydrateScope = statsReceiver.scope("tweetHydrator") - private[this] val outerTweetsScope = hydrateScope.scope("outerTweets") - private[this] val innerTweetsScope = hydrateScope.scope("innerTweets") - - private[this] val totalCounter = outerTweetsScope.counter(Total) - private[this] val totalInnerCounter = innerTweetsScope.counter(Total) - - /** - * Hydrates zero or more tweets from the given seq of tweet IDs. Returns requested tweets ordered - * by tweetIds and out of order inner tweet ids. - * - * Inner tweets that were also requested as outer tweets are returned as outer tweets. - * - * Note that some tweet may not be hydrated due to hydration errors or because they are deleted. - * Consequently, the size of output is <= size of input. That is the intended usage pattern. - */ - def hydrate( - viewerId: Option[UserId], - tweetIds: Seq[TweetId], - fieldsToHydrate: Set[TweetInclude] = TweetyPieClient.CoreTweetFields, - includeQuotedTweets: Boolean = false - ): Future[HydratedTweets] = { - if (tweetIds.isEmpty) { - TweetHydrator.EmptyHydratedTweetsFuture - } else { - val tweetStateMapFuture = tweetyPieClient.getHydratedTweetFields( - tweetIds, - viewerId, - fieldsToHydrate, - safetyLevel = Some(SafetyLevel.FilterNone), - bypassVisibilityFiltering = true, - includeSourceTweets = false, - includeQuotedTweets = includeQuotedTweets, - ignoreTweetSuppression = true - ) - - tweetStateMapFuture.map { tweetStateMap => - val innerTweetIdSet = tweetStateMap.keySet -- tweetIds.toSet - - val hydratedTweets = - HydratedTweetUtils.extractAndOrder(tweetIds ++ innerTweetIdSet.toSeq, tweetStateMap) - val (outer, inner) = hydratedTweets.partition { tweet => - !innerTweetIdSet.contains(tweet.tweetId) - } - - totalCounter.incr(outer.size) - totalInnerCounter.incr(inner.size) - HydratedTweets(outer, inner) - } - } - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetMediaFeatureExtractor.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetMediaFeatureExtractor.docx new file mode 100644 index 000000000..407409d0c Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetMediaFeatureExtractor.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetMediaFeatureExtractor.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetMediaFeatureExtractor.scala deleted file mode 100644 index 9c62a9d9d..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetMediaFeatureExtractor.scala +++ /dev/null @@ -1,272 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.mediaservices.commons.tweetmedia.thriftscala.MediaInfo -import com.twitter.tweetypie.thriftscala.MediaEntity -import com.twitter.tweetypie.thriftscala.MediaTag -import com.twitter.tweetypie.{thriftscala => tweetypie} -import scala.collection.Map -import com.twitter.mediaservices.commons.mediainformation.thriftscala.ColorPaletteItem -import com.twitter.mediaservices.commons.mediainformation.thriftscala.Face -import com.twitter.timelineranker.recap.model.ContentFeatures - -object TweetMediaFeaturesExtractor { - - // Method to overload for content features. - def addMediaFeaturesFromTweet( - inputFeatures: ContentFeatures, - tweet: tweetypie.Tweet, - enableTweetMediaHydration: Boolean - ): ContentFeatures = { - val featuresWithMediaEntity = tweet.media - .map { mediaEntities => - val sizeFeatures = getSizeFeatures(mediaEntities) - val playbackFeatures = getPlaybackFeatures(mediaEntities) - val mediaWidths = sizeFeatures.map(_.width.toShort) - val mediaHeights = sizeFeatures.map(_.height.toShort) - val resizeMethods = sizeFeatures.map(_.resizeMethod.toShort) - val faceMapAreas = getFaceMapAreas(mediaEntities) - val sortedColorPalette = getSortedColorPalette(mediaEntities) - val stickerFeatures = getStickerFeatures(mediaEntities) - val mediaOriginProviders = getMediaOriginProviders(mediaEntities) - val isManaged = getIsManaged(mediaEntities) - val is360 = getIs360(mediaEntities) - val viewCount = getViewCount(mediaEntities) - val userDefinedProductMetadataFeatures = - getUserDefinedProductMetadataFeatures(mediaEntities) - val isMonetizable = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isMonetizable)) - val isEmbeddable = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.isEmbeddable)) - val hasSelectedPreviewImage = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasSelectedPreviewImage)) - val hasTitle = getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasTitle)) - val hasDescription = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasDescription)) - val hasVisitSiteCallToAction = getOptBooleanFromSeqOpt( - userDefinedProductMetadataFeatures.map(_.hasVisitSiteCallToAction)) - val hasAppInstallCallToAction = getOptBooleanFromSeqOpt( - userDefinedProductMetadataFeatures.map(_.hasAppInstallCallToAction)) - val hasWatchNowCallToAction = - getOptBooleanFromSeqOpt(userDefinedProductMetadataFeatures.map(_.hasWatchNowCallToAction)) - - inputFeatures.copy( - videoDurationMs = playbackFeatures.durationMs, - bitRate = playbackFeatures.bitRate, - aspectRatioNum = playbackFeatures.aspectRatioNum, - aspectRatioDen = playbackFeatures.aspectRatioDen, - widths = Some(mediaWidths), - heights = Some(mediaHeights), - resizeMethods = Some(resizeMethods), - faceAreas = Some(faceMapAreas), - dominantColorRed = sortedColorPalette.headOption.map(_.rgb.red), - dominantColorBlue = sortedColorPalette.headOption.map(_.rgb.blue), - dominantColorGreen = sortedColorPalette.headOption.map(_.rgb.green), - dominantColorPercentage = sortedColorPalette.headOption.map(_.percentage), - numColors = Some(sortedColorPalette.size.toShort), - stickerIds = Some(stickerFeatures), - mediaOriginProviders = Some(mediaOriginProviders), - isManaged = Some(isManaged), - is360 = Some(is360), - viewCount = viewCount, - isMonetizable = isMonetizable, - isEmbeddable = isEmbeddable, - hasSelectedPreviewImage = hasSelectedPreviewImage, - hasTitle = hasTitle, - hasDescription = hasDescription, - hasVisitSiteCallToAction = hasVisitSiteCallToAction, - hasAppInstallCallToAction = hasAppInstallCallToAction, - hasWatchNowCallToAction = hasWatchNowCallToAction - ) - } - .getOrElse(inputFeatures) - - val featuresWithMediaTags = tweet.mediaTags - .map { mediaTags => - val mediaTagScreenNames = getMediaTagScreenNames(mediaTags.tagMap) - val numMediaTags = mediaTagScreenNames.size - - featuresWithMediaEntity.copy( - mediaTagScreenNames = Some(mediaTagScreenNames), - numMediaTags = Some(numMediaTags.toShort) - ) - } - .getOrElse(featuresWithMediaEntity) - - if (enableTweetMediaHydration) { - featuresWithMediaTags - .copy(media = tweet.media) - } else { - featuresWithMediaTags - } - } - - // Extracts height, width and resize method of photo/video. - private def getSizeFeatures(mediaEntities: Seq[MediaEntity]): Seq[MediaSizeFeatures] = { - mediaEntities.map { mediaEntity => - mediaEntity.sizes.foldLeft(MediaSizeFeatures(0, 0, 0))((accDimensions, dimensions) => - MediaSizeFeatures( - width = math.max(dimensions.width, accDimensions.width), - height = math.max(dimensions.height, accDimensions.height), - resizeMethod = math.max(dimensions.resizeMethod.getValue, accDimensions.resizeMethod) - )) - } - } - - // Extracts media playback features - private def getPlaybackFeatures(mediaEntities: Seq[MediaEntity]): PlaybackFeatures = { - val allPlaybackFeatures = mediaEntities - .flatMap { mediaEntity => - mediaEntity.mediaInfo map { - case videoEntity: MediaInfo.VideoInfo => - PlaybackFeatures( - durationMs = Some(videoEntity.videoInfo.durationMillis), - bitRate = videoEntity.videoInfo.variants.maxBy(_.bitRate).bitRate, - aspectRatioNum = Some(videoEntity.videoInfo.aspectRatio.numerator), - aspectRatioDen = Some(videoEntity.videoInfo.aspectRatio.denominator) - ) - case gifEntity: MediaInfo.AnimatedGifInfo => - PlaybackFeatures( - durationMs = None, - bitRate = gifEntity.animatedGifInfo.variants.maxBy(_.bitRate).bitRate, - aspectRatioNum = Some(gifEntity.animatedGifInfo.aspectRatio.numerator), - aspectRatioDen = Some(gifEntity.animatedGifInfo.aspectRatio.denominator) - ) - case _ => PlaybackFeatures(None, None, None, None) - } - } - .collect { - case playbackFeatures: PlaybackFeatures => playbackFeatures - } - - if (allPlaybackFeatures.nonEmpty) allPlaybackFeatures.maxBy(_.durationMs) - else PlaybackFeatures(None, None, None, None) - } - - private def getMediaTagScreenNames(tagMap: Map[Long, Seq[MediaTag]]): Seq[String] = - tagMap.values - .flatMap(seqMediaTag => seqMediaTag.flatMap(_.screenName)) - .toSeq - - // Areas of the faces identified in the media entities - private def getFaceMapAreas(mediaEntities: Seq[MediaEntity]): Seq[Int] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - faceData <- metadata.faceData - faces <- faceData.faces - } yield { - faces - .getOrElse("orig", Seq.empty[Face]) - .flatMap(f => f.boundingBox.map(bb => bb.width * bb.height)) - } - }.flatten - - // All ColorPalettes in the media sorted by the percentage in descending order - private def getSortedColorPalette(mediaEntities: Seq[MediaEntity]): Seq[ColorPaletteItem] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - colorInfo <- metadata.colorInfo - } yield { - colorInfo.palette - } - }.flatten.sortBy(_.percentage).reverse - - // Id's of stickers applied by the user - private def getStickerFeatures(mediaEntities: Seq[MediaEntity]): Seq[Long] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - stickerInfo <- metadata.stickerInfo - } yield { - stickerInfo.stickers.map(_.id) - } - }.flatten - - // 3rd party media providers. eg. giphy for gifs - private def getMediaOriginProviders(mediaEntities: Seq[MediaEntity]): Seq[String] = - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - mediaOrigin <- metadata.foundMediaOrigin - } yield { - mediaOrigin.provider - } - - private def getIsManaged(mediaEntities: Seq[MediaEntity]): Boolean = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - managementInfo <- metadata.managementInfo - } yield { - managementInfo.managed - } - }.contains(true) - - private def getIs360(mediaEntities: Seq[MediaEntity]): Boolean = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - info360 <- metadata.info360 - } yield { - info360.is360 - } - }.contains(Some(true)) - - private def getViewCount(mediaEntities: Seq[MediaEntity]): Option[Long] = { - for { - mediaEntity <- mediaEntities - metadata <- mediaEntity.additionalMetadata.toSeq - engagementInfo <- metadata.engagementInfo - viewCounts <- engagementInfo.viewCount - } yield { - viewCounts - } - }.reduceOption(_ max _) - - // metadata defined by the user when uploading the image - private def getUserDefinedProductMetadataFeatures( - mediaEntities: Seq[MediaEntity] - ): Seq[UserDefinedProductMetadataFeatures] = - for { - mediaEntity <- mediaEntities - userDefinedMetadata <- mediaEntity.metadata - } yield { - UserDefinedProductMetadataFeatures( - isMonetizable = userDefinedMetadata.monetizable, - isEmbeddable = userDefinedMetadata.embeddable, - hasSelectedPreviewImage = Some(userDefinedMetadata.previewImage.nonEmpty), - hasTitle = userDefinedMetadata.title.map(_.nonEmpty), - hasDescription = userDefinedMetadata.description.map(_.nonEmpty), - hasVisitSiteCallToAction = userDefinedMetadata.callToActions.map(_.visitSite.nonEmpty), - hasAppInstallCallToAction = userDefinedMetadata.callToActions.map(_.appInstall.nonEmpty), - hasWatchNowCallToAction = userDefinedMetadata.callToActions.map(_.watchNow.nonEmpty) - ) - } - - private def getOptBooleanFromSeqOpt( - seqOpt: Seq[Option[Boolean]], - default: Boolean = false - ): Option[Boolean] = Some( - seqOpt.exists(boolOpt => boolOpt.contains(true)) - ) - -} - -case class MediaSizeFeatures(width: Int, height: Int, resizeMethod: Int) - -case class PlaybackFeatures( - durationMs: Option[Int], - bitRate: Option[Int], - aspectRatioNum: Option[Short], - aspectRatioDen: Option[Short]) - -case class UserDefinedProductMetadataFeatures( - isMonetizable: Option[Boolean], - isEmbeddable: Option[Boolean], - hasSelectedPreviewImage: Option[Boolean], - hasTitle: Option[Boolean], - hasDescription: Option[Boolean], - hasVisitSiteCallToAction: Option[Boolean], - hasAppInstallCallToAction: Option[Boolean], - hasWatchNowCallToAction: Option[Boolean]) diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetTextFeaturesExtractor.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetTextFeaturesExtractor.docx new file mode 100644 index 000000000..ae3f1007b Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetTextFeaturesExtractor.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetTextFeaturesExtractor.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetTextFeaturesExtractor.scala deleted file mode 100644 index 653ff9646..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetTextFeaturesExtractor.scala +++ /dev/null @@ -1,199 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.common.text.tagger.UniversalPOS -import com.twitter.common.text.token.attribute.TokenType -import com.twitter.common_internal.text.pipeline.TwitterTextNormalizer -import com.twitter.common_internal.text.pipeline.TwitterTextTokenizer -import com.twitter.common_internal.text.version.PenguinVersion -import com.twitter.search.common.util.text.LanguageIdentifierHelper -import com.twitter.search.common.util.text.PhraseExtractor -import com.twitter.search.common.util.text.TokenizerHelper -import com.twitter.search.common.util.text.TokenizerResult -import com.twitter.timelineranker.recap.model.ContentFeatures -import com.twitter.tweetypie.{thriftscala => tweetypie} -import com.twitter.util.Try -import java.util.Locale -import scala.collection.JavaConversions._ - -object TweetTextFeaturesExtractor { - - private[this] val threadLocaltokenizer = new ThreadLocal[Option[TwitterTextTokenizer]] { - override protected def initialValue(): Option[TwitterTextTokenizer] = - Try { - val normalizer = new TwitterTextNormalizer.Builder(penguinVersion).build - TokenizerHelper - .getTokenizerBuilder(penguinVersion) - .enablePOSTagger - .enableStopwordFilterWithNormalizer(normalizer) - .setStopwordResourcePath("com/twitter/ml/feature/generator/stopwords_extended_{LANG}.txt") - .enableStemmer - .build - }.toOption - } - - val penguinVersion: PenguinVersion = PenguinVersion.PENGUIN_6 - - def addTextFeaturesFromTweet( - inputFeatures: ContentFeatures, - tweet: tweetypie.Tweet, - hydratePenguinTextFeatures: Boolean, - hydrateTokens: Boolean, - hydrateTweetText: Boolean - ): ContentFeatures = { - tweet.coreData - .map { coreData => - val tweetText = coreData.text - val hasQuestion = hasQuestionCharacter(tweetText) - val length = getLength(tweetText).toShort - val numCaps = getCaps(tweetText).toShort - val numWhiteSpaces = getSpaces(tweetText).toShort - val numNewlines = Some(getNumNewlines(tweetText)) - val tweetTextOpt = getTweetText(tweetText, hydrateTweetText) - - if (hydratePenguinTextFeatures) { - val locale = getLocale(tweetText) - val tokenizerOpt = threadLocaltokenizer.get - - val tokenizerResult = tokenizerOpt.flatMap { tokenizer => - tokenizeWithPosTagger(tokenizer, locale, tweetText) - } - - val normalizedTokensOpt = if (hydrateTokens) { - tokenizerOpt.flatMap { tokenizer => - tokenizedStringsWithNormalizerAndStemmer(tokenizer, locale, tweetText) - } - } else None - - val emoticonTokensOpt = tokenizerResult.map(getEmoticons) - val emojiTokensOpt = tokenizerResult.map(getEmojis) - val posUnigramsOpt = tokenizerResult.map(getPosUnigrams) - val posBigramsOpt = posUnigramsOpt.map(getPosBigrams) - val tokensOpt = normalizedTokensOpt - - inputFeatures.copy( - emojiTokens = emojiTokensOpt, - emoticonTokens = emoticonTokensOpt, - hasQuestion = hasQuestion, - length = length, - numCaps = numCaps, - numWhiteSpaces = numWhiteSpaces, - numNewlines = numNewlines, - posUnigrams = posUnigramsOpt.map(_.toSet), - posBigrams = posBigramsOpt.map(_.toSet), - tokens = tokensOpt.map(_.toSeq), - tweetText = tweetTextOpt - ) - } else { - inputFeatures.copy( - hasQuestion = hasQuestion, - length = length, - numCaps = numCaps, - numWhiteSpaces = numWhiteSpaces, - numNewlines = numNewlines, - tweetText = tweetTextOpt - ) - } - } - .getOrElse(inputFeatures) - } - - private def tokenizeWithPosTagger( - tokenizer: TwitterTextTokenizer, - locale: Locale, - text: String - ): Option[TokenizerResult] = { - tokenizer.enableStemmer(false) - tokenizer.enableStopwordFilter(false) - - Try { TokenizerHelper.tokenizeTweet(tokenizer, text, locale) }.toOption - } - - private def tokenizedStringsWithNormalizerAndStemmer( - tokenizer: TwitterTextTokenizer, - locale: Locale, - text: String - ): Option[Seq[String]] = { - tokenizer.enableStemmer(true) - tokenizer.enableStopwordFilter(true) - - Try { tokenizer.tokenizeToStrings(text, locale).toSeq }.toOption - } - - def getLocale(text: String): Locale = LanguageIdentifierHelper.identifyLanguage(text) - - def getTokens(tokenizerResult: TokenizerResult): List[String] = - tokenizerResult.rawSequence.getTokenStrings().toList - - def getEmoticons(tokenizerResult: TokenizerResult): Set[String] = - tokenizerResult.smileys.toSet - - def getEmojis(tokenizerResult: TokenizerResult): Set[String] = - tokenizerResult.rawSequence.getTokenStringsOf(TokenType.EMOJI).toSet - - def getPhrases(tokenizerResult: TokenizerResult, locale: Locale): Set[String] = { - PhraseExtractor.getPhrases(tokenizerResult.rawSequence, locale).map(_.toString).toSet - } - - def getPosUnigrams(tokenizerResult: TokenizerResult): List[String] = - tokenizerResult.tokenSequence.getTokens.toList - .map { token => - Option(token.getPartOfSpeech) - .map(_.toString) - .getOrElse(UniversalPOS.X.toString) // UniversalPOS.X is unknown POS tag - } - - def getPosBigrams(tagsList: List[String]): List[String] = { - if (tagsList.nonEmpty) { - tagsList - .zip(tagsList.tail) - .map(tagPair => Seq(tagPair._1, tagPair._2).mkString(" ")) - } else { - tagsList - } - } - - def getLength(text: String): Int = - text.codePointCount(0, text.length()) - - def getCaps(text: String): Int = text.count(Character.isUpperCase) - - def getSpaces(text: String): Int = text.count(Character.isWhitespace) - - def hasQuestionCharacter(text: String): Boolean = { - // List based on https://unicode-search.net/unicode-namesearch.pl?term=question - val QUESTION_MARK_CHARS = Seq( - "\u003F", - "\u00BF", - "\u037E", - "\u055E", - "\u061F", - "\u1367", - "\u1945", - "\u2047", - "\u2048", - "\u2049", - "\u2753", - "\u2754", - "\u2CFA", - "\u2CFB", - "\u2E2E", - "\uA60F", - "\uA6F7", - "\uFE16", - "\uFE56", - "\uFF1F", - "\u1114", - "\u1E95" - ) - QUESTION_MARK_CHARS.exists(text.contains) - } - - def getNumNewlines(text: String): Short = { - val newlineRegex = "\r\n|\r|\n".r - newlineRegex.findAllIn(text).length.toShort - } - - private[this] def getTweetText(tweetText: String, hydrateTweetText: Boolean): Option[String] = { - if (hydrateTweetText) Some(tweetText) else None - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilter.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilter.docx new file mode 100644 index 000000000..aa433fbc4 Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilter.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilter.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilter.scala deleted file mode 100644 index aad7a19ea..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilter.scala +++ /dev/null @@ -1,493 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Level -import com.twitter.logging.Logger -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelines.model.tweet.HydratedTweet -import com.twitter.timelines.util.stats.BooleanObserver -import com.twitter.timelines.util.stats.RequestStats -import scala.collection.mutable - -object TweetFilters extends Enumeration { - // Filters independent of users or their follow graph. - val DuplicateRetweets: Value = Value - val DuplicateTweets: Value = Value - val NullcastTweets: Value = Value - val Replies: Value = Value - val Retweets: Value = Value - - // Filters that depend on users or their follow graph. - val DirectedAtNotFollowedUsers: Value = Value - val NonReplyDirectedAtNotFollowedUsers: Value = Value - val TweetsFromNotFollowedUsers: Value = Value - val ExtendedReplies: Value = Value - val NotQualifiedExtendedReplies: Value = Value - val NotValidExpandedExtendedReplies: Value = Value - val NotQualifiedReverseExtendedReplies: Value = Value - val RecommendedRepliesToNotFollowedUsers: Value = Value - - val None: TweetFilters.ValueSet = ValueSet.empty - - val UserDependent: ValueSet = ValueSet( - NonReplyDirectedAtNotFollowedUsers, - DirectedAtNotFollowedUsers, - TweetsFromNotFollowedUsers, - ExtendedReplies, - NotQualifiedExtendedReplies, - NotValidExpandedExtendedReplies, - NotQualifiedReverseExtendedReplies, - RecommendedRepliesToNotFollowedUsers - ) - - val UserIndependent: ValueSet = ValueSet( - DuplicateRetweets, - DuplicateTweets, - NullcastTweets, - Replies, - Retweets - ) - require( - (UserDependent ++ UserIndependent) == TweetFilters.values, - "UserIndependent and UserDependent should contain all possible filters" - ) - - private[util] type FilterMethod = - (HydratedTweet, TweetsPostFilterParams, MutableState) => Boolean - - case class MutableState( - seenTweetIds: mutable.Map[TweetId, Int] = mutable.Map.empty[TweetId, Int].withDefaultValue(0)) { - def isSeen(tweetId: TweetId): Boolean = { - val seen = seenTweetIds(tweetId) >= 1 - incrementIf0(tweetId) - seen - } - - def incrementIf0(key: TweetId): Unit = { - if (seenTweetIds(key) == 0) { - seenTweetIds(key) = 1 - } - } - - def incrementThenGetCount(key: TweetId): Int = { - seenTweetIds(key) += 1 - seenTweetIds(key) - } - } -} - -case class TweetsPostFilterParams( - userId: UserId, - followedUserIds: Seq[UserId], - inNetworkUserIds: Seq[UserId], - mutedUserIds: Set[UserId], - numRetweetsAllowed: Int, - loggingPrefix: String = "", - sourceTweets: Seq[HydratedTweet] = Nil) { - lazy val sourceTweetsById: Map[TweetId, HydratedTweet] = - sourceTweets.map(tweet => tweet.tweetId -> tweet).toMap -} - -/** - * Performs post-filtering on tweets obtained from search. - * - * Search currently does not perform certain steps or performs them inadequately. - * This class addresses those shortcomings by post-processing hydrated search results. - */ -abstract class TweetsPostFilterBase( - filters: TweetFilters.ValueSet, - logger: Logger, - statsReceiver: StatsReceiver) - extends RequestStats { - import TweetFilters.FilterMethod - import TweetFilters.MutableState - - private[this] val baseScope = statsReceiver.scope("filter") - private[this] val directedAtNotFollowedCounter = baseScope.counter("directedAtNotFollowed") - private[this] val nonReplyDirectedAtNotFollowedObserver = - BooleanObserver(baseScope.scope("nonReplyDirectedAtNotFollowed")) - private[this] val dupRetweetCounter = baseScope.counter("dupRetweet") - private[this] val dupTweetCounter = baseScope.counter("dupTweet") - private[this] val notFollowedCounter = baseScope.counter("notFollowed") - private[this] val nullcastCounter = baseScope.counter("nullcast") - private[this] val repliesCounter = baseScope.counter("replies") - private[this] val retweetsCounter = baseScope.counter("retweets") - private[this] val extendedRepliesCounter = baseScope.counter("extendedReplies") - private[this] val notQualifiedExtendedRepliesObserver = - BooleanObserver(baseScope.scope("notQualifiedExtendedReplies")) - private[this] val notValidExpandedExtendedRepliesObserver = - BooleanObserver(baseScope.scope("notValidExpandedExtendedReplies")) - private[this] val notQualifiedReverseExtendedRepliesCounter = - baseScope.counter("notQualifiedReverseExtendedReplies") - private[this] val recommendedRepliesToNotFollowedUsersObserver = - BooleanObserver(baseScope.scope("recommendedRepliesToNotFollowedUsers")) - - private[this] val totalCounter = baseScope.counter(Total) - private[this] val resultCounter = baseScope.counter("result") - - // Used for debugging. Its values should remain false for prod use. - private[this] val alwaysLog = false - - val applicableFilters: Seq[FilterMethod] = Filters.getApplicableFilters(filters) - - protected def filter( - tweets: Seq[HydratedTweet], - params: TweetsPostFilterParams - ): Seq[HydratedTweet] = { - val invocationState = MutableState() - val result = tweets.reverseIterator - .filterNot { tweet => applicableFilters.exists(_(tweet, params, invocationState)) } - .toSeq - .reverse - totalCounter.incr(tweets.size) - resultCounter.incr(result.size) - result - } - - object Filters { - case class FilterData(kind: TweetFilters.Value, method: FilterMethod) - private val allFilters = Seq[FilterData]( - FilterData(TweetFilters.DuplicateTweets, isDuplicateTweet), - FilterData(TweetFilters.DuplicateRetweets, isDuplicateRetweet), - FilterData(TweetFilters.DirectedAtNotFollowedUsers, isDirectedAtNonFollowedUser), - FilterData( - TweetFilters.NonReplyDirectedAtNotFollowedUsers, - isNonReplyDirectedAtNonFollowedUser - ), - FilterData(TweetFilters.NullcastTweets, isNullcast), - FilterData(TweetFilters.Replies, isReply), - FilterData(TweetFilters.Retweets, isRetweet), - FilterData(TweetFilters.TweetsFromNotFollowedUsers, isFromNonFollowedUser), - FilterData(TweetFilters.ExtendedReplies, isExtendedReply), - FilterData(TweetFilters.NotQualifiedExtendedReplies, isNotQualifiedExtendedReply), - FilterData(TweetFilters.NotValidExpandedExtendedReplies, isNotValidExpandedExtendedReply), - FilterData( - TweetFilters.NotQualifiedReverseExtendedReplies, - isNotQualifiedReverseExtendedReply), - FilterData( - TweetFilters.RecommendedRepliesToNotFollowedUsers, - isRecommendedRepliesToNotFollowedUsers) - ) - - def getApplicableFilters(filters: TweetFilters.ValueSet): Seq[FilterMethod] = { - require(allFilters.map(_.kind).toSet == TweetFilters.values) - allFilters.filter(data => filters.contains(data.kind)).map(_.method) - } - - private def isNullcast( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - if (tweet.isNullcast) { - nullcastCounter.incr() - log( - Level.ERROR, - () => s"${params.loggingPrefix}:: Found nullcast tweet: tweet-id: ${tweet.tweetId}" - ) - true - } else { - false - } - } - - private def isReply( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - if (tweet.hasReply) { - repliesCounter.incr() - log(Level.OFF, () => s"${params.loggingPrefix}:: Removed reply: tweet-id: ${tweet.tweetId}") - true - } else { - false - } - } - - private def isRetweet( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - if (tweet.isRetweet) { - retweetsCounter.incr() - log( - Level.OFF, - () => s"${params.loggingPrefix}:: Removed retweet: tweet-id: ${tweet.tweetId}" - ) - true - } else { - false - } - } - - private def isFromNonFollowedUser( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - if ((tweet.userId != params.userId) && !params.inNetworkUserIds.contains(tweet.userId)) { - notFollowedCounter.incr() - log( - Level.ERROR, - () => - s"${params.loggingPrefix}:: Found tweet from not-followed user: ${tweet.tweetId} from ${tweet.userId}" - ) - true - } else { - false - } - } - - private def isDirectedAtNonFollowedUser( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - tweet.directedAtUser.exists { directedAtUserId => - val shouldFilterOut = (tweet.userId != params.userId) && !params.inNetworkUserIds - .contains(directedAtUserId) - // We do not log here because search is known to not handle this case. - if (shouldFilterOut) { - log( - Level.OFF, - () => - s"${params.loggingPrefix}:: Found tweet: ${tweet.tweetId} directed-at not-followed user: $directedAtUserId" - ) - directedAtNotFollowedCounter.incr() - } - shouldFilterOut - } - } - - private def isNonReplyDirectedAtNonFollowedUser( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - tweet.directedAtUser.exists { directedAtUserId => - val shouldFilterOut = !tweet.hasReply && - (tweet.userId != params.userId) && - !params.inNetworkUserIds.contains(directedAtUserId) - // We do not log here because search is known to not handle this case. - if (nonReplyDirectedAtNotFollowedObserver(shouldFilterOut)) { - log( - Level.OFF, - () => - s"${params.loggingPrefix}:: Found non-reply tweet: ${tweet.tweetId} directed-at not-followed user: $directedAtUserId" - ) - } - shouldFilterOut - } - } - - /** - * Determines whether the given tweet has already been seen. - */ - private def isDuplicateTweet( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - val shouldFilterOut = invocationState.isSeen(tweet.tweetId) - if (shouldFilterOut) { - dupTweetCounter.incr() - log(Level.ERROR, () => s"${params.loggingPrefix}:: Duplicate tweet found: ${tweet.tweetId}") - } - shouldFilterOut - } - - /** - * If the given tweet is a retweet, determines whether the source tweet - * of that retweet has already been seen. - */ - private def isDuplicateRetweet( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - invocationState.incrementIf0(tweet.tweetId) - tweet.sourceTweetId.exists { sourceTweetId => - val seenCount = invocationState.incrementThenGetCount(sourceTweetId) - val shouldFilterOut = seenCount > params.numRetweetsAllowed - if (shouldFilterOut) { - // We do not log here because search is known to not handle this case. - dupRetweetCounter.incr() - log( - Level.OFF, - () => - s"${params.loggingPrefix}:: Found dup retweet: ${tweet.tweetId} (source tweet: $sourceTweetId), count: $seenCount" - ) - } - shouldFilterOut - } - } - - private def isExtendedReply( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - val shouldFilterOut = ExtendedRepliesFilter.isExtendedReply( - tweet, - params.followedUserIds - ) - if (shouldFilterOut) { - extendedRepliesCounter.incr() - log( - Level.DEBUG, - () => s"${params.loggingPrefix}:: extended reply to be filtered: ${tweet.tweetId}" - ) - } - shouldFilterOut - } - - private def isNotQualifiedExtendedReply( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - val shouldFilterOut = ExtendedRepliesFilter.isNotQualifiedExtendedReply( - tweet, - params.userId, - params.followedUserIds, - params.mutedUserIds, - params.sourceTweetsById - ) - if (notQualifiedExtendedRepliesObserver(shouldFilterOut)) { - log( - Level.DEBUG, - () => - s"${params.loggingPrefix}:: non qualified extended reply to be filtered: ${tweet.tweetId}" - ) - } - shouldFilterOut - } - - private def isNotValidExpandedExtendedReply( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - val shouldFilterOut = ExtendedRepliesFilter.isNotValidExpandedExtendedReply( - tweet, - params.userId, - params.followedUserIds, - params.mutedUserIds, - params.sourceTweetsById - ) - if (notValidExpandedExtendedRepliesObserver(shouldFilterOut)) { - log( - Level.DEBUG, - () => - s"${params.loggingPrefix}:: non qualified extended reply to be filtered: ${tweet.tweetId}" - ) - } - shouldFilterOut - } - - private def isRecommendedRepliesToNotFollowedUsers( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - val shouldFilterOut = RecommendedRepliesFilter.isRecommendedReplyToNotFollowedUser( - tweet, - params.userId, - params.followedUserIds, - params.mutedUserIds - ) - if (recommendedRepliesToNotFollowedUsersObserver(shouldFilterOut)) { - log( - Level.DEBUG, - () => - s"${params.loggingPrefix}:: non qualified recommended reply to be filtered: ${tweet.tweetId}" - ) - } - shouldFilterOut - } - - //For now this filter is meant to be used only with reply tweets from the inReplyToUserId query - private def isNotQualifiedReverseExtendedReply( - tweet: HydratedTweet, - params: TweetsPostFilterParams, - invocationState: MutableState - ): Boolean = { - val shouldFilterOut = !ReverseExtendedRepliesFilter.isQualifiedReverseExtendedReply( - tweet, - params.userId, - params.followedUserIds, - params.mutedUserIds, - params.sourceTweetsById - ) - - if (shouldFilterOut) { - notQualifiedReverseExtendedRepliesCounter.incr() - log( - Level.DEBUG, - () => - s"${params.loggingPrefix}:: non qualified reverse extended reply to be filtered: ${tweet.tweetId}" - ) - } - shouldFilterOut - } - - private def log(level: Level, message: () => String): Unit = { - if (alwaysLog || ((level != Level.OFF) && logger.isLoggable(level))) { - val updatedLevel = if (alwaysLog) Level.INFO else level - logger.log(updatedLevel, message()) - } - } - } -} - -class TweetsPostFilter(filters: TweetFilters.ValueSet, logger: Logger, statsReceiver: StatsReceiver) - extends TweetsPostFilterBase(filters, logger, statsReceiver) { - - def apply( - userId: UserId, - followedUserIds: Seq[UserId], - inNetworkUserIds: Seq[UserId], - mutedUserIds: Set[UserId], - tweets: Seq[HydratedTweet], - numRetweetsAllowed: Int = 1, - sourceTweets: Seq[HydratedTweet] = Nil - ): Seq[HydratedTweet] = { - val loggingPrefix = s"userId: $userId" - val params = TweetsPostFilterParams( - userId = userId, - followedUserIds = followedUserIds, - inNetworkUserIds = inNetworkUserIds, - mutedUserIds = mutedUserIds, - numRetweetsAllowed = numRetweetsAllowed, - loggingPrefix = loggingPrefix, - sourceTweets = sourceTweets - ) - super.filter(tweets, params) - } -} - -class TweetsPostFilterUserIndependent( - filters: TweetFilters.ValueSet, - logger: Logger, - statsReceiver: StatsReceiver) - extends TweetsPostFilterBase(filters, logger, statsReceiver) { - - require( - (filters -- TweetFilters.UserIndependent).isEmpty, - "Only user independent filters are supported" - ) - - def apply(tweets: Seq[HydratedTweet], numRetweetsAllowed: Int = 1): Seq[HydratedTweet] = { - val params = TweetsPostFilterParams( - userId = 0L, - followedUserIds = Seq.empty, - inNetworkUserIds = Seq.empty, - mutedUserIds = Set.empty, - numRetweetsAllowed - ) - super.filter(tweets, params) - } -} diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilterBasedOnSearchMetadata.docx b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilterBasedOnSearchMetadata.docx new file mode 100644 index 000000000..4ebba038f Binary files /dev/null and b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilterBasedOnSearchMetadata.docx differ diff --git a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilterBasedOnSearchMetadata.scala b/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilterBasedOnSearchMetadata.scala deleted file mode 100644 index 56a7a77a4..000000000 --- a/timelineranker/server/src/main/scala/com/twitter/timelineranker/util/TweetsPostFilterBasedOnSearchMetadata.scala +++ /dev/null @@ -1,170 +0,0 @@ -package com.twitter.timelineranker.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Level -import com.twitter.logging.Logger -import com.twitter.search.earlybird.thriftscala.ThriftSearchResult -import com.twitter.timelines.model.TweetId -import com.twitter.timelines.model.UserId -import com.twitter.timelines.util.stats.RequestStats -import scala.collection.mutable - -object TweetFiltersBasedOnSearchMetadata extends Enumeration { - val DuplicateRetweets: Value = Value - val DuplicateTweets: Value = Value - - val None: TweetFiltersBasedOnSearchMetadata.ValueSet = ValueSet.empty - - private[util] type FilterBasedOnSearchMetadataMethod = - (ThriftSearchResult, TweetsPostFilterBasedOnSearchMetadataParams, MutableState) => Boolean - - case class MutableState( - seenTweetIds: mutable.Map[TweetId, Int] = mutable.Map.empty[TweetId, Int].withDefaultValue(0)) { - def isSeen(tweetId: TweetId): Boolean = { - val seen = seenTweetIds(tweetId) >= 1 - incrementIf0(tweetId) - seen - } - - def incrementIf0(key: TweetId): Unit = { - if (seenTweetIds(key) == 0) { - seenTweetIds(key) = 1 - } - } - - def incrementThenGetCount(key: TweetId): Int = { - seenTweetIds(key) += 1 - seenTweetIds(key) - } - } -} - -case class TweetsPostFilterBasedOnSearchMetadataParams( - userId: UserId, - inNetworkUserIds: Seq[UserId], - numRetweetsAllowed: Int, - loggingPrefix: String = "") - -/** - * Performs post-filtering on tweets obtained from search using metadata returned from search. - * - * Search currently does not perform certain steps while searching, so this class addresses those - * shortcomings by post-processing search results using the returned metadata. - */ -class TweetsPostFilterBasedOnSearchMetadata( - filters: TweetFiltersBasedOnSearchMetadata.ValueSet, - logger: Logger, - statsReceiver: StatsReceiver) - extends RequestStats { - import TweetFiltersBasedOnSearchMetadata.FilterBasedOnSearchMetadataMethod - import TweetFiltersBasedOnSearchMetadata.MutableState - - private[this] val baseScope = statsReceiver.scope("filter_based_on_search_metadata") - private[this] val dupRetweetCounter = baseScope.counter("dupRetweet") - private[this] val dupTweetCounter = baseScope.counter("dupTweet") - - private[this] val totalCounter = baseScope.counter(Total) - private[this] val resultCounter = baseScope.counter("result") - - // Used for debugging. Its values should remain false for prod use. - private[this] val alwaysLog = false - - val applicableFilters: Seq[FilterBasedOnSearchMetadataMethod] = - FiltersBasedOnSearchMetadata.getApplicableFilters(filters) - - def apply( - userId: UserId, - inNetworkUserIds: Seq[UserId], - tweets: Seq[ThriftSearchResult], - numRetweetsAllowed: Int = 1 - ): Seq[ThriftSearchResult] = { - val loggingPrefix = s"userId: $userId" - val params = TweetsPostFilterBasedOnSearchMetadataParams( - userId = userId, - inNetworkUserIds = inNetworkUserIds, - numRetweetsAllowed = numRetweetsAllowed, - loggingPrefix = loggingPrefix, - ) - filter(tweets, params) - } - - protected def filter( - tweets: Seq[ThriftSearchResult], - params: TweetsPostFilterBasedOnSearchMetadataParams - ): Seq[ThriftSearchResult] = { - val invocationState = MutableState() - val result = tweets.reverseIterator - .filterNot { tweet => applicableFilters.exists(_(tweet, params, invocationState)) } - .toSeq - .reverse - totalCounter.incr(tweets.size) - resultCounter.incr(result.size) - result - } - - object FiltersBasedOnSearchMetadata { - case class FilterData( - kind: TweetFiltersBasedOnSearchMetadata.Value, - method: FilterBasedOnSearchMetadataMethod) - private val allFilters = Seq[FilterData]( - FilterData(TweetFiltersBasedOnSearchMetadata.DuplicateTweets, isDuplicateTweet), - FilterData(TweetFiltersBasedOnSearchMetadata.DuplicateRetweets, isDuplicateRetweet) - ) - - def getApplicableFilters( - filters: TweetFiltersBasedOnSearchMetadata.ValueSet - ): Seq[FilterBasedOnSearchMetadataMethod] = { - require(allFilters.map(_.kind).toSet == TweetFiltersBasedOnSearchMetadata.values) - allFilters.filter(data => filters.contains(data.kind)).map(_.method) - } - - /** - * Determines whether the given tweet has already been seen. - */ - private def isDuplicateTweet( - tweet: ThriftSearchResult, - params: TweetsPostFilterBasedOnSearchMetadataParams, - invocationState: MutableState - ): Boolean = { - val shouldFilterOut = invocationState.isSeen(tweet.id) - if (shouldFilterOut) { - dupTweetCounter.incr() - log(Level.ERROR, () => s"${params.loggingPrefix}:: Duplicate tweet found: ${tweet.id}") - } - shouldFilterOut - } - - /** - * If the given tweet is a retweet, determines whether the source tweet - * of that retweet has already been seen. - */ - private def isDuplicateRetweet( - tweet: ThriftSearchResult, - params: TweetsPostFilterBasedOnSearchMetadataParams, - invocationState: MutableState - ): Boolean = { - invocationState.incrementIf0(tweet.id) - SearchResultUtil.getRetweetSourceTweetId(tweet).exists { sourceTweetId => - val seenCount = invocationState.incrementThenGetCount(sourceTweetId) - val shouldFilterOut = seenCount > params.numRetweetsAllowed - if (shouldFilterOut) { - // We do not log here because search is known to not handle this case. - dupRetweetCounter.incr() - log( - Level.OFF, - () => - s"${params.loggingPrefix}:: Found dup retweet: ${tweet.id} (source tweet: $sourceTweetId), count: $seenCount" - ) - } - shouldFilterOut - } - } - - private def log(level: Level, message: () => String): Unit = { - if (alwaysLog || ((level != Level.OFF) && logger.isLoggable(level))) { - val updatedLevel = if (alwaysLog) Level.INFO else level - logger.log(updatedLevel, message()) - } - } - } -}