Delete timelineranker directory
This commit is contained in:
dogemanttv
GitHub
parent
db1f0f75b6
commit
502f1f9d11
|
|
@ -1,13 +0,0 @@
|
|||
# TimelineRanker
|
||||
|
||||
**TimelineRanker** (TLR) is a legacy service that provides relevance-scored tweets from the Earlybird Search Index and User Tweet Entity Graph (UTEG) service. Despite its name, it no longer performs heavy ranking or model-based ranking itself; it only uses relevance scores from the Search Index for ranked tweet endpoints.
|
||||
|
||||
The following is a list of major services that Timeline Ranker interacts with:
|
||||
|
||||
- **Earlybird-root-superroot (a.k.a Search):** Timeline Ranker calls the Search Index's super root to fetch a list of Tweets.
|
||||
- **User Tweet Entity Graph (UTEG):** Timeline Ranker calls UTEG to fetch a list of tweets liked by the users you follow.
|
||||
- **Socialgraph:** Timeline Ranker calls Social Graph Service to obtain the follow graph and user states such as blocked, muted, retweets muted, etc.
|
||||
- **TweetyPie:** Timeline Ranker hydrates tweets by calling TweetyPie to post-filter tweets based on certain hydrated fields.
|
||||
- **Manhattan:** Timeline Ranker hydrates some tweet features (e.g., user languages) from Manhattan.
|
||||
|
||||
**Home Mixer** calls Timeline Ranker to fetch tweets from the Earlybird Search Index and User Tweet Entity Graph (UTEG) service to power both the For You and Following Home Timelines. Timeline Ranker performs light ranking based on Earlybird tweet candidate scores and truncates to the number of candidates requested by Home Mixer based on these scores.
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
target(
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"timelineranker/client/builder/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
# TimelineRanker client
|
||||
|
||||
Library for creating a client to talk to TLR. It contains a ClientBuilder implementation
|
||||
with some preferred settings for clients.
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["com/twitter/timelineranker/client/*.scala"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finagle/finagle-core/src/main",
|
||||
"finagle/finagle-stats",
|
||||
"finagle/finagle-thrift/src/main/java",
|
||||
"servo/client/src/main/scala/com/twitter/servo/client",
|
||||
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
||||
"src/thrift/com/twitter/timelineranker/server/model:thrift-scala",
|
||||
"timelineranker/common:model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,195 +0,0 @@
|
|||
package com.twitter.timelineranker.client
|
||||
|
||||
import com.twitter.finagle.SourcedException
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelineranker.model._
|
||||
import com.twitter.timelines.util.stats.RequestStats
|
||||
import com.twitter.timelines.util.stats.RequestStatsReceiver
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Throw
|
||||
import com.twitter.util.Try
|
||||
|
||||
case class TimelineRankerException(message: String)
|
||||
extends Exception(message)
|
||||
with SourcedException {
|
||||
serviceName = "timelineranker"
|
||||
}
|
||||
|
||||
/**
|
||||
* A timeline ranker client whose methods accept and produce model object instances
|
||||
* instead of thrift instances.
|
||||
*/
|
||||
class TimelineRankerClient(
|
||||
private val client: thrift.TimelineRanker.MethodPerEndpoint,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends RequestStats {
|
||||
|
||||
private[this] val baseScope = statsReceiver.scope("timelineRankerClient")
|
||||
private[this] val timelinesRequestStats = RequestStatsReceiver(baseScope.scope("timelines"))
|
||||
private[this] val recycledTweetRequestStats = RequestStatsReceiver(
|
||||
baseScope.scope("recycledTweet"))
|
||||
private[this] val recapHydrationRequestStats = RequestStatsReceiver(
|
||||
baseScope.scope("recapHydration"))
|
||||
private[this] val recapAuthorRequestStats = RequestStatsReceiver(baseScope.scope("recapAuthor"))
|
||||
private[this] val entityTweetsRequestStats = RequestStatsReceiver(baseScope.scope("entityTweets"))
|
||||
private[this] val utegLikedByTweetsRequestStats = RequestStatsReceiver(
|
||||
baseScope.scope("utegLikedByTweets"))
|
||||
|
||||
private[this] def fetchRecapQueryResultHead(
|
||||
results: Seq[Try[CandidateTweetsResult]]
|
||||
): CandidateTweetsResult = {
|
||||
results.head match {
|
||||
case Return(result) => result
|
||||
case Throw(e) => throw e
|
||||
}
|
||||
}
|
||||
|
||||
private[this] def tryResults[Req, Rep](
|
||||
reqs: Seq[Req],
|
||||
stats: RequestStatsReceiver,
|
||||
findError: Req => Option[thrift.TimelineError],
|
||||
)(
|
||||
getRep: (Req, RequestStatsReceiver) => Try[Rep]
|
||||
): Seq[Try[Rep]] = {
|
||||
reqs.map { req =>
|
||||
findError(req) match {
|
||||
case Some(error) if error.reason.exists { _ == thrift.ErrorReason.OverCapacity } =>
|
||||
// bubble up over capacity error, server shall handle it
|
||||
stats.onFailure(error)
|
||||
Throw(error)
|
||||
case Some(error) =>
|
||||
stats.onFailure(error)
|
||||
Throw(TimelineRankerException(error.message))
|
||||
case None =>
|
||||
getRep(req, stats)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private[this] def tryCandidateTweetsResults(
|
||||
responses: Seq[thrift.GetCandidateTweetsResponse],
|
||||
requestScopedStats: RequestStatsReceiver
|
||||
): Seq[Try[CandidateTweetsResult]] = {
|
||||
def errorInResponse(
|
||||
response: thrift.GetCandidateTweetsResponse
|
||||
): Option[thrift.TimelineError] = {
|
||||
response.error
|
||||
}
|
||||
|
||||
tryResults(
|
||||
responses,
|
||||
requestScopedStats,
|
||||
errorInResponse
|
||||
) { (response, stats) =>
|
||||
stats.onSuccess()
|
||||
Return(CandidateTweetsResult.fromThrift(response))
|
||||
}
|
||||
}
|
||||
|
||||
def getTimeline(query: TimelineQuery): Future[Try[Timeline]] = {
|
||||
getTimelines(Seq(query)).map(_.head)
|
||||
}
|
||||
|
||||
def getTimelines(queries: Seq[TimelineQuery]): Future[Seq[Try[Timeline]]] = {
|
||||
def errorInResponse(response: thrift.GetTimelineResponse): Option[thrift.TimelineError] = {
|
||||
response.error
|
||||
}
|
||||
val thriftQueries = queries.map(_.toThrift)
|
||||
timelinesRequestStats.latency {
|
||||
client.getTimelines(thriftQueries).map { responses =>
|
||||
tryResults(
|
||||
responses,
|
||||
timelinesRequestStats,
|
||||
errorInResponse
|
||||
) { (response, stats) =>
|
||||
response.timeline match {
|
||||
case Some(timeline) =>
|
||||
stats.onSuccess()
|
||||
Return(Timeline.fromThrift(timeline))
|
||||
// Should not really happen.
|
||||
case None =>
|
||||
val tlrException =
|
||||
TimelineRankerException("No timeline returned even when no error occurred.")
|
||||
stats.onFailure(tlrException)
|
||||
Throw(tlrException)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getRecycledTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
getRecycledTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
|
||||
}
|
||||
|
||||
def getRecycledTweetCandidates(
|
||||
queries: Seq[RecapQuery]
|
||||
): Future[Seq[Try[CandidateTweetsResult]]] = {
|
||||
val thriftQueries = queries.map(_.toThriftRecapQuery)
|
||||
recycledTweetRequestStats.latency {
|
||||
client.getRecycledTweetCandidates(thriftQueries).map {
|
||||
tryCandidateTweetsResults(_, recycledTweetRequestStats)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def hydrateTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
hydrateTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
|
||||
}
|
||||
|
||||
def hydrateTweetCandidates(queries: Seq[RecapQuery]): Future[Seq[Try[CandidateTweetsResult]]] = {
|
||||
val thriftQueries = queries.map(_.toThriftRecapHydrationQuery)
|
||||
recapHydrationRequestStats.latency {
|
||||
client.hydrateTweetCandidates(thriftQueries).map {
|
||||
tryCandidateTweetsResults(_, recapHydrationRequestStats)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getRecapCandidatesFromAuthors(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
getRecapCandidatesFromAuthors(Seq(query)).map(fetchRecapQueryResultHead)
|
||||
}
|
||||
|
||||
def getRecapCandidatesFromAuthors(
|
||||
queries: Seq[RecapQuery]
|
||||
): Future[Seq[Try[CandidateTweetsResult]]] = {
|
||||
val thriftQueries = queries.map(_.toThriftRecapQuery)
|
||||
recapAuthorRequestStats.latency {
|
||||
client.getRecapCandidatesFromAuthors(thriftQueries).map {
|
||||
tryCandidateTweetsResults(_, recapAuthorRequestStats)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getEntityTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
getEntityTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
|
||||
}
|
||||
|
||||
def getEntityTweetCandidates(
|
||||
queries: Seq[RecapQuery]
|
||||
): Future[Seq[Try[CandidateTweetsResult]]] = {
|
||||
val thriftQueries = queries.map(_.toThriftEntityTweetsQuery)
|
||||
entityTweetsRequestStats.latency {
|
||||
client.getEntityTweetCandidates(thriftQueries).map {
|
||||
tryCandidateTweetsResults(_, entityTweetsRequestStats)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getUtegLikedByTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
getUtegLikedByTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
|
||||
}
|
||||
|
||||
def getUtegLikedByTweetCandidates(
|
||||
queries: Seq[RecapQuery]
|
||||
): Future[Seq[Try[CandidateTweetsResult]]] = {
|
||||
val thriftQueries = queries.map(_.toThriftUtegLikedByTweetsQuery)
|
||||
utegLikedByTweetsRequestStats.latency {
|
||||
client.getUtegLikedByTweetCandidates(thriftQueries).map {
|
||||
tryCandidateTweetsResults(_, utegLikedByTweetsRequestStats)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
package com.twitter.timelineranker.client
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.builder.ClientBuilder
|
||||
import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.mtls.client.MtlsClientBuilder._
|
||||
import com.twitter.finagle.param.OppTls
|
||||
import com.twitter.finagle.service.RetryPolicy
|
||||
import com.twitter.finagle.service.RetryPolicy._
|
||||
import com.twitter.finagle.ssl.OpportunisticTls
|
||||
import com.twitter.finagle.thrift.ThriftClientRequest
|
||||
import com.twitter.servo.client.Environment.Local
|
||||
import com.twitter.servo.client.Environment.Staging
|
||||
import com.twitter.servo.client.Environment.Production
|
||||
import com.twitter.servo.client.Environment
|
||||
import com.twitter.servo.client.FinagleClientBuilder
|
||||
import com.twitter.util.Try
|
||||
import com.twitter.util.Duration
|
||||
|
||||
sealed trait TimelineRankerClientBuilderBase {
|
||||
def DefaultName: String = "timelineranker"
|
||||
|
||||
def DefaultProdDest: String
|
||||
|
||||
def DefaultProdRequestTimeout: Duration = 2.seconds
|
||||
def DefaultProdTimeout: Duration = 3.seconds
|
||||
def DefaultProdRetryPolicy: RetryPolicy[Try[Nothing]] =
|
||||
tries(2, TimeoutAndWriteExceptionsOnly orElse ChannelClosedExceptionsOnly)
|
||||
|
||||
def DefaultLocalTcpConnectTimeout: Duration = 1.second
|
||||
def DefaultLocalConnectTimeout: Duration = 1.second
|
||||
def DefaultLocalRetryPolicy: RetryPolicy[Try[Nothing]] = tries(2, TimeoutAndWriteExceptionsOnly)
|
||||
|
||||
def apply(
|
||||
finagleClientBuilder: FinagleClientBuilder,
|
||||
environment: Environment,
|
||||
name: String = DefaultName,
|
||||
serviceIdentifier: ServiceIdentifier = EmptyServiceIdentifier,
|
||||
opportunisticTlsOpt: Option[OpportunisticTls.Level] = None,
|
||||
): ClientBuilder.Complete[ThriftClientRequest, Array[Byte]] = {
|
||||
val defaultBuilder = finagleClientBuilder.thriftMuxClientBuilder(name)
|
||||
val destination = getDestOverride(environment)
|
||||
|
||||
val partialClient = environment match {
|
||||
case Production | Staging =>
|
||||
defaultBuilder
|
||||
.requestTimeout(DefaultProdRequestTimeout)
|
||||
.timeout(DefaultProdTimeout)
|
||||
.retryPolicy(DefaultProdRetryPolicy)
|
||||
.daemon(daemonize = true)
|
||||
.dest(destination)
|
||||
.mutualTls(serviceIdentifier)
|
||||
case Local =>
|
||||
defaultBuilder
|
||||
.tcpConnectTimeout(DefaultLocalTcpConnectTimeout)
|
||||
.connectTimeout(DefaultLocalConnectTimeout)
|
||||
.retryPolicy(DefaultLocalRetryPolicy)
|
||||
.failFast(enabled = false)
|
||||
.daemon(daemonize = false)
|
||||
.dest(destination)
|
||||
.mutualTls(serviceIdentifier)
|
||||
}
|
||||
|
||||
opportunisticTlsOpt match {
|
||||
case Some(_) =>
|
||||
val opportunisticTlsParam = OppTls(level = opportunisticTlsOpt)
|
||||
partialClient
|
||||
.configured(opportunisticTlsParam)
|
||||
case None => partialClient
|
||||
}
|
||||
}
|
||||
|
||||
private def getDestOverride(environment: Environment): String = {
|
||||
val defaultDest = DefaultProdDest
|
||||
environment match {
|
||||
// Allow overriding the target TimelineRanker instance in staging.
|
||||
// This is typically useful for redline testing of TimelineRanker.
|
||||
case Staging =>
|
||||
sys.props.getOrElse("target.timelineranker.instance", defaultDest)
|
||||
case _ =>
|
||||
defaultDest
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object TimelineRankerClientBuilder extends TimelineRankerClientBuilderBase {
|
||||
override def DefaultProdDest: String = "/s/timelineranker/timelineranker"
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
target(
|
||||
name = "adapter",
|
||||
dependencies = ["timelineranker/common/src/main/scala/com/twitter/timelineranker/adapter"],
|
||||
)
|
||||
|
||||
target(
|
||||
name = "model",
|
||||
dependencies = ["timelineranker/common/src/main/scala/com/twitter/timelineranker/model"],
|
||||
)
|
||||
|
||||
target(
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":adapter",
|
||||
":model",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
target(
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core",
|
||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"timelineranker/common:model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clientconfig",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,139 +0,0 @@
|
|||
package com.twitter.timelineranker.adapter
|
||||
|
||||
import com.twitter.timelineranker.model._
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelineservice.model.TimelineId
|
||||
import com.twitter.timelineservice.model.core
|
||||
import com.twitter.timelineservice.{model => tls}
|
||||
import com.twitter.timelineservice.{thriftscala => tlsthrift}
|
||||
import com.twitter.timelineservice.model.core._
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Throw
|
||||
import com.twitter.util.Try
|
||||
|
||||
/**
|
||||
* Enables TLR model objects to be converted to/from TLS model/thrift objects.
|
||||
*/
|
||||
object TimelineServiceAdapter {
|
||||
def toTlrQuery(
|
||||
id: Long,
|
||||
tlsRange: tls.TimelineRange,
|
||||
getTweetsFromArchiveIndex: Boolean = true
|
||||
): ReverseChronTimelineQuery = {
|
||||
val timelineId = TimelineId(id, TimelineKind.home)
|
||||
val maxCount = tlsRange.maxCount
|
||||
val tweetIdRange = tlsRange.cursor.map { cursor =>
|
||||
TweetIdRange(
|
||||
fromId = cursor.tweetIdBounds.bottom,
|
||||
toId = cursor.tweetIdBounds.top
|
||||
)
|
||||
}
|
||||
val options = ReverseChronTimelineQueryOptions(
|
||||
getTweetsFromArchiveIndex = getTweetsFromArchiveIndex
|
||||
)
|
||||
ReverseChronTimelineQuery(timelineId, Some(maxCount), tweetIdRange, Some(options))
|
||||
}
|
||||
|
||||
def toTlsQuery(query: ReverseChronTimelineQuery): tls.TimelineQuery = {
|
||||
val tlsRange = toTlsRange(query.range, query.maxCount)
|
||||
tls.TimelineQuery(
|
||||
id = query.id.id,
|
||||
kind = query.id.kind,
|
||||
range = tlsRange
|
||||
)
|
||||
}
|
||||
|
||||
def toTlsRange(range: Option[TimelineRange], maxCount: Option[Int]): tls.TimelineRange = {
|
||||
val cursor = range.map {
|
||||
case tweetIdRange: TweetIdRange =>
|
||||
RequestCursor(
|
||||
top = tweetIdRange.toId.map(CursorState.fromTweetId),
|
||||
bottom = tweetIdRange.fromId.map(core.CursorState.fromTweetId)
|
||||
)
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(s"Only TweetIdRange is supported. Found: $range")
|
||||
}
|
||||
maxCount
|
||||
.map { count => tls.TimelineRange(cursor, count) }
|
||||
.getOrElse(tls.TimelineRange(cursor))
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts TLS timeline to a Try of TLR timeline.
|
||||
*
|
||||
* TLS timeline not only contains timeline entries/attributes but also the retrieval state;
|
||||
* whereas TLR timeline only has entries/attributes. Therefore, the TLS timeline is
|
||||
* mapped to a Try[Timeline] where the Try part captures retrieval state and
|
||||
* Timeline captures entries/attributes.
|
||||
*/
|
||||
def toTlrTimelineTry(tlsTimeline: tls.Timeline[tls.TimelineEntry]): Try[Timeline] = {
|
||||
require(
|
||||
tlsTimeline.kind == TimelineKind.home,
|
||||
s"Only home timelines are supported. Found: ${tlsTimeline.kind}"
|
||||
)
|
||||
|
||||
tlsTimeline.state match {
|
||||
case Some(TimelineHit) | None =>
|
||||
val tweetEnvelopes = tlsTimeline.entries.map {
|
||||
case tweet: tls.Tweet =>
|
||||
TimelineEntryEnvelope(Tweet(tweet.tweetId))
|
||||
case entry =>
|
||||
throw new Exception(s"Only tweet timelines are supported. Found: $entry")
|
||||
}
|
||||
Return(Timeline(TimelineId(tlsTimeline.id, tlsTimeline.kind), tweetEnvelopes))
|
||||
case Some(TimelineNotFound) | Some(TimelineUnavailable) =>
|
||||
Throw(new tls.core.TimelineUnavailableException(tlsTimeline.id, Some(tlsTimeline.kind)))
|
||||
}
|
||||
}
|
||||
|
||||
def toTlsTimeline(timeline: Timeline): tls.Timeline[tls.Tweet] = {
|
||||
val entries = timeline.entries.map { entry =>
|
||||
entry.entry match {
|
||||
case tweet: Tweet => tls.Tweet(tweet.id)
|
||||
case entry: HydratedTweetEntry => tls.Tweet.fromThrift(entry.tweet)
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Only tweet timelines are supported. Found: ${entry.entry}"
|
||||
)
|
||||
}
|
||||
}
|
||||
tls.Timeline(
|
||||
id = timeline.id.id,
|
||||
kind = timeline.id.kind,
|
||||
entries = entries
|
||||
)
|
||||
}
|
||||
|
||||
def toTweetIds(timeline: tlsthrift.Timeline): Seq[TweetId] = {
|
||||
timeline.entries.map {
|
||||
case tlsthrift.TimelineEntry.Tweet(tweet) =>
|
||||
tweet.statusId
|
||||
case entry =>
|
||||
throw new IllegalArgumentException(s"Only tweet timelines are supported. Found: ${entry}")
|
||||
}
|
||||
}
|
||||
|
||||
def toTweetIds(timeline: Timeline): Seq[TweetId] = {
|
||||
timeline.entries.map { entry =>
|
||||
entry.entry match {
|
||||
case tweet: Tweet => tweet.id
|
||||
case entry: HydratedTweetEntry => entry.tweet.id
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Only tweet timelines are supported. Found: ${entry.entry}"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def toHydratedTweets(timeline: Timeline): Seq[HydratedTweet] = {
|
||||
timeline.entries.map { entry =>
|
||||
entry.entry match {
|
||||
case hydratedTweet: HydratedTweet => hydratedTweet
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(s"Expected hydrated tweet. Found: ${entry.entry}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"src/java/com/twitter/common/text/language:locale-util",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"src/thrift/com/twitter/search/common:features-scala",
|
||||
"src/thrift/com/twitter/timelineranker/server/model:thrift-scala",
|
||||
"timelines:config-api-base",
|
||||
"timelines/src/main/scala/com/twitter/timelines/common/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/candidate",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||
],
|
||||
exports = [
|
||||
"timelines:config-api-base",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.tweetypie.thriftscala
|
||||
|
||||
object CandidateTweet {
|
||||
val DefaultFeatures: ThriftTweetFeatures = ThriftTweetFeatures()
|
||||
|
||||
def fromThrift(candidate: thrift.CandidateTweet): CandidateTweet = {
|
||||
val tweet: thriftscala.Tweet = candidate.tweet.getOrElse(
|
||||
throw new IllegalArgumentException(s"CandidateTweet.tweet must have a value")
|
||||
)
|
||||
val features = candidate.features.getOrElse(
|
||||
throw new IllegalArgumentException(s"CandidateTweet.features must have a value")
|
||||
)
|
||||
|
||||
CandidateTweet(HydratedTweet(tweet), features)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A candidate Tweet and associated information.
|
||||
* Model object for CandidateTweet thrift struct.
|
||||
*/
|
||||
case class CandidateTweet(hydratedTweet: HydratedTweet, features: ThriftTweetFeatures) {
|
||||
|
||||
def toThrift: thrift.CandidateTweet = {
|
||||
thrift.CandidateTweet(
|
||||
tweet = Some(hydratedTweet.tweet),
|
||||
features = Some(features)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.util.Future
|
||||
|
||||
object CandidateTweetsResult {
|
||||
val Empty: CandidateTweetsResult = CandidateTweetsResult(Nil, Nil)
|
||||
val EmptyFuture: Future[CandidateTweetsResult] = Future.value(Empty)
|
||||
val EmptyCandidateTweet: Seq[CandidateTweet] = Seq.empty[CandidateTweet]
|
||||
|
||||
def fromThrift(response: thrift.GetCandidateTweetsResponse): CandidateTweetsResult = {
|
||||
val candidates = response.candidates
|
||||
.map(_.map(CandidateTweet.fromThrift))
|
||||
.getOrElse(EmptyCandidateTweet)
|
||||
val sourceTweets = response.sourceTweets
|
||||
.map(_.map(CandidateTweet.fromThrift))
|
||||
.getOrElse(EmptyCandidateTweet)
|
||||
if (sourceTweets.nonEmpty) {
|
||||
require(candidates.nonEmpty, "sourceTweets cannot have a value if candidates list is empty.")
|
||||
}
|
||||
CandidateTweetsResult(candidates, sourceTweets)
|
||||
}
|
||||
}
|
||||
|
||||
case class CandidateTweetsResult(
|
||||
candidates: Seq[CandidateTweet],
|
||||
sourceTweets: Seq[CandidateTweet]) {
|
||||
|
||||
def toThrift: thrift.GetCandidateTweetsResponse = {
|
||||
val thriftCandidates = candidates.map(_.toThrift)
|
||||
val thriftSourceTweets = sourceTweets.map(_.toThrift)
|
||||
thrift.GetCandidateTweetsResponse(
|
||||
candidates = Some(thriftCandidates),
|
||||
sourceTweets = Some(thriftSourceTweets)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.tweetypie.{thriftscala => tweetypie}
|
||||
|
||||
/**
|
||||
* Enables HydratedTweet entries to be included in a Timeline.
|
||||
*/
|
||||
class HydratedTweetEntry(tweet: tweetypie.Tweet) extends HydratedTweet(tweet) with TimelineEntry {
|
||||
|
||||
def this(hydratedTweet: HydratedTweet) = this(hydratedTweet.tweet)
|
||||
|
||||
override def toTimelineEntryThrift: thrift.TimelineEntry = {
|
||||
thrift.TimelineEntry.TweetypieTweet(tweet)
|
||||
}
|
||||
|
||||
override def throwIfInvalid(): Unit = {
|
||||
// No validation performed.
|
||||
}
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.common.text.language.LocaleUtil
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
object Language {
|
||||
|
||||
def fromThrift(lang: thrift.Language): Language = {
|
||||
require(lang.language.isDefined, "language can't be None")
|
||||
require(lang.scope.isDefined, "scope can't be None")
|
||||
Language(lang.language.get, LanguageScope.fromThrift(lang.scope.get))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a language and the scope that it relates to.
|
||||
*/
|
||||
case class Language(language: String, scope: LanguageScope.Value) {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def toThrift: thrift.Language = {
|
||||
val scopeOption = Some(LanguageScope.toThrift(scope))
|
||||
thrift.Language(Some(language), scopeOption)
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
val result = LocaleUtil.getLocaleOf(language)
|
||||
require(result != LocaleUtil.UNKNOWN, s"Language ${language} is unsupported")
|
||||
}
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
/**
|
||||
* Represents what this language is associated with.
|
||||
* For example, "user" is one of the scopes and "event"
|
||||
* could be another scope.
|
||||
*/
|
||||
object LanguageScope extends Enumeration {
|
||||
|
||||
// User scope means that the language is the user's language.
|
||||
val User: Value = Value(thrift.LanguageScope.User.value)
|
||||
|
||||
// Event scope means that the language is the event's language.
|
||||
val Event: Value = Value(thrift.LanguageScope.Event.value)
|
||||
|
||||
// list of all LanguageScope values
|
||||
val All: ValueSet = LanguageScope.ValueSet(User, Event)
|
||||
|
||||
def apply(scope: thrift.LanguageScope): LanguageScope.Value = {
|
||||
scope match {
|
||||
case thrift.LanguageScope.User =>
|
||||
User
|
||||
case thrift.LanguageScope.Event =>
|
||||
Event
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(s"Unsupported language scope: $scope")
|
||||
}
|
||||
}
|
||||
|
||||
def fromThrift(scope: thrift.LanguageScope): LanguageScope.Value = {
|
||||
apply(scope)
|
||||
}
|
||||
|
||||
def toThrift(scope: LanguageScope.Value): thrift.LanguageScope = {
|
||||
scope match {
|
||||
case LanguageScope.User =>
|
||||
thrift.LanguageScope.User
|
||||
case LanguageScope.Event =>
|
||||
thrift.LanguageScope.Event
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(s"Unsupported language scope: $scope")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,184 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.timelines.util.SnowflakeSortIndexHelper
|
||||
import com.twitter.tweetypie.{thriftscala => tweetypie}
|
||||
|
||||
object PartiallyHydratedTweet {
|
||||
private val InvalidValue = "Invalid value"
|
||||
|
||||
/**
|
||||
* Creates an instance of PartiallyHydratedTweet based on the given search result.
|
||||
*/
|
||||
def fromSearchResult(result: ThriftSearchResult): PartiallyHydratedTweet = {
|
||||
val tweetId = result.id
|
||||
val metadata = result.metadata.getOrElse(
|
||||
throw new IllegalArgumentException(
|
||||
s"cannot initialize PartiallyHydratedTweet $tweetId without ThriftSearchResult metadata."
|
||||
)
|
||||
)
|
||||
|
||||
val extraMetadataOpt = metadata.extraMetadata
|
||||
|
||||
val userId = metadata.fromUserId
|
||||
|
||||
// The value of referencedTweetAuthorId and sharedStatusId is only considered valid if it is greater than 0.
|
||||
val referencedTweetAuthorId =
|
||||
if (metadata.referencedTweetAuthorId > 0) Some(metadata.referencedTweetAuthorId) else None
|
||||
val sharedStatusId = if (metadata.sharedStatusId > 0) Some(metadata.sharedStatusId) else None
|
||||
|
||||
val isRetweet = metadata.isRetweet.getOrElse(false)
|
||||
val retweetSourceTweetId = if (isRetweet) sharedStatusId else None
|
||||
val retweetSourceUserId = if (isRetweet) referencedTweetAuthorId else None
|
||||
|
||||
// The fields sharedStatusId and referencedTweetAuthorId have overloaded meaning when
|
||||
// this tweet is not a retweet (for retweet, there is only 1 meaning).
|
||||
// When not a retweet,
|
||||
// if referencedTweetAuthorId and sharedStatusId are both set, it is considered a reply
|
||||
// if referencedTweetAuthorId is set and sharedStatusId is not set, it is a directed at tweet.
|
||||
// References: SEARCH-8561 and SEARCH-13142
|
||||
val inReplyToTweetId = if (!isRetweet) sharedStatusId else None
|
||||
val inReplyToUserId = if (!isRetweet) referencedTweetAuthorId else None
|
||||
val isReply = metadata.isReply.contains(true)
|
||||
|
||||
val quotedTweetId = extraMetadataOpt.flatMap(_.quotedTweetId)
|
||||
val quotedUserId = extraMetadataOpt.flatMap(_.quotedUserId)
|
||||
|
||||
val isNullcast = metadata.isNullcast.contains(true)
|
||||
|
||||
val conversationId = extraMetadataOpt.flatMap(_.conversationId)
|
||||
|
||||
// Root author id for the user who posts an exclusive tweet
|
||||
val exclusiveConversationAuthorId = extraMetadataOpt.flatMap(_.exclusiveConversationAuthorId)
|
||||
|
||||
// Card URI associated with an attached card to this tweet, if it contains one
|
||||
val cardUri = extraMetadataOpt.flatMap(_.cardUri)
|
||||
|
||||
val tweet = makeTweetyPieTweet(
|
||||
tweetId,
|
||||
userId,
|
||||
inReplyToTweetId,
|
||||
inReplyToUserId,
|
||||
retweetSourceTweetId,
|
||||
retweetSourceUserId,
|
||||
quotedTweetId,
|
||||
quotedUserId,
|
||||
isNullcast,
|
||||
isReply,
|
||||
conversationId,
|
||||
exclusiveConversationAuthorId,
|
||||
cardUri
|
||||
)
|
||||
new PartiallyHydratedTweet(tweet)
|
||||
}
|
||||
|
||||
def makeTweetyPieTweet(
|
||||
tweetId: TweetId,
|
||||
userId: UserId,
|
||||
inReplyToTweetId: Option[TweetId],
|
||||
inReplyToUserId: Option[TweetId],
|
||||
retweetSourceTweetId: Option[TweetId],
|
||||
retweetSourceUserId: Option[UserId],
|
||||
quotedTweetId: Option[TweetId],
|
||||
quotedUserId: Option[UserId],
|
||||
isNullcast: Boolean,
|
||||
isReply: Boolean,
|
||||
conversationId: Option[Long],
|
||||
exclusiveConversationAuthorId: Option[Long] = None,
|
||||
cardUri: Option[String] = None
|
||||
): tweetypie.Tweet = {
|
||||
val isDirectedAt = inReplyToUserId.isDefined
|
||||
val isRetweet = retweetSourceTweetId.isDefined && retweetSourceUserId.isDefined
|
||||
|
||||
val reply = if (isReply) {
|
||||
Some(
|
||||
tweetypie.Reply(
|
||||
inReplyToStatusId = inReplyToTweetId,
|
||||
inReplyToUserId = inReplyToUserId.getOrElse(0L) // Required
|
||||
)
|
||||
)
|
||||
} else None
|
||||
|
||||
val directedAt = if (isDirectedAt) {
|
||||
Some(
|
||||
tweetypie.DirectedAtUser(
|
||||
userId = inReplyToUserId.get,
|
||||
screenName = "" // not available from search
|
||||
)
|
||||
)
|
||||
} else None
|
||||
|
||||
val share = if (isRetweet) {
|
||||
Some(
|
||||
tweetypie.Share(
|
||||
sourceStatusId = retweetSourceTweetId.get,
|
||||
sourceUserId = retweetSourceUserId.get,
|
||||
parentStatusId =
|
||||
retweetSourceTweetId.get // Not always correct (eg, retweet of a retweet).
|
||||
)
|
||||
)
|
||||
} else None
|
||||
|
||||
val quotedTweet =
|
||||
for {
|
||||
tweetId <- quotedTweetId
|
||||
userId <- quotedUserId
|
||||
} yield tweetypie.QuotedTweet(tweetId = tweetId, userId = userId)
|
||||
|
||||
val coreData = tweetypie.TweetCoreData(
|
||||
userId = userId,
|
||||
text = InvalidValue,
|
||||
createdVia = InvalidValue,
|
||||
createdAtSecs = SnowflakeSortIndexHelper.idToTimestamp(tweetId).inSeconds,
|
||||
directedAtUser = directedAt,
|
||||
reply = reply,
|
||||
share = share,
|
||||
nullcast = isNullcast,
|
||||
conversationId = conversationId
|
||||
)
|
||||
|
||||
// Hydrate exclusiveTweetControl which determines whether the user is able to view an exclusive / SuperFollow tweet.
|
||||
val exclusiveTweetControl = exclusiveConversationAuthorId.map { authorId =>
|
||||
tweetypie.ExclusiveTweetControl(conversationAuthorId = authorId)
|
||||
}
|
||||
|
||||
val cardReference = cardUri.map { cardUriFromEB =>
|
||||
tweetypie.CardReference(cardUri = cardUriFromEB)
|
||||
}
|
||||
|
||||
tweetypie.Tweet(
|
||||
id = tweetId,
|
||||
quotedTweet = quotedTweet,
|
||||
coreData = Some(coreData),
|
||||
exclusiveTweetControl = exclusiveTweetControl,
|
||||
cardReference = cardReference
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an instance of HydratedTweet that is hydrated using search result
|
||||
* (instead of being hydrated using TweetyPie service).
|
||||
*
|
||||
* Not all fields are available using search therefore such fields if accessed
|
||||
* throw UnsupportedOperationException to ensure that they are not inadvertently
|
||||
* accessed and relied upon.
|
||||
*/
|
||||
class PartiallyHydratedTweet(tweet: tweetypie.Tweet) extends HydratedTweet(tweet) {
|
||||
override def parentTweetId: Option[TweetId] = throw notSupported("parentTweetId")
|
||||
override def mentionedUserIds: Seq[UserId] = throw notSupported("mentionedUserIds")
|
||||
override def takedownCountryCodes: Set[String] = throw notSupported("takedownCountryCodes")
|
||||
override def hasMedia: Boolean = throw notSupported("hasMedia")
|
||||
override def isNarrowcast: Boolean = throw notSupported("isNarrowcast")
|
||||
override def hasTakedown: Boolean = throw notSupported("hasTakedown")
|
||||
override def isNsfw: Boolean = throw notSupported("isNsfw")
|
||||
override def isNsfwUser: Boolean = throw notSupported("isNsfwUser")
|
||||
override def isNsfwAdmin: Boolean = throw notSupported("isNsfwAdmin")
|
||||
|
||||
private def notSupported(name: String): UnsupportedOperationException = {
|
||||
new UnsupportedOperationException(s"Not supported: $name")
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.TweetId
|
||||
|
||||
object PriorSeenEntries {
|
||||
def fromThrift(entries: thrift.PriorSeenEntries): PriorSeenEntries = {
|
||||
PriorSeenEntries(seenEntries = entries.seenEntries)
|
||||
}
|
||||
}
|
||||
|
||||
case class PriorSeenEntries(seenEntries: Seq[TweetId]) {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def toThrift: thrift.PriorSeenEntries = {
|
||||
thrift.PriorSeenEntries(seenEntries = seenEntries)
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
// No validation performed.
|
||||
}
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelineservice.model.TimelineId
|
||||
|
||||
case class RankedTimelineQuery(
|
||||
override val id: TimelineId,
|
||||
override val maxCount: Option[Int] = None,
|
||||
override val range: Option[TimelineRange] = None,
|
||||
override val options: Option[RankedTimelineQueryOptions] = None)
|
||||
extends TimelineQuery(thrift.TimelineQueryType.Ranked, id, maxCount, range, options) {
|
||||
|
||||
throwIfInvalid()
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
object RankedTimelineQueryOptions {
|
||||
def fromThrift(options: thrift.RankedTimelineQueryOptions): RankedTimelineQueryOptions = {
|
||||
RankedTimelineQueryOptions(
|
||||
seenEntries = options.seenEntries.map(PriorSeenEntries.fromThrift)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
case class RankedTimelineQueryOptions(seenEntries: Option[PriorSeenEntries])
|
||||
extends TimelineQueryOptions {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def toThrift: thrift.RankedTimelineQueryOptions = {
|
||||
thrift.RankedTimelineQueryOptions(seenEntries = seenEntries.map(_.toThrift))
|
||||
}
|
||||
|
||||
def toTimelineQueryOptionsThrift: thrift.TimelineQueryOptions = {
|
||||
thrift.TimelineQueryOptions.RankedTimelineQueryOptions(toThrift)
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
seenEntries.foreach(_.throwIfInvalid)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,278 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelines.model.candidate.CandidateTweetSourceId
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.common.model._
|
||||
import com.twitter.timelines.earlybird.common.options.EarlybirdOptions
|
||||
import com.twitter.timelines.earlybird.common.utils.SearchOperator
|
||||
import com.twitter.timelines.configapi.{
|
||||
DependencyProvider => ConfigApiDependencyProvider,
|
||||
FutureDependencyProvider => ConfigApiFutureDependencyProvider,
|
||||
_
|
||||
}
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.timelineservice.DeviceContext
|
||||
|
||||
object RecapQuery {
|
||||
|
||||
val EngagedTweetsSupportedTweetKindOption: TweetKindOption.ValueSet = TweetKindOption(
|
||||
includeReplies = false,
|
||||
includeRetweets = false,
|
||||
includeExtendedReplies = false,
|
||||
includeOriginalTweetsAndQuotes = true
|
||||
)
|
||||
|
||||
val DefaultSearchOperator: SearchOperator.Value = SearchOperator.Exclude
|
||||
def fromThrift(query: thrift.RecapQuery): RecapQuery = {
|
||||
|
||||
RecapQuery(
|
||||
userId = query.userId,
|
||||
maxCount = query.maxCount,
|
||||
range = query.range.map(TimelineRange.fromThrift),
|
||||
options = query.options
|
||||
.map(options => TweetKindOption.fromThrift(options.to[Set]))
|
||||
.getOrElse(TweetKindOption.None),
|
||||
searchOperator = query.searchOperator
|
||||
.map(SearchOperator.fromThrift)
|
||||
.getOrElse(DefaultSearchOperator),
|
||||
earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift),
|
||||
deviceContext = query.deviceContext.map(DeviceContext.fromThrift),
|
||||
authorIds = query.authorIds,
|
||||
excludedTweetIds = query.excludedTweetIds,
|
||||
searchClientSubId = query.searchClientSubId,
|
||||
candidateTweetSourceId =
|
||||
query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift),
|
||||
hydratesContentFeatures = query.hydratesContentFeatures
|
||||
)
|
||||
}
|
||||
|
||||
def fromThrift(query: thrift.RecapHydrationQuery): RecapQuery = {
|
||||
require(query.tweetIds.nonEmpty, "tweetIds must be non-empty")
|
||||
|
||||
RecapQuery(
|
||||
userId = query.userId,
|
||||
tweetIds = Some(query.tweetIds),
|
||||
searchOperator = DefaultSearchOperator,
|
||||
earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift),
|
||||
deviceContext = query.deviceContext.map(DeviceContext.fromThrift),
|
||||
candidateTweetSourceId =
|
||||
query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift),
|
||||
hydratesContentFeatures = query.hydratesContentFeatures
|
||||
)
|
||||
}
|
||||
|
||||
def fromThrift(query: thrift.EngagedTweetsQuery): RecapQuery = {
|
||||
val options = query.tweetKindOptions
|
||||
.map(tweetKindOptions => TweetKindOption.fromThrift(tweetKindOptions.to[Set]))
|
||||
.getOrElse(TweetKindOption.None)
|
||||
|
||||
if (!(options.isEmpty ||
|
||||
(options == EngagedTweetsSupportedTweetKindOption))) {
|
||||
throw new IllegalArgumentException(s"Unsupported TweetKindOption value: $options")
|
||||
}
|
||||
|
||||
RecapQuery(
|
||||
userId = query.userId,
|
||||
maxCount = query.maxCount,
|
||||
range = query.range.map(TimelineRange.fromThrift),
|
||||
options = options,
|
||||
searchOperator = DefaultSearchOperator,
|
||||
earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift),
|
||||
deviceContext = query.deviceContext.map(DeviceContext.fromThrift),
|
||||
authorIds = query.userIds,
|
||||
excludedTweetIds = query.excludedTweetIds,
|
||||
)
|
||||
}
|
||||
|
||||
def fromThrift(query: thrift.EntityTweetsQuery): RecapQuery = {
|
||||
require(
|
||||
query.semanticCoreIds.isDefined,
|
||||
"entities(semanticCoreIds) can't be None"
|
||||
)
|
||||
val options = query.tweetKindOptions
|
||||
.map(tweetKindOptions => TweetKindOption.fromThrift(tweetKindOptions.to[Set]))
|
||||
.getOrElse(TweetKindOption.None)
|
||||
|
||||
RecapQuery(
|
||||
userId = query.userId,
|
||||
maxCount = query.maxCount,
|
||||
range = query.range.map(TimelineRange.fromThrift),
|
||||
options = options,
|
||||
searchOperator = DefaultSearchOperator,
|
||||
earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift),
|
||||
deviceContext = query.deviceContext.map(DeviceContext.fromThrift),
|
||||
excludedTweetIds = query.excludedTweetIds,
|
||||
semanticCoreIds = query.semanticCoreIds.map(_.map(SemanticCoreAnnotation.fromThrift).toSet),
|
||||
hashtags = query.hashtags.map(_.toSet),
|
||||
languages = query.languages.map(_.map(Language.fromThrift).toSet),
|
||||
candidateTweetSourceId =
|
||||
query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift),
|
||||
includeNullcastTweets = query.includeNullcastTweets,
|
||||
includeTweetsFromArchiveIndex = query.includeTweetsFromArchiveIndex,
|
||||
authorIds = query.authorIds,
|
||||
hydratesContentFeatures = query.hydratesContentFeatures
|
||||
)
|
||||
}
|
||||
|
||||
def fromThrift(query: thrift.UtegLikedByTweetsQuery): RecapQuery = {
|
||||
val options = query.tweetKindOptions
|
||||
.map(tweetKindOptions => TweetKindOption.fromThrift(tweetKindOptions.to[Set]))
|
||||
.getOrElse(TweetKindOption.None)
|
||||
|
||||
RecapQuery(
|
||||
userId = query.userId,
|
||||
maxCount = query.maxCount,
|
||||
range = query.range.map(TimelineRange.fromThrift),
|
||||
options = options,
|
||||
earlybirdOptions = query.earlybirdOptions.map(EarlybirdOptions.fromThrift),
|
||||
deviceContext = query.deviceContext.map(DeviceContext.fromThrift),
|
||||
excludedTweetIds = query.excludedTweetIds,
|
||||
utegLikedByTweetsOptions = for {
|
||||
utegCount <- query.utegCount
|
||||
weightedFollowings <- query.weightedFollowings.map(_.toMap)
|
||||
} yield {
|
||||
UtegLikedByTweetsOptions(
|
||||
utegCount = utegCount,
|
||||
isInNetwork = query.isInNetwork,
|
||||
weightedFollowings = weightedFollowings
|
||||
)
|
||||
},
|
||||
candidateTweetSourceId =
|
||||
query.candidateTweetSourceId.flatMap(CandidateTweetSourceId.fromThrift),
|
||||
hydratesContentFeatures = query.hydratesContentFeatures
|
||||
)
|
||||
}
|
||||
|
||||
val paramGate: (Param[Boolean] => Gate[RecapQuery]) = HasParams.paramGate
|
||||
|
||||
type DependencyProvider[+T] = ConfigApiDependencyProvider[RecapQuery, T]
|
||||
object DependencyProvider extends DependencyProviderFunctions[RecapQuery]
|
||||
|
||||
type FutureDependencyProvider[+T] = ConfigApiFutureDependencyProvider[RecapQuery, T]
|
||||
object FutureDependencyProvider extends FutureDependencyProviderFunctions[RecapQuery]
|
||||
}
|
||||
|
||||
/**
|
||||
* Model object corresponding to RecapQuery thrift struct.
|
||||
*/
|
||||
case class RecapQuery(
|
||||
userId: UserId,
|
||||
maxCount: Option[Int] = None,
|
||||
range: Option[TimelineRange] = None,
|
||||
options: TweetKindOption.ValueSet = TweetKindOption.None,
|
||||
searchOperator: SearchOperator.Value = RecapQuery.DefaultSearchOperator,
|
||||
earlybirdOptions: Option[EarlybirdOptions] = None,
|
||||
deviceContext: Option[DeviceContext] = None,
|
||||
authorIds: Option[Seq[UserId]] = None,
|
||||
tweetIds: Option[Seq[TweetId]] = None,
|
||||
semanticCoreIds: Option[Set[SemanticCoreAnnotation]] = None,
|
||||
hashtags: Option[Set[String]] = None,
|
||||
languages: Option[Set[Language]] = None,
|
||||
excludedTweetIds: Option[Seq[TweetId]] = None,
|
||||
// options used only for yml tweets
|
||||
utegLikedByTweetsOptions: Option[UtegLikedByTweetsOptions] = None,
|
||||
searchClientSubId: Option[String] = None,
|
||||
override val params: Params = Params.Empty,
|
||||
candidateTweetSourceId: Option[CandidateTweetSourceId.Value] = None,
|
||||
includeNullcastTweets: Option[Boolean] = None,
|
||||
includeTweetsFromArchiveIndex: Option[Boolean] = None,
|
||||
hydratesContentFeatures: Option[Boolean] = None)
|
||||
extends HasParams {
|
||||
|
||||
override def toString: String = {
|
||||
s"RecapQuery(userId: $userId, maxCount: $maxCount, range: $range, options: $options, searchOperator: $searchOperator, " +
|
||||
s"earlybirdOptions: $earlybirdOptions, deviceContext: $deviceContext, authorIds: $authorIds, " +
|
||||
s"tweetIds: $tweetIds, semanticCoreIds: $semanticCoreIds, hashtags: $hashtags, languages: $languages, excludedTweetIds: $excludedTweetIds, " +
|
||||
s"utegLikedByTweetsOptions: $utegLikedByTweetsOptions, searchClientSubId: $searchClientSubId, " +
|
||||
s"params: $params, candidateTweetSourceId: $candidateTweetSourceId, includeNullcastTweets: $includeNullcastTweets, " +
|
||||
s"includeTweetsFromArchiveIndex: $includeTweetsFromArchiveIndex), hydratesContentFeatures: $hydratesContentFeatures"
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
def noDuplicates[T <: Traversable[_]](elements: T) = {
|
||||
elements.toSet.size == elements.size
|
||||
}
|
||||
|
||||
maxCount.foreach { max => require(max > 0, "maxCount must be a positive integer") }
|
||||
range.foreach(_.throwIfInvalid())
|
||||
earlybirdOptions.foreach(_.throwIfInvalid())
|
||||
tweetIds.foreach { ids => require(ids.nonEmpty, "tweetIds must be nonEmpty if present") }
|
||||
semanticCoreIds.foreach(_.foreach(_.throwIfInvalid()))
|
||||
languages.foreach(_.foreach(_.throwIfInvalid()))
|
||||
languages.foreach { langs =>
|
||||
require(langs.nonEmpty, "languages must be nonEmpty if present")
|
||||
require(noDuplicates(langs.map(_.language)), "languages must be unique")
|
||||
}
|
||||
}
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def toThriftRecapQuery: thrift.RecapQuery = {
|
||||
val thriftOptions = Some(TweetKindOption.toThrift(options))
|
||||
thrift.RecapQuery(
|
||||
userId,
|
||||
maxCount,
|
||||
range.map(_.toTimelineRangeThrift),
|
||||
deprecatedMinCount = None,
|
||||
thriftOptions,
|
||||
earlybirdOptions.map(_.toThrift),
|
||||
deviceContext.map(_.toThrift),
|
||||
authorIds,
|
||||
excludedTweetIds,
|
||||
Some(SearchOperator.toThrift(searchOperator)),
|
||||
searchClientSubId,
|
||||
candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift)
|
||||
)
|
||||
}
|
||||
|
||||
def toThriftRecapHydrationQuery: thrift.RecapHydrationQuery = {
|
||||
require(tweetIds.isDefined && tweetIds.get.nonEmpty, "tweetIds must be present")
|
||||
thrift.RecapHydrationQuery(
|
||||
userId,
|
||||
tweetIds.get,
|
||||
earlybirdOptions.map(_.toThrift),
|
||||
deviceContext.map(_.toThrift),
|
||||
candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift)
|
||||
)
|
||||
}
|
||||
|
||||
def toThriftEntityTweetsQuery: thrift.EntityTweetsQuery = {
|
||||
val thriftTweetKindOptions = Some(TweetKindOption.toThrift(options))
|
||||
thrift.EntityTweetsQuery(
|
||||
userId = userId,
|
||||
maxCount = maxCount,
|
||||
range = range.map(_.toTimelineRangeThrift),
|
||||
tweetKindOptions = thriftTweetKindOptions,
|
||||
earlybirdOptions = earlybirdOptions.map(_.toThrift),
|
||||
deviceContext = deviceContext.map(_.toThrift),
|
||||
excludedTweetIds = excludedTweetIds,
|
||||
semanticCoreIds = semanticCoreIds.map(_.map(_.toThrift)),
|
||||
hashtags = hashtags,
|
||||
languages = languages.map(_.map(_.toThrift)),
|
||||
candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift),
|
||||
includeNullcastTweets = includeNullcastTweets,
|
||||
includeTweetsFromArchiveIndex = includeTweetsFromArchiveIndex,
|
||||
authorIds = authorIds
|
||||
)
|
||||
}
|
||||
|
||||
def toThriftUtegLikedByTweetsQuery: thrift.UtegLikedByTweetsQuery = {
|
||||
|
||||
val thriftTweetKindOptions = Some(TweetKindOption.toThrift(options))
|
||||
thrift.UtegLikedByTweetsQuery(
|
||||
userId = userId,
|
||||
maxCount = maxCount,
|
||||
utegCount = utegLikedByTweetsOptions.map(_.utegCount),
|
||||
range = range.map(_.toTimelineRangeThrift),
|
||||
tweetKindOptions = thriftTweetKindOptions,
|
||||
earlybirdOptions = earlybirdOptions.map(_.toThrift),
|
||||
deviceContext = deviceContext.map(_.toThrift),
|
||||
excludedTweetIds = excludedTweetIds,
|
||||
isInNetwork = utegLikedByTweetsOptions.map(_.isInNetwork).get,
|
||||
weightedFollowings = utegLikedByTweetsOptions.map(_.weightedFollowings),
|
||||
candidateTweetSourceId = candidateTweetSourceId.flatMap(CandidateTweetSourceId.toThrift)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelineservice.model.TimelineId
|
||||
|
||||
object ReverseChronTimelineQuery {
|
||||
def fromTimelineQuery(query: TimelineQuery): ReverseChronTimelineQuery = {
|
||||
query match {
|
||||
case q: ReverseChronTimelineQuery => q
|
||||
case _ => throw new IllegalArgumentException(s"Unsupported query type: $query")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case class ReverseChronTimelineQuery(
|
||||
override val id: TimelineId,
|
||||
override val maxCount: Option[Int] = None,
|
||||
override val range: Option[TimelineRange] = None,
|
||||
override val options: Option[ReverseChronTimelineQueryOptions] = None)
|
||||
extends TimelineQuery(thrift.TimelineQueryType.ReverseChron, id, maxCount, range, options) {
|
||||
|
||||
throwIfInvalid()
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
object ReverseChronTimelineQueryOptions {
|
||||
val Default: ReverseChronTimelineQueryOptions = ReverseChronTimelineQueryOptions()
|
||||
|
||||
def fromThrift(
|
||||
options: thrift.ReverseChronTimelineQueryOptions
|
||||
): ReverseChronTimelineQueryOptions = {
|
||||
ReverseChronTimelineQueryOptions(
|
||||
getTweetsFromArchiveIndex = options.getTweetsFromArchiveIndex
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
case class ReverseChronTimelineQueryOptions(getTweetsFromArchiveIndex: Boolean = true)
|
||||
extends TimelineQueryOptions {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def toThrift: thrift.ReverseChronTimelineQueryOptions = {
|
||||
thrift.ReverseChronTimelineQueryOptions(getTweetsFromArchiveIndex = getTweetsFromArchiveIndex)
|
||||
}
|
||||
|
||||
def toTimelineQueryOptionsThrift: thrift.TimelineQueryOptions = {
|
||||
thrift.TimelineQueryOptions.ReverseChronTimelineQueryOptions(toThrift)
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {}
|
||||
}
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.util.Time
|
||||
|
||||
object TimeRange {
|
||||
val default: TimeRange = TimeRange(None, None)
|
||||
|
||||
def fromThrift(range: thrift.TimeRange): TimeRange = {
|
||||
TimeRange(
|
||||
from = range.fromMs.map(Time.fromMilliseconds),
|
||||
to = range.toMs.map(Time.fromMilliseconds)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
case class TimeRange(from: Option[Time], to: Option[Time]) extends TimelineRange {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
(from, to) match {
|
||||
case (Some(fromTime), Some(toTime)) =>
|
||||
require(fromTime <= toTime, "from-time must be less than or equal to-time.")
|
||||
case _ => // valid, do nothing.
|
||||
}
|
||||
}
|
||||
|
||||
def toThrift: thrift.TimeRange = {
|
||||
thrift.TimeRange(
|
||||
fromMs = from.map(_.inMilliseconds),
|
||||
toMs = to.map(_.inMilliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
def toTimelineRangeThrift: thrift.TimelineRange = {
|
||||
thrift.TimelineRange.TimeRange(toThrift)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.timelineservice.model.TimelineId
|
||||
import com.twitter.timelineservice.model.core.TimelineKind
|
||||
|
||||
object Timeline {
|
||||
def empty(id: TimelineId): Timeline = {
|
||||
Timeline(id, Nil)
|
||||
}
|
||||
|
||||
def fromThrift(timeline: thrift.Timeline): Timeline = {
|
||||
Timeline(
|
||||
id = TimelineId.fromThrift(timeline.id),
|
||||
entries = timeline.entries.map(TimelineEntryEnvelope.fromThrift)
|
||||
)
|
||||
}
|
||||
|
||||
def throwIfIdInvalid(id: TimelineId): Unit = {
|
||||
// Note: if we support timelines other than TimelineKind.home, we need to update
|
||||
// the implementation of userId method here and in TimelineQuery class.
|
||||
require(id.kind == TimelineKind.home, s"Expected TimelineKind.home, found: ${id.kind}")
|
||||
}
|
||||
}
|
||||
|
||||
case class Timeline(id: TimelineId, entries: Seq[TimelineEntryEnvelope]) {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def userId: UserId = {
|
||||
id.id
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
Timeline.throwIfIdInvalid(id)
|
||||
entries.foreach(_.throwIfInvalid())
|
||||
}
|
||||
|
||||
def toThrift: thrift.Timeline = {
|
||||
thrift.Timeline(
|
||||
id = id.toThrift,
|
||||
entries = entries.map(_.toThrift)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
object TimelineEntry {
|
||||
def fromThrift(entry: thrift.TimelineEntry): TimelineEntry = {
|
||||
entry match {
|
||||
case thrift.TimelineEntry.Tweet(e) => Tweet.fromThrift(e)
|
||||
case thrift.TimelineEntry.TweetypieTweet(e) => new HydratedTweetEntry(e)
|
||||
case _ => throw new IllegalArgumentException(s"Unsupported type: $entry")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait TimelineEntry {
|
||||
def toTimelineEntryThrift: thrift.TimelineEntry
|
||||
def throwIfInvalid(): Unit
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
object TimelineEntryEnvelope {
|
||||
def fromThrift(entryEnvelope: thrift.TimelineEntryEnvelope): TimelineEntryEnvelope = {
|
||||
TimelineEntryEnvelope(
|
||||
entry = TimelineEntry.fromThrift(entryEnvelope.entry)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
case class TimelineEntryEnvelope(entry: TimelineEntry) {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def toThrift: thrift.TimelineEntryEnvelope = {
|
||||
thrift.TimelineEntryEnvelope(entry.toTimelineEntryThrift)
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
entry.throwIfInvalid()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.timelineservice.model.TimelineId
|
||||
|
||||
object TimelineQuery {
|
||||
def fromThrift(query: thrift.TimelineQuery): TimelineQuery = {
|
||||
val queryType = query.queryType
|
||||
val id = TimelineId.fromThrift(query.timelineId)
|
||||
val maxCount = query.maxCount
|
||||
val range = query.range.map(TimelineRange.fromThrift)
|
||||
val options = query.options.map(TimelineQueryOptions.fromThrift)
|
||||
|
||||
queryType match {
|
||||
case thrift.TimelineQueryType.Ranked =>
|
||||
val rankedOptions = getRankedOptions(options)
|
||||
RankedTimelineQuery(id, maxCount, range, rankedOptions)
|
||||
|
||||
case thrift.TimelineQueryType.ReverseChron =>
|
||||
val reverseChronOptions = getReverseChronOptions(options)
|
||||
ReverseChronTimelineQuery(id, maxCount, range, reverseChronOptions)
|
||||
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(s"Unsupported query type: $queryType")
|
||||
}
|
||||
}
|
||||
|
||||
def getRankedOptions(
|
||||
options: Option[TimelineQueryOptions]
|
||||
): Option[RankedTimelineQueryOptions] = {
|
||||
options.map {
|
||||
case o: RankedTimelineQueryOptions => o
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
"Only RankedTimelineQueryOptions are supported when queryType is TimelineQueryType.Ranked"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def getReverseChronOptions(
|
||||
options: Option[TimelineQueryOptions]
|
||||
): Option[ReverseChronTimelineQueryOptions] = {
|
||||
options.map {
|
||||
case o: ReverseChronTimelineQueryOptions => o
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
"Only ReverseChronTimelineQueryOptions are supported when queryType is TimelineQueryType.ReverseChron"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
abstract class TimelineQuery(
|
||||
private val queryType: thrift.TimelineQueryType,
|
||||
val id: TimelineId,
|
||||
val maxCount: Option[Int],
|
||||
val range: Option[TimelineRange],
|
||||
val options: Option[TimelineQueryOptions]) {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def userId: UserId = {
|
||||
id.id
|
||||
}
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
Timeline.throwIfIdInvalid(id)
|
||||
range.foreach(_.throwIfInvalid())
|
||||
options.foreach(_.throwIfInvalid())
|
||||
}
|
||||
|
||||
def toThrift: thrift.TimelineQuery = {
|
||||
thrift.TimelineQuery(
|
||||
queryType = queryType,
|
||||
timelineId = id.toThrift,
|
||||
maxCount = maxCount,
|
||||
range = range.map(_.toTimelineRangeThrift),
|
||||
options = options.map(_.toTimelineQueryOptionsThrift)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
object TimelineQueryOptions {
|
||||
def fromThrift(options: thrift.TimelineQueryOptions): TimelineQueryOptions = {
|
||||
options match {
|
||||
case thrift.TimelineQueryOptions.RankedTimelineQueryOptions(r) =>
|
||||
RankedTimelineQueryOptions.fromThrift(r)
|
||||
case thrift.TimelineQueryOptions.ReverseChronTimelineQueryOptions(r) =>
|
||||
ReverseChronTimelineQueryOptions.fromThrift(r)
|
||||
case _ => throw new IllegalArgumentException(s"Unsupported type: $options")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait TimelineQueryOptions {
|
||||
def toTimelineQueryOptionsThrift: thrift.TimelineQueryOptions
|
||||
def throwIfInvalid(): Unit
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
object TimelineRange {
|
||||
def fromThrift(range: thrift.TimelineRange): TimelineRange = {
|
||||
range match {
|
||||
case thrift.TimelineRange.TimeRange(r) => TimeRange.fromThrift(r)
|
||||
case thrift.TimelineRange.TweetIdRange(r) => TweetIdRange.fromThrift(r)
|
||||
case _ => throw new IllegalArgumentException(s"Unsupported type: $range")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait TimelineRange {
|
||||
def toTimelineRangeThrift: thrift.TimelineRange
|
||||
def throwIfInvalid(): Unit
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.search.earlybird.thriftscala._
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.model.UserId
|
||||
|
||||
object Tweet {
|
||||
def fromThrift(tweet: thrift.Tweet): Tweet = {
|
||||
Tweet(id = tweet.id)
|
||||
}
|
||||
}
|
||||
|
||||
case class Tweet(
|
||||
id: TweetId,
|
||||
userId: Option[UserId] = None,
|
||||
sourceTweetId: Option[TweetId] = None,
|
||||
sourceUserId: Option[UserId] = None)
|
||||
extends TimelineEntry {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def throwIfInvalid(): Unit = {}
|
||||
|
||||
def toThrift: thrift.Tweet = {
|
||||
thrift.Tweet(
|
||||
id = id,
|
||||
userId = userId,
|
||||
sourceTweetId = sourceTweetId,
|
||||
sourceUserId = sourceUserId)
|
||||
}
|
||||
|
||||
def toTimelineEntryThrift: thrift.TimelineEntry = {
|
||||
thrift.TimelineEntry.Tweet(toThrift)
|
||||
}
|
||||
|
||||
def toThriftSearchResult: ThriftSearchResult = {
|
||||
val metadata = ThriftSearchResultMetadata(
|
||||
resultType = ThriftSearchResultType.Recency,
|
||||
fromUserId = userId match {
|
||||
case Some(id) => id
|
||||
case None => 0L
|
||||
},
|
||||
isRetweet =
|
||||
if (sourceUserId.isDefined || sourceUserId.isDefined) Some(true)
|
||||
else
|
||||
None,
|
||||
sharedStatusId = sourceTweetId match {
|
||||
case Some(id) => id
|
||||
case None => 0L
|
||||
},
|
||||
referencedTweetAuthorId = sourceUserId match {
|
||||
case Some(id) => id
|
||||
case None => 0L
|
||||
}
|
||||
)
|
||||
ThriftSearchResult(
|
||||
id = id,
|
||||
metadata = Some(metadata)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.TweetId
|
||||
|
||||
object TweetIdRange {
|
||||
val default: TweetIdRange = TweetIdRange(None, None)
|
||||
val empty: TweetIdRange = TweetIdRange(Some(0L), Some(0L))
|
||||
|
||||
def fromThrift(range: thrift.TweetIdRange): TweetIdRange = {
|
||||
TweetIdRange(fromId = range.fromId, toId = range.toId)
|
||||
}
|
||||
|
||||
def fromTimelineRange(range: TimelineRange): TweetIdRange = {
|
||||
range match {
|
||||
case r: TweetIdRange => r
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(s"Only Tweet ID range is supported. Found: $range")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A range of Tweet IDs with exclusive bounds.
|
||||
*/
|
||||
case class TweetIdRange(fromId: Option[TweetId] = None, toId: Option[TweetId] = None)
|
||||
extends TimelineRange {
|
||||
|
||||
throwIfInvalid()
|
||||
|
||||
def throwIfInvalid(): Unit = {
|
||||
(fromId, toId) match {
|
||||
case (Some(fromTweetId), Some(toTweetId)) =>
|
||||
require(fromTweetId <= toTweetId, "fromId must be less than or equal to toId.")
|
||||
case _ => // valid, do nothing.
|
||||
}
|
||||
}
|
||||
|
||||
def toThrift: thrift.TweetIdRange = {
|
||||
thrift.TweetIdRange(fromId = fromId, toId = toId)
|
||||
}
|
||||
|
||||
def toTimelineRangeThrift: thrift.TimelineRange = {
|
||||
thrift.TimelineRange.TweetIdRange(toThrift)
|
||||
}
|
||||
|
||||
def isEmpty: Boolean = {
|
||||
(fromId, toId) match {
|
||||
case (Some(fromId), Some(toId)) if fromId == toId => true
|
||||
case _ => false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
package com.twitter.timelineranker.model
|
||||
|
||||
import com.twitter.timelines.model.UserId
|
||||
|
||||
case class UtegLikedByTweetsOptions(
|
||||
utegCount: Int,
|
||||
isInNetwork: Boolean,
|
||||
weightedFollowings: Map[UserId, Double])
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
target(
|
||||
dependencies = [
|
||||
"timelineranker/config",
|
||||
"timelineranker/server/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "bundle",
|
||||
basename = "timelineranker-server-package-dist",
|
||||
binary = "timelineranker/server/src/main/scala:bin",
|
||||
bundles = [bundle(
|
||||
fileset = ["config/**/*"],
|
||||
owning_target = "timelineranker/server/config:files",
|
||||
)],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
resources(
|
||||
sources = ["**/*.yml"],
|
||||
)
|
||||
|
||||
# Created for Bazel compatibility.
|
||||
# In Bazel, loose files must be part of a target to be included into a bundle.
|
||||
# See also http://go/bazel-compatibility/bundle_does_not_match_any_files
|
||||
files(
|
||||
name = "files",
|
||||
sources = [
|
||||
"!BUILD",
|
||||
"**/*",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
# Deciders that can be used to control load on TLR or its backends.
|
||||
enable_max_concurrency_limiting:
|
||||
comment: "When enabled, limit maxConcurrency filter. Note: Requires system property maxConcurrency to be set."
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to testing / debugging.
|
||||
enable_routing_to_ranker_dev_proxy:
|
||||
comment: "Route dark traffic to the TimelineRanker development proxy. 100% means ~100% of requests to a host."
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to authorization.
|
||||
client_request_authorization:
|
||||
comment: "Enable client request authorization and rate limiting"
|
||||
default_availability: 10000
|
||||
client_write_whitelist:
|
||||
comment: "Enable authorization of write protected requests from only whitelisted clients"
|
||||
default_availability: 0
|
||||
allow_timeline_mixer_recap_prod:
|
||||
comment: "Allow requests from production TimelineMixer/recap"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_recycled_prod:
|
||||
comment: "Allow requests from production TimelineMixer/recycled"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_hydrate_prod:
|
||||
comment: "Allow requests from production TimelineMixer/hydrate"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_hydrate_recos_prod:
|
||||
comment: "Allow requests from production TimelineMixer/hydrate_recos"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_seed_authors_prod:
|
||||
comment: "Allow requests from production TimelineMixer/seed_author_ids"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_simcluster_prod:
|
||||
comment: "Allow requests from production TimelineMixer/simcluster"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_entity_tweets_prod:
|
||||
comment: "Allow requests from production TimelineMixer/entity_tweets"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_list_prod:
|
||||
comment: "Allow requests from production TimelineMixer/list"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_list_tweet_prod:
|
||||
comment: "Allow requests from production TimelineMixer/list_tweet"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_uteg_liked_by_tweets_prod:
|
||||
comment: "Allow requests from production TimelineMixer/uteg_liked_by_tweets"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_community_prod:
|
||||
comment: "Allow requests from production TimelineMixer/community"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_community_tweet_prod:
|
||||
comment: "Allow requests from production TimelineMixer/community_tweet"
|
||||
default_availability: 10000
|
||||
allow_timeline_scorer_recommended_trend_tweet_prod:
|
||||
comment: "Allow requests from production TimelineMixer/recommended_trend_tweet"
|
||||
default_availability: 10000
|
||||
|
||||
allow_timeline_scorer_rec_topic_tweets_prod:
|
||||
comment: "Allow requests from production TimelineScorer/rec_topic_tweets"
|
||||
default_availability: 10000
|
||||
allow_timeline_scorer_popular_topic_tweets_prod:
|
||||
comment: "Allow requests from production TimelineScorer/popular_topic_tweets"
|
||||
default_availability: 10000
|
||||
|
||||
allow_timelinescorer_hydrate_tweet_scoring_prod:
|
||||
comment: "Allow requests from production TimelineScorer/hydrate_tweet_scoring"
|
||||
default_availability: 10000
|
||||
|
||||
allow_timeline_mixer_staging:
|
||||
comment: "Allow requests from staging TimelineMixer"
|
||||
default_availability: 10000
|
||||
allow_timeline_ranker_warmup:
|
||||
comment: "Allow warmup requests from the TLR cluster"
|
||||
default_availability: 10000
|
||||
allow_timeline_ranker_proxy:
|
||||
comment: "Allow warmup requests from the TimelineRanker proxy"
|
||||
default_availability: 10000
|
||||
allow_timeline_service_prod:
|
||||
comment: "Allow requests from production TimelineService"
|
||||
default_availability: 10000
|
||||
allow_timeline_service_staging:
|
||||
comment: "Allow requests from staging TimelineService"
|
||||
default_availability: 10000
|
||||
rate_limit_override_unknown:
|
||||
comment: "Override the rate limit for unknown clients"
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to reverse-chron home timeline materialization.
|
||||
multiplier_of_materialization_tweets_fetched:
|
||||
comment: "Multiplier applied to the number of tweets fetched from search expressed as percentage. 100 means 100%. It can be used to fetch more than the number tweets requested by a caller (to improve similarity) or to fetch less than requested to reduce load."
|
||||
default_availability: 100
|
||||
enable_backfill_filtered_entries:
|
||||
comment: "Controls whether to back-fill timeline entries that get filtered out by TweetsPostFilter during home timeline materialization."
|
||||
default_availability: 0
|
||||
tweets_filtering_lossage_threshold:
|
||||
comment: "If back-filling filtered entries is enabled and if percentage of tweets that get filtered out exceeds this value then we will issue a second call to get more tweets. Default value 2000 == 20%"
|
||||
default_availability: 2000
|
||||
tweets_filtering_lossage_limit:
|
||||
comment: "We need to ensure that the number of tweets requested by the second call are not unbounded (for example, if everything is filtered out in the first call) therefore we limit the actual filtered out percentage to be no greater than the value below. Default value 6000 == 60%. That is, even if the actual lossage is 90% we will consider it to be only 60% for the purpose of back-filling."
|
||||
default_availability: 6000
|
||||
supplement_follows_with_real_graph:
|
||||
comment: "Whether to fetch additional follows from RealGraph for users with more than the max follows fetched from SGS during home timeline materialization."
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to recap.
|
||||
recap_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for recap Tweets. Otherwise those features are not set"
|
||||
default_availability: 10000
|
||||
recap_max_count_multiplier:
|
||||
comment: "We multiply maxCount (caller supplied value) by this multiplier and fetch those many candidates from search so that we are left with sufficient number of candidates after hydration and filtering. 100 == 1.0"
|
||||
default_availability: 100
|
||||
recap_enable_extra_sorting_in_results:
|
||||
comment: "If TLR will do extra sorting in search results"
|
||||
default_availability: 10000
|
||||
|
||||
# Deciders related to recycled tweets.
|
||||
recycled_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for recycled Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
recycled_max_count_multiplier:
|
||||
comment: "We multiply maxCount (caller supplied value) by this multiplier and fetch those many candidates from search so that we are left with sufficient number of candidates after hydration and filtering. 100 == 1.0"
|
||||
default_availability: 100
|
||||
|
||||
# Deciders related to entity tweets.
|
||||
entity_tweets_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for entity Tweets. Otherwise those features are not set"
|
||||
default_availability: 10000
|
||||
|
||||
# Deciders related to both recap and recycled tweets
|
||||
enable_real_graph_users:
|
||||
comment: "This is used only if user follows >= 1000. If true, expands user seedset with real graph users and recent followed users. Otherwise, user seedset only includes followed users."
|
||||
default_availability: 0
|
||||
max_real_graph_and_followed_users:
|
||||
comment: "Maximum number of combined real graph users and recent followed users in the user seedset for recap and recycled tweets if enable_real_graph_users is true and only_real_graph_users is false. This is upper bounded by 2000."
|
||||
default_availability: 1000
|
||||
|
||||
# Deciders related to recap author
|
||||
recap_author_enable_new_pipeline:
|
||||
comment: "Enable new recap author pipeline"
|
||||
default_availability: 0
|
||||
recap_author_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for PYLE Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to recap hydration(rectweet+ranked organic).
|
||||
recap_hydration_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for rectweet+ranked organic Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to uteg liked by tweets
|
||||
uteg_liked_by_tweets_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for rectweet+recycled utegLikedBy Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
resources(
|
||||
sources = [
|
||||
"*.xml",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
<configuration>
|
||||
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
|
||||
<property name="async_queue_size" value="${queue.size:-50000}"/>
|
||||
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
|
||||
<property name="SERVICE_OUTPUT" value="${log.service.output:-server.log}"/>
|
||||
<property name="DEBUG_TRANSCRIPTS_OUTPUT"
|
||||
value="${log.debug_transcripts.output:-debug_transcripts.log}"/>
|
||||
<property name="DEFAULT_SERVICE_PATTERN"
|
||||
value="%5p [%d{yyyyMMdd-HH:mm:ss.SSS}] %logger{0}: %m%n"/>
|
||||
<!-- JUL/JDK14 to Logback bridge -->
|
||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
|
||||
<resetJUL>true</resetJUL>
|
||||
</contextListener>
|
||||
|
||||
<!-- Service Log -->
|
||||
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${SERVICE_OUTPUT}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<!-- daily rollover -->
|
||||
<fileNamePattern>${SERVICE_OUTPUT}.%d.%i.gz</fileNamePattern>
|
||||
<maxFileSize>500MB</maxFileSize>
|
||||
<!-- keep 21 days' worth of history -->
|
||||
<maxHistory>21</maxHistory>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_SERVICE_PATTERN}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- debug transcripts -->
|
||||
<appender name="DEBUG-TRANSCRIPTS" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${DEBUG_TRANSCRIPTS_OUTPUT}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<!-- daily rollover -->
|
||||
<fileNamePattern>${DEBUG_TRANSCRIPTS_OUTPUT}.%d.%i.gz</fileNamePattern>
|
||||
<maxFileSize>500MB</maxFileSize>
|
||||
<!-- keep 21 days' worth of history -->
|
||||
<maxHistory>21</maxHistory>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_SERVICE_PATTERN}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- LogLens/splunk -->
|
||||
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>loglens</category>
|
||||
<index>${log.lens.index:-timelineranker}</index>
|
||||
<tag>${log.lens.tag}</tag>
|
||||
<encoder>
|
||||
<pattern>%msg%n</pattern>
|
||||
</encoder>
|
||||
<filter class="com.twitter.strato.logging.logback.RegexFilter">
|
||||
<forLogger>manhattan-client</forLogger>
|
||||
<excludeRegex>.*InvalidRequest.*</excludeRegex>
|
||||
</filter>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Primary Async Appenders -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<appender name="ASYNC-SERVICE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="SERVICE"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-DEBUG-TRANSCRIPTS" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="DEBUG-TRANSCRIPTS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS"/>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Package Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Per-Package Config -->
|
||||
<logger name="OptimisticLockingCache" level="off"/>
|
||||
<logger name="ZkSession" level="info"/>
|
||||
<logger name="com.twitter" level="info"/>
|
||||
<logger name="com.twitter.decider.StoreDecider" level="warn"/>
|
||||
<logger name="com.twitter.distributedlog.client" level="warn"/>
|
||||
<logger name="com.twitter.finagle.liveness" level="warn"/>
|
||||
<logger name="com.twitter.finagle.mtls.authorization.config.AccessControlListConfiguration" level="warn"/>
|
||||
<logger name="com.twitter.finagle.mux" level="warn"/>
|
||||
<logger name="com.twitter.finagle.serverset2" level="warn"/>
|
||||
<logger name="com.twitter.finatra.kafka.common.kerberoshelpers" level="warn"/>
|
||||
<logger name="com.twitter.finatra.kafka.utils.BootstrapServerUtils" level="warn"/>
|
||||
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
|
||||
<logger name="com.twitter.server.coordinate" level="error"/>
|
||||
<logger name="com.twitter.wilyns" level="warn"/>
|
||||
<logger name="com.twitter.zookeeper.client" level="info"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
|
||||
<logger name="manhattan-client" level="warn"/>
|
||||
<logger name="org.apache.kafka.clients.NetworkClient" level="error"/>
|
||||
<logger name="org.apache.kafka.clients.consumer.internals" level="error"/>
|
||||
<logger name="org.apache.kafka.clients.producer.internals" level="error"/>
|
||||
<logger name="org.apache.kafka.common.network" level="warn"/>
|
||||
<logger name="org.apache.zookeeper" level="error"/>
|
||||
<logger name="org.apache.zookeeper.ClientCnxn" level="warn"/>
|
||||
|
||||
<!-- Root Config -->
|
||||
<root level="${log_level:-INFO}">
|
||||
<appender-ref ref="ASYNC-SERVICE"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</root>
|
||||
|
||||
<!-- debug transcripts: logger name MUST be c.t.timelines.util.debuglog.DebugLog.DebugTranscriptsLog -->
|
||||
<logger name="DebugTranscripts" level="info">
|
||||
<appender-ref ref="ASYNC-DEBUG-TRANSCRIPTS"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</logger>
|
||||
</configuration>
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
target(
|
||||
dependencies = [
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/server",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/source",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "bin",
|
||||
basename = "timelineranker-server",
|
||||
main = "com.twitter.timelineranker.server.Main",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":scala",
|
||||
"3rdparty/jvm/org/slf4j:jcl-over-slf4j", # [1]
|
||||
"3rdparty/jvm/org/slf4j:log4j-over-slf4j", # [1]
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", # [2]
|
||||
"strato/src/main/scala/com/twitter/strato/logging/logback", # [2]
|
||||
"timelineranker/server/src/main/resources", # [2]
|
||||
"twitter-server/logback-classic/src/main/scala", #[2]
|
||||
],
|
||||
)
|
||||
|
||||
# [1] bridge other logging implementations to slf4j-api in addition to JUL
|
||||
# https://docbird.twitter.biz/core_libraries_guide/logging/twitter_server.html
|
||||
# without these, c.t.l.Logger become silent/null logger since no proper
|
||||
# configuration can be found. This can be removed once there are no
|
||||
# depdency from service to c.t.l.Logger
|
||||
#
|
||||
# [2] incur logback implementation
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"cortex-core/thrift/src/main/thrift:thrift-scala",
|
||||
"cortex-tweet-annotate/service/src/main/thrift:thrift-scala",
|
||||
"finagle/finagle-memcached/src/main/scala",
|
||||
"mediaservices/commons/src/main/thrift:thrift-scala",
|
||||
"servo/repo",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/ml/api:data-scala",
|
||||
"src/thrift/com/twitter/ml/prediction_service:prediction_service-scala",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-logging/src/main/scala",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
package com.twitter.timelineranker.clients
|
||||
|
||||
import com.twitter.cortex_core.thriftscala.ModelName
|
||||
import com.twitter.cortex_tweet_annotate.thriftscala._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.mediaservices.commons.mediainformation.thriftscala.CalibrationLevel
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.timelines.util.stats.RequestStats
|
||||
import com.twitter.timelines.util.stats.ScopedFactory
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
|
||||
object CortexTweetQueryServiceClient {
|
||||
private[this] val logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
/**
|
||||
* A tweet is considered safe if Cortex NSFA model gives it a score that is above the threshold.
|
||||
* Both the score and the threshold are returned in a response from getTweetSignalByIds endpoint.
|
||||
*/
|
||||
private def getSafeTweet(
|
||||
request: TweetSignalRequest,
|
||||
response: ModelResponseResult
|
||||
): Option[TweetId] = {
|
||||
val tweetId = request.tweetId
|
||||
response match {
|
||||
case ModelResponseResult(ModelResponseState.Success, Some(tid), Some(modelResponse), _) =>
|
||||
val prediction = modelResponse.predictions.flatMap(_.headOption)
|
||||
val score = prediction.map(_.score.score)
|
||||
val highRecallBucket = prediction.flatMap(_.calibrationBuckets).flatMap { buckets =>
|
||||
buckets.find(_.description.contains(CalibrationLevel.HighRecall))
|
||||
}
|
||||
val threshold = highRecallBucket.map(_.threshold)
|
||||
(score, threshold) match {
|
||||
case (Some(s), Some(t)) if (s > t) =>
|
||||
Some(tid)
|
||||
case (Some(s), Some(t)) =>
|
||||
logger.ifDebug(
|
||||
s"Cortex NSFA score for tweet $tweetId is $s (threshold is $t), removing as unsafe."
|
||||
)
|
||||
None
|
||||
case _ =>
|
||||
logger.ifDebug(s"Unexpected response, removing tweet $tweetId as unsafe.")
|
||||
None
|
||||
}
|
||||
case _ =>
|
||||
logger.ifWarning(
|
||||
s"Cortex tweet NSFA call was not successful, removing tweet $tweetId as unsafe."
|
||||
)
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enables calling cortex tweet query service to get NSFA scores on the tweet.
|
||||
*/
|
||||
class CortexTweetQueryServiceClient(
|
||||
cortexClient: CortexTweetQueryService.MethodPerEndpoint,
|
||||
requestScope: RequestScope,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends RequestStats {
|
||||
import CortexTweetQueryServiceClient._
|
||||
|
||||
private[this] val logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
private[this] val getTweetSignalByIdsRequestStats =
|
||||
requestScope.stats("cortex", statsReceiver, suffix = Some("getTweetSignalByIds"))
|
||||
private[this] val getTweetSignalByIdsRequestScopedStatsReceiver =
|
||||
getTweetSignalByIdsRequestStats.scopedStatsReceiver
|
||||
|
||||
private[this] val failedCortexTweetQueryServiceScope =
|
||||
getTweetSignalByIdsRequestScopedStatsReceiver.scope(Failures)
|
||||
private[this] val failedCortexTweetQueryServiceCallCounter =
|
||||
failedCortexTweetQueryServiceScope.counter("failOpen")
|
||||
|
||||
private[this] val cortexTweetQueryServiceFailOpenHandler = new FailOpenHandler(
|
||||
getTweetSignalByIdsRequestScopedStatsReceiver
|
||||
)
|
||||
|
||||
def getSafeTweets(tweetIds: Seq[TweetId]): Future[Seq[TweetId]] = {
|
||||
val requests = tweetIds.map { id => TweetSignalRequest(id, ModelName.TweetToNsfa) }
|
||||
val results = cortexClient
|
||||
.getTweetSignalByIds(
|
||||
GetTweetSignalByIdsRequest(requests)
|
||||
)
|
||||
.map(_.results)
|
||||
|
||||
cortexTweetQueryServiceFailOpenHandler(
|
||||
results.map { responses =>
|
||||
requests.zip(responses).flatMap {
|
||||
case (request, response) =>
|
||||
getSafeTweet(request, response)
|
||||
}
|
||||
}
|
||||
) { _ =>
|
||||
failedCortexTweetQueryServiceCallCounter.incr()
|
||||
logger.ifWarning(s"Cortex tweet NSFA call failed, considering tweets $tweetIds as unsafe.")
|
||||
Future.value(Seq())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class ScopedCortexTweetQueryServiceClientFactory(
|
||||
cortexClient: CortexTweetQueryService.MethodPerEndpoint,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends ScopedFactory[CortexTweetQueryServiceClient] {
|
||||
|
||||
override def scope(scope: RequestScope): CortexTweetQueryServiceClient = {
|
||||
new CortexTweetQueryServiceClient(cortexClient, scope, statsReceiver)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
package com.twitter.timelineranker.clients
|
||||
|
||||
import com.twitter.finagle.memcached.{Client => FinagleMemcacheClient}
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.servo.cache.FinagleMemcache
|
||||
import com.twitter.servo.cache.MemcacheCache
|
||||
import com.twitter.servo.cache.ObservableMemcache
|
||||
import com.twitter.servo.cache.Serializer
|
||||
import com.twitter.servo.cache.StatsReceiverCacheObserver
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.timelines.util.stats.ScopedFactory
|
||||
import com.twitter.util.Duration
|
||||
|
||||
/**
|
||||
* Factory to create a servo Memcache-backed Cache object. Clients are required to provide a
|
||||
* serializer/deserializer for keys and values.
|
||||
*/
|
||||
class MemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver) {
|
||||
private[this] val logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
def apply[K, V](
|
||||
keySerializer: K => String,
|
||||
valueSerializer: Serializer[V],
|
||||
ttl: Duration
|
||||
): MemcacheCache[K, V] = {
|
||||
new MemcacheCache[K, V](
|
||||
memcache = new ObservableMemcache(
|
||||
new FinagleMemcache(memcacheClient),
|
||||
new StatsReceiverCacheObserver(statsReceiver, 1000, logger)
|
||||
),
|
||||
ttl = ttl,
|
||||
serializer = valueSerializer,
|
||||
transformKey = keySerializer
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class ScopedMemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver)
|
||||
extends ScopedFactory[MemcacheFactory] {
|
||||
|
||||
override def scope(scope: RequestScope): MemcacheFactory = {
|
||||
new MemcacheFactory(
|
||||
memcacheClient,
|
||||
statsReceiver.scope("memcache", scope.scope)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/algebird:bijection",
|
||||
"3rdparty/jvm/com/twitter/bijection:core",
|
||||
"3rdparty/jvm/com/twitter/bijection:netty",
|
||||
"3rdparty/jvm/com/twitter/bijection:scrooge",
|
||||
"3rdparty/jvm/com/twitter/bijection:thrift",
|
||||
"3rdparty/jvm/com/twitter/bijection:util",
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"finagle/finagle-stats",
|
||||
"scrooge/scrooge-core/src/main/scala",
|
||||
"src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits",
|
||||
"src/thrift/com/twitter/timelines/content_features:thrift-scala",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/memcache_common",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
package com.twitter.timelineranker.clients.content_features_cache
|
||||
|
||||
import com.twitter.bijection.Injection
|
||||
import com.twitter.bijection.scrooge.CompactScalaCodec
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.clients.memcache_common._
|
||||
import com.twitter.timelines.content_features.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Duration
|
||||
|
||||
/**
|
||||
* Content features will be stored by tweetId
|
||||
*/
|
||||
class ContentFeaturesMemcacheBuilder(
|
||||
config: StorehausMemcacheConfig,
|
||||
ttl: Duration,
|
||||
statsReceiver: StatsReceiver) {
|
||||
private[this] val scalaToThriftInjection: Injection[ContentFeatures, thrift.ContentFeatures] =
|
||||
Injection.build[ContentFeatures, thrift.ContentFeatures](_.toThrift)(
|
||||
ContentFeatures.tryFromThrift)
|
||||
|
||||
private[this] val thriftToBytesInjection: Injection[thrift.ContentFeatures, Array[Byte]] =
|
||||
CompactScalaCodec(thrift.ContentFeatures)
|
||||
|
||||
private[this] implicit val valueInjection: Injection[ContentFeatures, Array[Byte]] =
|
||||
scalaToThriftInjection.andThen(thriftToBytesInjection)
|
||||
|
||||
private[this] val underlyingBuilder =
|
||||
new MemcacheStoreBuilder[TweetId, ContentFeatures](
|
||||
config = config,
|
||||
scopeName = "contentFeaturesCache",
|
||||
statsReceiver = statsReceiver,
|
||||
ttl = ttl
|
||||
)
|
||||
|
||||
def build(): Store[TweetId, ContentFeatures] = underlyingBuilder.build()
|
||||
}
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"finagle/finagle-core/src/main",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"src/thrift/com/twitter/search/common:constants-scala",
|
||||
"src/thrift/com/twitter/search/common:features-scala",
|
||||
"src/thrift/com/twitter/service/metastore/gen:thrift-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/common/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/candidate",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/bounds",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-logging/src/main/scala/com/twitter/logging",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.CandidateTweet
|
||||
import com.twitter.timelineranker.model.CandidateTweetsResult
|
||||
import com.twitter.util.Future
|
||||
|
||||
class CandidateGenerationTransform(statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[HydratedCandidatesAndFeaturesEnvelope, CandidateTweetsResult] {
|
||||
private[this] val numCandidateTweetsStat = statsReceiver.stat("numCandidateTweets")
|
||||
private[this] val numSourceTweetsStat = statsReceiver.stat("numSourceTweets")
|
||||
|
||||
override def apply(
|
||||
candidatesAndFeaturesEnvelope: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[CandidateTweetsResult] = {
|
||||
val hydratedTweets = candidatesAndFeaturesEnvelope.candidateEnvelope.hydratedTweets.outerTweets
|
||||
|
||||
if (hydratedTweets.nonEmpty) {
|
||||
val candidates = hydratedTweets.map { hydratedTweet =>
|
||||
CandidateTweet(hydratedTweet, candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId))
|
||||
}
|
||||
numCandidateTweetsStat.add(candidates.size)
|
||||
|
||||
val sourceTweets =
|
||||
candidatesAndFeaturesEnvelope.candidateEnvelope.sourceHydratedTweets.outerTweets.map {
|
||||
hydratedTweet =>
|
||||
CandidateTweet(
|
||||
hydratedTweet,
|
||||
candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId))
|
||||
}
|
||||
numSourceTweetsStat.add(sourceTweets.size)
|
||||
|
||||
Future.value(CandidateTweetsResult(candidates, sourceTweets))
|
||||
} else {
|
||||
Future.value(CandidateTweetsResult.Empty)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.contentfeatures.ContentFeaturesProvider
|
||||
import com.twitter.timelineranker.core.FutureDependencyTransformer
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.util.SearchResultUtil._
|
||||
import com.twitter.timelineranker.util.CachingContentFeaturesProvider
|
||||
import com.twitter.timelineranker.util.TweetHydrator
|
||||
import com.twitter.timelineranker.util.TweetypieContentFeaturesProvider
|
||||
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.timelines.configapi
|
||||
import com.twitter.timelines.util.FutureUtils
|
||||
|
||||
class ContentFeaturesHydrationTransformBuilder(
|
||||
tweetyPieClient: TweetyPieClient,
|
||||
contentFeaturesCache: Store[TweetId, ContentFeatures],
|
||||
enableContentFeaturesGate: Gate[RecapQuery],
|
||||
enableTokensInContentFeaturesGate: Gate[RecapQuery],
|
||||
enableTweetTextInContentFeaturesGate: Gate[RecapQuery],
|
||||
enableConversationControlContentFeaturesGate: Gate[RecapQuery],
|
||||
enableTweetMediaHydrationGate: Gate[RecapQuery],
|
||||
hydrateInReplyToTweets: Boolean,
|
||||
statsReceiver: StatsReceiver) {
|
||||
val scopedStatsReceiver: StatsReceiver = statsReceiver.scope("ContentFeaturesHydrationTransform")
|
||||
val tweetHydrator: TweetHydrator = new TweetHydrator(tweetyPieClient, scopedStatsReceiver)
|
||||
val tweetypieContentFeaturesProvider: ContentFeaturesProvider =
|
||||
new TweetypieContentFeaturesProvider(
|
||||
tweetHydrator,
|
||||
enableContentFeaturesGate,
|
||||
enableTokensInContentFeaturesGate,
|
||||
enableTweetTextInContentFeaturesGate,
|
||||
enableConversationControlContentFeaturesGate,
|
||||
enableTweetMediaHydrationGate,
|
||||
scopedStatsReceiver
|
||||
)
|
||||
|
||||
val cachingContentFeaturesProvider: ContentFeaturesProvider = new CachingContentFeaturesProvider(
|
||||
underlying = tweetypieContentFeaturesProvider,
|
||||
contentFeaturesCache = contentFeaturesCache,
|
||||
statsReceiver = scopedStatsReceiver
|
||||
)
|
||||
|
||||
val contentFeaturesProvider: configapi.FutureDependencyTransformer[RecapQuery, Seq[TweetId], Map[
|
||||
TweetId,
|
||||
ContentFeatures
|
||||
]] = FutureDependencyTransformer.partition(
|
||||
gate = enableContentFeaturesGate,
|
||||
ifTrue = cachingContentFeaturesProvider,
|
||||
ifFalse = tweetypieContentFeaturesProvider
|
||||
)
|
||||
|
||||
lazy val contentFeaturesHydrationTransform: ContentFeaturesHydrationTransform =
|
||||
new ContentFeaturesHydrationTransform(
|
||||
contentFeaturesProvider,
|
||||
enableContentFeaturesGate,
|
||||
hydrateInReplyToTweets
|
||||
)
|
||||
def build(): ContentFeaturesHydrationTransform = contentFeaturesHydrationTransform
|
||||
}
|
||||
|
||||
class ContentFeaturesHydrationTransform(
|
||||
contentFeaturesProvider: ContentFeaturesProvider,
|
||||
enableContentFeaturesGate: Gate[RecapQuery],
|
||||
hydrateInReplyToTweets: Boolean)
|
||||
extends FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
] {
|
||||
override def apply(
|
||||
request: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
if (enableContentFeaturesGate(request.candidateEnvelope.query)) {
|
||||
val searchResults = request.candidateEnvelope.searchResults
|
||||
|
||||
val sourceTweetIdMap = searchResults.map { searchResult =>
|
||||
(searchResult.id, getRetweetSourceTweetId(searchResult).getOrElse(searchResult.id))
|
||||
}.toMap
|
||||
|
||||
val inReplyToTweetIds = if (hydrateInReplyToTweets) {
|
||||
searchResults.flatMap(getInReplyToTweetId)
|
||||
} else {
|
||||
Seq.empty
|
||||
}
|
||||
|
||||
val tweetIdsToHydrate = (sourceTweetIdMap.values ++ inReplyToTweetIds).toSeq.distinct
|
||||
|
||||
val contentFeaturesMapFuture = if (tweetIdsToHydrate.nonEmpty) {
|
||||
contentFeaturesProvider(request.candidateEnvelope.query, tweetIdsToHydrate)
|
||||
} else {
|
||||
FutureUtils.EmptyMap[TweetId, ContentFeatures]
|
||||
}
|
||||
|
||||
Future.value(
|
||||
request.copy(
|
||||
contentFeaturesFuture = contentFeaturesMapFuture,
|
||||
tweetSourceTweetMap = sourceTweetIdMap,
|
||||
inReplyToTweetIds = inReplyToTweetIds.toSet
|
||||
)
|
||||
)
|
||||
} else {
|
||||
Future.value(request)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Create a CandidateEnvelope based on the incoming RecapQuery
|
||||
*/
|
||||
object CreateCandidateEnvelopeTransform extends FutureArrow[RecapQuery, CandidateEnvelope] {
|
||||
override def apply(query: RecapQuery): Future[CandidateEnvelope] = {
|
||||
Future.value(CandidateEnvelope(query))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Fetches all data required for feature hydration and generates the HydratedCandidatesAndFeaturesEnvelope
|
||||
* @param tweetHydrationAndFilteringPipeline Pipeline which fetches the candidate tweets, hydrates and filters them
|
||||
* @param languagesService Fetch user languages, required for feature hydration
|
||||
* @param userProfileInfoService Fetch user profile info, required for feature hydration
|
||||
*/
|
||||
class FeatureHydrationDataTransform(
|
||||
tweetHydrationAndFilteringPipeline: FutureArrow[RecapQuery, CandidateEnvelope],
|
||||
languagesService: UserLanguagesTransform,
|
||||
userProfileInfoService: UserProfileInfoTransform)
|
||||
extends FutureArrow[RecapQuery, HydratedCandidatesAndFeaturesEnvelope] {
|
||||
override def apply(request: RecapQuery): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
Future
|
||||
.join(
|
||||
languagesService(request),
|
||||
userProfileInfoService(request),
|
||||
tweetHydrationAndFilteringPipeline(request)).map {
|
||||
case (languages, userProfileInfo, transformedCandidateEnvelope) =>
|
||||
HydratedCandidatesAndFeaturesEnvelope(
|
||||
transformedCandidateEnvelope,
|
||||
languages,
|
||||
userProfileInfo)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,198 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.parameters.recap.RecapQueryContext
|
||||
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams.RecycledMaxFollowedUsersEnableAntiDilutionParam
|
||||
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
|
||||
import com.twitter.timelines.earlybird.common.options.AuthorScoreAdjustments
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Transform which conditionally augments follow graph data using the real graph,
|
||||
* derived from the earlybirdOptions passed in the query
|
||||
*
|
||||
* @param followGraphDataProvider data provider to be used for fetching updated mutual follow info
|
||||
* @param maxFollowedUsersProvider max number of users to return
|
||||
* @param enableRealGraphUsersProvider should we augment using real graph data?
|
||||
* @param maxRealGraphAndFollowedUsersProvider max combined users to return, overrides maxFollowedUsersProvider above
|
||||
* @param statsReceiver scoped stats received
|
||||
*/
|
||||
class FollowAndRealGraphCombiningTransform(
|
||||
followGraphDataProvider: FollowGraphDataProvider,
|
||||
maxFollowedUsersProvider: DependencyProvider[Int],
|
||||
enableRealGraphUsersProvider: DependencyProvider[Boolean],
|
||||
maxRealGraphAndFollowedUsersProvider: DependencyProvider[Int],
|
||||
imputeRealGraphAuthorWeightsProvider: DependencyProvider[Boolean],
|
||||
imputeRealGraphAuthorWeightsPercentileProvider: DependencyProvider[Int],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
// Number of authors in the seedset after mixing followed users and real graph users
|
||||
// Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is true and onlyRealGraphUsers is false
|
||||
val numFollowAndRealGraphUsersStat: Stat = statsReceiver.stat("numFollowAndRealGraphUsers")
|
||||
val numFollowAndRealGraphUsersFromFollowGraphStat =
|
||||
statsReceiver.scope("numFollowAndRealGraphUsers").stat("FollowGraphUsers")
|
||||
val numFollowAndRealGraphUsersFromRealGraphStat =
|
||||
statsReceiver.scope("numFollowAndRealGraphUsers").stat("RealGraphUsers")
|
||||
val numFollowAndRealGraphUsersFromRealGraphCounter =
|
||||
statsReceiver.scope("numFollowAndRealGraphUsers").counter()
|
||||
|
||||
// Number of authors in the seedset with only followed users
|
||||
// Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is false
|
||||
val numFollowedUsersStat: Stat = statsReceiver.stat("numFollowedUsers")
|
||||
|
||||
// Number of authors in the seedset with only followed users
|
||||
// Only have this stat if user follows < maxFollowedUsers
|
||||
val numFollowedUsersLessThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersLessThanMax")
|
||||
val numFollowedUsersLessThanMaxCounter =
|
||||
statsReceiver.scope("numFollowedUsersLessThanMax").counter()
|
||||
val numFollowedUsersMoreThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersMoreThanMax")
|
||||
val numFollowedUsersMoreThanMaxCounter =
|
||||
statsReceiver.scope("numFollowedUsersMoreThanMax").counter()
|
||||
|
||||
val realGraphAuthorWeightsSumProdStat: Stat = statsReceiver.stat("realGraphAuthorWeightsSumProd")
|
||||
val realGraphAuthorWeightsSumMinExpStat: Stat =
|
||||
statsReceiver.stat("realGraphAuthorWeightsSumMinExp")
|
||||
val realGraphAuthorWeightsSumP50ExpStat: Stat =
|
||||
statsReceiver.stat("realGraphAuthorWeightsSumP50Exp")
|
||||
val realGraphAuthorWeightsSumP95ExpStat: Stat =
|
||||
statsReceiver.stat("realGraphAuthorWeightsSumP95Exp")
|
||||
val numAuthorsWithoutRealgraphScoreStat: Stat =
|
||||
statsReceiver.stat("numAuthorsWithoutRealgraphScore")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val realGraphData = envelope.query.earlybirdOptions
|
||||
.map(_.authorScoreAdjustments.authorScoreMap)
|
||||
.getOrElse(Map.empty)
|
||||
|
||||
Future
|
||||
.join(
|
||||
envelope.followGraphData.followedUserIdsFuture,
|
||||
envelope.followGraphData.mutedUserIdsFuture
|
||||
).map {
|
||||
case (followedUserIds, mutedUserIds) =>
|
||||
// Anti-dilution param for DDG-16198
|
||||
val recycledMaxFollowedUsersEnableAntiDilutionParamProvider =
|
||||
DependencyProvider.from(RecycledMaxFollowedUsersEnableAntiDilutionParam)
|
||||
|
||||
val maxFollowedUsers = {
|
||||
if (followedUserIds.size > RecapQueryContext.MaxFollowedUsers.default && recycledMaxFollowedUsersEnableAntiDilutionParamProvider(
|
||||
envelope.query)) {
|
||||
// trigger experiment
|
||||
maxFollowedUsersProvider(envelope.query)
|
||||
} else {
|
||||
maxFollowedUsersProvider(envelope.query)
|
||||
}
|
||||
}
|
||||
|
||||
val filteredRealGraphUserIds = realGraphData.keySet
|
||||
.filterNot(mutedUserIds)
|
||||
.take(maxFollowedUsers)
|
||||
.toSeq
|
||||
|
||||
val filteredFollowedUserIds = followedUserIds.filterNot(mutedUserIds)
|
||||
|
||||
if (followedUserIds.size < maxFollowedUsers) {
|
||||
numFollowedUsersLessThanMaxStat.add(filteredFollowedUserIds.size)
|
||||
// stats
|
||||
numFollowedUsersLessThanMaxCounter.incr()
|
||||
(filteredFollowedUserIds, false)
|
||||
} else {
|
||||
numFollowedUsersMoreThanMaxStat.add(filteredFollowedUserIds.size)
|
||||
numFollowedUsersMoreThanMaxCounter.incr()
|
||||
if (enableRealGraphUsersProvider(envelope.query)) {
|
||||
val maxRealGraphAndFollowedUsersNum =
|
||||
maxRealGraphAndFollowedUsersProvider(envelope.query)
|
||||
require(
|
||||
maxRealGraphAndFollowedUsersNum >= maxFollowedUsers,
|
||||
"maxRealGraphAndFollowedUsers must be greater than or equal to maxFollowedUsers."
|
||||
)
|
||||
val recentFollowedUsersNum = RecapQueryContext.MaxFollowedUsers.bounds
|
||||
.apply(maxRealGraphAndFollowedUsersNum - filteredRealGraphUserIds.size)
|
||||
|
||||
val recentFollowedUsers =
|
||||
filteredFollowedUserIds
|
||||
.filterNot(filteredRealGraphUserIds.contains)
|
||||
.take(recentFollowedUsersNum)
|
||||
|
||||
val filteredFollowAndRealGraphUserIds =
|
||||
recentFollowedUsers ++ filteredRealGraphUserIds
|
||||
|
||||
// Track the size of recentFollowedUsers from SGS
|
||||
numFollowAndRealGraphUsersFromFollowGraphStat.add(recentFollowedUsers.size)
|
||||
// Track the size of filteredRealGraphUserIds from real graph dataset.
|
||||
numFollowAndRealGraphUsersFromRealGraphStat.add(filteredRealGraphUserIds.size)
|
||||
|
||||
numFollowAndRealGraphUsersFromRealGraphCounter.incr()
|
||||
|
||||
numFollowAndRealGraphUsersStat.add(filteredFollowAndRealGraphUserIds.size)
|
||||
|
||||
(filteredFollowAndRealGraphUserIds, true)
|
||||
} else {
|
||||
numFollowedUsersStat.add(followedUserIds.size)
|
||||
(filteredFollowedUserIds, false)
|
||||
}
|
||||
}
|
||||
}.map {
|
||||
case (updatedFollowSeq, shouldUpdateMutualFollows) =>
|
||||
val updatedMutualFollowing = if (shouldUpdateMutualFollows) {
|
||||
followGraphDataProvider.getMutuallyFollowingUserIds(
|
||||
envelope.query.userId,
|
||||
updatedFollowSeq)
|
||||
} else {
|
||||
envelope.followGraphData.mutuallyFollowingUserIdsFuture
|
||||
}
|
||||
|
||||
val followGraphData = envelope.followGraphData.copy(
|
||||
followedUserIdsFuture = Future.value(updatedFollowSeq),
|
||||
mutuallyFollowingUserIdsFuture = updatedMutualFollowing
|
||||
)
|
||||
|
||||
val authorIdsWithRealgraphScore = realGraphData.keySet
|
||||
val authorIdsWithoutRealgraphScores =
|
||||
updatedFollowSeq.filterNot(authorIdsWithRealgraphScore.contains)
|
||||
|
||||
//stat for logging the percentage of users' followings that do not have a realgraph score
|
||||
if (updatedFollowSeq.nonEmpty)
|
||||
numAuthorsWithoutRealgraphScoreStat.add(
|
||||
authorIdsWithoutRealgraphScores.size / updatedFollowSeq.size * 100)
|
||||
|
||||
if (imputeRealGraphAuthorWeightsProvider(envelope.query) && realGraphData.nonEmpty) {
|
||||
val imputedScorePercentile =
|
||||
imputeRealGraphAuthorWeightsPercentileProvider(envelope.query) / 100.0
|
||||
val existingAuthorIdScores = realGraphData.values.toList.sorted
|
||||
val imputedScoreIndex = Math.min(
|
||||
existingAuthorIdScores.length - 1,
|
||||
(existingAuthorIdScores.length * imputedScorePercentile).toInt)
|
||||
val imputedScore = existingAuthorIdScores(imputedScoreIndex)
|
||||
|
||||
val updatedAuthorScoreMap = realGraphData ++ authorIdsWithoutRealgraphScores
|
||||
.map(_ -> imputedScore).toMap
|
||||
imputedScorePercentile match {
|
||||
case 0.0 =>
|
||||
realGraphAuthorWeightsSumMinExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
|
||||
case 0.5 =>
|
||||
realGraphAuthorWeightsSumP50ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
|
||||
case 0.95 =>
|
||||
realGraphAuthorWeightsSumP95ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
|
||||
case _ =>
|
||||
}
|
||||
val earlybirdOptionsWithUpdatedAuthorScoreMap = envelope.query.earlybirdOptions
|
||||
.map(_.copy(authorScoreAdjustments = AuthorScoreAdjustments(updatedAuthorScoreMap)))
|
||||
val updatedQuery =
|
||||
envelope.query.copy(earlybirdOptions = earlybirdOptionsWithUpdatedAuthorScoreMap)
|
||||
envelope.copy(query = updatedQuery, followGraphData = followGraphData)
|
||||
} else {
|
||||
envelope.query.earlybirdOptions
|
||||
.map(_.authorScoreAdjustments.authorScoreMap.values.sum.toFloat).foreach {
|
||||
realGraphAuthorWeightsSumProdStat.add(_)
|
||||
}
|
||||
envelope.copy(followGraphData = followGraphData)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
|
||||
import com.twitter.util.Future
|
||||
|
||||
class FollowGraphDataTransform(
|
||||
followGraphDataProvider: FollowGraphDataProvider,
|
||||
maxFollowedUsersProvider: DependencyProvider[Int])
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
|
||||
val followGraphData = followGraphDataProvider.getAsync(
|
||||
envelope.query.userId,
|
||||
maxFollowedUsersProvider(envelope.query)
|
||||
)
|
||||
|
||||
Future.value(envelope.copy(followGraphData = followGraphData))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Transform that explicitly hydrates candidate tweets and fetches source tweets in parallel
|
||||
* and then joins the results back into the original Envelope
|
||||
* @param candidateTweetHydration Pipeline that hydrates candidate tweets
|
||||
* @param sourceTweetHydration Pipeline that fetches and hydrates source tweets
|
||||
*/
|
||||
class HydrateTweetsAndSourceTweetsInParallelTransform(
|
||||
candidateTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope],
|
||||
sourceTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope])
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
Future
|
||||
.join(
|
||||
candidateTweetHydration(envelope),
|
||||
sourceTweetHydration(envelope)
|
||||
).map {
|
||||
case (candidateTweetEnvelope, sourceTweetEnvelope) =>
|
||||
envelope.copy(
|
||||
hydratedTweets = candidateTweetEnvelope.hydratedTweets,
|
||||
sourceSearchResults = sourceTweetEnvelope.sourceSearchResults,
|
||||
sourceHydratedTweets = sourceTweetEnvelope.sourceHydratedTweets
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedTweets
|
||||
import com.twitter.timelineranker.util.TweetFilters
|
||||
import com.twitter.timelineranker.util.TweetsPostFilter
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.util.Future
|
||||
|
||||
object HydratedTweetsFilterTransform {
|
||||
val EmptyFollowGraphDataTuple: (Seq[UserId], Seq[UserId], Set[UserId]) =
|
||||
(Seq.empty[UserId], Seq.empty[UserId], Set.empty[UserId])
|
||||
val DefaultNumRetweetsAllowed = 1
|
||||
|
||||
// Number of duplicate retweets (including the first one) allowed.
|
||||
// For example,
|
||||
// If there are 7 retweets of a given tweet, the following value will cause 5 of them
|
||||
// to be returned after filtering and the additional 2 will be filtered out.
|
||||
val NumDuplicateRetweetsAllowed = 5
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform which takes TweetFilters ValueSets for inner and outer tweets and uses
|
||||
* TweetsPostFilter to filter down the HydratedTweets using the supplied filters
|
||||
*
|
||||
* @param useFollowGraphData - use follow graph for filtering; otherwise only does filtering
|
||||
* independent of follow graph data
|
||||
* @param useSourceTweets - only needed when filtering extended replies
|
||||
* @param statsReceiver - scoped stats receiver
|
||||
*/
|
||||
class HydratedTweetsFilterTransform(
|
||||
outerFilters: TweetFilters.ValueSet,
|
||||
innerFilters: TweetFilters.ValueSet,
|
||||
useFollowGraphData: Boolean,
|
||||
useSourceTweets: Boolean,
|
||||
statsReceiver: StatsReceiver,
|
||||
numRetweetsAllowed: Int = HydratedTweetsFilterTransform.DefaultNumRetweetsAllowed)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import HydratedTweetsFilterTransform._
|
||||
|
||||
val logger: Logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
if (outerFilters == TweetFilters.None) {
|
||||
Future.value(envelope)
|
||||
} else {
|
||||
val tweetsPostOuterFilter = new TweetsPostFilter(outerFilters, logger, statsReceiver)
|
||||
val tweetsPostInnerFilter = new TweetsPostFilter(innerFilters, logger, statsReceiver)
|
||||
|
||||
val graphData = if (useFollowGraphData) {
|
||||
Future.join(
|
||||
envelope.followGraphData.followedUserIdsFuture,
|
||||
envelope.followGraphData.inNetworkUserIdsFuture,
|
||||
envelope.followGraphData.mutedUserIdsFuture
|
||||
)
|
||||
} else {
|
||||
Future.value(EmptyFollowGraphDataTuple)
|
||||
}
|
||||
|
||||
val sourceTweets = if (useSourceTweets) {
|
||||
envelope.sourceHydratedTweets.outerTweets
|
||||
} else {
|
||||
Nil
|
||||
}
|
||||
|
||||
graphData.map {
|
||||
case (followedUserIds, inNetworkUserIds, mutedUserIds) =>
|
||||
val outerTweets = tweetsPostOuterFilter(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedUserIds,
|
||||
inNetworkUserIds = inNetworkUserIds,
|
||||
mutedUserIds = mutedUserIds,
|
||||
tweets = envelope.hydratedTweets.outerTweets,
|
||||
numRetweetsAllowed = numRetweetsAllowed,
|
||||
sourceTweets = sourceTweets
|
||||
)
|
||||
val innerTweets = tweetsPostInnerFilter(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedUserIds,
|
||||
inNetworkUserIds = inNetworkUserIds,
|
||||
mutedUserIds = mutedUserIds,
|
||||
// inner tweets refers to quoted tweets not source tweets, and special rulesets
|
||||
// in birdherd handle visibility of viewer to inner tweet author for these tweets.
|
||||
tweets = envelope.hydratedTweets.innerTweets,
|
||||
numRetweetsAllowed = numRetweetsAllowed,
|
||||
sourceTweets = sourceTweets
|
||||
)
|
||||
|
||||
envelope.copy(hydratedTweets = HydratedTweets(outerTweets, innerTweets))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator
|
||||
import com.twitter.util.Future
|
||||
|
||||
object InNetworkTweetsSearchFeaturesHydrationTransform
|
||||
extends FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
] {
|
||||
override def apply(
|
||||
request: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
Future
|
||||
.join(
|
||||
request.candidateEnvelope.followGraphData.followedUserIdsFuture,
|
||||
request.candidateEnvelope.followGraphData.mutuallyFollowingUserIdsFuture
|
||||
).map {
|
||||
case (followedIds, mutuallyFollowingIds) =>
|
||||
val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate(
|
||||
searcherUserId = request.candidateEnvelope.query.userId,
|
||||
searcherProfileInfo = request.userProfileInfo,
|
||||
followedUserIds = followedIds,
|
||||
mutuallyFollowingUserIds = mutuallyFollowingIds,
|
||||
userLanguages = request.userLanguages,
|
||||
uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode),
|
||||
searchResults = request.candidateEnvelope.searchResults,
|
||||
sourceTweetSearchResults = request.candidateEnvelope.sourceSearchResults,
|
||||
tweets = request.candidateEnvelope.hydratedTweets.outerTweets,
|
||||
sourceTweets = request.candidateEnvelope.sourceHydratedTweets.outerTweets
|
||||
)
|
||||
|
||||
request.copy(features = featuresByTweetId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.CandidateTweet
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Time
|
||||
import scala.util.Random
|
||||
|
||||
/**
|
||||
* picks up one or more random tweets and sets its tweetFeatures.isRandomTweet field to true.
|
||||
*/
|
||||
class MarkRandomTweetTransform(
|
||||
includeRandomTweetProvider: DependencyProvider[Boolean],
|
||||
randomGenerator: Random = new Random(Time.now.inMilliseconds),
|
||||
includeSingleRandomTweetProvider: DependencyProvider[Boolean],
|
||||
probabilityRandomTweetProvider: DependencyProvider[Double])
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val includeRandomTweet = includeRandomTweetProvider(envelope.query)
|
||||
val includeSingleRandomTweet = includeSingleRandomTweetProvider(envelope.query)
|
||||
val probabilityRandomTweet = probabilityRandomTweetProvider(envelope.query)
|
||||
val searchResults = envelope.searchResults
|
||||
|
||||
if (!includeRandomTweet || searchResults.isEmpty) { // random tweet off
|
||||
Future.value(envelope)
|
||||
} else if (includeSingleRandomTweet) { // pick only one
|
||||
val randomIdx = randomGenerator.nextInt(searchResults.size)
|
||||
val randomTweet = searchResults(randomIdx)
|
||||
val randomTweetWithFlag = randomTweet.copy(
|
||||
tweetFeatures = randomTweet.tweetFeatures
|
||||
.orElse(Some(CandidateTweet.DefaultFeatures))
|
||||
.map(_.copy(isRandomTweet = Some(true)))
|
||||
)
|
||||
val updatedSearchResults = searchResults.updated(randomIdx, randomTweetWithFlag)
|
||||
|
||||
Future.value(envelope.copy(searchResults = updatedSearchResults))
|
||||
} else { // pick tweets with perTweetProbability
|
||||
val updatedSearchResults = searchResults.map { result =>
|
||||
if (randomGenerator.nextDouble() < probabilityRandomTweet) {
|
||||
result.copy(
|
||||
tweetFeatures = result.tweetFeatures
|
||||
.orElse(Some(CandidateTweet.DefaultFeatures))
|
||||
.map(_.copy(isRandomTweet = Some(true))))
|
||||
|
||||
} else
|
||||
result
|
||||
}
|
||||
|
||||
Future.value(envelope.copy(searchResults = updatedSearchResults))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.util.Future
|
||||
|
||||
object OutOfNetworkRepliesToUserIdSearchResultsTransform {
|
||||
val DefaultMaxTweetCount = 100
|
||||
}
|
||||
|
||||
// Requests search results for out-of-network replies to a user Id
|
||||
class OutOfNetworkRepliesToUserIdSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
statsReceiver: StatsReceiver,
|
||||
logSearchDebugInfo: Boolean = true)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
private[this] val maxCountStat = statsReceiver.stat("maxCount")
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val maxCount = envelope.query.maxCount
|
||||
.getOrElse(OutOfNetworkRepliesToUserIdSearchResultsTransform.DefaultMaxTweetCount)
|
||||
maxCountStat.add(maxCount)
|
||||
|
||||
envelope.followGraphData.followedUserIdsFuture
|
||||
.flatMap {
|
||||
case followedIds =>
|
||||
searchClient
|
||||
.getOutOfNetworkRepliesToUserId(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedIds.toSet,
|
||||
maxCount = maxCount,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
logSearchDebugInfo
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
results.foreach { result =>
|
||||
val earlybirdScoreX100 =
|
||||
result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100
|
||||
earlybirdScoreX100Stat.add(earlybirdScoreX100)
|
||||
}
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator
|
||||
import com.twitter.util.Future
|
||||
|
||||
object OutOfNetworkTweetsSearchFeaturesHydrationTransform
|
||||
extends FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
] {
|
||||
override def apply(
|
||||
request: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate(
|
||||
searcherUserId = request.candidateEnvelope.query.userId,
|
||||
searcherProfileInfo = request.userProfileInfo,
|
||||
followedUserIds = Seq.empty,
|
||||
mutuallyFollowingUserIds = Set.empty,
|
||||
userLanguages = request.userLanguages,
|
||||
uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode),
|
||||
searchResults = request.candidateEnvelope.searchResults,
|
||||
sourceTweetSearchResults = Seq.empty,
|
||||
tweets = request.candidateEnvelope.hydratedTweets.outerTweets,
|
||||
sourceTweets = Seq.empty
|
||||
)
|
||||
|
||||
Future.value(request.copy(features = featuresByTweetId))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
|
||||
trait RecapHydrationSearchResultsTransformBase
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
protected def statsReceiver: StatsReceiver
|
||||
protected def searchClient: SearchClient
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
|
||||
def tweetIdsToHydrate(envelope: CandidateEnvelope): Seq[TweetId]
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
searchClient
|
||||
.getTweetsScoredForRecap(
|
||||
envelope.query.userId,
|
||||
tweetIdsToHydrate(envelope),
|
||||
envelope.query.earlybirdOptions
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.model.TweetIdRange
|
||||
import com.twitter.timelineranker.parameters.recap.RecapParams
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Fetch recap/recycled search results using the search client
|
||||
* and populate them into the CandidateEnvelope
|
||||
*/
|
||||
class RecapSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
maxCountProvider: DependencyProvider[Int],
|
||||
returnAllResultsProvider: DependencyProvider[Boolean],
|
||||
relevanceOptionsMaxHitsToProcessProvider: DependencyProvider[Int],
|
||||
enableExcludeSourceTweetIdsProvider: DependencyProvider[Boolean],
|
||||
enableSettingTweetTypesWithTweetKindOptionProvider: DependencyProvider[Boolean],
|
||||
perRequestSearchClientIdProvider: DependencyProvider[Option[String]],
|
||||
relevanceSearchProvider: DependencyProvider[Boolean] =
|
||||
DependencyProvider.from(RecapParams.EnableRelevanceSearchParam),
|
||||
statsReceiver: StatsReceiver,
|
||||
logSearchDebugInfo: Boolean = true)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
private[this] val maxCountStat = statsReceiver.stat("maxCount")
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
private[this] val excludedTweetIdsStat = statsReceiver.stat("excludedTweetIds")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val maxCount = maxCountProvider(envelope.query)
|
||||
maxCountStat.add(maxCount)
|
||||
|
||||
val excludedTweetIdsOpt = envelope.query.excludedTweetIds
|
||||
excludedTweetIdsOpt.foreach { excludedTweetIds =>
|
||||
excludedTweetIdsStat.add(excludedTweetIds.size)
|
||||
}
|
||||
|
||||
val tweetIdRange = envelope.query.range
|
||||
.map(TweetIdRange.fromTimelineRange)
|
||||
.getOrElse(TweetIdRange.default)
|
||||
|
||||
val beforeTweetIdExclusive = tweetIdRange.toId
|
||||
val afterTweetIdExclusive = tweetIdRange.fromId
|
||||
|
||||
val returnAllResults = returnAllResultsProvider(envelope.query)
|
||||
val relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcessProvider(envelope.query)
|
||||
|
||||
Future
|
||||
.join(
|
||||
envelope.followGraphData.followedUserIdsFuture,
|
||||
envelope.followGraphData.retweetsMutedUserIdsFuture
|
||||
).flatMap {
|
||||
case (followedIds, retweetsMutedIds) =>
|
||||
val followedIdsIncludingSelf = followedIds.toSet + envelope.query.userId
|
||||
|
||||
searchClient
|
||||
.getUsersTweetsForRecap(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedIdsIncludingSelf,
|
||||
retweetsMutedUserIds = retweetsMutedIds,
|
||||
maxCount = maxCount,
|
||||
tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options),
|
||||
searchOperator = envelope.query.searchOperator,
|
||||
beforeTweetIdExclusive = beforeTweetIdExclusive,
|
||||
afterTweetIdExclusive = afterTweetIdExclusive,
|
||||
enableSettingTweetTypesWithTweetKindOption =
|
||||
enableSettingTweetTypesWithTweetKindOptionProvider(envelope.query),
|
||||
excludedTweetIds = excludedTweetIdsOpt,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
getOnlyProtectedTweets = false,
|
||||
logSearchDebugInfo = logSearchDebugInfo,
|
||||
returnAllResults = returnAllResults,
|
||||
enableExcludeSourceTweetIdsQuery =
|
||||
enableExcludeSourceTweetIdsProvider(envelope.query),
|
||||
relevanceSearch = relevanceSearchProvider(envelope.query),
|
||||
searchClientId = perRequestSearchClientIdProvider(envelope.query),
|
||||
relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcess
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.util.SearchResultUtil
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Truncate the search results by score. Assumes that the search results are sorted in
|
||||
* score-descending order unless extraSortBeforeTruncation is set to true.
|
||||
*
|
||||
* This transform has two main use cases:
|
||||
*
|
||||
* - when returnAllResults is set to true, earlybird returns (numResultsPerShard * number of shards)
|
||||
* results. this transform is then used to further truncate the result, so that the size will be the
|
||||
* same as when returnAllResults is set to false.
|
||||
*
|
||||
* - we retrieve extra number of results from earlybird, as specified in MaxCountMultiplierParam,
|
||||
* so that we are left with sufficient number of candidates after hydration and filtering.
|
||||
* this transform will be used to get rid of extra results we ended up not using.
|
||||
*/
|
||||
class RecapSearchResultsTruncationTransform(
|
||||
extraSortBeforeTruncationGate: DependencyProvider[Boolean],
|
||||
maxCountProvider: DependencyProvider[Int],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
private[this] val postTruncationSizeStat = statsReceiver.stat("postTruncationSize")
|
||||
private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val sortBeforeTruncation = extraSortBeforeTruncationGate(envelope.query)
|
||||
val maxCount = maxCountProvider(envelope.query)
|
||||
val searchResults = envelope.searchResults
|
||||
|
||||
// set aside results that are marked by isRandomTweet field
|
||||
val (randomTweetSeq, searchResultsExcludingRandom) = searchResults.partition { result =>
|
||||
result.tweetFeatures.flatMap(_.isRandomTweet).getOrElse(false)
|
||||
}
|
||||
|
||||
// sort and truncate searchResults other than the random tweet
|
||||
val maxCountExcludingRandom = Math.max(0, maxCount - randomTweetSeq.size)
|
||||
|
||||
val truncatedResultsExcludingRandom =
|
||||
if (sortBeforeTruncation || searchResultsExcludingRandom.size > maxCountExcludingRandom) {
|
||||
val sorted = if (sortBeforeTruncation) {
|
||||
searchResultsExcludingRandom.sortWith(
|
||||
SearchResultUtil.getScore(_) > SearchResultUtil.getScore(_))
|
||||
} else searchResultsExcludingRandom
|
||||
sorted.take(maxCountExcludingRandom)
|
||||
} else searchResultsExcludingRandom
|
||||
|
||||
// put back the random tweet set aside previously
|
||||
val allTruncatedResults = truncatedResultsExcludingRandom ++ randomTweetSeq
|
||||
|
||||
// stats
|
||||
postTruncationSizeStat.add(allTruncatedResults.size)
|
||||
allTruncatedResults.foreach { result =>
|
||||
val earlybirdScoreX100 =
|
||||
result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100
|
||||
earlybirdScoreX100Stat.add(earlybirdScoreX100)
|
||||
}
|
||||
|
||||
Future.value(envelope.copy(searchResults = allTruncatedResults))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
import scala.collection.mutable
|
||||
|
||||
/**
|
||||
* Remove duplicate search results and order them reverse-chron.
|
||||
*/
|
||||
object SearchResultDedupAndSortingTransform
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val seenTweetIds = mutable.Set.empty[TweetId]
|
||||
val dedupedResults = envelope.searchResults
|
||||
.filter(result => seenTweetIds.add(result.id))
|
||||
.sortBy(_.id)(Ordering[TweetId].reverse)
|
||||
|
||||
val transformedEnvelope = envelope.copy(searchResults = dedupedResults)
|
||||
Future.value(transformedEnvelope)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.util.SourceTweetsUtil
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
|
||||
object SourceTweetsSearchResultsTransform {
|
||||
val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult]
|
||||
val EmptySearchResultsFuture: Future[Seq[ThriftSearchResult]] = Future.value(EmptySearchResults)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch source tweets for a given set of search results
|
||||
* Collects ids of source tweets, including extended reply and reply source tweets if needed,
|
||||
* fetches those tweets from search and populates them into the envelope
|
||||
*/
|
||||
class SourceTweetsSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
failOpenHandler: FailOpenHandler,
|
||||
hydrateReplyRootTweetProvider: DependencyProvider[Boolean],
|
||||
perRequestSourceSearchClientIdProvider: DependencyProvider[Option[String]],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import SourceTweetsSearchResultsTransform._
|
||||
|
||||
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
failOpenHandler {
|
||||
envelope.followGraphData.followedUserIdsFuture.flatMap { followedUserIds =>
|
||||
// NOTE: tweetIds are pre-computed as a performance optimisation.
|
||||
val searchResultsTweetIds = envelope.searchResults.map(_.id).toSet
|
||||
val sourceTweetIds = SourceTweetsUtil.getSourceTweetIds(
|
||||
searchResults = envelope.searchResults,
|
||||
searchResultsTweetIds = searchResultsTweetIds,
|
||||
followedUserIds = followedUserIds,
|
||||
shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query),
|
||||
statsReceiver = scopedStatsReceiver
|
||||
)
|
||||
if (sourceTweetIds.isEmpty) {
|
||||
EmptySearchResultsFuture
|
||||
} else {
|
||||
searchClient.getTweetsScoredForRecap(
|
||||
userId = envelope.query.userId,
|
||||
tweetIds = sourceTweetIds,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
logSearchDebugInfo = false,
|
||||
searchClientId = perRequestSourceSearchClientIdProvider(envelope.query)
|
||||
)
|
||||
}
|
||||
}
|
||||
} { _: Throwable => EmptySearchResultsFuture }.map { sourceSearchResults =>
|
||||
envelope.copy(sourceSearchResults = sourceSearchResults)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* trims searchResults to match with hydratedTweets
|
||||
* (if we previously filtered out hydrated tweets, this transform filters the search result set
|
||||
* down to match the hydrated tweets.)
|
||||
*/
|
||||
object TrimToMatchHydratedTweetsTransform
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val filteredSearchResults =
|
||||
trimSearchResults(envelope.searchResults, envelope.hydratedTweets.outerTweets)
|
||||
val filteredSourceSearchResults =
|
||||
trimSearchResults(envelope.sourceSearchResults, envelope.sourceHydratedTweets.outerTweets)
|
||||
|
||||
Future.value(
|
||||
envelope.copy(
|
||||
searchResults = filteredSearchResults,
|
||||
sourceSearchResults = filteredSourceSearchResults
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private def trimSearchResults(
|
||||
searchResults: Seq[ThriftSearchResult],
|
||||
hydratedTweets: Seq[HydratedTweet]
|
||||
): Seq[ThriftSearchResult] = {
|
||||
val filteredTweetIds = hydratedTweets.map(_.tweetId).toSet
|
||||
searchResults.filter(result => filteredTweetIds.contains(result.id))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.util.SourceTweetsUtil
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* trims elements of the envelope other than the searchResults
|
||||
* (i.e. sourceSearchResults, hydratedTweets, sourceHydratedTweets) to match with searchResults.
|
||||
*/
|
||||
class TrimToMatchSearchResultsTransform(
|
||||
hydrateReplyRootTweetProvider: DependencyProvider[Boolean],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val searchResults = envelope.searchResults
|
||||
val searchResultsIds = searchResults.map(_.id).toSet
|
||||
|
||||
// Trim rest of the seqs to match top search results.
|
||||
val hydratedTweets = envelope.hydratedTweets.outerTweets
|
||||
val topHydratedTweets = hydratedTweets.filter(ht => searchResultsIds.contains(ht.tweetId))
|
||||
|
||||
envelope.followGraphData.followedUserIdsFuture.map { followedUserIds =>
|
||||
val sourceTweetIdsOfTopResults =
|
||||
SourceTweetsUtil
|
||||
.getSourceTweetIds(
|
||||
searchResults = searchResults,
|
||||
searchResultsTweetIds = searchResultsIds,
|
||||
followedUserIds = followedUserIds,
|
||||
shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query),
|
||||
statsReceiver = scopedStatsReceiver
|
||||
).toSet
|
||||
val sourceTweetSearchResultsForTopN =
|
||||
envelope.sourceSearchResults.filter(r => sourceTweetIdsOfTopResults.contains(r.id))
|
||||
val hydratedSourceTweetsForTopN =
|
||||
envelope.sourceHydratedTweets.outerTweets.filter(ht =>
|
||||
sourceTweetIdsOfTopResults.contains(ht.tweetId))
|
||||
|
||||
val hydratedTweetsForEnvelope = envelope.hydratedTweets.copy(outerTweets = topHydratedTweets)
|
||||
val hydratedSourceTweetsForEnvelope =
|
||||
envelope.sourceHydratedTweets.copy(outerTweets = hydratedSourceTweetsForTopN)
|
||||
|
||||
envelope.copy(
|
||||
hydratedTweets = hydratedTweetsForEnvelope,
|
||||
searchResults = searchResults,
|
||||
sourceHydratedTweets = hydratedSourceTweetsForEnvelope,
|
||||
sourceSearchResults = sourceTweetSearchResultsForTopN
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.IndividualRequestTimeoutException
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedTweets
|
||||
import com.twitter.timelineranker.model.PartiallyHydratedTweet
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.util.Future
|
||||
|
||||
object TweetHydrationTransform {
|
||||
val EmptyHydratedTweets: HydratedTweets =
|
||||
HydratedTweets(Seq.empty[HydratedTweet], Seq.empty[HydratedTweet])
|
||||
val EmptyHydratedTweetsFuture: Future[HydratedTweets] = Future.value(EmptyHydratedTweets)
|
||||
}
|
||||
|
||||
object CandidateTweetHydrationTransform extends TweetHydrationTransform {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
hydrate(
|
||||
searchResults = envelope.searchResults,
|
||||
envelope = envelope
|
||||
).map { tweets => envelope.copy(hydratedTweets = tweets) }
|
||||
}
|
||||
}
|
||||
|
||||
object SourceTweetHydrationTransform extends TweetHydrationTransform {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
hydrate(
|
||||
searchResults = envelope.sourceSearchResults,
|
||||
envelope = envelope
|
||||
).map { tweets => envelope.copy(sourceHydratedTweets = tweets) }
|
||||
}
|
||||
}
|
||||
|
||||
// Static IRTE to indicate timeout in tweet hydrator. Placeholder timeout duration of 0 millis is used
|
||||
// since we are only concerned with the source of the exception.
|
||||
object TweetHydrationTimeoutException extends IndividualRequestTimeoutException(0.millis) {
|
||||
serviceName = "tweetHydrator"
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform which hydrates tweets in the CandidateEnvelope
|
||||
**/
|
||||
trait TweetHydrationTransform extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
import TweetHydrationTransform._
|
||||
|
||||
protected def hydrate(
|
||||
searchResults: Seq[ThriftSearchResult],
|
||||
envelope: CandidateEnvelope
|
||||
): Future[HydratedTweets] = {
|
||||
if (searchResults.nonEmpty) {
|
||||
Future.value(
|
||||
HydratedTweets(searchResults.map(PartiallyHydratedTweet.fromSearchResult))
|
||||
)
|
||||
} else {
|
||||
EmptyHydratedTweetsFuture
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelineranker.parameters.recap.RecapParams
|
||||
import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsParams
|
||||
import com.twitter.timelineranker.util.TweetFilters
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
import com.twitter.util.Future
|
||||
import scala.collection.mutable
|
||||
|
||||
object TweetKindOptionHydratedTweetsFilterTransform {
|
||||
private[common] val enableExpandedExtendedRepliesGate: Gate[RecapQuery] =
|
||||
RecapQuery.paramGate(RecapParams.EnableExpandedExtendedRepliesFilterParam)
|
||||
|
||||
private[common] val excludeRecommendedRepliesToNonFollowedUsersGate: Gate[RecapQuery] =
|
||||
RecapQuery.paramGate(
|
||||
UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeRecommendedRepliesToNonFollowedUsersParam)
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter hydrated tweets dynamically based on TweetKindOptions in the query.
|
||||
*/
|
||||
class TweetKindOptionHydratedTweetsFilterTransform(
|
||||
useFollowGraphData: Boolean,
|
||||
useSourceTweets: Boolean,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import TweetKindOptionHydratedTweetsFilterTransform._
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val filters = convertToFilters(envelope)
|
||||
|
||||
val filterTransform = if (filters == TweetFilters.ValueSet.empty) {
|
||||
FutureArrow.identity[CandidateEnvelope]
|
||||
} else {
|
||||
new HydratedTweetsFilterTransform(
|
||||
outerFilters = filters,
|
||||
innerFilters = TweetFilters.None,
|
||||
useFollowGraphData = useFollowGraphData,
|
||||
useSourceTweets = useSourceTweets,
|
||||
statsReceiver = statsReceiver,
|
||||
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
|
||||
)
|
||||
}
|
||||
|
||||
filterTransform.apply(envelope)
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the given query options to equivalent TweetFilter values.
|
||||
*
|
||||
* Note:
|
||||
* -- The semantic of TweetKindOption is opposite of that of TweetFilters.
|
||||
* TweetKindOption values are of the form IncludeX. That is, they result in X being added.
|
||||
* TweetFilters values specify what to exclude.
|
||||
* -- IncludeExtendedReplies requires IncludeReplies to be also specified to be effective.
|
||||
*/
|
||||
private[common] def convertToFilters(envelope: CandidateEnvelope): TweetFilters.ValueSet = {
|
||||
val queryOptions = envelope.query.options
|
||||
val filters = mutable.Set.empty[TweetFilters.Value]
|
||||
if (queryOptions.contains(TweetKindOption.IncludeReplies)) {
|
||||
if (excludeRecommendedRepliesToNonFollowedUsersGate(
|
||||
envelope.query) && envelope.query.utegLikedByTweetsOptions.isDefined) {
|
||||
filters += TweetFilters.RecommendedRepliesToNotFollowedUsers
|
||||
} else if (queryOptions.contains(TweetKindOption.IncludeExtendedReplies)) {
|
||||
if (enableExpandedExtendedRepliesGate(envelope.query)) {
|
||||
filters += TweetFilters.NotValidExpandedExtendedReplies
|
||||
} else {
|
||||
filters += TweetFilters.NotQualifiedExtendedReplies
|
||||
}
|
||||
} else {
|
||||
filters += TweetFilters.ExtendedReplies
|
||||
}
|
||||
} else {
|
||||
filters += TweetFilters.Replies
|
||||
}
|
||||
if (!queryOptions.contains(TweetKindOption.IncludeRetweets)) {
|
||||
filters += TweetFilters.Retweets
|
||||
}
|
||||
TweetFilters.ValueSet.empty ++ filters
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.search.common.constants.thriftscala.ThriftLanguage
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.clients.manhattan.LanguageUtils
|
||||
import com.twitter.timelines.clients.manhattan.UserMetadataClient
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.service.metastore.gen.thriftscala.UserLanguages
|
||||
|
||||
object UserLanguagesTransform {
|
||||
val EmptyUserLanguagesFuture: Future[UserLanguages] =
|
||||
Future.value(UserMetadataClient.EmptyUserLanguages)
|
||||
}
|
||||
|
||||
/**
|
||||
* FutureArrow which fetches user languages
|
||||
* It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets
|
||||
*/
|
||||
class UserLanguagesTransform(handler: FailOpenHandler, userMetadataClient: UserMetadataClient)
|
||||
extends FutureArrow[RecapQuery, Seq[ThriftLanguage]] {
|
||||
override def apply(request: RecapQuery): Future[Seq[ThriftLanguage]] = {
|
||||
import UserLanguagesTransform._
|
||||
|
||||
handler {
|
||||
userMetadataClient.getUserLanguages(request.userId)
|
||||
} { _: Throwable => EmptyUserLanguagesFuture }
|
||||
}.map(LanguageUtils.computeLanguages(_))
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
|
||||
import com.twitter.timelines.clients.gizmoduck.UserProfileInfo
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
|
||||
object UserProfileInfoTransform {
|
||||
val EmptyUserProfileInfo: UserProfileInfo = UserProfileInfo(None, None, None, None)
|
||||
val EmptyUserProfileInfoFuture: Future[UserProfileInfo] = Future.value(EmptyUserProfileInfo)
|
||||
}
|
||||
|
||||
/**
|
||||
* FutureArrow which fetches user profile info
|
||||
* It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets
|
||||
*/
|
||||
class UserProfileInfoTransform(handler: FailOpenHandler, gizmoduckClient: GizmoduckClient)
|
||||
extends FutureArrow[RecapQuery, UserProfileInfo] {
|
||||
import UserProfileInfoTransform._
|
||||
override def apply(request: RecapQuery): Future[UserProfileInfo] = {
|
||||
handler {
|
||||
gizmoduckClient.getProfileInfo(request.userId).map { profileInfoOpt =>
|
||||
profileInfoOpt.getOrElse(EmptyUserProfileInfo)
|
||||
}
|
||||
} { _: Throwable => EmptyUserProfileInfoFuture }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedTweets
|
||||
import com.twitter.timelines.visibility.VisibilityEnforcer
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Transform which uses an instance of a VisiblityEnforcer to filter down HydratedTweets
|
||||
*/
|
||||
class VisibilityEnforcingTransform(visibilityEnforcer: VisibilityEnforcer)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
visibilityEnforcer.apply(Some(envelope.query.userId), envelope.hydratedTweets.outerTweets).map {
|
||||
visibleTweets =>
|
||||
val innerTweets = envelope.hydratedTweets.innerTweets
|
||||
envelope.copy(
|
||||
hydratedTweets = HydratedTweets(outerTweets = visibleTweets, innerTweets = innerTweets))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"3rdparty/jvm/org/apache/thrift:libthrift",
|
||||
"abdecider",
|
||||
"cortex-tweet-annotate/service/src/main/thrift:thrift-scala",
|
||||
"decider",
|
||||
"featureswitches/featureswitches-core/src/main/scala",
|
||||
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
|
||||
"finagle/finagle-core/src/main",
|
||||
"finagle/finagle-memcached/src/main/scala",
|
||||
"finagle/finagle-stats",
|
||||
"finagle/finagle-thrift/src/main/java",
|
||||
"finagle/finagle-thrift/src/main/scala",
|
||||
"finagle/finagle-thriftmux/src/main/scala",
|
||||
"loglens/loglens-logging/src/main/scala",
|
||||
"merlin/util/src/main/scala",
|
||||
"servo/decider",
|
||||
"servo/request/src/main/scala",
|
||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||
"src/thrift/com/twitter/manhattan:v1-scala",
|
||||
"src/thrift/com/twitter/merlin:thrift-scala",
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
||||
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
||||
"src/thrift/com/twitter/timelineservice:thrift-scala",
|
||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
"timelineranker/server/config",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/clients",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines:authorization",
|
||||
"timelines:config",
|
||||
"timelines:features",
|
||||
"timelines:util",
|
||||
"timelines:visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/memcache_common",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/socialgraph",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/strato/realgraph",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config/configapi",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/logging",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||
"twitter-server-internal",
|
||||
"util/util-app",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
exports = [
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.util.Duration
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Information about a single method call.
|
||||
*
|
||||
* The purpose of this class is to allow one to express a call graph and latency associated with each (sub)call.
|
||||
* Once a call graph is defined, calling getOverAllLatency() off the top level call returns total time taken by that call.
|
||||
* That value can then be compared with the timeout budget allocated to that call to see if the
|
||||
* value fits within the overall timeout budget of that call.
|
||||
*
|
||||
* This is useful in case of a complex call graph where it is hard to mentally estimate the effect on
|
||||
* overall latency when updating timeout value of one or more sub-calls.
|
||||
*
|
||||
* @param methodName name of the called method.
|
||||
* @param latency P999 Latency incurred in calling a service if the method calls an external service. Otherwise 0.
|
||||
* @param dependsOn Other calls that this call depends on.
|
||||
*/
|
||||
case class Call(
|
||||
methodName: String,
|
||||
latency: Duration = 0.milliseconds,
|
||||
dependsOn: Seq[Call] = Nil) {
|
||||
|
||||
/**
|
||||
* Latency incurred in this call as well as recursively all calls this call depends on.
|
||||
*/
|
||||
def getOverAllLatency: Duration = {
|
||||
val dependencyLatency = if (dependsOn.isEmpty) {
|
||||
0.milliseconds
|
||||
} else {
|
||||
dependsOn.map(_.getOverAllLatency).max
|
||||
}
|
||||
latency + dependencyLatency
|
||||
}
|
||||
|
||||
/**
|
||||
* Call paths starting at this call and recursively traversing all dependencies.
|
||||
* Typically used for debugging or logging.
|
||||
*/
|
||||
def getLatencyPaths: String = {
|
||||
val sb = new StringBuilder
|
||||
getLatencyPaths(sb, 1)
|
||||
sb.toString
|
||||
}
|
||||
|
||||
def getLatencyPaths(sb: StringBuilder, level: Int): Unit = {
|
||||
sb.append(s"${getPrefix(level)} ${getLatencyString(getOverAllLatency)} $methodName\n")
|
||||
if ((latency > 0.milliseconds) && !dependsOn.isEmpty) {
|
||||
sb.append(s"${getPrefix(level + 1)} ${getLatencyString(latency)} self\n")
|
||||
}
|
||||
dependsOn.foreach(_.getLatencyPaths(sb, level + 1))
|
||||
}
|
||||
|
||||
private def getLatencyString(latencyValue: Duration): String = {
|
||||
val latencyMs = latencyValue.inUnit(TimeUnit.MILLISECONDS)
|
||||
f"[$latencyMs%3d]"
|
||||
}
|
||||
|
||||
private def getPrefix(level: Int): String = {
|
||||
" " * (level * 4) + "--"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Information about the getRecapTweetCandidates call.
|
||||
*
|
||||
* Acronyms used:
|
||||
* : call internal to TLR
|
||||
* EB : Earlybird (search super root)
|
||||
* GZ : Gizmoduck
|
||||
* MH : Manhattan
|
||||
* SGS : Social graph service
|
||||
*
|
||||
* The latency values are based on p9999 values observed over 1 week.
|
||||
*/
|
||||
object GetRecycledTweetCandidatesCall {
|
||||
val getUserProfileInfo: Call = Call("GZ.getUserProfileInfo", 200.milliseconds)
|
||||
val getUserLanguages: Call = Call("MH.getUserLanguages", 300.milliseconds) // p99: 15ms
|
||||
|
||||
val getFollowing: Call = Call("SGS.getFollowing", 250.milliseconds) // p99: 75ms
|
||||
val getMutuallyFollowing: Call =
|
||||
Call("SGS.getMutuallyFollowing", 400.milliseconds, Seq(getFollowing)) // p99: 100
|
||||
val getVisibilityProfiles: Call =
|
||||
Call("SGS.getVisibilityProfiles", 400.milliseconds, Seq(getFollowing)) // p99: 100
|
||||
val getVisibilityData: Call = Call(
|
||||
"getVisibilityData",
|
||||
dependsOn = Seq(getFollowing, getMutuallyFollowing, getVisibilityProfiles)
|
||||
)
|
||||
val getTweetsForRecapRegular: Call =
|
||||
Call("EB.getTweetsForRecap(regular)", 500.milliseconds, Seq(getVisibilityData)) // p99: 250
|
||||
val getTweetsForRecapProtected: Call =
|
||||
Call("EB.getTweetsForRecap(protected)", 250.milliseconds, Seq(getVisibilityData)) // p99: 150
|
||||
val getSearchResults: Call =
|
||||
Call("getSearchResults", dependsOn = Seq(getTweetsForRecapRegular, getTweetsForRecapProtected))
|
||||
val getTweetsScoredForRecap: Call =
|
||||
Call("EB.getTweetsScoredForRecap", 400.milliseconds, Seq(getSearchResults)) // p99: 100
|
||||
|
||||
val hydrateSearchResults: Call = Call("hydrateSearchResults")
|
||||
val getSourceTweetSearchResults: Call =
|
||||
Call("getSourceTweetSearchResults", dependsOn = Seq(getSearchResults))
|
||||
val hydrateTweets: Call =
|
||||
Call("hydrateTweets", dependsOn = Seq(getSearchResults, hydrateSearchResults))
|
||||
val hydrateSourceTweets: Call =
|
||||
Call("hydrateSourceTweets", dependsOn = Seq(getSourceTweetSearchResults, hydrateSearchResults))
|
||||
val topLevel: Call = Call(
|
||||
"getRecapTweetCandidates",
|
||||
dependsOn = Seq(
|
||||
getUserProfileInfo,
|
||||
getUserLanguages,
|
||||
getVisibilityData,
|
||||
getSearchResults,
|
||||
hydrateSearchResults,
|
||||
hydrateSourceTweets
|
||||
)
|
||||
)
|
||||
}
|
||||
|
|
@ -1,287 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.timelineranker.decider.DeciderKey._
|
||||
import com.twitter.timelines.authorization.TrustedPermission
|
||||
import com.twitter.timelines.authorization.RateLimitingTrustedPermission
|
||||
import com.twitter.timelines.authorization.RateLimitingUntrustedPermission
|
||||
import com.twitter.timelines.authorization.ClientDetails
|
||||
|
||||
object ClientAccessPermissions {
|
||||
// We want timelineranker locked down for requests outside of what's defined here.
|
||||
val DefaultRateLimit = 0d
|
||||
|
||||
def unknown(name: String): ClientDetails = {
|
||||
ClientDetails(name, RateLimitingUntrustedPermission(RateLimitOverrideUnknown, DefaultRateLimit))
|
||||
}
|
||||
|
||||
val All: Seq[ClientDetails] = Seq(
|
||||
/**
|
||||
* Production clients for timelinemixer.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinemixer.recap.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecapProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.recycled.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate_recos.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.seed_author_ids.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.simcluster.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.entity_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* This client is whitelisted for timelinemixer only as it used by
|
||||
* List injection service which will not be migrated to timelinescorer.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinemixer.list.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerListProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.list_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerCommunityProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.uteg_liked_by_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* Production clients for timelinescorer. Most of these clients have their
|
||||
* equivalents under the timelinemixer scope (with exception of list injection
|
||||
* client).
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinescorer.recap.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecapProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.recycled.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate_recos.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.seed_author_ids.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.simcluster.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.entity_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.list_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.uteg_liked_by_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelineservice.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineServiceProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate_tweet_scoring.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerHydrateTweetScoringProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.community_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.recommended_trend_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerRecommendedTrendTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.rec_topic_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerRecTopicTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.popular_topic_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerPopularTopicTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* TimelineRanker utilities. Traffic proxy, warmups, and console.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelineranker.proxy",
|
||||
RateLimitingTrustedPermission(AllowTimelineRankerProxy),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
TimelineRankerConstants.WarmupClientName,
|
||||
RateLimitingTrustedPermission(AllowTimelineRankerWarmup),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
TimelineRankerConstants.ForwardedClientName,
|
||||
RateLimitingTrustedPermission(AllowTimelineRankerWarmup),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelineranker.console",
|
||||
RateLimitingUntrustedPermission(RateLimitOverrideUnknown, 1d),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* Staging clients.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinemixer.recap.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.recycled.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate_recos.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.seed_author_ids.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.simcluster.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.entity_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.list.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.list_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.community_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.recommended_trend_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.uteg_liked_by_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.entity_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate_tweet_scoring.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.rec_topic_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.popular_topic_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelineservice.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineServiceStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
)
|
||||
)
|
||||
}
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelineranker.clients.ScopedCortexTweetQueryServiceClientFactory
|
||||
import com.twitter.timelines.clients.gizmoduck.ScopedGizmoduckClientFactory
|
||||
import com.twitter.timelines.clients.manhattan.ScopedUserMetadataClientFactory
|
||||
import com.twitter.timelines.clients.socialgraph.ScopedSocialGraphClientFactory
|
||||
import com.twitter.timelines.clients.strato.realgraph.ScopedRealGraphClientFactory
|
||||
import com.twitter.timelines.clients.tweetypie.AdditionalFieldConfig
|
||||
import com.twitter.timelines.clients.tweetypie.ScopedTweetyPieClientFactory
|
||||
import com.twitter.timelines.visibility.VisibilityEnforcerFactory
|
||||
import com.twitter.timelines.visibility.VisibilityProfileHydratorFactory
|
||||
import com.twitter.tweetypie.thriftscala.{Tweet => TTweet}
|
||||
|
||||
class ClientWrapperFactories(config: RuntimeConfiguration) {
|
||||
private[this] val statsReceiver = config.statsReceiver
|
||||
|
||||
val cortexTweetQueryServiceClientFactory: ScopedCortexTweetQueryServiceClientFactory =
|
||||
new ScopedCortexTweetQueryServiceClientFactory(
|
||||
config.underlyingClients.cortexTweetQueryServiceClient,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val gizmoduckClientFactory: ScopedGizmoduckClientFactory = new ScopedGizmoduckClientFactory(
|
||||
config.underlyingClients.gizmoduckClient,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val socialGraphClientFactory: ScopedSocialGraphClientFactory = new ScopedSocialGraphClientFactory(
|
||||
config.underlyingClients.sgsClient,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val visibilityEnforcerFactory: VisibilityEnforcerFactory = new VisibilityEnforcerFactory(
|
||||
gizmoduckClientFactory,
|
||||
socialGraphClientFactory,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val tweetyPieAdditionalFieldsToDisable: Seq[Short] = Seq(
|
||||
TTweet.MediaTagsField.id,
|
||||
TTweet.SchedulingInfoField.id,
|
||||
TTweet.EscherbirdEntityAnnotationsField.id,
|
||||
TTweet.CardReferenceField.id,
|
||||
TTweet.SelfPermalinkField.id,
|
||||
TTweet.ExtendedTweetMetadataField.id,
|
||||
TTweet.CommunitiesField.id,
|
||||
TTweet.VisibleTextRangeField.id
|
||||
)
|
||||
|
||||
val tweetyPieHighQoSClientFactory: ScopedTweetyPieClientFactory =
|
||||
new ScopedTweetyPieClientFactory(
|
||||
tweetyPieClient = config.underlyingClients.tweetyPieHighQoSClient,
|
||||
additionalFieldConfig = AdditionalFieldConfig(
|
||||
fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap
|
||||
),
|
||||
includePartialResults = Gate.False,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val tweetyPieLowQoSClientFactory: ScopedTweetyPieClientFactory = new ScopedTweetyPieClientFactory(
|
||||
tweetyPieClient = config.underlyingClients.tweetyPieLowQoSClient,
|
||||
additionalFieldConfig = AdditionalFieldConfig(
|
||||
fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap
|
||||
),
|
||||
includePartialResults = Gate.False,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val userMetadataClientFactory: ScopedUserMetadataClientFactory =
|
||||
new ScopedUserMetadataClientFactory(
|
||||
config.underlyingClients.manhattanStarbuckClient,
|
||||
TimelineRankerConstants.ManhattanStarbuckAppId,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val visibilityProfileHydratorFactory: VisibilityProfileHydratorFactory =
|
||||
new VisibilityProfileHydratorFactory(
|
||||
gizmoduckClientFactory,
|
||||
socialGraphClientFactory,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val realGraphClientFactory =
|
||||
new ScopedRealGraphClientFactory(config.underlyingClients.stratoClient, statsReceiver)
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.model.TweetId
|
||||
class ClientWrappers(config: RuntimeConfiguration) {
|
||||
private[this] val backendClientConfiguration = config.underlyingClients
|
||||
|
||||
val contentFeaturesCache: Store[TweetId, ContentFeatures] =
|
||||
backendClientConfiguration.contentFeaturesCache
|
||||
}
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.conversions.PercentOps._
|
||||
import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService
|
||||
import com.twitter.finagle.ssl.OpportunisticTls
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finagle.util.DefaultTimer
|
||||
import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck}
|
||||
import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1}
|
||||
import com.twitter.merlin.thriftscala.UserRolesService
|
||||
import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph
|
||||
import com.twitter.socialgraph.thriftscala.SocialGraphService
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.strato.client.Strato
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.timelineranker.clients.content_features_cache.ContentFeaturesMemcacheBuilder
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.thriftscala.TimelineRanker
|
||||
import com.twitter.timelines.clients.memcache_common.StorehausMemcacheConfig
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelineservice.thriftscala.TimelineService
|
||||
import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie}
|
||||
import com.twitter.util.Timer
|
||||
import org.apache.thrift.protocol.TCompactProtocol
|
||||
|
||||
class DefaultUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver)
|
||||
extends UnderlyingClientConfiguration(flags, statsReceiver) { top =>
|
||||
|
||||
val timer: Timer = DefaultTimer
|
||||
val twCachePrefix = "/srv#/prod/local/cache"
|
||||
|
||||
override val cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[
|
||||
CortexTweetQueryService.ServicePerEndpoint,
|
||||
CortexTweetQueryService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("cortex-tweet-query", requireMutualTls = true)
|
||||
.dest("/s/cortex-tweet-annotate/cortex-tweet-query")
|
||||
.requestTimeout(200.milliseconds)
|
||||
.timeout(500.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override val gizmoduckClient: Gizmoduck.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[Gizmoduck.ServicePerEndpoint, Gizmoduck.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("gizmoduck", requireMutualTls = true)
|
||||
.dest("/s/gizmoduck/gizmoduck")
|
||||
.requestTimeout(400.milliseconds)
|
||||
.timeout(900.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override lazy val manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[ManhattanV1.ServicePerEndpoint, ManhattanV1.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("manhattan.starbuck", requireMutualTls = true)
|
||||
.dest("/s/manhattan/starbuck.native-thrift")
|
||||
.requestTimeout(600.millis)
|
||||
)
|
||||
}
|
||||
|
||||
override val sgsClient: SocialGraphService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[
|
||||
SocialGraphService.ServicePerEndpoint,
|
||||
SocialGraphService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("socialgraph", requireMutualTls = true)
|
||||
.dest("/s/socialgraph/socialgraph")
|
||||
.requestTimeout(350.milliseconds)
|
||||
.timeout(700.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override lazy val timelineRankerForwardingClient: TimelineRanker.FinagledClient =
|
||||
new TimelineRanker.FinagledClient(
|
||||
thriftMuxClientBuilder(
|
||||
TimelineRankerConstants.ForwardedClientName,
|
||||
ClientId(TimelineRankerConstants.ForwardedClientName),
|
||||
protocolFactoryOption = Some(new TCompactProtocol.Factory()),
|
||||
requireMutualTls = true
|
||||
).dest("/s/timelineranker/timelineranker:compactthrift").build(),
|
||||
protocolFactory = new TCompactProtocol.Factory()
|
||||
)
|
||||
|
||||
override val timelineServiceClient: TimelineService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[TimelineService.ServicePerEndpoint, TimelineService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("timelineservice", requireMutualTls = true)
|
||||
.dest("/s/timelineservice/timelineservice")
|
||||
.requestTimeout(600.milliseconds)
|
||||
.timeout(800.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override val tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("tweetypieHighQoS", requireMutualTls = true)
|
||||
.dest("/s/tweetypie/tweetypie")
|
||||
.requestTimeout(450.milliseconds)
|
||||
.timeout(800.milliseconds),
|
||||
maxExtraLoadPercent = Some(1.percent)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide less costly TweetPie client with the trade-off of reduced quality. Intended for use cases
|
||||
* which are not essential for successful completion of timeline requests. Currently this client differs
|
||||
* from the highQoS endpoint by having retries count set to 1 instead of 2.
|
||||
*/
|
||||
override val tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("tweetypieLowQoS", requireMutualTls = true)
|
||||
.dest("/s/tweetypie/tweetypie")
|
||||
.retryPolicy(mkRetryPolicy(1)) // override default value
|
||||
.requestTimeout(450.milliseconds)
|
||||
.timeout(800.milliseconds),
|
||||
maxExtraLoadPercent = Some(1.percent)
|
||||
)
|
||||
}
|
||||
|
||||
override val userRolesServiceClient: UserRolesService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[
|
||||
UserRolesService.ServicePerEndpoint,
|
||||
UserRolesService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("merlin", requireMutualTls = true)
|
||||
.dest("/s/merlin/merlin")
|
||||
.requestTimeout(1.second)
|
||||
)
|
||||
}
|
||||
|
||||
lazy val contentFeaturesCache: Store[TweetId, ContentFeatures] =
|
||||
new ContentFeaturesMemcacheBuilder(
|
||||
config = new StorehausMemcacheConfig(
|
||||
destName = s"$twCachePrefix/timelines_content_features:twemcaches",
|
||||
keyPrefix = "",
|
||||
requestTimeout = 50.milliseconds,
|
||||
numTries = 1,
|
||||
globalTimeout = 60.milliseconds,
|
||||
tcpConnectTimeout = 50.milliseconds,
|
||||
connectionAcquisitionTimeout = 25.milliseconds,
|
||||
numPendingRequests = 100,
|
||||
isReadOnly = false,
|
||||
serviceIdentifier = serviceIdentifier
|
||||
),
|
||||
ttl = 48.hours,
|
||||
statsReceiver
|
||||
).build
|
||||
|
||||
override val userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient =
|
||||
new UserTweetEntityGraph.FinagledClient(
|
||||
thriftMuxClientBuilder("user_tweet_entity_graph", requireMutualTls = true)
|
||||
.dest("/s/cassowary/user_tweet_entity_graph")
|
||||
.retryPolicy(mkRetryPolicy(2))
|
||||
.requestTimeout(300.milliseconds)
|
||||
.build()
|
||||
)
|
||||
|
||||
override val stratoClient: StratoClient =
|
||||
Strato.client.withMutualTls(serviceIdentifier, OpportunisticTls.Required).build()
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
|
||||
object RequestScopes {
|
||||
val HomeTimelineMaterialization: RequestScope = RequestScope("homeMaterialization")
|
||||
val InNetworkTweetSource: RequestScope = RequestScope("inNetworkTweet")
|
||||
val RecapHydrationSource: RequestScope = RequestScope("recapHydration")
|
||||
val RecapAuthorSource: RequestScope = RequestScope("recapAuthor")
|
||||
val ReverseChronHomeTimelineSource: RequestScope = RequestScope("reverseChronHomeTimelineSource")
|
||||
val EntityTweetsSource: RequestScope = RequestScope("entityTweets")
|
||||
val UtegLikedByTweetsSource: RequestScope = RequestScope("utegLikedByTweets")
|
||||
}
|
||||
|
|
@ -1,133 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.abdecider.ABDeciderFactory
|
||||
import com.twitter.abdecider.LoggingABDecider
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.featureswitches.Value
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.servo.util.Effect
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelines.authorization.TimelinesClientRequestAuthorizer
|
||||
import com.twitter.timelines.config._
|
||||
import com.twitter.timelines.config.configapi._
|
||||
import com.twitter.timelines.features._
|
||||
import com.twitter.timelines.util.ImpressionCountingABDecider
|
||||
import com.twitter.timelines.util.logging.Scribe
|
||||
import com.twitter.util.Await
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelines.model.UserId
|
||||
|
||||
trait ClientProvider {
|
||||
def clientWrappers: ClientWrappers
|
||||
def underlyingClients: UnderlyingClientConfiguration
|
||||
}
|
||||
|
||||
trait UtilityProvider {
|
||||
def abdecider: LoggingABDecider
|
||||
def clientRequestAuthorizer: TimelinesClientRequestAuthorizer
|
||||
def configStore: ConfigStore
|
||||
def decider: Decider
|
||||
def deciderGateBuilder: DeciderGateBuilder
|
||||
def employeeGate: UserRolesGate.EmployeeGate
|
||||
def configApiConfiguration: ConfigApiConfiguration
|
||||
def statsReceiver: StatsReceiver
|
||||
def whitelist: UserList
|
||||
}
|
||||
|
||||
trait RuntimeConfiguration extends ClientProvider with UtilityProvider with ConfigUtils {
|
||||
def isProd: Boolean
|
||||
def maxConcurrency: Int
|
||||
def clientEventScribe: Effect[String]
|
||||
def clientWrapperFactories: ClientWrapperFactories
|
||||
}
|
||||
|
||||
class RuntimeConfigurationImpl(
|
||||
flags: TimelineRankerFlags,
|
||||
configStoreFactory: DynamicConfigStoreFactory,
|
||||
val decider: Decider,
|
||||
val forcedFeatureValues: Map[String, Value] = Map.empty[String, Value],
|
||||
val statsReceiver: StatsReceiver)
|
||||
extends RuntimeConfiguration {
|
||||
|
||||
// Creates and initialize config store as early as possible so other parts could have a dependency on it for settings.
|
||||
override val configStore: DynamicConfigStore =
|
||||
configStoreFactory.createDcEnvAwareFileBasedConfigStore(
|
||||
relativeConfigFilePath = "timelines/timelineranker/service_settings.yml",
|
||||
dc = flags.getDatacenter,
|
||||
env = flags.getEnv,
|
||||
configBusConfig = ConfigBusProdConfig,
|
||||
onUpdate = ConfigStore.NullOnUpdateCallback,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
Await.result(configStore.init)
|
||||
|
||||
val environment: Env.Value = flags.getEnv
|
||||
override val isProd: Boolean = isProdEnv(environment)
|
||||
val datacenter: Datacenter.Value = flags.getDatacenter
|
||||
val abDeciderPath = "/usr/local/config/abdecider/abdecider.yml"
|
||||
override val maxConcurrency: Int = flags.maxConcurrency()
|
||||
|
||||
val deciderGateBuilder: DeciderGateBuilder = new DeciderGateBuilder(decider)
|
||||
|
||||
val clientRequestAuthorizer: TimelinesClientRequestAuthorizer =
|
||||
new TimelinesClientRequestAuthorizer(
|
||||
deciderGateBuilder = deciderGateBuilder,
|
||||
clientDetails = ClientAccessPermissions.All,
|
||||
unknownClientDetails = ClientAccessPermissions.unknown,
|
||||
clientAuthorizationGate =
|
||||
deciderGateBuilder.linearGate(DeciderKey.ClientRequestAuthorization),
|
||||
clientWriteWhitelistGate = deciderGateBuilder.linearGate(DeciderKey.ClientWriteWhitelist),
|
||||
globalCapacityQPS = flags.requestRateLimit(),
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
override val clientEventScribe = Scribe.clientEvent(isProd, statsReceiver)
|
||||
val abdecider: LoggingABDecider = new ImpressionCountingABDecider(
|
||||
abdecider = ABDeciderFactory.withScribeEffect(
|
||||
abDeciderYmlPath = abDeciderPath,
|
||||
scribeEffect = clientEventScribe,
|
||||
decider = None,
|
||||
environment = Some("production"),
|
||||
).buildWithLogging(),
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val underlyingClients: UnderlyingClientConfiguration = buildUnderlyingClientConfiguration
|
||||
|
||||
val clientWrappers: ClientWrappers = new ClientWrappers(this)
|
||||
override val clientWrapperFactories: ClientWrapperFactories = new ClientWrapperFactories(this)
|
||||
|
||||
private[this] val userRolesCacheFactory = new UserRolesCacheFactory(
|
||||
userRolesService = underlyingClients.userRolesServiceClient,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
override val whitelist: Whitelist = Whitelist(
|
||||
configStoreFactory = configStoreFactory,
|
||||
userRolesCacheFactory = userRolesCacheFactory,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
override val employeeGate: Gate[UserId] = UserRolesGate(
|
||||
userRolesCacheFactory.create(UserRoles.EmployeesRoleName)
|
||||
)
|
||||
|
||||
private[this] val featureRecipientFactory =
|
||||
new UserRolesCachingFeatureRecipientFactory(userRolesCacheFactory, statsReceiver)
|
||||
|
||||
override val configApiConfiguration: FeatureSwitchesV2ConfigApiConfiguration =
|
||||
FeatureSwitchesV2ConfigApiConfiguration(
|
||||
datacenter = flags.getDatacenter,
|
||||
serviceName = ServiceName.TimelineRanker,
|
||||
abdecider = abdecider,
|
||||
featureRecipientFactory = featureRecipientFactory,
|
||||
forcedValues = forcedFeatureValues,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
private[this] def buildUnderlyingClientConfiguration: UnderlyingClientConfiguration = {
|
||||
environment match {
|
||||
case Env.prod => new DefaultUnderlyingClientConfiguration(flags, statsReceiver)
|
||||
case _ => new StagingUnderlyingClientConfiguration(flags, statsReceiver)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
|
||||
class StagingUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver)
|
||||
extends DefaultUnderlyingClientConfiguration(flags, statsReceiver)
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
object TimelineRankerConstants {
|
||||
val ClientPrefix = "timelineranker."
|
||||
val ManhattanStarbuckAppId = "timelineranker"
|
||||
val WarmupClientName = "timelineranker.warmup"
|
||||
val ForwardedClientName = "timelineranker.forwarded"
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.app.Flags
|
||||
import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.timelines.config.CommonFlags
|
||||
import com.twitter.timelines.config.ConfigUtils
|
||||
import com.twitter.timelines.config.Datacenter
|
||||
import com.twitter.timelines.config.Env
|
||||
import com.twitter.timelines.config.ProvidesServiceIdentifier
|
||||
import java.net.InetSocketAddress
|
||||
import com.twitter.app.Flag
|
||||
|
||||
class TimelineRankerFlags(flag: Flags)
|
||||
extends CommonFlags(flag)
|
||||
with ConfigUtils
|
||||
with ProvidesServiceIdentifier {
|
||||
val dc: Flag[String] = flag(
|
||||
"dc",
|
||||
"smf1",
|
||||
"Name of data center in which this instance will execute"
|
||||
)
|
||||
val environment: Flag[String] = flag(
|
||||
"environment",
|
||||
"devel",
|
||||
"The mesos environment in which this instance will be running"
|
||||
)
|
||||
val maxConcurrency: Flag[Int] = flag(
|
||||
"maxConcurrency",
|
||||
200,
|
||||
"Maximum concurrent requests"
|
||||
)
|
||||
val servicePort: Flag[InetSocketAddress] = flag(
|
||||
"service.port",
|
||||
new InetSocketAddress(8287),
|
||||
"Port number that this thrift service will listen on"
|
||||
)
|
||||
val serviceCompactPort: Flag[InetSocketAddress] = flag(
|
||||
"service.compact.port",
|
||||
new InetSocketAddress(8288),
|
||||
"Port number that the TCompactProtocol-based thrift service will listen on"
|
||||
)
|
||||
|
||||
val serviceIdentifier: Flag[ServiceIdentifier] = flag[ServiceIdentifier](
|
||||
"service.identifier",
|
||||
EmptyServiceIdentifier,
|
||||
"service identifier for this service for use with mutual TLS, " +
|
||||
"format is expected to be -service.identifier=\"role:service:environment:zone\""
|
||||
)
|
||||
|
||||
val opportunisticTlsLevel = flag[String](
|
||||
"opportunistic.tls.level",
|
||||
"desired",
|
||||
"The server's OpportunisticTls level."
|
||||
)
|
||||
|
||||
val requestRateLimit: Flag[Double] = flag[Double](
|
||||
"requestRateLimit",
|
||||
1000.0,
|
||||
"Request rate limit to be used by the client request authorizer"
|
||||
)
|
||||
|
||||
val enableThriftmuxCompression = flag(
|
||||
"enableThriftmuxServerCompression",
|
||||
true,
|
||||
"build server with thriftmux compression enabled"
|
||||
)
|
||||
|
||||
def getDatacenter: Datacenter.Value = getDC(dc())
|
||||
def getEnv: Env.Value = getEnv(environment())
|
||||
override def getServiceIdentifier: ServiceIdentifier = serviceIdentifier()
|
||||
}
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService
|
||||
import com.twitter.finagle.Service
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.service.RetryPolicy
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finagle.thrift.ThriftClientRequest
|
||||
import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck}
|
||||
import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1}
|
||||
import com.twitter.merlin.thriftscala.UserRolesService
|
||||
import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph
|
||||
import com.twitter.search.earlybird.thriftscala.EarlybirdService
|
||||
import com.twitter.socialgraph.thriftscala.SocialGraphService
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.thriftscala.TimelineRanker
|
||||
import com.twitter.timelines.config.ConfigUtils
|
||||
import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelineservice.thriftscala.TimelineService
|
||||
import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie}
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Try
|
||||
import org.apache.thrift.protocol.TCompactProtocol
|
||||
|
||||
abstract class UnderlyingClientConfiguration(
|
||||
flags: TimelineRankerFlags,
|
||||
val statsReceiver: StatsReceiver)
|
||||
extends TimelinesUnderlyingClientConfiguration
|
||||
with ConfigUtils {
|
||||
|
||||
lazy val thriftClientId: ClientId = timelineRankerClientId()
|
||||
override lazy val serviceIdentifier: ServiceIdentifier = flags.getServiceIdentifier
|
||||
|
||||
def timelineRankerClientId(scope: Option[String] = None): ClientId = {
|
||||
clientIdWithScopeOpt("timelineranker", flags.getEnv, scope)
|
||||
}
|
||||
|
||||
def createEarlybirdClient(
|
||||
scope: String,
|
||||
requestTimeout: Duration,
|
||||
timeout: Duration,
|
||||
retryPolicy: RetryPolicy[Try[Nothing]]
|
||||
): EarlybirdService.MethodPerEndpoint = {
|
||||
val scopedName = s"earlybird/$scope"
|
||||
|
||||
methodPerEndpointClient[
|
||||
EarlybirdService.ServicePerEndpoint,
|
||||
EarlybirdService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder(
|
||||
scopedName,
|
||||
protocolFactoryOption = Some(new TCompactProtocol.Factory),
|
||||
requireMutualTls = true)
|
||||
.dest("/s/earlybird-root-superroot/root-superroot")
|
||||
.timeout(timeout)
|
||||
.requestTimeout(requestTimeout)
|
||||
.retryPolicy(retryPolicy)
|
||||
)
|
||||
}
|
||||
|
||||
def createEarlybirdRealtimeCgClient(
|
||||
scope: String,
|
||||
requestTimeout: Duration,
|
||||
timeout: Duration,
|
||||
retryPolicy: RetryPolicy[Try[Nothing]]
|
||||
): EarlybirdService.MethodPerEndpoint = {
|
||||
val scopedName = s"earlybird/$scope"
|
||||
|
||||
methodPerEndpointClient[
|
||||
EarlybirdService.ServicePerEndpoint,
|
||||
EarlybirdService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder(
|
||||
scopedName,
|
||||
protocolFactoryOption = Some(new TCompactProtocol.Factory),
|
||||
requireMutualTls = true)
|
||||
.dest("/s/earlybird-rootrealtimecg/root-realtime_cg")
|
||||
.timeout(timeout)
|
||||
.requestTimeout(requestTimeout)
|
||||
.retryPolicy(retryPolicy)
|
||||
)
|
||||
}
|
||||
|
||||
def cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint
|
||||
def gizmoduckClient: Gizmoduck.MethodPerEndpoint
|
||||
def manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint
|
||||
def sgsClient: SocialGraphService.MethodPerEndpoint
|
||||
def timelineRankerForwardingClient: TimelineRanker.FinagledClient
|
||||
def timelineServiceClient: TimelineService.MethodPerEndpoint
|
||||
def tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint
|
||||
def tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint
|
||||
def userRolesServiceClient: UserRolesService.MethodPerEndpoint
|
||||
def contentFeaturesCache: Store[TweetId, ContentFeatures]
|
||||
def userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient
|
||||
def stratoClient: StratoClient
|
||||
|
||||
def darkTrafficProxy: Option[Service[ThriftClientRequest, Array[Byte]]] = {
|
||||
if (flags.darkTrafficDestination.trim.nonEmpty) {
|
||||
Some(darkTrafficClient(flags.darkTrafficDestination))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
package com.twitter.timelineranker
|
||||
|
||||
import com.twitter.timelineranker.core.FutureDependencyTransformer
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.model.TweetId
|
||||
|
||||
package object contentfeatures {
|
||||
type ContentFeaturesProvider =
|
||||
FutureDependencyTransformer[Seq[TweetId], Map[TweetId, ContentFeatures]]
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"src/thrift/com/twitter/search/common:constants-scala",
|
||||
"src/thrift/com/twitter/search/common:features-scala",
|
||||
"timelineranker/common:model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
],
|
||||
exports = [
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.model.TweetId
|
||||
|
||||
object CandidateEnvelope {
|
||||
val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult]
|
||||
val EmptyHydratedTweets: HydratedTweets = HydratedTweets(Seq.empty, Seq.empty)
|
||||
val EmptyUtegResults: Map[TweetId, TweetRecommendation] = Map.empty[TweetId, TweetRecommendation]
|
||||
}
|
||||
|
||||
case class CandidateEnvelope(
|
||||
query: RecapQuery,
|
||||
searchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults,
|
||||
utegResults: Map[TweetId, TweetRecommendation] = CandidateEnvelope.EmptyUtegResults,
|
||||
hydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets,
|
||||
followGraphData: FollowGraphDataFuture = FollowGraphDataFuture.EmptyFollowGraphDataFuture,
|
||||
// The source tweets are
|
||||
// - the retweeted tweet, for retweets
|
||||
// - the inReplyTo tweet, for extended replies
|
||||
sourceSearchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults,
|
||||
sourceHydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets)
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.timelines.model.UserId
|
||||
|
||||
/**
|
||||
* Follow graph details of a given user. Includes users followed, but also followed users in various
|
||||
* states of mute.
|
||||
*
|
||||
* @param userId ID of a given user.
|
||||
* @param followedUserIds IDs of users who the given user follows.
|
||||
* @param mutuallyFollowingUserIds A subset of followedUserIds where followed users follow back the given user.
|
||||
* @param mutedUserIds A subset of followedUserIds that the given user has muted.
|
||||
* @param retweetsMutedUserIds A subset of followedUserIds whose retweets are muted by the given user.
|
||||
*/
|
||||
case class FollowGraphData(
|
||||
userId: UserId,
|
||||
followedUserIds: Seq[UserId],
|
||||
mutuallyFollowingUserIds: Set[UserId],
|
||||
mutedUserIds: Set[UserId],
|
||||
retweetsMutedUserIds: Set[UserId]) {
|
||||
val filteredFollowedUserIds: Seq[UserId] = followedUserIds.filterNot(mutedUserIds)
|
||||
val allUserIds: Seq[UserId] = filteredFollowedUserIds :+ userId
|
||||
val inNetworkUserIds: Seq[UserId] = followedUserIds :+ userId
|
||||
}
|
||||
|
||||
object FollowGraphData {
|
||||
val Empty: FollowGraphData = FollowGraphData(
|
||||
0L,
|
||||
Seq.empty[UserId],
|
||||
Set.empty[UserId],
|
||||
Set.empty[UserId],
|
||||
Set.empty[UserId]
|
||||
)
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Similar to FollowGraphData but makes available its fields as separate futures.
|
||||
*/
|
||||
case class FollowGraphDataFuture(
|
||||
userId: UserId,
|
||||
followedUserIdsFuture: Future[Seq[UserId]],
|
||||
mutuallyFollowingUserIdsFuture: Future[Set[UserId]],
|
||||
mutedUserIdsFuture: Future[Set[UserId]],
|
||||
retweetsMutedUserIdsFuture: Future[Set[UserId]]) {
|
||||
|
||||
def inNetworkUserIdsFuture: Future[Seq[UserId]] = followedUserIdsFuture.map(_ :+ userId)
|
||||
|
||||
def get(): Future[FollowGraphData] = {
|
||||
Future
|
||||
.join(
|
||||
followedUserIdsFuture,
|
||||
mutuallyFollowingUserIdsFuture,
|
||||
mutedUserIdsFuture,
|
||||
retweetsMutedUserIdsFuture
|
||||
)
|
||||
.map {
|
||||
case (followedUserIds, mutuallyFollowingUserIds, mutedUserIds, retweetsMutedUserIds) =>
|
||||
FollowGraphData(
|
||||
userId,
|
||||
followedUserIds,
|
||||
mutuallyFollowingUserIds,
|
||||
mutedUserIds,
|
||||
retweetsMutedUserIds
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object FollowGraphDataFuture {
|
||||
private def mkEmptyFuture(field: String) = {
|
||||
Future.exception(
|
||||
new IllegalStateException(s"FollowGraphDataFuture field $field accessed without being set")
|
||||
)
|
||||
}
|
||||
|
||||
val EmptyFollowGraphDataFuture: FollowGraphDataFuture = FollowGraphDataFuture(
|
||||
userId = 0L,
|
||||
followedUserIdsFuture = mkEmptyFuture("followedUserIdsFuture"),
|
||||
mutuallyFollowingUserIdsFuture = mkEmptyFuture("mutuallyFollowingUserIdsFuture"),
|
||||
mutedUserIdsFuture = mkEmptyFuture("mutedUserIdsFuture"),
|
||||
retweetsMutedUserIdsFuture = mkEmptyFuture("retweetsMutedUserIdsFuture")
|
||||
)
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.search.common.constants.thriftscala.ThriftLanguage
|
||||
import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.clients.gizmoduck.UserProfileInfo
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.util.FutureUtils
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class HydratedCandidatesAndFeaturesEnvelope(
|
||||
candidateEnvelope: CandidateEnvelope,
|
||||
userLanguages: Seq[ThriftLanguage],
|
||||
userProfileInfo: UserProfileInfo,
|
||||
features: Map[TweetId, ThriftTweetFeatures] = Map.empty,
|
||||
contentFeaturesFuture: Future[Map[TweetId, ContentFeatures]] = FutureUtils.EmptyMap,
|
||||
tweetSourceTweetMap: Map[TweetId, TweetId] = Map.empty,
|
||||
inReplyToTweetIds: Set[TweetId] = Set.empty)
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
|
||||
case class HydratedTweets(
|
||||
outerTweets: Seq[HydratedTweet],
|
||||
innerTweets: Seq[HydratedTweet] = Seq.empty)
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
package com.twitter.timelineranker
|
||||
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.configapi
|
||||
|
||||
package object core {
|
||||
type FutureDependencyTransformer[-U, +V] = configapi.FutureDependencyTransformer[RecapQuery, U, V]
|
||||
object FutureDependencyTransformer
|
||||
extends configapi.FutureDependencyTransformerFunctions[RecapQuery]
|
||||
|
||||
type DependencyTransformer[-U, +V] = configapi.DependencyTransformer[RecapQuery, U, V]
|
||||
object DependencyTransformer extends configapi.DependencyTransformerFunctions[RecapQuery]
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"servo/decider",
|
||||
"timelineranker/server/config",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
package com.twitter.timelineranker.decider
|
||||
|
||||
import com.twitter.servo.decider.DeciderKeyEnum
|
||||
|
||||
object DeciderKey extends DeciderKeyEnum {
|
||||
// Deciders that can be used to control load on TLR or its backends.
|
||||
val EnableMaxConcurrencyLimiting: Value = Value("enable_max_concurrency_limiting")
|
||||
|
||||
// Deciders related to testing / debugging.
|
||||
val EnableRoutingToRankerDevProxy: Value = Value("enable_routing_to_ranker_dev_proxy")
|
||||
|
||||
// Deciders related to authorization.
|
||||
val ClientRequestAuthorization: Value = Value("client_request_authorization")
|
||||
val ClientWriteWhitelist: Value = Value("client_write_whitelist")
|
||||
val AllowTimelineMixerRecapProd: Value = Value("allow_timeline_mixer_recap_prod")
|
||||
val AllowTimelineMixerRecycledProd: Value = Value("allow_timeline_mixer_recycled_prod")
|
||||
val AllowTimelineMixerHydrateProd: Value = Value("allow_timeline_mixer_hydrate_prod")
|
||||
val AllowTimelineMixerHydrateRecosProd: Value = Value("allow_timeline_mixer_hydrate_recos_prod")
|
||||
val AllowTimelineMixerSeedAuthorsProd: Value = Value("allow_timeline_mixer_seed_authors_prod")
|
||||
val AllowTimelineMixerSimclusterProd: Value = Value("allow_timeline_mixer_simcluster_prod")
|
||||
val AllowTimelineMixerEntityTweetsProd: Value = Value("allow_timeline_mixer_entity_tweets_prod")
|
||||
val AllowTimelineMixerListProd: Value = Value("allow_timeline_mixer_list_prod")
|
||||
val AllowTimelineMixerListTweetProd: Value = Value("allow_timeline_mixer_list_tweet_prod")
|
||||
val AllowTimelineMixerCommunityProd: Value = Value("allow_timeline_mixer_community_prod")
|
||||
val AllowTimelineMixerCommunityTweetProd: Value = Value(
|
||||
"allow_timeline_mixer_community_tweet_prod")
|
||||
val AllowTimelineScorerRecommendedTrendTweetProd: Value = Value(
|
||||
"allow_timeline_scorer_recommended_trend_tweet_prod")
|
||||
val AllowTimelineMixerUtegLikedByTweetsProd: Value = Value(
|
||||
"allow_timeline_mixer_uteg_liked_by_tweets_prod")
|
||||
val AllowTimelineMixerStaging: Value = Value("allow_timeline_mixer_staging")
|
||||
val AllowTimelineRankerProxy: Value = Value("allow_timeline_ranker_proxy")
|
||||
val AllowTimelineRankerWarmup: Value = Value("allow_timeline_ranker_warmup")
|
||||
val AllowTimelineScorerRecTopicTweetsProd: Value =
|
||||
Value("allow_timeline_scorer_rec_topic_tweets_prod")
|
||||
val AllowTimelineScorerPopularTopicTweetsProd: Value =
|
||||
Value("allow_timeline_scorer_popular_topic_tweets_prod")
|
||||
val AllowTimelineScorerHydrateTweetScoringProd: Value = Value(
|
||||
"allow_timelinescorer_hydrate_tweet_scoring_prod")
|
||||
val AllowTimelineServiceProd: Value = Value("allow_timeline_service_prod")
|
||||
val AllowTimelineServiceStaging: Value = Value("allow_timeline_service_staging")
|
||||
val RateLimitOverrideUnknown: Value = Value("rate_limit_override_unknown")
|
||||
|
||||
// Deciders related to reverse-chron home timeline materialization.
|
||||
val MultiplierOfMaterializationTweetsFetched: Value = Value(
|
||||
"multiplier_of_materialization_tweets_fetched"
|
||||
)
|
||||
val BackfillFilteredEntries: Value = Value("enable_backfill_filtered_entries")
|
||||
val TweetsFilteringLossageThreshold: Value = Value("tweets_filtering_lossage_threshold")
|
||||
val TweetsFilteringLossageLimit: Value = Value("tweets_filtering_lossage_limit")
|
||||
val SupplementFollowsWithRealGraph: Value = Value("supplement_follows_with_real_graph")
|
||||
|
||||
// Deciders related to recap.
|
||||
val RecapEnableContentFeaturesHydration: Value = Value("recap_enable_content_features_hydration")
|
||||
val RecapMaxCountMultiplier: Value = Value("recap_max_count_multiplier")
|
||||
val RecapEnableExtraSortingInResults: Value = Value("recap_enable_extra_sorting_in_results")
|
||||
|
||||
// Deciders related to recycled tweets.
|
||||
val RecycledMaxCountMultiplier: Value = Value("recycled_max_count_multiplier")
|
||||
val RecycledEnableContentFeaturesHydration: Value = Value(
|
||||
"recycled_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to entity tweets.
|
||||
val EntityTweetsEnableContentFeaturesHydration: Value = Value(
|
||||
"entity_tweets_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to both recap and recycled tweets
|
||||
val EnableRealGraphUsers: Value = Value("enable_real_graph_users")
|
||||
val MaxRealGraphAndFollowedUsers: Value = Value("max_real_graph_and_followed_users")
|
||||
|
||||
// Deciders related to recap author
|
||||
val RecapAuthorEnableNewPipeline: Value = Value("recap_author_enable_new_pipeline")
|
||||
val RecapAuthorEnableContentFeaturesHydration: Value = Value(
|
||||
"recap_author_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to recap hydration (rectweet and ranked organic).
|
||||
val RecapHydrationEnableContentFeaturesHydration: Value = Value(
|
||||
"recap_hydration_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to uteg liked by tweets
|
||||
val UtegLikedByTweetsEnableContentFeaturesHydration: Value = Value(
|
||||
"uteg_liked_by_tweets_enable_content_features_hydration")
|
||||
}
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
|
||||
"finagle/finagle-core/src/main",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/common",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/common/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.timelineranker.model.CandidateTweetsResult
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* A repository of entity tweets candidates.
|
||||
*
|
||||
* For now, it does not cache any results therefore forwards all calls to the underlying source.
|
||||
*/
|
||||
class EntityTweetsRepository(source: EntityTweetsSource) {
|
||||
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
source.get(query)
|
||||
}
|
||||
|
||||
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
|
||||
source.get(queries)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.timelineranker.config.RequestScopes
|
||||
import com.twitter.timelineranker.config.RuntimeConfiguration
|
||||
import com.twitter.timelineranker.parameters.ConfigBuilder
|
||||
import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder
|
||||
import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields
|
||||
import com.twitter.search.earlybird.thriftscala.EarlybirdService
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.util.Duration
|
||||
|
||||
class EntityTweetsRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder)
|
||||
extends CandidatesRepositoryBuilder(config) {
|
||||
|
||||
// Default client id for this repository if the upstream requests doesn't indicate one.
|
||||
override val clientSubId = "community"
|
||||
override val requestScope: RequestScope = RequestScopes.EntityTweetsSource
|
||||
override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet =
|
||||
SgsFollowGraphDataFields.ValueSet(
|
||||
SgsFollowGraphDataFields.FollowedUserIds,
|
||||
SgsFollowGraphDataFields.MutuallyFollowingUserIds,
|
||||
SgsFollowGraphDataFields.MutedUserIds
|
||||
)
|
||||
|
||||
/**
|
||||
* [1] timeout is derived from p9999 TLR <-> Earlybird latency and shall be less than
|
||||
* request timeout of timelineranker client within downstream timelinemixer, which is
|
||||
* 1s now
|
||||
*
|
||||
* [2] processing timeout is less than request timeout [1] with 100ms space for networking and
|
||||
* other times such as gc
|
||||
*/
|
||||
override val searchProcessingTimeout: Duration = 550.milliseconds // [2]
|
||||
override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint =
|
||||
config.underlyingClients.createEarlybirdClient(
|
||||
scope = scope,
|
||||
requestTimeout = 650.milliseconds, // [1]
|
||||
timeout = 900.milliseconds, // [1]
|
||||
retryPolicy = config.underlyingClients.DefaultRetryPolicy
|
||||
)
|
||||
|
||||
def apply(): EntityTweetsRepository = {
|
||||
val entityTweetsSource = new EntityTweetsSource(
|
||||
gizmoduckClient,
|
||||
searchClient,
|
||||
tweetyPieHighQoSClient,
|
||||
userMetadataClient,
|
||||
followGraphDataProvider,
|
||||
clientFactories.visibilityEnforcerFactory.apply(
|
||||
VisibilityRules,
|
||||
RequestScopes.EntityTweetsSource
|
||||
),
|
||||
config.underlyingClients.contentFeaturesCache,
|
||||
config.statsReceiver
|
||||
)
|
||||
|
||||
new EntityTweetsRepository(entityTweetsSource)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.TweetIdRange
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
|
||||
object EntityTweetsSearchResultsTransform {
|
||||
// If EntityTweetsQuery.maxCount is not specified, the following count is used.
|
||||
val DefaultEntityTweetsMaxTweetCount = 200
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch entity tweets search results using the search client
|
||||
* and populate them into the CandidateEnvelope
|
||||
*/
|
||||
class EntityTweetsSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
statsReceiver: StatsReceiver,
|
||||
logSearchDebugInfo: Boolean = false)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import EntityTweetsSearchResultsTransform._
|
||||
|
||||
private[this] val maxCountStat = statsReceiver.stat("maxCount")
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val maxCount = envelope.query.maxCount.getOrElse(DefaultEntityTweetsMaxTweetCount)
|
||||
maxCountStat.add(maxCount)
|
||||
|
||||
val tweetIdRange = envelope.query.range
|
||||
.map(TweetIdRange.fromTimelineRange)
|
||||
.getOrElse(TweetIdRange.default)
|
||||
|
||||
val beforeTweetIdExclusive = tweetIdRange.toId
|
||||
val afterTweetIdExclusive = tweetIdRange.fromId
|
||||
|
||||
val excludedTweetIds = envelope.query.excludedTweetIds.getOrElse(Seq.empty[TweetId]).toSet
|
||||
val languages = envelope.query.languages.map(_.map(_.language))
|
||||
|
||||
envelope.followGraphData.inNetworkUserIdsFuture.flatMap { inNetworkUserIds =>
|
||||
searchClient
|
||||
.getEntityTweets(
|
||||
userId = Some(envelope.query.userId),
|
||||
followedUserIds = inNetworkUserIds.toSet,
|
||||
maxCount = maxCount,
|
||||
beforeTweetIdExclusive = beforeTweetIdExclusive,
|
||||
afterTweetIdExclusive = afterTweetIdExclusive,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
semanticCoreIds = envelope.query.semanticCoreIds,
|
||||
hashtags = envelope.query.hashtags,
|
||||
languages = languages,
|
||||
tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options),
|
||||
searchOperator = envelope.query.searchOperator,
|
||||
excludedTweetIds = excludedTweetIds,
|
||||
logSearchDebugInfo = logSearchDebugInfo,
|
||||
includeNullcastTweets = envelope.query.includeNullcastTweets.getOrElse(false),
|
||||
includeTweetsFromArchiveIndex =
|
||||
envelope.query.includeTweetsFromArchiveIndex.getOrElse(false),
|
||||
authorIds = envelope.query.authorIds.map(_.toSet)
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.common._
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.model._
|
||||
import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsParams._
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform
|
||||
import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform
|
||||
import com.twitter.timelineranker.util.TweetFilters
|
||||
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
|
||||
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
|
||||
import com.twitter.timelines.clients.manhattan.UserMetadataClient
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.timelines.util.stats.RequestStatsReceiver
|
||||
import com.twitter.timelines.visibility.VisibilityEnforcer
|
||||
import com.twitter.util.Future
|
||||
|
||||
class EntityTweetsSource(
|
||||
gizmoduckClient: GizmoduckClient,
|
||||
searchClient: SearchClient,
|
||||
tweetyPieClient: TweetyPieClient,
|
||||
userMetadataClient: UserMetadataClient,
|
||||
followGraphDataProvider: FollowGraphDataProvider,
|
||||
visibilityEnforcer: VisibilityEnforcer,
|
||||
contentFeaturesCache: Store[TweetId, ContentFeatures],
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
private[this] val baseScope = statsReceiver.scope("entityTweetsSource")
|
||||
private[this] val requestStats = RequestStatsReceiver(baseScope)
|
||||
|
||||
private[this] val failOpenScope = baseScope.scope("failOpen")
|
||||
private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo")
|
||||
private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages")
|
||||
|
||||
private[this] val followGraphDataTransform = new FollowGraphDataTransform(
|
||||
followGraphDataProvider = followGraphDataProvider,
|
||||
maxFollowedUsersProvider = DependencyProvider.from(MaxFollowedUsersParam)
|
||||
)
|
||||
private[this] val fetchSearchResultsTransform = new EntityTweetsSearchResultsTransform(
|
||||
searchClient = searchClient,
|
||||
statsReceiver = baseScope
|
||||
)
|
||||
private[this] val userProfileInfoTransform =
|
||||
new UserProfileInfoTransform(userProfileHandler, gizmoduckClient)
|
||||
private[this] val languagesTransform =
|
||||
new UserLanguagesTransform(userLanguagesHandler, userMetadataClient)
|
||||
|
||||
private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform(
|
||||
visibilityEnforcer
|
||||
)
|
||||
|
||||
private[this] val filters = TweetFilters.ValueSet(
|
||||
TweetFilters.DuplicateTweets,
|
||||
TweetFilters.DuplicateRetweets
|
||||
)
|
||||
|
||||
private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform(
|
||||
outerFilters = filters,
|
||||
innerFilters = TweetFilters.None,
|
||||
useFollowGraphData = false,
|
||||
useSourceTweets = false,
|
||||
statsReceiver = baseScope,
|
||||
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
|
||||
)
|
||||
|
||||
private[this] val contentFeaturesHydrationTransform =
|
||||
new ContentFeaturesHydrationTransformBuilder(
|
||||
tweetyPieClient = tweetyPieClient,
|
||||
contentFeaturesCache = contentFeaturesCache,
|
||||
enableContentFeaturesGate = RecapQuery.paramGate(EnableContentFeaturesHydrationParam),
|
||||
enableTokensInContentFeaturesGate =
|
||||
RecapQuery.paramGate(EnableTokensInContentFeaturesHydrationParam),
|
||||
enableTweetTextInContentFeaturesGate =
|
||||
RecapQuery.paramGate(EnableTweetTextInContentFeaturesHydrationParam),
|
||||
enableConversationControlContentFeaturesGate =
|
||||
RecapQuery.paramGate(EnableConversationControlInContentFeaturesHydrationParam),
|
||||
enableTweetMediaHydrationGate = RecapQuery.paramGate(EnableTweetMediaHydrationParam),
|
||||
hydrateInReplyToTweets = false,
|
||||
statsReceiver = baseScope
|
||||
).build()
|
||||
|
||||
private[this] def hydratesContentFeatures(
|
||||
hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Boolean =
|
||||
hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true)
|
||||
|
||||
private[this] val contentFeaturesTransformer = FutureArrow.choose(
|
||||
predicate = hydratesContentFeatures,
|
||||
ifTrue = contentFeaturesHydrationTransform
|
||||
.andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform)
|
||||
.andThen(CopyContentFeaturesIntoHydratedTweetsTransform),
|
||||
ifFalse = FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
](Future.value) // empty transformer
|
||||
)
|
||||
|
||||
private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope)
|
||||
|
||||
private[this] val hydrationAndFilteringPipeline =
|
||||
CreateCandidateEnvelopeTransform
|
||||
.andThen(followGraphDataTransform) // Fetch follow graph data
|
||||
.andThen(fetchSearchResultsTransform) // fetch search results
|
||||
.andThen(SearchResultDedupAndSortingTransform) // dedup and order search results
|
||||
.andThen(CandidateTweetHydrationTransform) // hydrate search results
|
||||
.andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones
|
||||
.andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter
|
||||
.andThen(
|
||||
TrimToMatchHydratedTweetsTransform
|
||||
) // trim search result set to match filtered hydrated tweets (this needs to be accurate for feature hydration)
|
||||
|
||||
// runs the main pipeline in parallel with fetching user profile info and user languages
|
||||
private[this] val featureHydrationDataTransform =
|
||||
new FeatureHydrationDataTransform(
|
||||
hydrationAndFilteringPipeline,
|
||||
languagesTransform,
|
||||
userProfileInfoTransform
|
||||
)
|
||||
|
||||
private[this] val tweetFeaturesHydrationTransform =
|
||||
OutOfNetworkTweetsSearchFeaturesHydrationTransform
|
||||
.andThen(contentFeaturesTransformer)
|
||||
|
||||
private[this] val featureHydrationPipeline =
|
||||
featureHydrationDataTransform
|
||||
.andThen(tweetFeaturesHydrationTransform)
|
||||
.andThen(candidateGenerationTransform)
|
||||
|
||||
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
requestStats.addEventStats {
|
||||
featureHydrationPipeline(query)
|
||||
}
|
||||
}
|
||||
|
||||
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
|
||||
Future.collect(queries.map(get))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
|
||||
"finagle/finagle-core/src/main",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/common",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.timelineranker.in_network_tweets
|
||||
|
||||
import com.twitter.timelineranker.model.CandidateTweetsResult
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* A repository of in-network tweet candidates.
|
||||
* For now, it does not cache any results therefore forwards all calls to the underlying source.
|
||||
*/
|
||||
class InNetworkTweetRepository(
|
||||
source: InNetworkTweetSource,
|
||||
realtimeCGSource: InNetworkTweetSource) {
|
||||
|
||||
private[this] val enableRealtimeCGProvider =
|
||||
DependencyProvider.from(InNetworkTweetParams.EnableEarlybirdRealtimeCgMigrationParam)
|
||||
|
||||
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
if (enableRealtimeCGProvider(query)) {
|
||||
realtimeCGSource.get(query)
|
||||
} else {
|
||||
source.get(query)
|
||||
}
|
||||
}
|
||||
|
||||
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
|
||||
Future.collect(queries.map(query => get(query)))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
package com.twitter.timelineranker.in_network_tweets
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.service.RetryPolicy
|
||||
import com.twitter.search.earlybird.thriftscala.EarlybirdService
|
||||
import com.twitter.timelineranker.config.RequestScopes
|
||||
import com.twitter.timelineranker.config.RuntimeConfiguration
|
||||
import com.twitter.timelineranker.parameters.ConfigBuilder
|
||||
import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder
|
||||
import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.timelines.visibility.model.CheckedUserActorType
|
||||
import com.twitter.timelines.visibility.model.ExclusionReason
|
||||
import com.twitter.timelines.visibility.model.VisibilityCheckStatus
|
||||
import com.twitter.timelines.visibility.model.VisibilityCheckUser
|
||||
import com.twitter.util.Duration
|
||||
|
||||
object InNetworkTweetRepositoryBuilder {
|
||||
val VisibilityRuleExclusions: Set[ExclusionReason] = Set[ExclusionReason](
|
||||
ExclusionReason(
|
||||
CheckedUserActorType(Some(false), VisibilityCheckUser.SourceUser),
|
||||
Set(VisibilityCheckStatus.Blocked)
|
||||
)
|
||||
)
|
||||
|
||||
private val EarlybirdTimeout = 600.milliseconds
|
||||
private val EarlybirdRequestTimeout = 600.milliseconds
|
||||
|
||||
/**
|
||||
* The timeouts below are only used for the Earlybird Cluster Migration
|
||||
*/
|
||||
private val EarlybirdRealtimeCGTimeout = 600.milliseconds
|
||||
private val EarlybirdRealtimeCGRequestTimeout = 600.milliseconds
|
||||
}
|
||||
|
||||
class InNetworkTweetRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder)
|
||||
extends CandidatesRepositoryBuilder(config) {
|
||||
import InNetworkTweetRepositoryBuilder._
|
||||
|
||||
override val clientSubId = "recycled_tweets"
|
||||
override val requestScope: RequestScope = RequestScopes.InNetworkTweetSource
|
||||
override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet =
|
||||
SgsFollowGraphDataFields.ValueSet(
|
||||
SgsFollowGraphDataFields.FollowedUserIds,
|
||||
SgsFollowGraphDataFields.MutuallyFollowingUserIds,
|
||||
SgsFollowGraphDataFields.MutedUserIds,
|
||||
SgsFollowGraphDataFields.RetweetsMutedUserIds
|
||||
)
|
||||
override val searchProcessingTimeout: Duration = 200.milliseconds
|
||||
|
||||
override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint =
|
||||
config.underlyingClients.createEarlybirdClient(
|
||||
scope = scope,
|
||||
requestTimeout = EarlybirdRequestTimeout,
|
||||
timeout = EarlybirdTimeout,
|
||||
retryPolicy = RetryPolicy.Never
|
||||
)
|
||||
|
||||
private lazy val searchClientForSourceTweets =
|
||||
newSearchClient(clientId = clientSubId + "_source_tweets")
|
||||
|
||||
/** The RealtimeCG clients below are only used for the Earlybird Cluster Migration */
|
||||
private def earlybirdRealtimeCGClient(scope: String): EarlybirdService.MethodPerEndpoint =
|
||||
config.underlyingClients.createEarlybirdRealtimeCgClient(
|
||||
scope = scope,
|
||||
requestTimeout = EarlybirdRealtimeCGRequestTimeout,
|
||||
timeout = EarlybirdRealtimeCGTimeout,
|
||||
retryPolicy = RetryPolicy.Never
|
||||
)
|
||||
private val realtimeCGClientSubId = "realtime_cg_recycled_tweets"
|
||||
private lazy val searchRealtimeCGClient =
|
||||
newSearchClient(earlybirdRealtimeCGClient, clientId = realtimeCGClientSubId)
|
||||
|
||||
def apply(): InNetworkTweetRepository = {
|
||||
val inNetworkTweetSource = new InNetworkTweetSource(
|
||||
gizmoduckClient,
|
||||
searchClient,
|
||||
searchClientForSourceTweets,
|
||||
tweetyPieHighQoSClient,
|
||||
userMetadataClient,
|
||||
followGraphDataProvider,
|
||||
config.underlyingClients.contentFeaturesCache,
|
||||
clientFactories.visibilityEnforcerFactory.apply(
|
||||
VisibilityRules,
|
||||
RequestScopes.InNetworkTweetSource,
|
||||
reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions
|
||||
),
|
||||
config.statsReceiver
|
||||
)
|
||||
|
||||
val inNetworkTweetRealtimeCGSource = new InNetworkTweetSource(
|
||||
gizmoduckClient,
|
||||
searchRealtimeCGClient,
|
||||
searchClientForSourceTweets, // do not migrate source_tweets as they are sharded by TweetID
|
||||
tweetyPieHighQoSClient,
|
||||
userMetadataClient,
|
||||
followGraphDataProvider,
|
||||
config.underlyingClients.contentFeaturesCache,
|
||||
clientFactories.visibilityEnforcerFactory.apply(
|
||||
VisibilityRules,
|
||||
RequestScopes.InNetworkTweetSource,
|
||||
reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions
|
||||
),
|
||||
config.statsReceiver.scope("replacementRealtimeCG")
|
||||
)
|
||||
|
||||
new InNetworkTweetRepository(inNetworkTweetSource, inNetworkTweetRealtimeCGSource)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue