mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-12-24 03:01:52 +01:00
[docx] split commit for file 1200
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
7ed90fef05
commit
c80f53f99d
Binary file not shown.
@ -1,27 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.transforms.modify_social_proof
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.GatedTransform
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class RemoveAccountProofTransform @Inject() (statsReceiver: StatsReceiver)
|
||||
extends GatedTransform[HasClientContext with HasParams, CandidateUser] {
|
||||
|
||||
private val stats = statsReceiver.scope(this.getClass.getSimpleName)
|
||||
private val removedProofsCounter = stats.counter("num_removed_proofs")
|
||||
|
||||
override def transform(
|
||||
target: HasClientContext with HasParams,
|
||||
items: Seq[CandidateUser]
|
||||
): Stitch[Seq[CandidateUser]] =
|
||||
Stitch.value(items.map { candidate =>
|
||||
removedProofsCounter.incr()
|
||||
candidate.copy(reason = None)
|
||||
})
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/constants",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,24 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.transforms.ranker_id
|
||||
|
||||
import com.google.inject.Inject
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.follow_recommendations.common.base.GatedTransform
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.Score
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
|
||||
/**
|
||||
* This class appends each candidate's rankerIds with the RandomRankerId.
|
||||
* This is primarily for determining if a candidate was generated via random shuffling.
|
||||
*/
|
||||
@Singleton
|
||||
class RandomRankerIdTransform @Inject() () extends GatedTransform[HasParams, CandidateUser] {
|
||||
|
||||
override def transform(
|
||||
target: HasParams,
|
||||
candidates: Seq[CandidateUser]
|
||||
): Stitch[Seq[CandidateUser]] = {
|
||||
Stitch.value(candidates.map(_.addScore(Score.RandomScore)))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,20 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.transforms.recommendation_flow_identifier
|
||||
|
||||
import com.google.inject.Inject
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.HasRecommendationFlowIdentifier
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
class AddRecommendationFlowIdentifierTransform @Inject()
|
||||
extends Transform[HasRecommendationFlowIdentifier, CandidateUser] {
|
||||
|
||||
override def transform(
|
||||
target: HasRecommendationFlowIdentifier,
|
||||
items: Seq[CandidateUser]
|
||||
): Stitch[Seq[CandidateUser]] = {
|
||||
Stitch.value(items.map { candidateUser =>
|
||||
candidateUser.copy(recommendationFlowIdentifier = target.recommendationFlowIdentifier)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
@ -1,18 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/constants",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,76 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.transforms.tracking_token
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.HasDisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.Session
|
||||
import com.twitter.follow_recommendations.common.models.TrackingToken
|
||||
import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap
|
||||
import com.twitter.hermit.model.Algorithm
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.util.logging.Logging
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* This transform adds the tracking token for all candidates
|
||||
* Since this happens in the same request, we use the same trace-id for all candidates
|
||||
* There are no RPC calls in this transform so it's safe to chain it with `andThen` at the end of
|
||||
* all other product-specific transforms
|
||||
*/
|
||||
@Singleton
|
||||
class TrackingTokenTransform @Inject() (baseStatsReceiver: StatsReceiver)
|
||||
extends Transform[HasDisplayLocation with HasClientContext, CandidateUser]
|
||||
with Logging {
|
||||
|
||||
def profileResults(
|
||||
target: HasDisplayLocation with HasClientContext,
|
||||
candidates: Seq[CandidateUser]
|
||||
) = {
|
||||
// Metrics to track # results per candidate source
|
||||
val stats = baseStatsReceiver.scope(target.displayLocation.toString + "/final_results")
|
||||
stats.stat("total").add(candidates.size)
|
||||
|
||||
stats.counter(target.displayLocation.toString).incr()
|
||||
|
||||
val flattenedCandidates: Seq[(CandidateSourceIdentifier, CandidateUser)] = for {
|
||||
candidate <- candidates
|
||||
identifier <- candidate.getPrimaryCandidateSource
|
||||
} yield (identifier, candidate)
|
||||
val candidatesGroupedBySource: Map[CandidateSourceIdentifier, Seq[CandidateUser]] =
|
||||
flattenedCandidates.groupBy(_._1).mapValues(_.map(_._2))
|
||||
candidatesGroupedBySource map {
|
||||
case (source, candidates) => stats.stat(source.name).add(candidates.size)
|
||||
}
|
||||
}
|
||||
|
||||
override def transform(
|
||||
target: HasDisplayLocation with HasClientContext,
|
||||
candidates: Seq[CandidateUser]
|
||||
): Stitch[Seq[CandidateUser]] = {
|
||||
profileResults(target, candidates)
|
||||
|
||||
Stitch.value(
|
||||
target.getOptionalUserId
|
||||
.map { _ =>
|
||||
candidates.map {
|
||||
candidate =>
|
||||
val token = Some(TrackingToken(
|
||||
sessionId = Session.getSessionId,
|
||||
displayLocation = Some(target.displayLocation),
|
||||
controllerData = None,
|
||||
algorithmId = candidate.userCandidateSourceDetails.flatMap(_.primaryCandidateSource
|
||||
.flatMap { identifier =>
|
||||
Algorithm.withNameOpt(identifier.name).flatMap(AlgorithmToFeedbackTokenMap.get)
|
||||
})
|
||||
))
|
||||
candidate.copy(trackingToken = token)
|
||||
}
|
||||
}.getOrElse(candidates))
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,138 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.transforms.weighted_sampling
|
||||
import com.twitter.follow_recommendations.common.base.GatedTransform
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.HasDebugOptions
|
||||
import com.twitter.follow_recommendations.common.models.Score
|
||||
import com.twitter.follow_recommendations.common.models.Scores
|
||||
import com.twitter.follow_recommendations.common.rankers.common.RankerId
|
||||
import com.twitter.follow_recommendations.common.rankers.utils.Utils
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class SamplingTransform @Inject() ()
|
||||
extends GatedTransform[HasClientContext with HasParams with HasDebugOptions, CandidateUser] {
|
||||
|
||||
val name: String = this.getClass.getSimpleName
|
||||
|
||||
/*
|
||||
Description: This function takes in a set of candidate users and ranks them for a who-to-follow
|
||||
request by sampling from the Placket-Luce distribution
|
||||
(https://cran.rstudio.com/web/packages/PlackettLuce/vignettes/Overview.html) with a three
|
||||
variations. The first variation is that the scores of the candidates are multiplied by
|
||||
multiplicativeFactor before sampling. The second variation is that the scores are
|
||||
exponentiated before sampling. The third variation is that depending on how many who-to-follow
|
||||
positions are being requested, the first k positions are reserved for the candidates with the
|
||||
highest scores (and they are sorted in decreasing order of score) and the remaining positions
|
||||
are sampled from a Placket-Luce. We use the efficient algorithm proposed in this blog
|
||||
https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051
|
||||
to sample from a Plackett-Luce. Because of numerical stability reasons, before sampling from this
|
||||
distribution, (1) we subtract off the maximum score from all the scores and (2) if after
|
||||
this subtraction and multiplication by the multiplicative factor the resulting score is <= -10,
|
||||
we force the candidate's transformed score under the above algorithm to be 0 (so r^(1/w) = 0)
|
||||
where r is a random number and w is the transformed score.
|
||||
|
||||
inputs:
|
||||
- target: HasClientContext (WTF request)
|
||||
- candidates: sequence of CandidateUsers (users that need to be ranked from a who-to-follow
|
||||
request) each of which has a score
|
||||
|
||||
inputs accessed through feature switches, i.e. through target.params (see the following file:
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/
|
||||
transforms/weighted_sampling/SamplingTransformParams.scala"):
|
||||
- topKFixed: the first k positions of the who-to-follow ranking correspond to the users with the k
|
||||
highest scores and are not sampled from the Placket-Luce distribution
|
||||
- multiplicativeFactor: multiplicativeFactor is used to transform the scores of each candidate by
|
||||
multiplying that user's score by multiplicativeFactor
|
||||
|
||||
output:
|
||||
- Sequence of CandidateUser whose order represents the ranking of users in a who-to-follow request
|
||||
This ranking is sampled from a Placket-Luce distribution.
|
||||
*/
|
||||
override def transform(
|
||||
target: HasClientContext with HasParams with HasDebugOptions,
|
||||
candidates: Seq[CandidateUser]
|
||||
): Stitch[Seq[CandidateUser]] = {
|
||||
|
||||
// the first k positions of the who-to-follow ranking correspond to the users with the k
|
||||
// highest scores and are not sampled from the Placket-Luce distribution
|
||||
val topKFixed = target.params(SamplingTransformParams.TopKFixed)
|
||||
|
||||
// multiplicativeFactor is used to transform the scores of each candidate by
|
||||
// multiplying that user's score by multiplicativeFactor
|
||||
val multiplicativeFactor = target.params(SamplingTransformParams.MultiplicativeFactor)
|
||||
|
||||
// sort candidates by their score
|
||||
val candidatesSorted = candidates.sortBy(-1 * _.score.getOrElse(0.0))
|
||||
|
||||
// pick the top K candidates by score and the remaining candidates
|
||||
val (topKFixedCandidates, candidatesOutsideOfTopK) =
|
||||
candidatesSorted.zipWithIndex.partition { case (value, index) => index < topKFixed }
|
||||
|
||||
val randomNumGenerator =
|
||||
new scala.util.Random(target.getRandomizationSeed.getOrElse(System.currentTimeMillis))
|
||||
|
||||
// we need to subtract the maximum score off the scores for numerical stability reasons
|
||||
// subtracting the max score off does not effect the underlying distribution we are sampling
|
||||
// the candidates from
|
||||
// we need the if statement since you cannot take the max of an empty sequence
|
||||
val maximum_score = if (candidatesOutsideOfTopK.nonEmpty) {
|
||||
candidatesOutsideOfTopK.map(x => x._1.score.getOrElse(0.0)).max
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
|
||||
// for candidates in candidatesOutsideOfTopK, we transform their score by subtracting off
|
||||
// maximum_score and then multiply by multiplicativeFactor
|
||||
val candidatesOutsideOfTopKTransformedScore = candidatesOutsideOfTopK.map(x =>
|
||||
(x._1, multiplicativeFactor * (x._1.score.getOrElse(0.0) - maximum_score)))
|
||||
|
||||
// for each candidate with score transformed and clip score w, sample a random number r,
|
||||
// create a new score r^(1/w) and sort the candidates to get the final ranking.
|
||||
// for numerical stability reasons if the score is <=-10, we force r^(1/w) = 0.
|
||||
// this samples the candidates from the modified Plackett-Luce distribution. See
|
||||
// https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051
|
||||
|
||||
val candidatesOutsideOfTopKSampled = candidatesOutsideOfTopKTransformedScore
|
||||
.map(x =>
|
||||
(
|
||||
x._1,
|
||||
if (x._2 <= -10.0)
|
||||
0.0
|
||||
else
|
||||
scala.math.pow(
|
||||
randomNumGenerator.nextFloat(),
|
||||
1 / (scala.math
|
||||
.exp(x._2))))).sortBy(-1 * _._2)
|
||||
|
||||
val topKCandidates: Seq[CandidateUser] = topKFixedCandidates.map(_._1)
|
||||
|
||||
val scribeRankingInfo: Boolean =
|
||||
target.params(SamplingTransformParams.ScribeRankingInfoInSamplingTransform)
|
||||
|
||||
val transformedCandidates: Seq[CandidateUser] = if (scribeRankingInfo) {
|
||||
val topKCandidatesWithRankingInfo: Seq[CandidateUser] =
|
||||
Utils.addRankingInfo(topKCandidates, name)
|
||||
val candidatesOutsideOfTopKSampledWithRankingInfo: Seq[CandidateUser] =
|
||||
candidatesOutsideOfTopKSampled.zipWithIndex.map {
|
||||
case ((candidate, score), rank) =>
|
||||
val newScore = Seq(Score(score, Some(RankerId.PlacketLuceSamplingTransformer)))
|
||||
val newScores: Option[Scores] = candidate.scores
|
||||
.map { scores =>
|
||||
scores.copy(scores = scores.scores ++ newScore)
|
||||
}.orElse(Some(Scores(newScore, Some(RankerId.PlacketLuceSamplingTransformer))))
|
||||
val globalRank = rank + topKFixed + 1
|
||||
candidate.addInfoPerRankingStage(name, newScores, globalRank)
|
||||
}
|
||||
|
||||
topKCandidatesWithRankingInfo ++ candidatesOutsideOfTopKSampledWithRankingInfo
|
||||
} else {
|
||||
topKCandidates ++ candidatesOutsideOfTopKSampled.map(_._1)
|
||||
}
|
||||
|
||||
Stitch.value(transformedCandidates)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,19 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.transforms.weighted_sampling
|
||||
|
||||
import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class SamplingTransformFSConfig @Inject() () extends FeatureSwitchConfig {
|
||||
override val intFSParams: Seq[FSBoundedParam[Int]] = Seq(SamplingTransformParams.TopKFixed)
|
||||
|
||||
override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq(
|
||||
SamplingTransformParams.MultiplicativeFactor)
|
||||
|
||||
override val booleanFSParams: Seq[FSParam[Boolean]] = Seq(
|
||||
SamplingTransformParams.ScribeRankingInfoInSamplingTransform)
|
||||
}
|
||||
Binary file not shown.
@ -1,25 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.transforms.weighted_sampling
|
||||
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
|
||||
object SamplingTransformParams {
|
||||
|
||||
case object TopKFixed // indicates how many of the fisrt K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score
|
||||
extends FSBoundedParam[Int](
|
||||
name = "post_nux_ml_flow_weighted_sampling_top_k_fixed",
|
||||
default = 0,
|
||||
min = 0,
|
||||
max = 100)
|
||||
|
||||
case object MultiplicativeFactor // CandidateUser.score gets transformed to multiplicativeFactor*CandidateUser.score before sampling from the Plackett-Luce distribution
|
||||
extends FSBoundedParam[Double](
|
||||
name = "post_nux_ml_flow_weighted_sampling_multiplicative_factor",
|
||||
default = 1.0,
|
||||
min = -1000.0,
|
||||
max = 1000.0)
|
||||
|
||||
case object ScribeRankingInfoInSamplingTransform
|
||||
extends FSParam[Boolean]("sampling_transform_scribe_ranking_info", false)
|
||||
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
scala_library(
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request",
|
||||
"stitch/stitch-core",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,22 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.utils
|
||||
|
||||
object CollectionUtil {
|
||||
|
||||
/**
|
||||
* Transposes a sequence of sequences. As opposed to the Scala collection library version
|
||||
* of transpose, the sequences do not have to have the same length.
|
||||
*
|
||||
* Example:
|
||||
* transpose(immutable.Seq(immutable.Seq(1,2,3), immutable.Seq(4,5), immutable.Seq(6,7)))
|
||||
* => immutable.Seq(immutable.Seq(1, 4, 6), immutable.Seq(2, 5, 7), immutable.Seq(3))
|
||||
*
|
||||
* @param seq a sequence of sequences
|
||||
* @tparam A the type of elements in the seq
|
||||
* @return the transposed sequence of sequences
|
||||
*/
|
||||
def transposeLazy[A](seq: Seq[Seq[A]]): Stream[Seq[A]] =
|
||||
seq.filter(_.nonEmpty) match {
|
||||
case Nil => Stream.empty
|
||||
case ys => ys.map(_.head) #:: transposeLazy(ys.map(_.tail))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,27 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.utils
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.Product
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.Product
|
||||
|
||||
object DisplayLocationProductConverterUtil {
|
||||
def productToDisplayLocation(product: Product): DisplayLocation = {
|
||||
product match {
|
||||
case Product.MagicRecs => DisplayLocation.MagicRecs
|
||||
case _ =>
|
||||
throw UnconvertibleProductMixerProductException(
|
||||
s"Cannot convert Product Mixer Product ${product.identifier.name} into a FRS DisplayLocation.")
|
||||
}
|
||||
}
|
||||
|
||||
def displayLocationToProduct(displayLocation: DisplayLocation): Product = {
|
||||
displayLocation match {
|
||||
case DisplayLocation.MagicRecs => Product.MagicRecs
|
||||
case _ =>
|
||||
throw UnconvertibleProductMixerProductException(
|
||||
s"Cannot convert DisplayLocation ${displayLocation.toFsName} into a Product Mixer Product.")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case class UnconvertibleProductMixerProductException(message: String) extends Exception(message)
|
||||
Binary file not shown.
@ -1,51 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.utils
|
||||
|
||||
object MergeUtil {
|
||||
|
||||
/**
|
||||
* Takes a seq of items which have weights. Returns an infinite stream of each item
|
||||
* by their weights. All weights need to be greater than or equal to zero. In addition,
|
||||
* the sum of weights should be greater than zero.
|
||||
*
|
||||
* Example usage of this function:
|
||||
* Input weighted Item {{CS1, 3}, {CS2, 2}, {CS3, 5}}
|
||||
* Output stream: (CS1, CS1, CS1, CS2, CS2, CS3, CS3, CS3, CS3, CS3, CS1, CS1, CS1, CS2,...}
|
||||
*
|
||||
* @param items items
|
||||
* @param weighted provides weights for items
|
||||
* @tparam T type of item
|
||||
*
|
||||
* @return Stream of Ts
|
||||
*/
|
||||
def weightedRoundRobin[T](
|
||||
items: Seq[T]
|
||||
)(
|
||||
implicit weighted: Weighted[T]
|
||||
): Stream[T] = {
|
||||
if (items.isEmpty) {
|
||||
Stream.empty
|
||||
} else {
|
||||
val weights = items.map { i => weighted(i) }
|
||||
assert(
|
||||
weights.forall {
|
||||
_ >= 0
|
||||
},
|
||||
"Negative weight exists for sampling")
|
||||
val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail
|
||||
assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive")
|
||||
|
||||
var weightIdx = 0
|
||||
var weight = 0
|
||||
|
||||
def next(): Stream[T] = {
|
||||
val tmpIdx = weightIdx
|
||||
weight = weight + 1
|
||||
weight = if (weight >= weights(weightIdx)) 0 else weight
|
||||
weightIdx = if (weight == 0) weightIdx + 1 else weightIdx
|
||||
weightIdx = if (weightIdx == weights.length) 0 else weightIdx
|
||||
items(tmpIdx) #:: next()
|
||||
}
|
||||
next()
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,88 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.utils
|
||||
import scala.util.Random
|
||||
|
||||
object RandomUtil {
|
||||
|
||||
/**
|
||||
* Takes a seq of items which have weights. Returns an infinite stream that is
|
||||
* sampled with replacement using the weights for each item. All weights need
|
||||
* to be greater than or equal to zero. In addition, the sum of weights
|
||||
* should be greater than zero.
|
||||
*
|
||||
* @param items items
|
||||
* @param weighted provides weights for items
|
||||
* @tparam T type of item
|
||||
* @return Stream of Ts
|
||||
*/
|
||||
def weightedRandomSamplingWithReplacement[T](
|
||||
items: Seq[T],
|
||||
random: Option[Random] = None
|
||||
)(
|
||||
implicit weighted: Weighted[T]
|
||||
): Stream[T] = {
|
||||
if (items.isEmpty) {
|
||||
Stream.empty
|
||||
} else {
|
||||
val weights = items.map { i => weighted(i) }
|
||||
assert(weights.forall { _ >= 0 }, "Negative weight exists for sampling")
|
||||
val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail
|
||||
assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive")
|
||||
val cumulativeProbability = cumulativeWeight map (_ / cumulativeWeight.last)
|
||||
def next(): Stream[T] = {
|
||||
val rand = random.getOrElse(Random).nextDouble()
|
||||
val idx = cumulativeProbability.indexWhere(_ >= rand)
|
||||
items(if (idx == -1) items.length - 1 else idx) #:: next()
|
||||
}
|
||||
next()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a seq of items and their weights. Returns a lazy weighted shuffle of
|
||||
* the elements in the list. All weights should be greater than zero.
|
||||
*
|
||||
* @param items items
|
||||
* @param weighted provides weights for items
|
||||
* @tparam T type of item
|
||||
* @return Stream of Ts
|
||||
*/
|
||||
def weightedRandomShuffle[T](
|
||||
items: Seq[T],
|
||||
random: Option[Random] = None
|
||||
)(
|
||||
implicit weighted: Weighted[T]
|
||||
): Stream[T] = {
|
||||
assert(items.forall { i => weighted(i) > 0 }, "Non-positive weight exists for shuffling")
|
||||
def next(it: Seq[T]): Stream[T] = {
|
||||
if (it.isEmpty)
|
||||
Stream.empty
|
||||
else {
|
||||
val cumulativeWeight = it.scanLeft(0.0)((acc: Double, curr: T) => acc + weighted(curr)).tail
|
||||
val cutoff = random.getOrElse(Random).nextDouble() * cumulativeWeight.last
|
||||
val idx = cumulativeWeight.indexWhere(_ >= cutoff)
|
||||
val (left, right) = it.splitAt(idx)
|
||||
it(if (idx == -1) it.size - 1 else idx) #:: next(left ++ right.drop(1))
|
||||
}
|
||||
}
|
||||
next(items)
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes a seq of items and a weight function, returns a lazy weighted shuffle of
|
||||
* the elements in the list.The weight function is based on the rank of the element
|
||||
* in the original lst.
|
||||
* @param items
|
||||
* @param rankToWeight
|
||||
* @param random
|
||||
* @tparam T
|
||||
* @return
|
||||
*/
|
||||
def weightedRandomShuffleByRank[T](
|
||||
items: Seq[T],
|
||||
rankToWeight: Int => Double,
|
||||
random: Option[Random] = None
|
||||
): Stream[T] = {
|
||||
val candWeights = items.zipWithIndex.map { case (item, rank) => (item, rankToWeight(rank)) }
|
||||
RandomUtil.weightedRandomShuffle(candWeights, random).map(_._1)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,50 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.utils
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.StatsUtil
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.TimeoutException
|
||||
|
||||
object RescueWithStatsUtils {
|
||||
def rescueWithStats[T](
|
||||
s: Stitch[Seq[T]],
|
||||
stats: StatsReceiver,
|
||||
source: String
|
||||
): Stitch[Seq[T]] = {
|
||||
StatsUtil.profileStitchSeqResults(s, stats.scope(source)).rescue {
|
||||
case _: Exception => Stitch.Nil
|
||||
}
|
||||
}
|
||||
|
||||
def rescueOptionalWithStats[T](
|
||||
s: Stitch[Option[T]],
|
||||
stats: StatsReceiver,
|
||||
source: String
|
||||
): Stitch[Option[T]] = {
|
||||
StatsUtil.profileStitchOptionalResults(s, stats.scope(source)).rescue {
|
||||
case _: Exception => Stitch.None
|
||||
}
|
||||
}
|
||||
|
||||
def rescueWithStatsWithin[T](
|
||||
s: Stitch[Seq[T]],
|
||||
stats: StatsReceiver,
|
||||
source: String,
|
||||
timeout: Duration
|
||||
): Stitch[Seq[T]] = {
|
||||
val hydratedScopeSource = stats.scope(source)
|
||||
StatsUtil
|
||||
.profileStitchSeqResults(
|
||||
s.within(timeout)(com.twitter.finagle.util.DefaultTimer),
|
||||
hydratedScopeSource)
|
||||
.rescue {
|
||||
case _: TimeoutException =>
|
||||
hydratedScopeSource.counter("timeout").incr()
|
||||
Stitch.Nil
|
||||
case _: Exception =>
|
||||
hydratedScopeSource.counter("exception").incr()
|
||||
Stitch.Nil
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,14 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.utils
|
||||
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Time
|
||||
|
||||
object UserSignupUtil {
|
||||
def signupTime(hasClientContext: HasClientContext): Option[Time] =
|
||||
hasClientContext.clientContext.userId.flatMap(SnowflakeId.timeFromIdOpt)
|
||||
|
||||
def userSignupAge(hasClientContext: HasClientContext): Option[Duration] =
|
||||
signupTime(hasClientContext).map(Time.now - _)
|
||||
}
|
||||
Binary file not shown.
@ -1,21 +0,0 @@
|
||||
package com.twitter.follow_recommendations.common.utils
|
||||
|
||||
/**
|
||||
* Typeclass for any Recommendation type that has a weight
|
||||
*
|
||||
*/
|
||||
trait Weighted[-Rec] {
|
||||
def apply(rec: Rec): Double
|
||||
}
|
||||
|
||||
object Weighted {
|
||||
implicit object WeightedTuple extends Weighted[(_, Double)] {
|
||||
override def apply(rec: (_, Double)): Double = rec._2
|
||||
}
|
||||
|
||||
def fromFunction[Rec](f: Rec => Double): Weighted[Rec] = {
|
||||
new Weighted[Rec] {
|
||||
override def apply(rec: Rec): Double = f(rec)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
resources(
|
||||
sources = [
|
||||
"*.tsv",
|
||||
"*.xml",
|
||||
"**/*",
|
||||
"config/*.yml",
|
||||
],
|
||||
)
|
||||
|
||||
# Created for Bazel compatibility.
|
||||
# In Bazel, loose files must be part of a target to be included into a bundle.
|
||||
files(
|
||||
name = "frs_resources",
|
||||
sources = [
|
||||
"*.tsv",
|
||||
"*.xml",
|
||||
"*.yml",
|
||||
"**/*",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,129 +0,0 @@
|
||||
enable_recommendations:
|
||||
comment: Proportion of requests where we return an actual response as part. Decreasing the value will increase the portion of empty responses (in order to disable the service) as part of the graceful degradation.
|
||||
default_availability: 10000
|
||||
enable_score_user_candidates:
|
||||
comment: Proportion of requests where score user candidates from the scoreUserCandidates endpoint
|
||||
default_availability: 10000
|
||||
enable_profile_sidebar_product:
|
||||
comment: Proportion of requests where we return an actual response for profile sidebar product
|
||||
default_availability: 10000
|
||||
enable_magic_recs_product:
|
||||
comment: Proportion of requests where we return an actual response for magic recs product
|
||||
default_availability: 10000
|
||||
enable_rux_landing_page_product:
|
||||
comment: Proportion of requests where we return an actual response for rux landing page product
|
||||
default_availability: 10000
|
||||
enable_rux_pymk_product:
|
||||
comment: Proportion of requests where we return an actual response for rux pymk product
|
||||
default_availability: 10000
|
||||
enable_profile_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for profile bonus follow product
|
||||
default_availability: 10000
|
||||
enable_election_explore_wtf_product:
|
||||
comment: Proportion of requests where we return an actual response for election explore wtf product
|
||||
default_availability: 10000
|
||||
enable_cluster_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for cluster follow product
|
||||
default_availability: 10000
|
||||
enable_home_timeline_product:
|
||||
comment: Proportion of requests where we return an actual response for htl wtf product
|
||||
default_availability: 10000
|
||||
enable_htl_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for htl bonus follow product
|
||||
default_availability: 10000
|
||||
enable_explore_tab_product:
|
||||
comment: Proportion of requests where we return an actual response for explore tab product
|
||||
default_availability: 10000
|
||||
enable_sidebar_product:
|
||||
comment: Proportion of requests where we return an actual response for sidebar product
|
||||
default_availability: 10000
|
||||
enable_campaign_form_product:
|
||||
comment: Proportion of requests where we return an actual response for campaign form product
|
||||
default_availability: 10000
|
||||
enable_reactive_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for reactive follow product
|
||||
default_availability: 10000
|
||||
enable_nux_pymk_product:
|
||||
comment: Proportion of requests where we return an actual response for nux pymk product
|
||||
default_availability: 10000
|
||||
enable_nux_interests_product:
|
||||
comment: Proportion of requests where we return an actual response for nux interests product
|
||||
default_availability: 10000
|
||||
enable_nux_topic_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for nux topic-based bonus follow product
|
||||
default_availability: 10000
|
||||
enable_india_covid19_curated_accounts_wtf_product:
|
||||
comment: Proportion of requests where we return an actual response for india covid19 curated accounts wtf product
|
||||
default_availability: 10000
|
||||
enable_ab_upload_product:
|
||||
comment: Proportion of requests where we return an actual response for the address book upload product
|
||||
default_availability: 10000
|
||||
enable_people_plus_plus_product:
|
||||
comment: Proportion of requests where we return an actual response for the PeoplePlusPlus/Connect Tab product
|
||||
default_availability: 10000
|
||||
enable_tweet_notification_recs_product:
|
||||
comment: Proportion of requests where we return an actual response for the Tweet Notification Recommendations product
|
||||
default_availability: 10000
|
||||
enable_profile_device_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for the ProfileDeviceFollow product
|
||||
default_availability: 10000
|
||||
enable_diffy_module_dark_reading:
|
||||
comment: Percentage of dark read traffic routed to diffy thrift
|
||||
default_availability: 0
|
||||
enable_recos_backfill_product:
|
||||
comment: Proportion of requests where we return an actual response for the RecosBackfill product
|
||||
default_availability: 10000
|
||||
enable_post_nux_follow_task_product:
|
||||
comment: Proportion of requests where we return an actual response for post NUX follow task product
|
||||
default_availability: 10000
|
||||
enable_curated_space_hosts_product:
|
||||
comment: Proportion of requests where we return an actual response for curated space hosts product
|
||||
default_availability: 10000
|
||||
enable_nux_geo_category_product:
|
||||
comment: Proportion of requests where we return an actual response for nux geo category product
|
||||
default_availability: 10000
|
||||
enable_nux_interests_category_product:
|
||||
comment: Proportion of requests where we return an actual response for nux interests category product
|
||||
default_availability: 10000
|
||||
enable_nux_pymk_category_product:
|
||||
comment: Proportion of requests where we return an actual response for nux pymk category product
|
||||
default_availability: 10000
|
||||
enable_home_timeline_tweet_recs_product:
|
||||
comment: Proportion of requests where we return an actual response for the Home Timeline Tweet Recs product
|
||||
default_availability: 10000
|
||||
enable_htl_bulk_friend_follows_product:
|
||||
comment: Proportion of requests where we return an actual response for the HTL bulk friend follows product
|
||||
default_availability: 10000
|
||||
enable_nux_auto_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for the NUX auto follow product
|
||||
default_availability: 10000
|
||||
enable_search_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for search bonus follow product
|
||||
default_availability: 10000
|
||||
enable_fetch_user_in_request_builder:
|
||||
comment: Proportion of requests where we fetch user object from gizmoduck in request builder
|
||||
default_availability: 0
|
||||
enable_product_mixer_magic_recs_product:
|
||||
comment: Proportion of requests where we enable the product mixer magic recs product
|
||||
default_availability: 10000
|
||||
enable_home_timeline_reverse_chron_product:
|
||||
comment: Proportion of requests where we return an actual response for Home timeline reverse chron product
|
||||
default_availability: 10000
|
||||
enable_product_mixer_pipeline_magic_recs_dark_read:
|
||||
comment: Compare product mixer pipeline responses to current FRS pipeline responses for Magic Recs
|
||||
default_availability: 0
|
||||
enable_experimental_caching:
|
||||
comment: Proportion of requests we use experimental caching for data caching
|
||||
default_availability: 0
|
||||
enable_distributed_caching:
|
||||
comment: Proportion of requests we use a distributed cache cluster for data caching
|
||||
default_availability: 10000
|
||||
enable_gizmoduck_caching:
|
||||
comment: Proportion of requests we use a distributed cache cluster for data caching in Gizmoduck
|
||||
default_availability: 10000
|
||||
enable_traffic_dark_reading:
|
||||
comment: Proportion of requests where we replicate the request for traffic dark reading
|
||||
default_availability: 0
|
||||
enable_graph_feature_service_requests:
|
||||
comment: Proportion of requests where we allow request calls to Graph Feature Service
|
||||
default_availability: 10000
|
||||
Binary file not shown.
@ -1,133 +0,0 @@
|
||||
<configuration>
|
||||
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Service Config -->
|
||||
<!-- ===================================================== -->
|
||||
<property name="DEFAULT_SERVICE_PATTERN"
|
||||
value="%-16X{traceId} %-12X{serviceIdentifier:--} %-16X{method} %-12X{product:--} %-25logger{0} %msg"/>
|
||||
|
||||
<property name="DEFAULT_ACCESS_PATTERN"
|
||||
value="%msg %-12X{serviceIdentifier:--} %X{traceId} %X{product:--}"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Common Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- JUL/JDK14 to Logback bridge -->
|
||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
|
||||
<resetJUL>true</resetJUL>
|
||||
</contextListener>
|
||||
|
||||
<!-- Service Log (Rollover every 50MB, max 5 logs) -->
|
||||
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.service.output}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
||||
<fileNamePattern>${log.service.output}.%i</fileNamePattern>
|
||||
<minIndex>1</minIndex>
|
||||
<maxIndex>5</maxIndex>
|
||||
</rollingPolicy>
|
||||
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
||||
<maxFileSize>50MB</maxFileSize>
|
||||
</triggeringPolicy>
|
||||
<encoder>
|
||||
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- Access Log (Rollover every 50MB, max 5 logs) -->
|
||||
<appender name="ACCESS" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.access.output}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
||||
<fileNamePattern>${log.access.output}.%i</fileNamePattern>
|
||||
<minIndex>1</minIndex>
|
||||
<maxIndex>5</maxIndex>
|
||||
</rollingPolicy>
|
||||
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
||||
<maxFileSize>50MB</maxFileSize>
|
||||
</triggeringPolicy>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_ACCESS_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!--LogLens -->
|
||||
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>${log.lens.category}</category>
|
||||
<index>${log.lens.index}</index>
|
||||
<tag>${log.lens.tag}/service</tag>
|
||||
<encoder>
|
||||
<pattern>%msg</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- LogLens Access -->
|
||||
<appender name="LOGLENS-ACCESS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>${log.lens.category}</category>
|
||||
<index>${log.lens.index}</index>
|
||||
<tag>${log.lens.tag}/access</tag>
|
||||
<encoder>
|
||||
<pattern>%msg</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Primary Async Appenders -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<property name="async_queue_size" value="${queue.size:-50000}"/>
|
||||
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
|
||||
|
||||
<appender name="ASYNC-SERVICE" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="SERVICE"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-ACCESS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="ACCESS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS-ACCESS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS-ACCESS"/>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Package Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Per-Package Config -->
|
||||
<logger name="com.twitter" level="info"/>
|
||||
<logger name="com.twitter.wilyns" level="warn"/>
|
||||
<logger name="com.twitter.finagle.mux" level="warn"/>
|
||||
<logger name="com.twitter.finagle.serverset2" level="warn"/>
|
||||
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
|
||||
|
||||
<!-- Root Config -->
|
||||
<root level="${log_level:-INFO}">
|
||||
<appender-ref ref="ASYNC-SERVICE"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</root>
|
||||
|
||||
<!-- Access Logging -->
|
||||
<logger name="com.twitter.finatra.thrift.filters.AccessLoggingFilter"
|
||||
level="info"
|
||||
additivity="false">
|
||||
<appender-ref ref="ASYNC-ACCESS"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS-ACCESS"/>
|
||||
</logger>
|
||||
|
||||
</configuration>
|
||||
@ -1,8 +0,0 @@
|
||||
# OWNER = jdeng
|
||||
# Date = 20141223_153423
|
||||
# Training Size = 16744473
|
||||
# Testing Size = 16767335
|
||||
# trained with ElasticNetCV alpha=0.05 cv_folds=5 best_lambda=1.0E-7
|
||||
# num base features: 10
|
||||
# num nonzero weights: 30
|
||||
{bias:-5.67151,featureMetadataMap:["fwd_email":{metadata:{featureWeight:{weight:2.47389}}},"rev_phone":{metadata:{featureWeight:{weight:1.88433}}},"mutual_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:47.0,weight:6.31809},{left:11.0,right:16.0,weight:4.52959},{left:31.0,right:47.0,weight:5.7101},{right:2.0,weight:0.383515},{left:24.0,right:31.0,weight:5.26515},{left:3.0,right:4.0,weight:2.91751},{left:2.0,right:3.0,weight:2.22851},{left:4.0,right:5.0,weight:3.28515},{left:8.0,right:11.0,weight:4.14731},{left:5.0,right:8.0,weight:3.73588},{left:16.0,right:24.0,weight:4.90908}]}}},"fwd_phone":{metadata:{featureWeight:{weight:2.07327}}},"fwd_email_path":{metadata:{featureWeight:{weight:0.961773}}},"rev_phone_path":{metadata:{featureWeight:{weight:0.354484}}},"low_tweepcred_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:4.0,right:5.0,weight:0.177209},{left:7.0,right:8.0,weight:0.12378},{left:3.0,right:4.0,weight:0.197566},{left:5.0,right:6.0,weight:0.15867},{left:2.0,right:3.0,weight:0.196539},{right:2.0,weight:0.1805},{left:75.0,weight:-0.424598},{left:6.0,right:7.0,weight:0.143698},{left:10.0,right:13.0,weight:0.0458502},{left:8.0,right:10.0,weight:0.0919314},{left:13.0,right:75.0,weight:-0.111484}]}}},"rev_email_path":{metadata:{featureWeight:{weight:0.654451}}},"rev_email":{metadata:{featureWeight:{weight:2.33859}}},"fwd_phone_path":{metadata:{featureWeight:{weight:0.210418}}}]}
|
||||
Binary file not shown.
@ -1 +0,0 @@
|
||||
{input:{context:"discover.prod",startDateTime:"",endDateTime:"",trainingFeatures:["STP_FEATURES":["fwd_email","mutual_follow_path","fwd_email_path","rev_phone_path","low_tweepcred_follow_path","rev_phone","fwd_phone","rev_email_path","rev_email","fwd_phone_path"]],engagementActions:["click","favorite","open_link","open","send_tweet","send_reply","retweet","reply","profile_click","follow"],impressionActions:["discard","results","impression"],dataFormat:1,dataPath:"",isLabeled:0},sample:{positiveSampleRatio:1.0,negativeSampleRatio:1.0,sampleType:1},split:{trainingDataSplitSize:0.5,testingDataSplitSize:0.5,splitType:2},transform:{},filter:{featureOptions:[]},join:{engagementRules:["discover"],contentIdType:"tweet",groupBucketSize:3600000},discretize:{}}
|
||||
Binary file not shown.
@ -1,48 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finagle/finagle-http/src/main/scala",
|
||||
"finagle/finagle-thriftmux/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra-internal/international/src/main/scala/com/twitter/finatra/international/modules",
|
||||
"finatra-internal/mtls-http/src/main/scala",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/http-core/src/main/java/com/twitter/finatra/http",
|
||||
"finatra/inject/inject-app/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"finatra/jackson/src/main/scala/com/twitter/finatra/jackson/modules",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato",
|
||||
"follow-recommendations-service/server/src/main/resources",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,118 +0,0 @@
|
||||
package com.twitter.follow_recommendations
|
||||
|
||||
import com.google.inject.Module
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.http.HttpServer
|
||||
import com.twitter.finatra.http.routing.HttpRouter
|
||||
import com.twitter.finatra.international.modules.I18nFactoryModule
|
||||
import com.twitter.finatra.international.modules.LanguagesModule
|
||||
import com.twitter.finatra.jackson.modules.ScalaObjectMapperModule
|
||||
import com.twitter.finatra.mtls.http.{Mtls => HttpMtls}
|
||||
import com.twitter.finatra.mtls.thriftmux.Mtls
|
||||
import com.twitter.finatra.thrift.ThriftServer
|
||||
import com.twitter.finatra.thrift.filters._
|
||||
import com.twitter.finagle.thrift.Protocols
|
||||
import com.twitter.finatra.thrift.routing.ThriftRouter
|
||||
import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookModule
|
||||
import com.twitter.follow_recommendations.common.clients.adserver.AdserverModule
|
||||
import com.twitter.follow_recommendations.common.clients.cache.MemcacheModule
|
||||
import com.twitter.follow_recommendations.common.clients.deepbirdv2.DeepBirdV2PredictionServiceClientModule
|
||||
import com.twitter.follow_recommendations.common.clients.email_storage_service.EmailStorageServiceModule
|
||||
import com.twitter.follow_recommendations.common.clients.geoduck.LocationServiceModule
|
||||
import com.twitter.follow_recommendations.common.clients.gizmoduck.GizmoduckModule
|
||||
import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureStoreModule
|
||||
import com.twitter.follow_recommendations.common.clients.impression_store.ImpressionStoreModule
|
||||
import com.twitter.follow_recommendations.common.clients.phone_storage_service.PhoneStorageServiceModule
|
||||
import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphModule
|
||||
import com.twitter.follow_recommendations.common.clients.strato.StratoClientModule
|
||||
import com.twitter.follow_recommendations.common.constants.ServiceConstants._
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.sources.HydrationSourcesModule
|
||||
import com.twitter.follow_recommendations.controllers.ThriftController
|
||||
import com.twitter.follow_recommendations.modules._
|
||||
import com.twitter.follow_recommendations.service.exceptions.UnknownLoggingExceptionMapper
|
||||
import com.twitter.follow_recommendations.services.FollowRecommendationsServiceWarmupHandler
|
||||
import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService
|
||||
import com.twitter.geoduck.service.common.clientmodules.ReverseGeocoderThriftClientModule
|
||||
import com.twitter.inject.thrift.filters.DarkTrafficFilter
|
||||
import com.twitter.inject.thrift.modules.ThriftClientIdModule
|
||||
import com.twitter.product_mixer.core.controllers.ProductMixerController
|
||||
import com.twitter.product_mixer.core.module.PipelineExecutionLoggerModule
|
||||
import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule
|
||||
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
|
||||
import com.twitter.product_mixer.core.product.guice.ProductScopeModule
|
||||
|
||||
object FollowRecommendationsServiceThriftServerMain extends FollowRecommendationsServiceThriftServer
|
||||
|
||||
class FollowRecommendationsServiceThriftServer
|
||||
extends ThriftServer
|
||||
with Mtls
|
||||
with HttpServer
|
||||
with HttpMtls {
|
||||
override val name: String = "follow-recommendations-service-server"
|
||||
|
||||
override val modules: Seq[Module] =
|
||||
Seq(
|
||||
ABDeciderModule,
|
||||
AddressbookModule,
|
||||
AdserverModule,
|
||||
ConfigApiModule,
|
||||
DeciderModule,
|
||||
DeepBirdV2PredictionServiceClientModule,
|
||||
DiffyModule,
|
||||
EmailStorageServiceModule,
|
||||
FeaturesSwitchesModule,
|
||||
FlagsModule,
|
||||
GizmoduckModule,
|
||||
GraphFeatureStoreModule,
|
||||
HydrationSourcesModule,
|
||||
I18nFactoryModule,
|
||||
ImpressionStoreModule,
|
||||
LanguagesModule,
|
||||
LocationServiceModule,
|
||||
MemcacheModule,
|
||||
PhoneStorageServiceModule,
|
||||
PipelineExecutionLoggerModule,
|
||||
ProductMixerFlagModule,
|
||||
ProductRegistryModule,
|
||||
new ProductScopeModule(),
|
||||
new ProductScopeStringCenterModule(),
|
||||
new ReverseGeocoderThriftClientModule,
|
||||
ScalaObjectMapperModule,
|
||||
ScorerModule,
|
||||
ScribeModule,
|
||||
SocialGraphModule,
|
||||
StratoClientModule,
|
||||
ThriftClientIdModule,
|
||||
TimerModule,
|
||||
)
|
||||
|
||||
def configureThrift(router: ThriftRouter): Unit = {
|
||||
router
|
||||
.filter[LoggingMDCFilter]
|
||||
.filter[TraceIdMDCFilter]
|
||||
.filter[ThriftMDCFilter]
|
||||
.filter[StatsFilter]
|
||||
.filter[AccessLoggingFilter]
|
||||
.filter[ExceptionMappingFilter]
|
||||
.exceptionMapper[UnknownLoggingExceptionMapper]
|
||||
.filter[DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]]
|
||||
.add[ThriftController]
|
||||
}
|
||||
|
||||
override def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server = {
|
||||
server.withProtocolFactory(
|
||||
Protocols.binaryFactory(
|
||||
stringLengthLimit = StringLengthLimit,
|
||||
containerLengthLimit = ContainerLengthLimit))
|
||||
}
|
||||
|
||||
override def configureHttp(router: HttpRouter): Unit = router.add(
|
||||
ProductMixerController[FollowRecommendationsThriftService.MethodPerEndpoint](
|
||||
this.injector,
|
||||
FollowRecommendationsThriftService.ExecutePipeline))
|
||||
|
||||
override def warmup(): Unit = {
|
||||
handle[FollowRecommendationsServiceWarmupHandler]()
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,9 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Action(text: String, actionURL: String) {
|
||||
lazy val toThrift: t.Action = {
|
||||
t.Action(text, actionURL)
|
||||
}
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"stringcenter/client",
|
||||
],
|
||||
exports = [
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,8 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.stringcenter.client.core.ExternalString
|
||||
|
||||
case class HeaderConfig(title: TitleConfig)
|
||||
case class TitleConfig(text: ExternalString)
|
||||
case class FooterConfig(actionConfig: Option[ActionConfig])
|
||||
case class ActionConfig(footerText: ExternalString, actionURL: String)
|
||||
Binary file not shown.
@ -1,13 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
trait FeedbackAction {
|
||||
def toThrift: t.FeedbackAction
|
||||
}
|
||||
|
||||
case class DismissUserId() extends FeedbackAction {
|
||||
override lazy val toThrift: t.FeedbackAction = {
|
||||
t.FeedbackAction.DismissUserId(t.DismissUserId())
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,9 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Footer(action: Option[Action]) {
|
||||
lazy val toThrift: t.Footer = {
|
||||
t.Footer(action.map(_.toThrift))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,9 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Header(title: Title) {
|
||||
lazy val toThrift: t.Header = {
|
||||
t.Header(title.toThrift)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,16 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
sealed trait Layout
|
||||
|
||||
case class UserListLayout(
|
||||
header: Option[HeaderConfig],
|
||||
userListOptions: UserListOptions,
|
||||
socialProofs: Option[Seq[SocialProof]],
|
||||
footer: Option[FooterConfig])
|
||||
extends Layout
|
||||
|
||||
case class CarouselLayout(
|
||||
header: Option[HeaderConfig],
|
||||
carouselOptions: CarouselOptions,
|
||||
socialProofs: Option[Seq[SocialProof]])
|
||||
extends Layout
|
||||
Binary file not shown.
@ -1,11 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
sealed trait RecommendationOptions
|
||||
|
||||
case class UserListOptions(
|
||||
userBioEnabled: Boolean,
|
||||
userBioTruncated: Boolean,
|
||||
userBioMaxLines: Option[Long],
|
||||
) extends RecommendationOptions
|
||||
|
||||
case class CarouselOptions() extends RecommendationOptions
|
||||
Binary file not shown.
@ -1,16 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.stringcenter.client.core.ExternalString
|
||||
|
||||
sealed trait SocialProof
|
||||
|
||||
case class GeoContextProof(popularInCountryText: ExternalString) extends SocialProof
|
||||
case class FollowedByUsersProof(text1: ExternalString, text2: ExternalString, textN: ExternalString)
|
||||
extends SocialProof
|
||||
|
||||
sealed trait SocialText {
|
||||
def text: String
|
||||
}
|
||||
|
||||
case class GeoSocialText(text: String) extends SocialText
|
||||
case class FollowedByUsersText(text: String) extends SocialText
|
||||
Binary file not shown.
@ -1,9 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Title(text: String) {
|
||||
lazy val toThrift: t.Title = {
|
||||
t.Title(text)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,47 +0,0 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
trait WTFPresentation {
|
||||
def toThrift: t.WTFPresentation
|
||||
}
|
||||
|
||||
case class UserList(
|
||||
userBioEnabled: Boolean,
|
||||
userBioTruncated: Boolean,
|
||||
userBioMaxLines: Option[Long],
|
||||
feedbackAction: Option[FeedbackAction])
|
||||
extends WTFPresentation {
|
||||
def toThrift: t.WTFPresentation = {
|
||||
t.WTFPresentation.UserBioList(
|
||||
t.UserList(userBioEnabled, userBioTruncated, userBioMaxLines, feedbackAction.map(_.toThrift)))
|
||||
}
|
||||
}
|
||||
|
||||
object UserList {
|
||||
def fromUserListOptions(
|
||||
userListOptions: UserListOptions
|
||||
): UserList = {
|
||||
UserList(
|
||||
userListOptions.userBioEnabled,
|
||||
userListOptions.userBioTruncated,
|
||||
userListOptions.userBioMaxLines,
|
||||
None)
|
||||
}
|
||||
}
|
||||
|
||||
case class Carousel(
|
||||
feedbackAction: Option[FeedbackAction])
|
||||
extends WTFPresentation {
|
||||
def toThrift: t.WTFPresentation = {
|
||||
t.WTFPresentation.Carousel(t.Carousel(feedbackAction.map(_.toThrift)))
|
||||
}
|
||||
}
|
||||
|
||||
object Carousel {
|
||||
def fromCarouselOptions(
|
||||
carouselOptions: CarouselOptions
|
||||
): Carousel = {
|
||||
Carousel(None)
|
||||
}
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,138 +0,0 @@
|
||||
package com.twitter.follow_recommendations.blenders
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.follow_recommendations.common.models.AdMetadata
|
||||
import com.twitter.follow_recommendations.common.models.Recommendation
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class PromotedAccountsBlender @Inject() (statsReceiver: StatsReceiver)
|
||||
extends Transform[Int, Recommendation]
|
||||
with Logging {
|
||||
|
||||
import PromotedAccountsBlender._
|
||||
val stats = statsReceiver.scope(Name)
|
||||
val inputOrganicAccounts = stats.counter(InputOrganic)
|
||||
val inputPromotedAccounts = stats.counter(InputPromoted)
|
||||
val outputOrganicAccounts = stats.counter(OutputOrganic)
|
||||
val outputPromotedAccounts = stats.counter(OutputPromoted)
|
||||
val promotedAccountsStats = stats.scope(NumPromotedAccounts)
|
||||
|
||||
override def transform(
|
||||
maxResults: Int,
|
||||
items: Seq[Recommendation]
|
||||
): Stitch[Seq[Recommendation]] = {
|
||||
val (promoted, organic) = items.partition(_.isPromotedAccount)
|
||||
val promotedIds = promoted.map(_.id).toSet
|
||||
val dedupedOrganic = organic.filterNot(u => promotedIds.contains(u.id))
|
||||
val blended = blendPromotedAccount(dedupedOrganic, promoted, maxResults)
|
||||
val (outputPromoted, outputOrganic) = blended.partition(_.isPromotedAccount)
|
||||
inputOrganicAccounts.incr(dedupedOrganic.size)
|
||||
inputPromotedAccounts.incr(promoted.size)
|
||||
outputOrganicAccounts.incr(outputOrganic.size)
|
||||
val size = outputPromoted.size
|
||||
outputPromotedAccounts.incr(size)
|
||||
if (size <= 5) {
|
||||
promotedAccountsStats.counter(outputPromoted.size.toString).incr()
|
||||
} else {
|
||||
promotedAccountsStats.counter(MoreThan5Promoted).incr()
|
||||
}
|
||||
Stitch.value(blended)
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge Promoted results and organic results. Promoted result dictates the position
|
||||
* in the merge list.
|
||||
*
|
||||
* merge a list of positioned users, aka. promoted, and a list of organic
|
||||
* users. The positioned promoted users are pre-sorted with regards to their
|
||||
* position ascendingly. Only requirement about position is to be within the
|
||||
* range, i.e, can not exceed the combined length if merge is successful, ok
|
||||
* to be at the last position, but not beyond.
|
||||
* For more detailed description of location position:
|
||||
* http://confluence.local.twitter.com/display/ADS/Promoted+Tweets+in+Timeline+Design+Document
|
||||
*/
|
||||
@VisibleForTesting
|
||||
private[blenders] def mergePromotedAccounts(
|
||||
organicUsers: Seq[Recommendation],
|
||||
promotedUsers: Seq[Recommendation]
|
||||
): Seq[Recommendation] = {
|
||||
def mergeAccountWithIndex(
|
||||
organicUsers: Seq[Recommendation],
|
||||
promotedUsers: Seq[Recommendation],
|
||||
index: Int
|
||||
): Stream[Recommendation] = {
|
||||
if (promotedUsers.isEmpty) organicUsers.toStream
|
||||
else {
|
||||
val promotedHead = promotedUsers.head
|
||||
val promotedTail = promotedUsers.tail
|
||||
promotedHead.adMetadata match {
|
||||
case Some(AdMetadata(position, _)) =>
|
||||
if (position < 0) mergeAccountWithIndex(organicUsers, promotedTail, index)
|
||||
else if (position == index)
|
||||
promotedHead #:: mergeAccountWithIndex(organicUsers, promotedTail, index)
|
||||
else if (organicUsers.isEmpty) organicUsers.toStream
|
||||
else {
|
||||
val organicHead = organicUsers.head
|
||||
val organicTail = organicUsers.tail
|
||||
organicHead #:: mergeAccountWithIndex(organicTail, promotedUsers, index + 1)
|
||||
}
|
||||
case _ =>
|
||||
logger.error("Unknown Candidate type in mergePromotedAccounts")
|
||||
Stream.empty
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mergeAccountWithIndex(organicUsers, promotedUsers, 0)
|
||||
}
|
||||
|
||||
private[this] def blendPromotedAccount(
|
||||
organic: Seq[Recommendation],
|
||||
promoted: Seq[Recommendation],
|
||||
maxResults: Int
|
||||
): Seq[Recommendation] = {
|
||||
|
||||
val merged = mergePromotedAccounts(organic, promoted)
|
||||
val mergedServed = merged.take(maxResults)
|
||||
val promotedServed = promoted.intersect(mergedServed)
|
||||
|
||||
if (isBlendPromotedNeeded(
|
||||
mergedServed.size - promotedServed.size,
|
||||
promotedServed.size,
|
||||
maxResults
|
||||
)) {
|
||||
mergedServed
|
||||
} else {
|
||||
organic.take(maxResults)
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
private[blenders] def isBlendPromotedNeeded(
|
||||
organicSize: Int,
|
||||
promotedSize: Int,
|
||||
maxResults: Int
|
||||
): Boolean = {
|
||||
(organicSize > 1) &&
|
||||
(promotedSize > 0) &&
|
||||
(promotedSize < organicSize) &&
|
||||
(promotedSize <= 2) &&
|
||||
(maxResults > 1)
|
||||
}
|
||||
}
|
||||
|
||||
object PromotedAccountsBlender {
|
||||
val Name = "promoted_accounts_blender"
|
||||
val InputOrganic = "input_organic_accounts"
|
||||
val InputPromoted = "input_promoted_accounts"
|
||||
val OutputOrganic = "output_organic_accounts"
|
||||
val OutputPromoted = "output_promoted_accounts"
|
||||
val NumPromotedAccounts = "num_promoted_accounts"
|
||||
val MoreThan5Promoted = "more_than_5"
|
||||
}
|
||||
@ -1,28 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
"configapi/configapi-featureswitches:v2",
|
||||
"featureswitches/featureswitches-core",
|
||||
"featureswitches/featureswitches-core:v2",
|
||||
"featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,16 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.timelines.configapi.CompositeConfig
|
||||
import com.twitter.timelines.configapi.Config
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ConfigBuilder @Inject() (
|
||||
deciderConfigs: DeciderConfigs,
|
||||
featureSwitchConfigs: FeatureSwitchConfigs) {
|
||||
// The order of configs added to `CompositeConfig` is important. The config will be matched with
|
||||
// the first possible rule. So, current setup will give priority to Deciders instead of FS
|
||||
def build(): Config =
|
||||
new CompositeConfig(Seq(deciderConfigs.config, featureSwitchConfigs.config))
|
||||
}
|
||||
Binary file not shown.
@ -1,52 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.decider.Recipient
|
||||
import com.twitter.decider.SimpleRecipient
|
||||
import com.twitter.follow_recommendations.configapi.deciders.DeciderKey
|
||||
import com.twitter.follow_recommendations.configapi.deciders.DeciderParams
|
||||
import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.timelines.configapi._
|
||||
import com.twitter.timelines.configapi.decider.DeciderSwitchOverrideValue
|
||||
import com.twitter.timelines.configapi.decider.GuestRecipient
|
||||
import com.twitter.timelines.configapi.decider.RecipientBuilder
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class DeciderConfigs @Inject() (deciderGateBuilder: DeciderGateBuilder) {
|
||||
val overrides: Seq[OptionalOverride[_]] = DeciderConfigs.ParamsToDeciderMap.map {
|
||||
case (params, deciderKey) =>
|
||||
params.optionalOverrideValue(
|
||||
DeciderSwitchOverrideValue(
|
||||
feature = deciderGateBuilder.keyToFeature(deciderKey),
|
||||
enabledValue = true,
|
||||
recipientBuilder = DeciderConfigs.UserOrGuestOrRequest
|
||||
)
|
||||
)
|
||||
}.toSeq
|
||||
|
||||
val config: BaseConfig = BaseConfigBuilder(overrides).build("FollowRecommendationServiceDeciders")
|
||||
}
|
||||
|
||||
object DeciderConfigs {
|
||||
val ParamsToDeciderMap = Map(
|
||||
DeciderParams.EnableRecommendations -> DeciderKey.EnableRecommendations,
|
||||
DeciderParams.EnableScoreUserCandidates -> DeciderKey.EnableScoreUserCandidates,
|
||||
HomeTimelineTweetRecsParams.EnableProduct -> DeciderKey.EnableHomeTimelineTweetRecsProduct,
|
||||
)
|
||||
|
||||
object UserOrGuestOrRequest extends RecipientBuilder {
|
||||
|
||||
def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match {
|
||||
case c: WithUserId if c.userId.isDefined =>
|
||||
c.userId.map(SimpleRecipient)
|
||||
case c: WithGuestId if c.guestId.isDefined =>
|
||||
c.guestId.map(GuestRecipient)
|
||||
case c: WithGuestId =>
|
||||
RecipientBuilder.Request(c)
|
||||
case _ =>
|
||||
throw new UndefinedUserIdNorGuestIDException(requestContext)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,138 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.base.SocialProofEnforcedCandidateSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoQualityFollowSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims.SimsSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStpSourceFsConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphFSConfig
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.sources.FeatureHydrationSourcesFSConfig
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRankerFSConfig
|
||||
import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig
|
||||
import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlowFSConfig
|
||||
import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateFSConfig
|
||||
import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateFSConfig
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.SgsPredicateFSConfig
|
||||
import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlowFSConfig
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.timelines.configapi.BaseConfigBuilder
|
||||
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class FeatureSwitchConfigs @Inject() (
|
||||
globalFeatureSwitchConfig: GlobalFeatureSwitchConfig,
|
||||
featureHydrationSourcesFSConfig: FeatureHydrationSourcesFSConfig,
|
||||
weightedCandidateSourceRankerFSConfig: WeightedCandidateSourceRankerFSConfig,
|
||||
// Flow related config
|
||||
contentRecommenderFlowFSConfig: ContentRecommenderFlowFSConfig,
|
||||
postNuxMlFlowFSConfig: PostNuxMlFlowFSConfig,
|
||||
// Candidate source related config
|
||||
crowdSearchAccountsFSConfig: CrowdSearchAccountsFSConfig,
|
||||
offlineStpSourceFsConfig: OfflineStpSourceFsConfig,
|
||||
onlineSTPSourceFSConfig: OnlineSTPSourceFSConfig,
|
||||
popGeoSourceFSConfig: PopGeoSourceFSConfig,
|
||||
popGeoQualityFollowFSConfig: PopGeoQualityFollowSourceFSConfig,
|
||||
realGraphOonFSConfig: RealGraphOonFSConfig,
|
||||
repeatedProfileVisitsFSConfig: RepeatedProfileVisitsFSConfig,
|
||||
recentEngagementSimilarUsersFSConfig: RecentEngagementSimilarUsersFSConfig,
|
||||
recentFollowingRecentFollowingExpansionSourceFSConfig: RecentFollowingRecentFollowingExpansionSourceFSConfig,
|
||||
simsExpansionFSConfig: SimsExpansionFSConfig,
|
||||
simsSourceFSConfig: SimsSourceFSConfig,
|
||||
socialProofEnforcedCandidateSourceFSConfig: SocialProofEnforcedCandidateSourceFSConfig,
|
||||
triangularLoopsFSConfig: TriangularLoopsFSConfig,
|
||||
userUserGraphFSConfig: UserUserGraphFSConfig,
|
||||
// Predicate related configs
|
||||
gizmoduckPredicateFSConfig: GizmoduckPredicateFSConfig,
|
||||
hssPredicateFSConfig: HssPredicateFSConfig,
|
||||
sgsPredicateFSConfig: SgsPredicateFSConfig,
|
||||
ppmiLocaleSourceFSConfig: PPMILocaleFollowSourceFSConfig,
|
||||
topOrganicFollowsAccountsFSConfig: TopOrganicFollowsAccountsFSConfig,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
val logger = Logger(classOf[FeatureSwitchConfigs])
|
||||
|
||||
val mergedFSConfig =
|
||||
FeatureSwitchConfig.merge(
|
||||
Seq(
|
||||
globalFeatureSwitchConfig,
|
||||
featureHydrationSourcesFSConfig,
|
||||
weightedCandidateSourceRankerFSConfig,
|
||||
// Flow related config
|
||||
contentRecommenderFlowFSConfig,
|
||||
postNuxMlFlowFSConfig,
|
||||
// Candidate source related config
|
||||
crowdSearchAccountsFSConfig,
|
||||
offlineStpSourceFsConfig,
|
||||
onlineSTPSourceFSConfig,
|
||||
popGeoSourceFSConfig,
|
||||
popGeoQualityFollowFSConfig,
|
||||
realGraphOonFSConfig,
|
||||
repeatedProfileVisitsFSConfig,
|
||||
recentEngagementSimilarUsersFSConfig,
|
||||
recentFollowingRecentFollowingExpansionSourceFSConfig,
|
||||
simsExpansionFSConfig,
|
||||
simsSourceFSConfig,
|
||||
socialProofEnforcedCandidateSourceFSConfig,
|
||||
triangularLoopsFSConfig,
|
||||
userUserGraphFSConfig,
|
||||
// Predicate related configs:
|
||||
gizmoduckPredicateFSConfig,
|
||||
hssPredicateFSConfig,
|
||||
sgsPredicateFSConfig,
|
||||
ppmiLocaleSourceFSConfig,
|
||||
topOrganicFollowsAccountsFSConfig,
|
||||
)
|
||||
)
|
||||
|
||||
/**
|
||||
* enum params have to be listed in this main file together as otherwise we'll have to pass in
|
||||
* some signature like `Seq[FSEnumParams[_]]` which are generics of generics and won't compile.
|
||||
* we only have enumFsParams from globalFeatureSwitchConfig at the moment
|
||||
*/
|
||||
val enumOverrides = globalFeatureSwitchConfig.enumFsParams.flatMap { enumParam =>
|
||||
FeatureSwitchOverrideUtil.getEnumFSOverrides(statsReceiver, logger, enumParam)
|
||||
}
|
||||
|
||||
val gatedOverrides = mergedFSConfig.gatedOverridesMap.flatMap {
|
||||
case (fsName, overrides) =>
|
||||
FeatureSwitchOverrideUtil.gatedOverrides(fsName, overrides: _*)
|
||||
}
|
||||
|
||||
val enumSeqOverrides = globalFeatureSwitchConfig.enumSeqFsParams.flatMap { enumSeqParam =>
|
||||
FeatureSwitchOverrideUtil.getEnumSeqFSOverrides(statsReceiver, logger, enumSeqParam)
|
||||
}
|
||||
|
||||
val overrides =
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBooleanFSOverrides(mergedFSConfig.booleanFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedIntFSOverrides(mergedFSConfig.intFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedLongFSOverrides(mergedFSConfig.longFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedDoubleFSOverrides(mergedFSConfig.doubleFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getDurationFSOverrides(mergedFSConfig.durationFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedOptionalDoubleOverrides(mergedFSConfig.optionalDoubleFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil.getStringSeqFSOverrides(mergedFSConfig.stringSeqFSParams: _*) ++
|
||||
enumOverrides ++
|
||||
gatedOverrides ++
|
||||
enumSeqOverrides
|
||||
|
||||
val config = BaseConfigBuilder(overrides).build("FollowRecommendationServiceFeatureSwitches")
|
||||
}
|
||||
Binary file not shown.
@ -1,49 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.AccountsFilteringAndRankingLogics
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.{
|
||||
AccountsFilteringAndRankingLogics => OrganicAccountsFilteringAndRankingLogics
|
||||
}
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersParams
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionSourceParams
|
||||
import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams.CandidateScorerIdParam
|
||||
import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.CandidateSourcesToFilter
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableCandidateParamHydrations
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableRecommendationFlowLogs
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepSocialUserCandidate
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepUserCandidate
|
||||
import com.twitter.timelines.configapi.FSName
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class GlobalFeatureSwitchConfig @Inject() () extends FeatureSwitchConfig {
|
||||
override val booleanFSParams: Seq[Param[Boolean] with FSName] = {
|
||||
Seq(
|
||||
EnableCandidateParamHydrations,
|
||||
KeepUserCandidate,
|
||||
KeepSocialUserCandidate,
|
||||
EnableGFSSocialProofTransform,
|
||||
EnableWhoToFollowProducts,
|
||||
EnableRecommendationFlowLogs
|
||||
)
|
||||
}
|
||||
|
||||
val enumFsParams =
|
||||
Seq(
|
||||
CandidateScorerIdParam,
|
||||
SimsExpansionSourceParams.Aggregator,
|
||||
RecentEngagementSimilarUsersParams.Aggregator,
|
||||
CandidateSourcesToFilter,
|
||||
)
|
||||
|
||||
val enumSeqFsParams =
|
||||
Seq(
|
||||
AccountsFilteringAndRankingLogics,
|
||||
OrganicAccountsFilteringAndRankingLogics
|
||||
)
|
||||
}
|
||||
Binary file not shown.
@ -1,29 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.servo.util.MemoizingStatsReceiver
|
||||
import com.twitter.timelines.configapi.Config
|
||||
import com.twitter.timelines.configapi.FeatureValue
|
||||
import com.twitter.timelines.configapi.Params
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ParamsFactory @Inject() (
|
||||
config: Config,
|
||||
requestContextFactory: RequestContextFactory,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi"))
|
||||
def apply(followRecommendationServiceRequestContext: RequestContext): Params =
|
||||
config(followRecommendationServiceRequestContext, stats)
|
||||
|
||||
def apply(
|
||||
clientContext: ClientContext,
|
||||
displayLocation: DisplayLocation,
|
||||
featureOverrides: Map[String, FeatureValue]
|
||||
): Params =
|
||||
apply(requestContextFactory(clientContext, displayLocation, featureOverrides))
|
||||
}
|
||||
Binary file not shown.
@ -1,19 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.timelines.configapi.BaseRequestContext
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.NullFeatureContext
|
||||
import com.twitter.timelines.configapi.GuestId
|
||||
import com.twitter.timelines.configapi.UserId
|
||||
import com.twitter.timelines.configapi.WithFeatureContext
|
||||
import com.twitter.timelines.configapi.WithGuestId
|
||||
import com.twitter.timelines.configapi.WithUserId
|
||||
|
||||
case class RequestContext(
|
||||
userId: Option[UserId],
|
||||
guestId: Option[GuestId],
|
||||
featureContext: FeatureContext = NullFeatureContext)
|
||||
extends BaseRequestContext
|
||||
with WithUserId
|
||||
with WithGuestId
|
||||
with WithFeatureContext
|
||||
Binary file not shown.
@ -1,74 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import com.google.inject.Inject
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.featureswitches.v2.FeatureSwitches
|
||||
import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient}
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.FeatureValue
|
||||
import com.twitter.timelines.configapi.ForcedFeatureContext
|
||||
import com.twitter.timelines.configapi.OrElseFeatureContext
|
||||
import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext
|
||||
import javax.inject.Singleton
|
||||
|
||||
/*
|
||||
* Request Context Factory is used to build RequestContext objects which are used
|
||||
* by the config api to determine the param overrides to apply to the request.
|
||||
* The param overrides are determined per request by configs which specify which
|
||||
* FS/Deciders/AB translate to what param overrides.
|
||||
*/
|
||||
@Singleton
|
||||
class RequestContextFactory @Inject() (featureSwitches: FeatureSwitches, decider: Decider) {
|
||||
def apply(
|
||||
clientContext: ClientContext,
|
||||
displayLocation: DisplayLocation,
|
||||
featureOverrides: Map[String, FeatureValue]
|
||||
): RequestContext = {
|
||||
val featureContext = getFeatureContext(clientContext, displayLocation, featureOverrides)
|
||||
RequestContext(clientContext.userId, clientContext.guestId, featureContext)
|
||||
}
|
||||
|
||||
private[configapi] def getFeatureContext(
|
||||
clientContext: ClientContext,
|
||||
displayLocation: DisplayLocation,
|
||||
featureOverrides: Map[String, FeatureValue]
|
||||
): FeatureContext = {
|
||||
val recipient =
|
||||
getFeatureSwitchRecipient(clientContext)
|
||||
.withCustomFields("display_location" -> displayLocation.toFsName)
|
||||
|
||||
// userAgeOpt is going to be set to None for logged out users and defaulted to Some(Int.MaxValue) for non-snowflake users
|
||||
val userAgeOpt = clientContext.userId.map { userId =>
|
||||
SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue)
|
||||
}
|
||||
val recipientWithAccountAge =
|
||||
userAgeOpt
|
||||
.map(age => recipient.withCustomFields("account_age_in_days" -> age)).getOrElse(recipient)
|
||||
|
||||
val results = featureSwitches.matchRecipient(recipientWithAccountAge)
|
||||
OrElseFeatureContext(
|
||||
ForcedFeatureContext(featureOverrides),
|
||||
new FeatureSwitchResultsFeatureContext(results))
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
private[configapi] def getFeatureSwitchRecipient(
|
||||
clientContext: ClientContext
|
||||
): FeatureSwitchRecipient = {
|
||||
FeatureSwitchRecipient(
|
||||
userId = clientContext.userId,
|
||||
userRoles = clientContext.userRoles,
|
||||
deviceId = clientContext.deviceId,
|
||||
guestId = clientContext.guestId,
|
||||
languageCode = clientContext.languageCode,
|
||||
countryCode = clientContext.countryCode,
|
||||
isVerified = None,
|
||||
clientApplicationId = clientContext.appId,
|
||||
isTwoffice = clientContext.isTwoffice
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,18 +0,0 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
"configapi/configapi-featureswitches:v2",
|
||||
"featureswitches/featureswitches-core",
|
||||
"featureswitches/featureswitches-core:v2",
|
||||
"featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,19 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi.candidates
|
||||
|
||||
import com.twitter.timelines.configapi.BaseRequestContext
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.NullFeatureContext
|
||||
import com.twitter.timelines.configapi.WithFeatureContext
|
||||
import com.twitter.timelines.configapi.WithUserId
|
||||
|
||||
/**
|
||||
* represent the context for a recommendation candidate (producer side)
|
||||
* @param userId id of the recommended user
|
||||
* @param featureContext feature context
|
||||
*/
|
||||
case class CandidateUserContext(
|
||||
override val userId: Option[Long],
|
||||
featureContext: FeatureContext = NullFeatureContext)
|
||||
extends BaseRequestContext
|
||||
with WithUserId
|
||||
with WithFeatureContext
|
||||
Binary file not shown.
@ -1,55 +0,0 @@
|
||||
package com.twitter.follow_recommendations.configapi.candidates
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import com.google.inject.Inject
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.featureswitches.v2.FeatureSwitches
|
||||
import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient}
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext
|
||||
import javax.inject.Named
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class CandidateUserContextFactory @Inject() (
|
||||
@Named(PRODUCER_SIDE_FEATURE_SWITCHES) featureSwitches: FeatureSwitches,
|
||||
decider: Decider) {
|
||||
def apply(
|
||||
candidateUser: CandidateUser,
|
||||
displayLocation: DisplayLocation
|
||||
): CandidateUserContext = {
|
||||
val featureContext = getFeatureContext(candidateUser, displayLocation)
|
||||
|
||||
CandidateUserContext(Some(candidateUser.id), featureContext)
|
||||
}
|
||||
|
||||
private[configapi] def getFeatureContext(
|
||||
candidateUser: CandidateUser,
|
||||
displayLocation: DisplayLocation
|
||||
): FeatureContext = {
|
||||
|
||||
val recipient = getFeatureSwitchRecipient(candidateUser).withCustomFields(
|
||||
"display_location" -> displayLocation.toFsName)
|
||||
new FeatureSwitchResultsFeatureContext(featureSwitches.matchRecipient(recipient))
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
private[configapi] def getFeatureSwitchRecipient(
|
||||
candidateUser: CandidateUser
|
||||
): FeatureSwitchRecipient = {
|
||||
FeatureSwitchRecipient(
|
||||
userId = Some(candidateUser.id),
|
||||
userRoles = None,
|
||||
deviceId = None,
|
||||
guestId = None,
|
||||
languageCode = None,
|
||||
countryCode = None,
|
||||
isVerified = None,
|
||||
clientApplicationId = None,
|
||||
isTwoffice = None
|
||||
)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user