diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.docx new file mode 100644 index 000000000..458f6fee5 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.scala deleted file mode 100644 index 8face1164..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.modify_social_proof - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RemoveAccountProofTransform @Inject() (statsReceiver: StatsReceiver) - extends GatedTransform[HasClientContext with HasParams, CandidateUser] { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val removedProofsCounter = stats.counter("num_removed_proofs") - - override def transform( - target: HasClientContext with HasParams, - items: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = - Stitch.value(items.map { candidate => - removedProofsCounter.incr() - candidate.copy(reason = None) - }) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD deleted file mode 100644 index d6dcd8522..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD.docx new file mode 100644 index 000000000..00883da59 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.docx new file mode 100644 index 000000000..1d1d5c2d1 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.scala deleted file mode 100644 index 03639da26..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.ranker_id - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This class appends each candidate's rankerIds with the RandomRankerId. - * This is primarily for determining if a candidate was generated via random shuffling. - */ -@Singleton -class RandomRankerIdTransform @Inject() () extends GatedTransform[HasParams, CandidateUser] { - - override def transform( - target: HasParams, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - Stitch.value(candidates.map(_.addScore(Score.RandomScore))) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.docx new file mode 100644 index 000000000..bc267dbb9 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.scala deleted file mode 100644 index 87c111a6c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.recommendation_flow_identifier - -import com.google.inject.Inject -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecommendationFlowIdentifier -import com.twitter.stitch.Stitch - -class AddRecommendationFlowIdentifierTransform @Inject() - extends Transform[HasRecommendationFlowIdentifier, CandidateUser] { - - override def transform( - target: HasRecommendationFlowIdentifier, - items: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - Stitch.value(items.map { candidateUser => - candidateUser.copy(recommendationFlowIdentifier = target.recommendationFlowIdentifier) - }) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD deleted file mode 100644 index 820e2df66..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD.docx new file mode 100644 index 000000000..43b93eb1c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD deleted file mode 100644 index d9b257348..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD.docx new file mode 100644 index 000000000..5ff65161c Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.docx new file mode 100644 index 000000000..7ca3802f0 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.scala deleted file mode 100644 index 5a30c9cb1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.tracking_token - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.Session -import com.twitter.follow_recommendations.common.models.TrackingToken -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * This transform adds the tracking token for all candidates - * Since this happens in the same request, we use the same trace-id for all candidates - * There are no RPC calls in this transform so it's safe to chain it with `andThen` at the end of - * all other product-specific transforms - */ -@Singleton -class TrackingTokenTransform @Inject() (baseStatsReceiver: StatsReceiver) - extends Transform[HasDisplayLocation with HasClientContext, CandidateUser] - with Logging { - - def profileResults( - target: HasDisplayLocation with HasClientContext, - candidates: Seq[CandidateUser] - ) = { - // Metrics to track # results per candidate source - val stats = baseStatsReceiver.scope(target.displayLocation.toString + "/final_results") - stats.stat("total").add(candidates.size) - - stats.counter(target.displayLocation.toString).incr() - - val flattenedCandidates: Seq[(CandidateSourceIdentifier, CandidateUser)] = for { - candidate <- candidates - identifier <- candidate.getPrimaryCandidateSource - } yield (identifier, candidate) - val candidatesGroupedBySource: Map[CandidateSourceIdentifier, Seq[CandidateUser]] = - flattenedCandidates.groupBy(_._1).mapValues(_.map(_._2)) - candidatesGroupedBySource map { - case (source, candidates) => stats.stat(source.name).add(candidates.size) - } - } - - override def transform( - target: HasDisplayLocation with HasClientContext, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - profileResults(target, candidates) - - Stitch.value( - target.getOptionalUserId - .map { _ => - candidates.map { - candidate => - val token = Some(TrackingToken( - sessionId = Session.getSessionId, - displayLocation = Some(target.displayLocation), - controllerData = None, - algorithmId = candidate.userCandidateSourceDetails.flatMap(_.primaryCandidateSource - .flatMap { identifier => - Algorithm.withNameOpt(identifier.name).flatMap(AlgorithmToFeedbackTokenMap.get) - }) - )) - candidate.copy(trackingToken = token) - } - }.getOrElse(candidates)) - - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD deleted file mode 100644 index 606e8edfa..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD.docx new file mode 100644 index 000000000..8bb34bee6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.docx new file mode 100644 index 000000000..cd0fd9cc6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.scala deleted file mode 100644 index 269a39a48..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.weighted_sampling -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.follow_recommendations.common.models.Scores -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SamplingTransform @Inject() () - extends GatedTransform[HasClientContext with HasParams with HasDebugOptions, CandidateUser] { - - val name: String = this.getClass.getSimpleName - - /* - Description: This function takes in a set of candidate users and ranks them for a who-to-follow - request by sampling from the Placket-Luce distribution - (https://cran.rstudio.com/web/packages/PlackettLuce/vignettes/Overview.html) with a three - variations. The first variation is that the scores of the candidates are multiplied by - multiplicativeFactor before sampling. The second variation is that the scores are - exponentiated before sampling. The third variation is that depending on how many who-to-follow - positions are being requested, the first k positions are reserved for the candidates with the - highest scores (and they are sorted in decreasing order of score) and the remaining positions - are sampled from a Placket-Luce. We use the efficient algorithm proposed in this blog - https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051 - to sample from a Plackett-Luce. Because of numerical stability reasons, before sampling from this - distribution, (1) we subtract off the maximum score from all the scores and (2) if after - this subtraction and multiplication by the multiplicative factor the resulting score is <= -10, - we force the candidate's transformed score under the above algorithm to be 0 (so r^(1/w) = 0) - where r is a random number and w is the transformed score. - - inputs: - - target: HasClientContext (WTF request) - - candidates: sequence of CandidateUsers (users that need to be ranked from a who-to-follow - request) each of which has a score - - inputs accessed through feature switches, i.e. through target.params (see the following file: - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/ - transforms/weighted_sampling/SamplingTransformParams.scala"): - - topKFixed: the first k positions of the who-to-follow ranking correspond to the users with the k - highest scores and are not sampled from the Placket-Luce distribution - - multiplicativeFactor: multiplicativeFactor is used to transform the scores of each candidate by - multiplying that user's score by multiplicativeFactor - - output: - - Sequence of CandidateUser whose order represents the ranking of users in a who-to-follow request - This ranking is sampled from a Placket-Luce distribution. - */ - override def transform( - target: HasClientContext with HasParams with HasDebugOptions, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - - // the first k positions of the who-to-follow ranking correspond to the users with the k - // highest scores and are not sampled from the Placket-Luce distribution - val topKFixed = target.params(SamplingTransformParams.TopKFixed) - - // multiplicativeFactor is used to transform the scores of each candidate by - // multiplying that user's score by multiplicativeFactor - val multiplicativeFactor = target.params(SamplingTransformParams.MultiplicativeFactor) - - // sort candidates by their score - val candidatesSorted = candidates.sortBy(-1 * _.score.getOrElse(0.0)) - - // pick the top K candidates by score and the remaining candidates - val (topKFixedCandidates, candidatesOutsideOfTopK) = - candidatesSorted.zipWithIndex.partition { case (value, index) => index < topKFixed } - - val randomNumGenerator = - new scala.util.Random(target.getRandomizationSeed.getOrElse(System.currentTimeMillis)) - - // we need to subtract the maximum score off the scores for numerical stability reasons - // subtracting the max score off does not effect the underlying distribution we are sampling - // the candidates from - // we need the if statement since you cannot take the max of an empty sequence - val maximum_score = if (candidatesOutsideOfTopK.nonEmpty) { - candidatesOutsideOfTopK.map(x => x._1.score.getOrElse(0.0)).max - } else { - 0.0 - } - - // for candidates in candidatesOutsideOfTopK, we transform their score by subtracting off - // maximum_score and then multiply by multiplicativeFactor - val candidatesOutsideOfTopKTransformedScore = candidatesOutsideOfTopK.map(x => - (x._1, multiplicativeFactor * (x._1.score.getOrElse(0.0) - maximum_score))) - - // for each candidate with score transformed and clip score w, sample a random number r, - // create a new score r^(1/w) and sort the candidates to get the final ranking. - // for numerical stability reasons if the score is <=-10, we force r^(1/w) = 0. - // this samples the candidates from the modified Plackett-Luce distribution. See - // https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051 - - val candidatesOutsideOfTopKSampled = candidatesOutsideOfTopKTransformedScore - .map(x => - ( - x._1, - if (x._2 <= -10.0) - 0.0 - else - scala.math.pow( - randomNumGenerator.nextFloat(), - 1 / (scala.math - .exp(x._2))))).sortBy(-1 * _._2) - - val topKCandidates: Seq[CandidateUser] = topKFixedCandidates.map(_._1) - - val scribeRankingInfo: Boolean = - target.params(SamplingTransformParams.ScribeRankingInfoInSamplingTransform) - - val transformedCandidates: Seq[CandidateUser] = if (scribeRankingInfo) { - val topKCandidatesWithRankingInfo: Seq[CandidateUser] = - Utils.addRankingInfo(topKCandidates, name) - val candidatesOutsideOfTopKSampledWithRankingInfo: Seq[CandidateUser] = - candidatesOutsideOfTopKSampled.zipWithIndex.map { - case ((candidate, score), rank) => - val newScore = Seq(Score(score, Some(RankerId.PlacketLuceSamplingTransformer))) - val newScores: Option[Scores] = candidate.scores - .map { scores => - scores.copy(scores = scores.scores ++ newScore) - }.orElse(Some(Scores(newScore, Some(RankerId.PlacketLuceSamplingTransformer)))) - val globalRank = rank + topKFixed + 1 - candidate.addInfoPerRankingStage(name, newScores, globalRank) - } - - topKCandidatesWithRankingInfo ++ candidatesOutsideOfTopKSampledWithRankingInfo - } else { - topKCandidates ++ candidatesOutsideOfTopKSampled.map(_._1) - } - - Stitch.value(transformedCandidates) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.docx new file mode 100644 index 000000000..01c151469 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.scala deleted file mode 100644 index b97251f93..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.weighted_sampling - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SamplingTransformFSConfig @Inject() () extends FeatureSwitchConfig { - override val intFSParams: Seq[FSBoundedParam[Int]] = Seq(SamplingTransformParams.TopKFixed) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - SamplingTransformParams.MultiplicativeFactor) - - override val booleanFSParams: Seq[FSParam[Boolean]] = Seq( - SamplingTransformParams.ScribeRankingInfoInSamplingTransform) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.docx new file mode 100644 index 000000000..a2d3f466d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala deleted file mode 100644 index 363487a9b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.weighted_sampling - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object SamplingTransformParams { - - case object TopKFixed // indicates how many of the fisrt K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score - extends FSBoundedParam[Int]( - name = "post_nux_ml_flow_weighted_sampling_top_k_fixed", - default = 0, - min = 0, - max = 100) - - case object MultiplicativeFactor // CandidateUser.score gets transformed to multiplicativeFactor*CandidateUser.score before sampling from the Plackett-Luce distribution - extends FSBoundedParam[Double]( - name = "post_nux_ml_flow_weighted_sampling_multiplicative_factor", - default = 1.0, - min = -1000.0, - max = 1000.0) - - case object ScribeRankingInfoInSamplingTransform - extends FSParam[Boolean]("sampling_transform_scribe_ranking_info", false) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD deleted file mode 100644 index 7075167e3..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "stitch/stitch-core", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD.docx new file mode 100644 index 000000000..db6271f55 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.docx new file mode 100644 index 000000000..ffdfde26d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.scala deleted file mode 100644 index db9a1f9f5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -object CollectionUtil { - - /** - * Transposes a sequence of sequences. As opposed to the Scala collection library version - * of transpose, the sequences do not have to have the same length. - * - * Example: - * transpose(immutable.Seq(immutable.Seq(1,2,3), immutable.Seq(4,5), immutable.Seq(6,7))) - * => immutable.Seq(immutable.Seq(1, 4, 6), immutable.Seq(2, 5, 7), immutable.Seq(3)) - * - * @param seq a sequence of sequences - * @tparam A the type of elements in the seq - * @return the transposed sequence of sequences - */ - def transposeLazy[A](seq: Seq[Seq[A]]): Stream[Seq[A]] = - seq.filter(_.nonEmpty) match { - case Nil => Stream.empty - case ys => ys.map(_.head) #:: transposeLazy(ys.map(_.tail)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.docx new file mode 100644 index 000000000..e9d2f3ec8 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.scala deleted file mode 100644 index 2f6db39b6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Product -import com.twitter.product_mixer.core.model.marshalling.request.Product - -object DisplayLocationProductConverterUtil { - def productToDisplayLocation(product: Product): DisplayLocation = { - product match { - case Product.MagicRecs => DisplayLocation.MagicRecs - case _ => - throw UnconvertibleProductMixerProductException( - s"Cannot convert Product Mixer Product ${product.identifier.name} into a FRS DisplayLocation.") - } - } - - def displayLocationToProduct(displayLocation: DisplayLocation): Product = { - displayLocation match { - case DisplayLocation.MagicRecs => Product.MagicRecs - case _ => - throw UnconvertibleProductMixerProductException( - s"Cannot convert DisplayLocation ${displayLocation.toFsName} into a Product Mixer Product.") - } - } -} - -case class UnconvertibleProductMixerProductException(message: String) extends Exception(message) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.docx new file mode 100644 index 000000000..fc993fd06 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.scala deleted file mode 100644 index 6aaee4c45..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -object MergeUtil { - - /** - * Takes a seq of items which have weights. Returns an infinite stream of each item - * by their weights. All weights need to be greater than or equal to zero. In addition, - * the sum of weights should be greater than zero. - * - * Example usage of this function: - * Input weighted Item {{CS1, 3}, {CS2, 2}, {CS3, 5}} - * Output stream: (CS1, CS1, CS1, CS2, CS2, CS3, CS3, CS3, CS3, CS3, CS1, CS1, CS1, CS2,...} - * - * @param items items - * @param weighted provides weights for items - * @tparam T type of item - * - * @return Stream of Ts - */ - def weightedRoundRobin[T]( - items: Seq[T] - )( - implicit weighted: Weighted[T] - ): Stream[T] = { - if (items.isEmpty) { - Stream.empty - } else { - val weights = items.map { i => weighted(i) } - assert( - weights.forall { - _ >= 0 - }, - "Negative weight exists for sampling") - val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail - assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive") - - var weightIdx = 0 - var weight = 0 - - def next(): Stream[T] = { - val tmpIdx = weightIdx - weight = weight + 1 - weight = if (weight >= weights(weightIdx)) 0 else weight - weightIdx = if (weight == 0) weightIdx + 1 else weightIdx - weightIdx = if (weightIdx == weights.length) 0 else weightIdx - items(tmpIdx) #:: next() - } - next() - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.docx new file mode 100644 index 000000000..1d7a4f8e5 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.scala deleted file mode 100644 index 9d66e8deb..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.follow_recommendations.common.utils -import scala.util.Random - -object RandomUtil { - - /** - * Takes a seq of items which have weights. Returns an infinite stream that is - * sampled with replacement using the weights for each item. All weights need - * to be greater than or equal to zero. In addition, the sum of weights - * should be greater than zero. - * - * @param items items - * @param weighted provides weights for items - * @tparam T type of item - * @return Stream of Ts - */ - def weightedRandomSamplingWithReplacement[T]( - items: Seq[T], - random: Option[Random] = None - )( - implicit weighted: Weighted[T] - ): Stream[T] = { - if (items.isEmpty) { - Stream.empty - } else { - val weights = items.map { i => weighted(i) } - assert(weights.forall { _ >= 0 }, "Negative weight exists for sampling") - val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail - assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive") - val cumulativeProbability = cumulativeWeight map (_ / cumulativeWeight.last) - def next(): Stream[T] = { - val rand = random.getOrElse(Random).nextDouble() - val idx = cumulativeProbability.indexWhere(_ >= rand) - items(if (idx == -1) items.length - 1 else idx) #:: next() - } - next() - } - } - - /** - * Takes a seq of items and their weights. Returns a lazy weighted shuffle of - * the elements in the list. All weights should be greater than zero. - * - * @param items items - * @param weighted provides weights for items - * @tparam T type of item - * @return Stream of Ts - */ - def weightedRandomShuffle[T]( - items: Seq[T], - random: Option[Random] = None - )( - implicit weighted: Weighted[T] - ): Stream[T] = { - assert(items.forall { i => weighted(i) > 0 }, "Non-positive weight exists for shuffling") - def next(it: Seq[T]): Stream[T] = { - if (it.isEmpty) - Stream.empty - else { - val cumulativeWeight = it.scanLeft(0.0)((acc: Double, curr: T) => acc + weighted(curr)).tail - val cutoff = random.getOrElse(Random).nextDouble() * cumulativeWeight.last - val idx = cumulativeWeight.indexWhere(_ >= cutoff) - val (left, right) = it.splitAt(idx) - it(if (idx == -1) it.size - 1 else idx) #:: next(left ++ right.drop(1)) - } - } - next(items) - } - - /** - * Takes a seq of items and a weight function, returns a lazy weighted shuffle of - * the elements in the list.The weight function is based on the rank of the element - * in the original lst. - * @param items - * @param rankToWeight - * @param random - * @tparam T - * @return - */ - def weightedRandomShuffleByRank[T]( - items: Seq[T], - rankToWeight: Int => Double, - random: Option[Random] = None - ): Stream[T] = { - val candWeights = items.zipWithIndex.map { case (item, rank) => (item, rankToWeight(rank)) } - RandomUtil.weightedRandomShuffle(candWeights, random).map(_._1) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.docx new file mode 100644 index 000000000..1bd1a01b6 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.scala deleted file mode 100644 index 8275228d6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.util.TimeoutException - -object RescueWithStatsUtils { - def rescueWithStats[T]( - s: Stitch[Seq[T]], - stats: StatsReceiver, - source: String - ): Stitch[Seq[T]] = { - StatsUtil.profileStitchSeqResults(s, stats.scope(source)).rescue { - case _: Exception => Stitch.Nil - } - } - - def rescueOptionalWithStats[T]( - s: Stitch[Option[T]], - stats: StatsReceiver, - source: String - ): Stitch[Option[T]] = { - StatsUtil.profileStitchOptionalResults(s, stats.scope(source)).rescue { - case _: Exception => Stitch.None - } - } - - def rescueWithStatsWithin[T]( - s: Stitch[Seq[T]], - stats: StatsReceiver, - source: String, - timeout: Duration - ): Stitch[Seq[T]] = { - val hydratedScopeSource = stats.scope(source) - StatsUtil - .profileStitchSeqResults( - s.within(timeout)(com.twitter.finagle.util.DefaultTimer), - hydratedScopeSource) - .rescue { - case _: TimeoutException => - hydratedScopeSource.counter("timeout").incr() - Stitch.Nil - case _: Exception => - hydratedScopeSource.counter("exception").incr() - Stitch.Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.docx new file mode 100644 index 000000000..d3497f43d Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.scala deleted file mode 100644 index 73d90a85b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.util.Duration -import com.twitter.util.Time - -object UserSignupUtil { - def signupTime(hasClientContext: HasClientContext): Option[Time] = - hasClientContext.clientContext.userId.flatMap(SnowflakeId.timeFromIdOpt) - - def userSignupAge(hasClientContext: HasClientContext): Option[Duration] = - signupTime(hasClientContext).map(Time.now - _) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.docx b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.docx new file mode 100644 index 000000000..3314b8072 Binary files /dev/null and b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.docx differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.scala deleted file mode 100644 index adb95e5f5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -/** - * Typeclass for any Recommendation type that has a weight - * - */ -trait Weighted[-Rec] { - def apply(rec: Rec): Double -} - -object Weighted { - implicit object WeightedTuple extends Weighted[(_, Double)] { - override def apply(rec: (_, Double)): Double = rec._2 - } - - def fromFunction[Rec](f: Rec => Double): Weighted[Rec] = { - new Weighted[Rec] { - override def apply(rec: Rec): Double = f(rec) - } - } -} diff --git a/follow-recommendations-service/server/src/main/resources/BUILD b/follow-recommendations-service/server/src/main/resources/BUILD deleted file mode 100644 index 610607ec5..000000000 --- a/follow-recommendations-service/server/src/main/resources/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -resources( - sources = [ - "*.tsv", - "*.xml", - "**/*", - "config/*.yml", - ], -) - -# Created for Bazel compatibility. -# In Bazel, loose files must be part of a target to be included into a bundle. -files( - name = "frs_resources", - sources = [ - "*.tsv", - "*.xml", - "*.yml", - "**/*", - ], -) diff --git a/follow-recommendations-service/server/src/main/resources/BUILD.docx b/follow-recommendations-service/server/src/main/resources/BUILD.docx new file mode 100644 index 000000000..573a69965 Binary files /dev/null and b/follow-recommendations-service/server/src/main/resources/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/resources/config/decider.docx b/follow-recommendations-service/server/src/main/resources/config/decider.docx new file mode 100644 index 000000000..51a0bc767 Binary files /dev/null and b/follow-recommendations-service/server/src/main/resources/config/decider.docx differ diff --git a/follow-recommendations-service/server/src/main/resources/config/decider.yml b/follow-recommendations-service/server/src/main/resources/config/decider.yml deleted file mode 100644 index a46626094..000000000 --- a/follow-recommendations-service/server/src/main/resources/config/decider.yml +++ /dev/null @@ -1,129 +0,0 @@ -enable_recommendations: - comment: Proportion of requests where we return an actual response as part. Decreasing the value will increase the portion of empty responses (in order to disable the service) as part of the graceful degradation. - default_availability: 10000 -enable_score_user_candidates: - comment: Proportion of requests where score user candidates from the scoreUserCandidates endpoint - default_availability: 10000 -enable_profile_sidebar_product: - comment: Proportion of requests where we return an actual response for profile sidebar product - default_availability: 10000 -enable_magic_recs_product: - comment: Proportion of requests where we return an actual response for magic recs product - default_availability: 10000 -enable_rux_landing_page_product: - comment: Proportion of requests where we return an actual response for rux landing page product - default_availability: 10000 -enable_rux_pymk_product: - comment: Proportion of requests where we return an actual response for rux pymk product - default_availability: 10000 -enable_profile_bonus_follow_product: - comment: Proportion of requests where we return an actual response for profile bonus follow product - default_availability: 10000 -enable_election_explore_wtf_product: - comment: Proportion of requests where we return an actual response for election explore wtf product - default_availability: 10000 -enable_cluster_follow_product: - comment: Proportion of requests where we return an actual response for cluster follow product - default_availability: 10000 -enable_home_timeline_product: - comment: Proportion of requests where we return an actual response for htl wtf product - default_availability: 10000 -enable_htl_bonus_follow_product: - comment: Proportion of requests where we return an actual response for htl bonus follow product - default_availability: 10000 -enable_explore_tab_product: - comment: Proportion of requests where we return an actual response for explore tab product - default_availability: 10000 -enable_sidebar_product: - comment: Proportion of requests where we return an actual response for sidebar product - default_availability: 10000 -enable_campaign_form_product: - comment: Proportion of requests where we return an actual response for campaign form product - default_availability: 10000 -enable_reactive_follow_product: - comment: Proportion of requests where we return an actual response for reactive follow product - default_availability: 10000 -enable_nux_pymk_product: - comment: Proportion of requests where we return an actual response for nux pymk product - default_availability: 10000 -enable_nux_interests_product: - comment: Proportion of requests where we return an actual response for nux interests product - default_availability: 10000 -enable_nux_topic_bonus_follow_product: - comment: Proportion of requests where we return an actual response for nux topic-based bonus follow product - default_availability: 10000 -enable_india_covid19_curated_accounts_wtf_product: - comment: Proportion of requests where we return an actual response for india covid19 curated accounts wtf product - default_availability: 10000 -enable_ab_upload_product: - comment: Proportion of requests where we return an actual response for the address book upload product - default_availability: 10000 -enable_people_plus_plus_product: - comment: Proportion of requests where we return an actual response for the PeoplePlusPlus/Connect Tab product - default_availability: 10000 -enable_tweet_notification_recs_product: - comment: Proportion of requests where we return an actual response for the Tweet Notification Recommendations product - default_availability: 10000 -enable_profile_device_follow_product: - comment: Proportion of requests where we return an actual response for the ProfileDeviceFollow product - default_availability: 10000 -enable_diffy_module_dark_reading: - comment: Percentage of dark read traffic routed to diffy thrift - default_availability: 0 -enable_recos_backfill_product: - comment: Proportion of requests where we return an actual response for the RecosBackfill product - default_availability: 10000 -enable_post_nux_follow_task_product: - comment: Proportion of requests where we return an actual response for post NUX follow task product - default_availability: 10000 -enable_curated_space_hosts_product: - comment: Proportion of requests where we return an actual response for curated space hosts product - default_availability: 10000 -enable_nux_geo_category_product: - comment: Proportion of requests where we return an actual response for nux geo category product - default_availability: 10000 -enable_nux_interests_category_product: - comment: Proportion of requests where we return an actual response for nux interests category product - default_availability: 10000 -enable_nux_pymk_category_product: - comment: Proportion of requests where we return an actual response for nux pymk category product - default_availability: 10000 -enable_home_timeline_tweet_recs_product: - comment: Proportion of requests where we return an actual response for the Home Timeline Tweet Recs product - default_availability: 10000 -enable_htl_bulk_friend_follows_product: - comment: Proportion of requests where we return an actual response for the HTL bulk friend follows product - default_availability: 10000 -enable_nux_auto_follow_product: - comment: Proportion of requests where we return an actual response for the NUX auto follow product - default_availability: 10000 -enable_search_bonus_follow_product: - comment: Proportion of requests where we return an actual response for search bonus follow product - default_availability: 10000 -enable_fetch_user_in_request_builder: - comment: Proportion of requests where we fetch user object from gizmoduck in request builder - default_availability: 0 -enable_product_mixer_magic_recs_product: - comment: Proportion of requests where we enable the product mixer magic recs product - default_availability: 10000 -enable_home_timeline_reverse_chron_product: - comment: Proportion of requests where we return an actual response for Home timeline reverse chron product - default_availability: 10000 -enable_product_mixer_pipeline_magic_recs_dark_read: - comment: Compare product mixer pipeline responses to current FRS pipeline responses for Magic Recs - default_availability: 0 -enable_experimental_caching: - comment: Proportion of requests we use experimental caching for data caching - default_availability: 0 -enable_distributed_caching: - comment: Proportion of requests we use a distributed cache cluster for data caching - default_availability: 10000 -enable_gizmoduck_caching: - comment: Proportion of requests we use a distributed cache cluster for data caching in Gizmoduck - default_availability: 10000 -enable_traffic_dark_reading: - comment: Proportion of requests where we replicate the request for traffic dark reading - default_availability: 0 -enable_graph_feature_service_requests: - comment: Proportion of requests where we allow request calls to Graph Feature Service - default_availability: 10000 diff --git a/follow-recommendations-service/server/src/main/resources/logback.docx b/follow-recommendations-service/server/src/main/resources/logback.docx new file mode 100644 index 000000000..1de3ab294 Binary files /dev/null and b/follow-recommendations-service/server/src/main/resources/logback.docx differ diff --git a/follow-recommendations-service/server/src/main/resources/logback.xml b/follow-recommendations-service/server/src/main/resources/logback.xml deleted file mode 100644 index 96348c8e4..000000000 --- a/follow-recommendations-service/server/src/main/resources/logback.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - - - - - - - - - - true - - - - - ${log.service.output} - - ${log.service.output}.%i - 1 - 5 - - - 50MB - - - %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n - - - - - - ${log.access.output} - - ${log.access.output}.%i - 1 - 5 - - - 50MB - - - ${DEFAULT_ACCESS_PATTERN}%n - - - - - - true - ${log.lens.category} - ${log.lens.index} - ${log.lens.tag}/service - - %msg - - - - - - true - ${log.lens.category} - ${log.lens.index} - ${log.lens.tag}/access - - %msg - - - - - - - - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel deleted file mode 100644 index a08d9723c..000000000 --- a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel +++ /dev/null @@ -1,8 +0,0 @@ -# OWNER = jdeng -# Date = 20141223_153423 -# Training Size = 16744473 -# Testing Size = 16767335 -# trained with ElasticNetCV alpha=0.05 cv_folds=5 best_lambda=1.0E-7 -# num base features: 10 -# num nonzero weights: 30 -{bias:-5.67151,featureMetadataMap:["fwd_email":{metadata:{featureWeight:{weight:2.47389}}},"rev_phone":{metadata:{featureWeight:{weight:1.88433}}},"mutual_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:47.0,weight:6.31809},{left:11.0,right:16.0,weight:4.52959},{left:31.0,right:47.0,weight:5.7101},{right:2.0,weight:0.383515},{left:24.0,right:31.0,weight:5.26515},{left:3.0,right:4.0,weight:2.91751},{left:2.0,right:3.0,weight:2.22851},{left:4.0,right:5.0,weight:3.28515},{left:8.0,right:11.0,weight:4.14731},{left:5.0,right:8.0,weight:3.73588},{left:16.0,right:24.0,weight:4.90908}]}}},"fwd_phone":{metadata:{featureWeight:{weight:2.07327}}},"fwd_email_path":{metadata:{featureWeight:{weight:0.961773}}},"rev_phone_path":{metadata:{featureWeight:{weight:0.354484}}},"low_tweepcred_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:4.0,right:5.0,weight:0.177209},{left:7.0,right:8.0,weight:0.12378},{left:3.0,right:4.0,weight:0.197566},{left:5.0,right:6.0,weight:0.15867},{left:2.0,right:3.0,weight:0.196539},{right:2.0,weight:0.1805},{left:75.0,weight:-0.424598},{left:6.0,right:7.0,weight:0.143698},{left:10.0,right:13.0,weight:0.0458502},{left:8.0,right:10.0,weight:0.0919314},{left:13.0,right:75.0,weight:-0.111484}]}}},"rev_email_path":{metadata:{featureWeight:{weight:0.654451}}},"rev_email":{metadata:{featureWeight:{weight:2.33859}}},"fwd_phone_path":{metadata:{featureWeight:{weight:0.210418}}}]} diff --git a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel.docx b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel.docx new file mode 100644 index 000000000..cd091df75 Binary files /dev/null and b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel.docx differ diff --git a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig deleted file mode 100644 index ab38990dd..000000000 --- a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig +++ /dev/null @@ -1 +0,0 @@ -{input:{context:"discover.prod",startDateTime:"",endDateTime:"",trainingFeatures:["STP_FEATURES":["fwd_email","mutual_follow_path","fwd_email_path","rev_phone_path","low_tweepcred_follow_path","rev_phone","fwd_phone","rev_email_path","rev_email","fwd_phone_path"]],engagementActions:["click","favorite","open_link","open","send_tweet","send_reply","retweet","reply","profile_click","follow"],impressionActions:["discard","results","impression"],dataFormat:1,dataPath:"",isLabeled:0},sample:{positiveSampleRatio:1.0,negativeSampleRatio:1.0,sampleType:1},split:{trainingDataSplitSize:0.5,testingDataSplitSize:0.5,splitType:2},transform:{},filter:{featureOptions:[]},join:{engagementRules:["discover"],contentIdType:"tweet",groupBucketSize:3600000},discretize:{}} diff --git a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig.docx b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig.docx new file mode 100644 index 000000000..91baa74ab Binary files /dev/null and b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD deleted file mode 100644 index 4dcbaab44..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD +++ /dev/null @@ -1,48 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra-internal/international/src/main/scala/com/twitter/finatra/international/modules", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-core/src/main/java/com/twitter/finatra/http", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client", - "finatra/jackson/src/main/scala/com/twitter/finatra/jackson/modules", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/server/src/main/resources", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD.docx new file mode 100644 index 000000000..1acb54278 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.docx new file mode 100644 index 000000000..2c5883218 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.scala deleted file mode 100644 index fba889c2f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.follow_recommendations - -import com.google.inject.Module -import com.twitter.finagle.ThriftMux -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.http.HttpServer -import com.twitter.finatra.http.routing.HttpRouter -import com.twitter.finatra.international.modules.I18nFactoryModule -import com.twitter.finatra.international.modules.LanguagesModule -import com.twitter.finatra.jackson.modules.ScalaObjectMapperModule -import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} -import com.twitter.finatra.mtls.thriftmux.Mtls -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters._ -import com.twitter.finagle.thrift.Protocols -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookModule -import com.twitter.follow_recommendations.common.clients.adserver.AdserverModule -import com.twitter.follow_recommendations.common.clients.cache.MemcacheModule -import com.twitter.follow_recommendations.common.clients.deepbirdv2.DeepBirdV2PredictionServiceClientModule -import com.twitter.follow_recommendations.common.clients.email_storage_service.EmailStorageServiceModule -import com.twitter.follow_recommendations.common.clients.geoduck.LocationServiceModule -import com.twitter.follow_recommendations.common.clients.gizmoduck.GizmoduckModule -import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureStoreModule -import com.twitter.follow_recommendations.common.clients.impression_store.ImpressionStoreModule -import com.twitter.follow_recommendations.common.clients.phone_storage_service.PhoneStorageServiceModule -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphModule -import com.twitter.follow_recommendations.common.clients.strato.StratoClientModule -import com.twitter.follow_recommendations.common.constants.ServiceConstants._ -import com.twitter.follow_recommendations.common.feature_hydration.sources.HydrationSourcesModule -import com.twitter.follow_recommendations.controllers.ThriftController -import com.twitter.follow_recommendations.modules._ -import com.twitter.follow_recommendations.service.exceptions.UnknownLoggingExceptionMapper -import com.twitter.follow_recommendations.services.FollowRecommendationsServiceWarmupHandler -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.geoduck.service.common.clientmodules.ReverseGeocoderThriftClientModule -import com.twitter.inject.thrift.filters.DarkTrafficFilter -import com.twitter.inject.thrift.modules.ThriftClientIdModule -import com.twitter.product_mixer.core.controllers.ProductMixerController -import com.twitter.product_mixer.core.module.PipelineExecutionLoggerModule -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule -import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule -import com.twitter.product_mixer.core.product.guice.ProductScopeModule - -object FollowRecommendationsServiceThriftServerMain extends FollowRecommendationsServiceThriftServer - -class FollowRecommendationsServiceThriftServer - extends ThriftServer - with Mtls - with HttpServer - with HttpMtls { - override val name: String = "follow-recommendations-service-server" - - override val modules: Seq[Module] = - Seq( - ABDeciderModule, - AddressbookModule, - AdserverModule, - ConfigApiModule, - DeciderModule, - DeepBirdV2PredictionServiceClientModule, - DiffyModule, - EmailStorageServiceModule, - FeaturesSwitchesModule, - FlagsModule, - GizmoduckModule, - GraphFeatureStoreModule, - HydrationSourcesModule, - I18nFactoryModule, - ImpressionStoreModule, - LanguagesModule, - LocationServiceModule, - MemcacheModule, - PhoneStorageServiceModule, - PipelineExecutionLoggerModule, - ProductMixerFlagModule, - ProductRegistryModule, - new ProductScopeModule(), - new ProductScopeStringCenterModule(), - new ReverseGeocoderThriftClientModule, - ScalaObjectMapperModule, - ScorerModule, - ScribeModule, - SocialGraphModule, - StratoClientModule, - ThriftClientIdModule, - TimerModule, - ) - - def configureThrift(router: ThriftRouter): Unit = { - router - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[StatsFilter] - .filter[AccessLoggingFilter] - .filter[ExceptionMappingFilter] - .exceptionMapper[UnknownLoggingExceptionMapper] - .filter[DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]] - .add[ThriftController] - } - - override def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server = { - server.withProtocolFactory( - Protocols.binaryFactory( - stringLengthLimit = StringLengthLimit, - containerLengthLimit = ContainerLengthLimit)) - } - - override def configureHttp(router: HttpRouter): Unit = router.add( - ProductMixerController[FollowRecommendationsThriftService.MethodPerEndpoint]( - this.injector, - FollowRecommendationsThriftService.ExecutePipeline)) - - override def warmup(): Unit = { - handle[FollowRecommendationsServiceWarmupHandler]() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.docx new file mode 100644 index 000000000..c41479990 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.scala deleted file mode 100644 index cd3d88967..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Action(text: String, actionURL: String) { - lazy val toThrift: t.Action = { - t.Action(text, actionURL) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD deleted file mode 100644 index 65d265480..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "stringcenter/client", - ], - exports = [ - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD.docx new file mode 100644 index 000000000..3fe46fb86 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.docx new file mode 100644 index 000000000..bc5aea577 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.scala deleted file mode 100644 index 3346a05e1..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.stringcenter.client.core.ExternalString - -case class HeaderConfig(title: TitleConfig) -case class TitleConfig(text: ExternalString) -case class FooterConfig(actionConfig: Option[ActionConfig]) -case class ActionConfig(footerText: ExternalString, actionURL: String) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.docx new file mode 100644 index 000000000..7d7fe2de1 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.scala deleted file mode 100644 index 25caa933a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -trait FeedbackAction { - def toThrift: t.FeedbackAction -} - -case class DismissUserId() extends FeedbackAction { - override lazy val toThrift: t.FeedbackAction = { - t.FeedbackAction.DismissUserId(t.DismissUserId()) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.docx new file mode 100644 index 000000000..393725f04 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.scala deleted file mode 100644 index f62368431..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Footer(action: Option[Action]) { - lazy val toThrift: t.Footer = { - t.Footer(action.map(_.toThrift)) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.docx new file mode 100644 index 000000000..5a1787b97 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.scala deleted file mode 100644 index 58c60c789..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Header(title: Title) { - lazy val toThrift: t.Header = { - t.Header(title.toThrift) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.docx new file mode 100644 index 000000000..e8ca1d34d Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.scala deleted file mode 100644 index f0dc9630a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -sealed trait Layout - -case class UserListLayout( - header: Option[HeaderConfig], - userListOptions: UserListOptions, - socialProofs: Option[Seq[SocialProof]], - footer: Option[FooterConfig]) - extends Layout - -case class CarouselLayout( - header: Option[HeaderConfig], - carouselOptions: CarouselOptions, - socialProofs: Option[Seq[SocialProof]]) - extends Layout diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.docx new file mode 100644 index 000000000..27753143e Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.scala deleted file mode 100644 index 72351e033..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -sealed trait RecommendationOptions - -case class UserListOptions( - userBioEnabled: Boolean, - userBioTruncated: Boolean, - userBioMaxLines: Option[Long], -) extends RecommendationOptions - -case class CarouselOptions() extends RecommendationOptions diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.docx new file mode 100644 index 000000000..e3ddb2398 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.scala deleted file mode 100644 index fd5878af8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.stringcenter.client.core.ExternalString - -sealed trait SocialProof - -case class GeoContextProof(popularInCountryText: ExternalString) extends SocialProof -case class FollowedByUsersProof(text1: ExternalString, text2: ExternalString, textN: ExternalString) - extends SocialProof - -sealed trait SocialText { - def text: String -} - -case class GeoSocialText(text: String) extends SocialText -case class FollowedByUsersText(text: String) extends SocialText diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.docx new file mode 100644 index 000000000..dc53595c5 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.scala deleted file mode 100644 index 3d128e7c8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Title(text: String) { - lazy val toThrift: t.Title = { - t.Title(text) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.docx new file mode 100644 index 000000000..ab1deb0d0 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.scala deleted file mode 100644 index 7cfda1846..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -trait WTFPresentation { - def toThrift: t.WTFPresentation -} - -case class UserList( - userBioEnabled: Boolean, - userBioTruncated: Boolean, - userBioMaxLines: Option[Long], - feedbackAction: Option[FeedbackAction]) - extends WTFPresentation { - def toThrift: t.WTFPresentation = { - t.WTFPresentation.UserBioList( - t.UserList(userBioEnabled, userBioTruncated, userBioMaxLines, feedbackAction.map(_.toThrift))) - } -} - -object UserList { - def fromUserListOptions( - userListOptions: UserListOptions - ): UserList = { - UserList( - userListOptions.userBioEnabled, - userListOptions.userBioTruncated, - userListOptions.userBioMaxLines, - None) - } -} - -case class Carousel( - feedbackAction: Option[FeedbackAction]) - extends WTFPresentation { - def toThrift: t.WTFPresentation = { - t.WTFPresentation.Carousel(t.Carousel(feedbackAction.map(_.toThrift))) - } -} - -object Carousel { - def fromCarouselOptions( - carouselOptions: CarouselOptions - ): Carousel = { - Carousel(None) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD deleted file mode 100644 index 81d912a99..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD.docx new file mode 100644 index 000000000..72274a428 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.docx new file mode 100644 index 000000000..1d9d11816 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.scala deleted file mode 100644 index 8516de53d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.follow_recommendations.blenders - -import com.google.common.annotations.VisibleForTesting -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.AdMetadata -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.inject.Logging -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PromotedAccountsBlender @Inject() (statsReceiver: StatsReceiver) - extends Transform[Int, Recommendation] - with Logging { - - import PromotedAccountsBlender._ - val stats = statsReceiver.scope(Name) - val inputOrganicAccounts = stats.counter(InputOrganic) - val inputPromotedAccounts = stats.counter(InputPromoted) - val outputOrganicAccounts = stats.counter(OutputOrganic) - val outputPromotedAccounts = stats.counter(OutputPromoted) - val promotedAccountsStats = stats.scope(NumPromotedAccounts) - - override def transform( - maxResults: Int, - items: Seq[Recommendation] - ): Stitch[Seq[Recommendation]] = { - val (promoted, organic) = items.partition(_.isPromotedAccount) - val promotedIds = promoted.map(_.id).toSet - val dedupedOrganic = organic.filterNot(u => promotedIds.contains(u.id)) - val blended = blendPromotedAccount(dedupedOrganic, promoted, maxResults) - val (outputPromoted, outputOrganic) = blended.partition(_.isPromotedAccount) - inputOrganicAccounts.incr(dedupedOrganic.size) - inputPromotedAccounts.incr(promoted.size) - outputOrganicAccounts.incr(outputOrganic.size) - val size = outputPromoted.size - outputPromotedAccounts.incr(size) - if (size <= 5) { - promotedAccountsStats.counter(outputPromoted.size.toString).incr() - } else { - promotedAccountsStats.counter(MoreThan5Promoted).incr() - } - Stitch.value(blended) - } - - /** - * Merge Promoted results and organic results. Promoted result dictates the position - * in the merge list. - * - * merge a list of positioned users, aka. promoted, and a list of organic - * users. The positioned promoted users are pre-sorted with regards to their - * position ascendingly. Only requirement about position is to be within the - * range, i.e, can not exceed the combined length if merge is successful, ok - * to be at the last position, but not beyond. - * For more detailed description of location position: - * http://confluence.local.twitter.com/display/ADS/Promoted+Tweets+in+Timeline+Design+Document - */ - @VisibleForTesting - private[blenders] def mergePromotedAccounts( - organicUsers: Seq[Recommendation], - promotedUsers: Seq[Recommendation] - ): Seq[Recommendation] = { - def mergeAccountWithIndex( - organicUsers: Seq[Recommendation], - promotedUsers: Seq[Recommendation], - index: Int - ): Stream[Recommendation] = { - if (promotedUsers.isEmpty) organicUsers.toStream - else { - val promotedHead = promotedUsers.head - val promotedTail = promotedUsers.tail - promotedHead.adMetadata match { - case Some(AdMetadata(position, _)) => - if (position < 0) mergeAccountWithIndex(organicUsers, promotedTail, index) - else if (position == index) - promotedHead #:: mergeAccountWithIndex(organicUsers, promotedTail, index) - else if (organicUsers.isEmpty) organicUsers.toStream - else { - val organicHead = organicUsers.head - val organicTail = organicUsers.tail - organicHead #:: mergeAccountWithIndex(organicTail, promotedUsers, index + 1) - } - case _ => - logger.error("Unknown Candidate type in mergePromotedAccounts") - Stream.empty - } - } - } - - mergeAccountWithIndex(organicUsers, promotedUsers, 0) - } - - private[this] def blendPromotedAccount( - organic: Seq[Recommendation], - promoted: Seq[Recommendation], - maxResults: Int - ): Seq[Recommendation] = { - - val merged = mergePromotedAccounts(organic, promoted) - val mergedServed = merged.take(maxResults) - val promotedServed = promoted.intersect(mergedServed) - - if (isBlendPromotedNeeded( - mergedServed.size - promotedServed.size, - promotedServed.size, - maxResults - )) { - mergedServed - } else { - organic.take(maxResults) - } - } - - @VisibleForTesting - private[blenders] def isBlendPromotedNeeded( - organicSize: Int, - promotedSize: Int, - maxResults: Int - ): Boolean = { - (organicSize > 1) && - (promotedSize > 0) && - (promotedSize < organicSize) && - (promotedSize <= 2) && - (maxResults > 1) - } -} - -object PromotedAccountsBlender { - val Name = "promoted_accounts_blender" - val InputOrganic = "input_organic_accounts" - val InputPromoted = "input_promoted_accounts" - val OutputOrganic = "output_organic_accounts" - val OutputPromoted = "output_promoted_accounts" - val NumPromotedAccounts = "num_promoted_accounts" - val MoreThan5Promoted = "more_than_5" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD deleted file mode 100644 index f64fcda69..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - "configapi/configapi-featureswitches:v2", - "featureswitches/featureswitches-core", - "featureswitches/featureswitches-core:v2", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD.docx new file mode 100644 index 000000000..b63223dd2 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.docx new file mode 100644 index 000000000..a620d40d7 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.scala deleted file mode 100644 index 818f4402c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.timelines.configapi.CompositeConfig -import com.twitter.timelines.configapi.Config -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ConfigBuilder @Inject() ( - deciderConfigs: DeciderConfigs, - featureSwitchConfigs: FeatureSwitchConfigs) { - // The order of configs added to `CompositeConfig` is important. The config will be matched with - // the first possible rule. So, current setup will give priority to Deciders instead of FS - def build(): Config = - new CompositeConfig(Seq(deciderConfigs.config, featureSwitchConfigs.config)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.docx new file mode 100644 index 000000000..8ea2a8c42 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.scala deleted file mode 100644 index 0154a1703..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.decider.Recipient -import com.twitter.decider.SimpleRecipient -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi._ -import com.twitter.timelines.configapi.decider.DeciderSwitchOverrideValue -import com.twitter.timelines.configapi.decider.GuestRecipient -import com.twitter.timelines.configapi.decider.RecipientBuilder -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class DeciderConfigs @Inject() (deciderGateBuilder: DeciderGateBuilder) { - val overrides: Seq[OptionalOverride[_]] = DeciderConfigs.ParamsToDeciderMap.map { - case (params, deciderKey) => - params.optionalOverrideValue( - DeciderSwitchOverrideValue( - feature = deciderGateBuilder.keyToFeature(deciderKey), - enabledValue = true, - recipientBuilder = DeciderConfigs.UserOrGuestOrRequest - ) - ) - }.toSeq - - val config: BaseConfig = BaseConfigBuilder(overrides).build("FollowRecommendationServiceDeciders") -} - -object DeciderConfigs { - val ParamsToDeciderMap = Map( - DeciderParams.EnableRecommendations -> DeciderKey.EnableRecommendations, - DeciderParams.EnableScoreUserCandidates -> DeciderKey.EnableScoreUserCandidates, - HomeTimelineTweetRecsParams.EnableProduct -> DeciderKey.EnableHomeTimelineTweetRecsProduct, - ) - - object UserOrGuestOrRequest extends RecipientBuilder { - - def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match { - case c: WithUserId if c.userId.isDefined => - c.userId.map(SimpleRecipient) - case c: WithGuestId if c.guestId.isDefined => - c.guestId.map(GuestRecipient) - case c: WithGuestId => - RecipientBuilder.Request(c) - case _ => - throw new UndefinedUserIdNorGuestIDException(requestContext) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.docx new file mode 100644 index 000000000..90f256e8f Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.scala deleted file mode 100644 index c7f7f6d9e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.base.SocialProofEnforcedCandidateSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoQualityFollowSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.sims.SimsSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStpSourceFsConfig -import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphFSConfig -import com.twitter.follow_recommendations.common.feature_hydration.sources.FeatureHydrationSourcesFSConfig -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRankerFSConfig -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlowFSConfig -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateFSConfig -import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateFSConfig -import com.twitter.follow_recommendations.common.predicates.sgs.SgsPredicateFSConfig -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlowFSConfig -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FeatureSwitchConfigs @Inject() ( - globalFeatureSwitchConfig: GlobalFeatureSwitchConfig, - featureHydrationSourcesFSConfig: FeatureHydrationSourcesFSConfig, - weightedCandidateSourceRankerFSConfig: WeightedCandidateSourceRankerFSConfig, - // Flow related config - contentRecommenderFlowFSConfig: ContentRecommenderFlowFSConfig, - postNuxMlFlowFSConfig: PostNuxMlFlowFSConfig, - // Candidate source related config - crowdSearchAccountsFSConfig: CrowdSearchAccountsFSConfig, - offlineStpSourceFsConfig: OfflineStpSourceFsConfig, - onlineSTPSourceFSConfig: OnlineSTPSourceFSConfig, - popGeoSourceFSConfig: PopGeoSourceFSConfig, - popGeoQualityFollowFSConfig: PopGeoQualityFollowSourceFSConfig, - realGraphOonFSConfig: RealGraphOonFSConfig, - repeatedProfileVisitsFSConfig: RepeatedProfileVisitsFSConfig, - recentEngagementSimilarUsersFSConfig: RecentEngagementSimilarUsersFSConfig, - recentFollowingRecentFollowingExpansionSourceFSConfig: RecentFollowingRecentFollowingExpansionSourceFSConfig, - simsExpansionFSConfig: SimsExpansionFSConfig, - simsSourceFSConfig: SimsSourceFSConfig, - socialProofEnforcedCandidateSourceFSConfig: SocialProofEnforcedCandidateSourceFSConfig, - triangularLoopsFSConfig: TriangularLoopsFSConfig, - userUserGraphFSConfig: UserUserGraphFSConfig, - // Predicate related configs - gizmoduckPredicateFSConfig: GizmoduckPredicateFSConfig, - hssPredicateFSConfig: HssPredicateFSConfig, - sgsPredicateFSConfig: SgsPredicateFSConfig, - ppmiLocaleSourceFSConfig: PPMILocaleFollowSourceFSConfig, - topOrganicFollowsAccountsFSConfig: TopOrganicFollowsAccountsFSConfig, - statsReceiver: StatsReceiver) { - - val logger = Logger(classOf[FeatureSwitchConfigs]) - - val mergedFSConfig = - FeatureSwitchConfig.merge( - Seq( - globalFeatureSwitchConfig, - featureHydrationSourcesFSConfig, - weightedCandidateSourceRankerFSConfig, - // Flow related config - contentRecommenderFlowFSConfig, - postNuxMlFlowFSConfig, - // Candidate source related config - crowdSearchAccountsFSConfig, - offlineStpSourceFsConfig, - onlineSTPSourceFSConfig, - popGeoSourceFSConfig, - popGeoQualityFollowFSConfig, - realGraphOonFSConfig, - repeatedProfileVisitsFSConfig, - recentEngagementSimilarUsersFSConfig, - recentFollowingRecentFollowingExpansionSourceFSConfig, - simsExpansionFSConfig, - simsSourceFSConfig, - socialProofEnforcedCandidateSourceFSConfig, - triangularLoopsFSConfig, - userUserGraphFSConfig, - // Predicate related configs: - gizmoduckPredicateFSConfig, - hssPredicateFSConfig, - sgsPredicateFSConfig, - ppmiLocaleSourceFSConfig, - topOrganicFollowsAccountsFSConfig, - ) - ) - - /** - * enum params have to be listed in this main file together as otherwise we'll have to pass in - * some signature like `Seq[FSEnumParams[_]]` which are generics of generics and won't compile. - * we only have enumFsParams from globalFeatureSwitchConfig at the moment - */ - val enumOverrides = globalFeatureSwitchConfig.enumFsParams.flatMap { enumParam => - FeatureSwitchOverrideUtil.getEnumFSOverrides(statsReceiver, logger, enumParam) - } - - val gatedOverrides = mergedFSConfig.gatedOverridesMap.flatMap { - case (fsName, overrides) => - FeatureSwitchOverrideUtil.gatedOverrides(fsName, overrides: _*) - } - - val enumSeqOverrides = globalFeatureSwitchConfig.enumSeqFsParams.flatMap { enumSeqParam => - FeatureSwitchOverrideUtil.getEnumSeqFSOverrides(statsReceiver, logger, enumSeqParam) - } - - val overrides = - FeatureSwitchOverrideUtil - .getBooleanFSOverrides(mergedFSConfig.booleanFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedIntFSOverrides(mergedFSConfig.intFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedLongFSOverrides(mergedFSConfig.longFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedDoubleFSOverrides(mergedFSConfig.doubleFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getDurationFSOverrides(mergedFSConfig.durationFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedOptionalDoubleOverrides(mergedFSConfig.optionalDoubleFSParams: _*) ++ - FeatureSwitchOverrideUtil.getStringSeqFSOverrides(mergedFSConfig.stringSeqFSParams: _*) ++ - enumOverrides ++ - gatedOverrides ++ - enumSeqOverrides - - val config = BaseConfigBuilder(overrides).build("FollowRecommendationServiceFeatureSwitches") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.docx new file mode 100644 index 000000000..85e31a202 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.scala deleted file mode 100644 index 537ed660c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.AccountsFilteringAndRankingLogics -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.{ - AccountsFilteringAndRankingLogics => OrganicAccountsFilteringAndRankingLogics -} -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersParams -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionSourceParams -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams.CandidateScorerIdParam -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.configapi.params.GlobalParams.CandidateSourcesToFilter -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableCandidateParamHydrations -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableRecommendationFlowLogs -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts -import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepSocialUserCandidate -import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepUserCandidate -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GlobalFeatureSwitchConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = { - Seq( - EnableCandidateParamHydrations, - KeepUserCandidate, - KeepSocialUserCandidate, - EnableGFSSocialProofTransform, - EnableWhoToFollowProducts, - EnableRecommendationFlowLogs - ) - } - - val enumFsParams = - Seq( - CandidateScorerIdParam, - SimsExpansionSourceParams.Aggregator, - RecentEngagementSimilarUsersParams.Aggregator, - CandidateSourcesToFilter, - ) - - val enumSeqFsParams = - Seq( - AccountsFilteringAndRankingLogics, - OrganicAccountsFilteringAndRankingLogics - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.docx new file mode 100644 index 000000000..a029a6acc Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.scala deleted file mode 100644 index 847d69962..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.timelines.configapi.Config -import com.twitter.timelines.configapi.FeatureValue -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ParamsFactory @Inject() ( - config: Config, - requestContextFactory: RequestContextFactory, - statsReceiver: StatsReceiver) { - - private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi")) - def apply(followRecommendationServiceRequestContext: RequestContext): Params = - config(followRecommendationServiceRequestContext, stats) - - def apply( - clientContext: ClientContext, - displayLocation: DisplayLocation, - featureOverrides: Map[String, FeatureValue] - ): Params = - apply(requestContextFactory(clientContext, displayLocation, featureOverrides)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.docx new file mode 100644 index 000000000..707af2ff8 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.scala deleted file mode 100644 index ebc8abf3c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.timelines.configapi.BaseRequestContext -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.NullFeatureContext -import com.twitter.timelines.configapi.GuestId -import com.twitter.timelines.configapi.UserId -import com.twitter.timelines.configapi.WithFeatureContext -import com.twitter.timelines.configapi.WithGuestId -import com.twitter.timelines.configapi.WithUserId - -case class RequestContext( - userId: Option[UserId], - guestId: Option[GuestId], - featureContext: FeatureContext = NullFeatureContext) - extends BaseRequestContext - with WithUserId - with WithGuestId - with WithFeatureContext diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.docx new file mode 100644 index 000000000..62dfe22d1 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.scala deleted file mode 100644 index 89d8617a3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient} -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.FeatureValue -import com.twitter.timelines.configapi.ForcedFeatureContext -import com.twitter.timelines.configapi.OrElseFeatureContext -import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext -import javax.inject.Singleton - -/* - * Request Context Factory is used to build RequestContext objects which are used - * by the config api to determine the param overrides to apply to the request. - * The param overrides are determined per request by configs which specify which - * FS/Deciders/AB translate to what param overrides. - */ -@Singleton -class RequestContextFactory @Inject() (featureSwitches: FeatureSwitches, decider: Decider) { - def apply( - clientContext: ClientContext, - displayLocation: DisplayLocation, - featureOverrides: Map[String, FeatureValue] - ): RequestContext = { - val featureContext = getFeatureContext(clientContext, displayLocation, featureOverrides) - RequestContext(clientContext.userId, clientContext.guestId, featureContext) - } - - private[configapi] def getFeatureContext( - clientContext: ClientContext, - displayLocation: DisplayLocation, - featureOverrides: Map[String, FeatureValue] - ): FeatureContext = { - val recipient = - getFeatureSwitchRecipient(clientContext) - .withCustomFields("display_location" -> displayLocation.toFsName) - - // userAgeOpt is going to be set to None for logged out users and defaulted to Some(Int.MaxValue) for non-snowflake users - val userAgeOpt = clientContext.userId.map { userId => - SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) - } - val recipientWithAccountAge = - userAgeOpt - .map(age => recipient.withCustomFields("account_age_in_days" -> age)).getOrElse(recipient) - - val results = featureSwitches.matchRecipient(recipientWithAccountAge) - OrElseFeatureContext( - ForcedFeatureContext(featureOverrides), - new FeatureSwitchResultsFeatureContext(results)) - } - - @VisibleForTesting - private[configapi] def getFeatureSwitchRecipient( - clientContext: ClientContext - ): FeatureSwitchRecipient = { - FeatureSwitchRecipient( - userId = clientContext.userId, - userRoles = clientContext.userRoles, - deviceId = clientContext.deviceId, - guestId = clientContext.guestId, - languageCode = clientContext.languageCode, - countryCode = clientContext.countryCode, - isVerified = None, - clientApplicationId = clientContext.appId, - isTwoffice = clientContext.isTwoffice - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD deleted file mode 100644 index 5470c9bf4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - "configapi/configapi-featureswitches:v2", - "featureswitches/featureswitches-core", - "featureswitches/featureswitches-core:v2", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD.docx new file mode 100644 index 000000000..a111adfea Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.docx new file mode 100644 index 000000000..50f894d66 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.scala deleted file mode 100644 index 6b954a911..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.twitter.timelines.configapi.BaseRequestContext -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.NullFeatureContext -import com.twitter.timelines.configapi.WithFeatureContext -import com.twitter.timelines.configapi.WithUserId - -/** - * represent the context for a recommendation candidate (producer side) - * @param userId id of the recommended user - * @param featureContext feature context - */ -case class CandidateUserContext( - override val userId: Option[Long], - featureContext: FeatureContext = NullFeatureContext) - extends BaseRequestContext - with WithUserId - with WithFeatureContext diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.docx new file mode 100644 index 000000000..16bbf40c7 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.scala deleted file mode 100644 index 4f30cf17a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient} -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class CandidateUserContextFactory @Inject() ( - @Named(PRODUCER_SIDE_FEATURE_SWITCHES) featureSwitches: FeatureSwitches, - decider: Decider) { - def apply( - candidateUser: CandidateUser, - displayLocation: DisplayLocation - ): CandidateUserContext = { - val featureContext = getFeatureContext(candidateUser, displayLocation) - - CandidateUserContext(Some(candidateUser.id), featureContext) - } - - private[configapi] def getFeatureContext( - candidateUser: CandidateUser, - displayLocation: DisplayLocation - ): FeatureContext = { - - val recipient = getFeatureSwitchRecipient(candidateUser).withCustomFields( - "display_location" -> displayLocation.toFsName) - new FeatureSwitchResultsFeatureContext(featureSwitches.matchRecipient(recipient)) - } - - @VisibleForTesting - private[configapi] def getFeatureSwitchRecipient( - candidateUser: CandidateUser - ): FeatureSwitchRecipient = { - FeatureSwitchRecipient( - userId = Some(candidateUser.id), - userRoles = None, - deviceId = None, - guestId = None, - languageCode = None, - countryCode = None, - isVerified = None, - clientApplicationId = None, - isTwoffice = None - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.docx new file mode 100644 index 000000000..e4e95fa84 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.scala deleted file mode 100644 index 5afd09a63..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.timelines.configapi.Config -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -/** - * CandidateParamsFactory is primarily used for "producer side" experiments, don't use it on consumer side experiments - */ -@Singleton -class CandidateUserParamsFactory[T <: HasParams with HasDisplayLocation] @Inject() ( - config: Config, - candidateContextFactory: CandidateUserContextFactory, - statsReceiver: StatsReceiver) { - private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi_candidate_params")) - def apply(candidateContext: CandidateUser, request: T): CandidateUser = { - if (candidateContext.params == Params.Invalid) { - if (request.params(GlobalParams.EnableCandidateParamHydrations)) { - candidateContext.copy(params = - config(candidateContextFactory(candidateContext, request.displayLocation), stats)) - } else { - candidateContext.copy(params = Params.Empty) - } - } else { - candidateContext - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.docx new file mode 100644 index 000000000..eab0080be Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.scala deleted file mode 100644 index deadd1d70..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging - -@Singleton -class HydrateCandidateParamsTransform[Target <: HasParams with HasDisplayLocation] @Inject() ( - candidateParamsFactory: CandidateUserParamsFactory[Target]) - extends Transform[Target, CandidateUser] - with Logging { - - def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - Stitch.value(candidates.map(candidateParamsFactory.apply(_, target))) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD deleted file mode 100644 index 6fee24f89..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD.docx new file mode 100644 index 000000000..a8b7903ed Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.docx new file mode 100644 index 000000000..5d1ab0e73 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.scala deleted file mode 100644 index 798b02670..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.follow_recommendations.configapi.common - -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.DefinedFeatureName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.ValueFeatureName -import com.twitter.timelines.configapi.BoundedParam -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.OptionalOverride -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -trait FeatureSwitchConfig { - def booleanFSParams: Seq[Param[Boolean] with FSName] = Nil - - def intFSParams: Seq[FSBoundedParam[Int]] = Nil - - def longFSParams: Seq[FSBoundedParam[Long]] = Nil - - def doubleFSParams: Seq[FSBoundedParam[Double]] = Nil - - def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Nil - - def optionalDoubleFSParams: Seq[ - (BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName) - ] = Nil - - def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Nil - - /** - * Apply overrides in list when the given FS Key is enabled. - * This override type does NOT work with experiments. Params here will be evaluated for every - * request IMMEDIATELY, not upon param.apply. If you would like to use an experiment pls use - * the primitive type or ENUM overrides. - */ - def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = Map.empty -} - -object FeatureSwitchConfig { - def merge(configs: Seq[FeatureSwitchConfig]): FeatureSwitchConfig = new FeatureSwitchConfig { - override def booleanFSParams: Seq[Param[Boolean] with FSName] = - configs.flatMap(_.booleanFSParams) - override def intFSParams: Seq[FSBoundedParam[Int]] = - configs.flatMap(_.intFSParams) - override def longFSParams: Seq[FSBoundedParam[Long]] = - configs.flatMap(_.longFSParams) - override def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = - configs.flatMap(_.durationFSParams) - override def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = - configs.flatMap(_.gatedOverridesMap).toMap - override def doubleFSParams: Seq[FSBoundedParam[Double]] = - configs.flatMap(_.doubleFSParams) - override def optionalDoubleFSParams: Seq[ - (BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName) - ] = - configs.flatMap(_.optionalDoubleFSParams) - override def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = - configs.flatMap(_.stringSeqFSParams) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD deleted file mode 100644 index e4982dc0f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD.docx new file mode 100644 index 000000000..b152adac1 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.docx new file mode 100644 index 000000000..a2400cb3c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.scala deleted file mode 100644 index f4c069a63..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.follow_recommendations.configapi.deciders - -import com.twitter.servo.decider.DeciderKeyEnum - -object DeciderKey extends DeciderKeyEnum { - val EnableDiffyModuleDarkReading = Value("enable_diffy_module_dark_reading") - val EnableRecommendations = Value("enable_recommendations") - val EnableScoreUserCandidates = Value("enable_score_user_candidates") - val EnableProfileSidebarProduct = Value("enable_profile_sidebar_product") - val EnableMagicRecsProduct = Value("enable_magic_recs_product") - val EnableRuxLandingPageProduct = Value("enable_rux_landing_page_product") - val EnableRuxPymkProduct = Value("enable_rux_pymk_product") - val EnableProfileBonusFollowProduct = Value("enable_profile_bonus_follow_product") - val EnableElectionExploreWtfProduct = Value("enable_election_explore_wtf_product") - val EnableClusterFollowProduct = Value("enable_cluster_follow_product") - val EnableHomeTimelineProduct = Value("enable_home_timeline_product") - val EnableHtlBonusFollowProduct = Value("enable_htl_bonus_follow_product") - val EnableExploreTabProduct = Value("enable_explore_tab_product") - val EnableSidebarProduct = Value("enable_sidebar_product") - val EnableNuxPymkProduct = Value("enable_nux_pymk_product") - val EnableNuxInterestsProduct = Value("enable_nux_interests_product") - val EnableNuxTopicBonusFollowProduct = Value("enable_nux_topic_bonus_follow_product") - val EnableCampaignFormProduct = Value("enable_campaign_form_product") - val EnableReactiveFollowProduct = Value("enable_reactive_follow_product") - val EnableIndiaCovid19CuratedAccountsWtfProduct = Value( - "enable_india_covid19_curated_accounts_wtf_product") - val EnableAbUploadProduct = Value("enable_ab_upload_product") - val EnablePeolePlusPlusProduct = Value("enable_people_plus_plus_product") - val EnableTweetNotificationRecsProduct = Value("enable_tweet_notification_recs_product") - val EnableProfileDeviceFollow = Value("enable_profile_device_follow_product") - val EnableRecosBackfillProduct = Value("enable_recos_backfill_product") - val EnablePostNuxFollowTaskProduct = Value("enable_post_nux_follow_task_product") - val EnableCuratedSpaceHostsProduct = Value("enable_curated_space_hosts_product") - val EnableNuxGeoCategoryProduct = Value("enable_nux_geo_category_product") - val EnableNuxInterestsCategoryProduct = Value("enable_nux_interests_category_product") - val EnableNuxPymkCategoryProduct = Value("enable_nux_pymk_category_product") - val EnableHomeTimelineTweetRecsProduct = Value("enable_home_timeline_tweet_recs_product") - val EnableHtlBulkFriendFollowsProduct = Value("enable_htl_bulk_friend_follows_product") - val EnableNuxAutoFollowProduct = Value("enable_nux_auto_follow_product") - val EnableSearchBonusFollowProduct = Value("enable_search_bonus_follow_product") - val EnableFetchUserInRequestBuilder = Value("enable_fetch_user_in_request_builder") - val EnableProductMixerMagicRecsProduct = Value("enable_product_mixer_magic_recs_product") - val EnableHomeTimelineReverseChronProduct = Value("enable_home_timeline_reverse_chron_product") - val EnableProductMixerPipelineMagicRecsDarkRead = Value( - "enable_product_mixer_pipeline_magic_recs_dark_read") - val EnableExperimentalCaching = Value("enable_experimental_caching") - val EnableDistributedCaching = Value("enable_distributed_caching") - val EnableGizmoduckCaching = Value("enable_gizmoduck_caching") - val EnableTrafficDarkReading = Value("enable_traffic_dark_reading") - val EnableGraphFeatureServiceRequests = Value("enable_graph_feature_service_requests") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.docx new file mode 100644 index 000000000..f66d3c482 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.scala deleted file mode 100644 index 07bf3e14d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.configapi.deciders - -import com.twitter.timelines.configapi.Param - -object DeciderParams { - object EnableRecommendations extends Param[Boolean](false) - object EnableScoreUserCandidates extends Param[Boolean](false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD deleted file mode 100644 index 1bb357b2c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - "configapi/configapi-featureswitches:v2", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD.docx new file mode 100644 index 000000000..1eb0f2af3 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.docx new file mode 100644 index 000000000..7f4e2fa87 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.scala deleted file mode 100644 index cf1905c41..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.follow_recommendations.configapi.params - -import com.twitter.follow_recommendations.models.CandidateSourceType -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam - -/** - * When adding Producer side experiments, make sure to register the FS Key in [[ProducerFeatureFilter]] - * in [[FeatureSwitchesModule]], otherwise, the FS will not work. - */ -object GlobalParams { - - object EnableCandidateParamHydrations - extends FSParam[Boolean]("frs_receiver_enable_candidate_params", false) - - object KeepUserCandidate - extends FSParam[Boolean]("frs_receiver_holdback_keep_user_candidate", true) - - object KeepSocialUserCandidate - extends FSParam[Boolean]("frs_receiver_holdback_keep_social_user_candidate", true) - - case object EnableGFSSocialProofTransform - extends FSParam("social_proof_transform_use_graph_feature_service", true) - - case object EnableWhoToFollowProducts extends FSParam("who_to_follow_product_enabled", true) - - case object CandidateSourcesToFilter - extends FSEnumParam[CandidateSourceType.type]( - "candidate_sources_type_filter_id", - CandidateSourceType.None, - CandidateSourceType) - - object EnableRecommendationFlowLogs - extends FSParam[Boolean]("frs_recommendation_flow_logs_enabled", false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD deleted file mode 100644 index 2a625d856..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "decider/src/main/scala", - "finagle/finagle-core/src/main", - "finatra/inject/inject-core/src/main/scala", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/exceptions", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/modules", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/response", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", - "scrooge/scrooge-core/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD.docx new file mode 100644 index 000000000..3df6cc833 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.docx new file mode 100644 index 000000000..e842e3685 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.scala deleted file mode 100644 index 1695c464d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.follow_recommendations.common.models._ -import com.twitter.follow_recommendations.configapi.ParamsFactory -import com.twitter.follow_recommendations.models.CandidateUserDebugParams -import com.twitter.follow_recommendations.models.FeatureValue -import com.twitter.follow_recommendations.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CandidateUserDebugParamsBuilder @Inject() (paramsFactory: ParamsFactory) { - def fromThrift(req: t.ScoringUserRequest): CandidateUserDebugParams = { - val clientContext = ClientContextConverter.fromThrift(req.clientContext) - val displayLocation = DisplayLocation.fromThrift(req.displayLocation) - - CandidateUserDebugParams(req.candidates.map { candidate => - candidate.userId -> paramsFactory( - clientContext, - displayLocation, - candidate.featureOverrides - .map(_.mapValues(FeatureValue.fromThrift).toMap).getOrElse(Map.empty)) - }.toMap) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.docx new file mode 100644 index 000000000..fb989b5cd Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.scala deleted file mode 100644 index bc21fd6a3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.DisplayContext -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecommendationRequestBuilder @Inject() ( - requestBuilderUserFetcher: RequestBuilderUserFetcher, - statsReceiver: StatsReceiver) { - private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) - private val isSoftUserCounter = scopedStats.counter("is_soft_user") - - def fromThrift(tRequest: t.RecommendationRequest): Stitch[RecommendationRequest] = { - requestBuilderUserFetcher.fetchUser(tRequest.clientContext.userId).map { userOpt => - val isSoftUser = userOpt.exists(_.userType == UserType.Soft) - if (isSoftUser) isSoftUserCounter.incr() - RecommendationRequest( - clientContext = ClientContextConverter.fromThrift(tRequest.clientContext), - displayLocation = DisplayLocation.fromThrift(tRequest.displayLocation), - displayContext = tRequest.displayContext.map(DisplayContext.fromThrift), - maxResults = tRequest.maxResults, - cursor = tRequest.cursor, - excludedIds = tRequest.excludedIds, - fetchPromotedContent = tRequest.fetchPromotedContent, - debugParams = tRequest.debugParams.map(DebugParams.fromThrift), - userLocationState = tRequest.userLocationState, - isSoftUser = isSoftUser - ) - } - - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.docx new file mode 100644 index 000000000..a9c4b41e3 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.scala deleted file mode 100644 index 3953b5ef3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.decider.Decider -import com.twitter.decider.SimpleRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RequestBuilderUserFetcher @Inject() ( - gizmoduck: Gizmoduck, - statsReceiver: StatsReceiver, - decider: Decider) { - private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) - - def fetchUser(userIdOpt: Option[Long]): Stitch[Option[User]] = { - userIdOpt match { - case Some(userId) if enableDecider(userId) => - val stitch = gizmoduck - .getUserById( - userId = userId, - context = LookupContext( - forUserId = Some(userId), - includeProtected = true, - includeSoftUsers = true - ) - ).map(user => Some(user)) - StatsUtil - .profileStitch(stitch, scopedStats) - .handle { - case _: Throwable => None - } - case _ => Stitch.None - } - } - - private def enableDecider(userId: Long): Boolean = { - decider.isAvailable( - DeciderKey.EnableFetchUserInRequestBuilder.toString, - Some(SimpleRecipient(userId))) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.docx new file mode 100644 index 000000000..52e3e1ebb Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.scala deleted file mode 100644 index 4a45a19f7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.ScoringUserRequest -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.stitch.Stitch - -@Singleton -class ScoringUserRequestBuilder @Inject() ( - requestBuilderUserFetcher: RequestBuilderUserFetcher, - candidateUserDebugParamsBuilder: CandidateUserDebugParamsBuilder, - statsReceiver: StatsReceiver) { - private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) - private val isSoftUserCounter = scopedStats.counter("is_soft_user") - - def fromThrift(req: t.ScoringUserRequest): Stitch[ScoringUserRequest] = { - requestBuilderUserFetcher.fetchUser(req.clientContext.userId).map { userOpt => - val isSoftUser = userOpt.exists(_.userType == UserType.Soft) - if (isSoftUser) isSoftUserCounter.incr() - - val candidateUsersParamsMap = candidateUserDebugParamsBuilder.fromThrift(req) - val candidates = req.candidates.map { candidate => - CandidateUser - .fromUserRecommendation(candidate).copy(params = - candidateUsersParamsMap.paramsMap.getOrElse(candidate.userId, Params.Invalid)) - } - - ScoringUserRequest( - clientContext = ClientContextConverter.fromThrift(req.clientContext), - displayLocation = DisplayLocation.fromThrift(req.displayLocation), - params = Params.Empty, - debugOptions = req.debugParams.map(DebugOptions.fromDebugParamsThrift), - recentFollowedUserIds = None, - recentFollowedByUserIds = None, - wtfImpressions = None, - similarToUserIds = Nil, - candidates = candidates, - debugParams = req.debugParams.map(DebugParams.fromThrift), - isSoftUser = isSoftUser - ) - } - } - -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.docx new file mode 100644 index 000000000..d11cc9b92 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.scala deleted file mode 100644 index f3014982e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.finatra.thrift.Controller -import com.twitter.follow_recommendations.configapi.ParamsFactory -import com.twitter.follow_recommendations.services.ProductPipelineSelector -import com.twitter.follow_recommendations.services.UserScoringService -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService._ -import com.twitter.stitch.Stitch -import javax.inject.Inject - -class ThriftController @Inject() ( - userScoringService: UserScoringService, - recommendationRequestBuilder: RecommendationRequestBuilder, - scoringUserRequestBuilder: ScoringUserRequestBuilder, - productPipelineSelector: ProductPipelineSelector, - paramsFactory: ParamsFactory) - extends Controller(FollowRecommendationsThriftService) { - - handle(GetRecommendations) { args: GetRecommendations.Args => - val stitch = recommendationRequestBuilder.fromThrift(args.request).flatMap { request => - val params = paramsFactory( - request.clientContext, - request.displayLocation, - request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty)) - productPipelineSelector.selectPipeline(request, params).map(_.toThrift) - } - Stitch.run(stitch) - } - - handle(ScoreUserCandidates) { args: ScoreUserCandidates.Args => - val stitch = scoringUserRequestBuilder.fromThrift(args.request).flatMap { request => - val params = paramsFactory( - request.clientContext, - request.displayLocation, - request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty)) - userScoringService.get(request.copy(params = params)).map(_.toThrift) - } - Stitch.run(stitch) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD deleted file mode 100644 index aa775c6a7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD.docx new file mode 100644 index 000000000..cc7949023 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.docx new file mode 100644 index 000000000..f2c2457f1 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.scala deleted file mode 100644 index dd9372484..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource -import com.twitter.follow_recommendations.common.base.IdentityRanker -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.ParamPredicate -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.RecommendationFlow -import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.base.TruePredicate -import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedAccountsCandidateSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate -import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PromotedAccountsFlow @Inject() ( - promotedAccountsCandidateSource: PromotedAccountsCandidateSource, - trackingTokenTransform: TrackingTokenTransform, - baseStatsReceiver: StatsReceiver, - @Flag("fetch_prod_promoted_accounts") fetchProductionPromotedAccounts: Boolean) - extends RecommendationFlow[PromotedAccountsFlowRequest, CandidateUser] { - - protected override def targetEligibility: Predicate[PromotedAccountsFlowRequest] = - new ParamPredicate[PromotedAccountsFlowRequest]( - PromotedAccountsFlowParams.TargetEligibility - ) - - protected override def candidateSources( - target: PromotedAccountsFlowRequest - ): Seq[CandidateSource[PromotedAccountsFlowRequest, CandidateUser]] = { - import EnrichedCandidateSource._ - val candidateSourceStats = statsReceiver.scope("candidate_sources") - val budget: Duration = target.params(PromotedAccountsFlowParams.FetchCandidateSourceBudget) - val candidateSources = Seq( - promotedAccountsCandidateSource - .mapKeys[PromotedAccountsFlowRequest](r => - Seq(r.toAdsRequest(fetchProductionPromotedAccounts))) - .mapValue(PromotedAccountsUtil.toCandidateUser) - ).map { candidateSource => - candidateSource - .failOpenWithin(budget, candidateSourceStats).observe(candidateSourceStats) - } - candidateSources - } - - protected override def preRankerCandidateFilter: Predicate[ - (PromotedAccountsFlowRequest, CandidateUser) - ] = { - val preRankerFilterStats = statsReceiver.scope("pre_ranker") - ExcludedUserIdPredicate.observe(preRankerFilterStats.scope("exclude_user_id_predicate")) - } - - /** - * rank the candidates - */ - protected override def selectRanker( - target: PromotedAccountsFlowRequest - ): Ranker[PromotedAccountsFlowRequest, CandidateUser] = { - new IdentityRanker[PromotedAccountsFlowRequest, CandidateUser] - } - - /** - * transform the candidates after ranking (e.g. dedupping, grouping and etc) - */ - protected override def postRankerTransform: Transform[ - PromotedAccountsFlowRequest, - CandidateUser - ] = { - new IdentityTransform[PromotedAccountsFlowRequest, CandidateUser] - } - - /** - * filter invalid candidates before returning the results. - * - * Some heavy filters e.g. SGS filter could be applied in this step - */ - protected override def validateCandidates: Predicate[ - (PromotedAccountsFlowRequest, CandidateUser) - ] = { - new TruePredicate[(PromotedAccountsFlowRequest, CandidateUser)] - } - - /** - * transform the candidates into results and return - */ - protected override def transformResults: Transform[PromotedAccountsFlowRequest, CandidateUser] = { - trackingTokenTransform - } - - /** - * configuration for recommendation results - */ - protected override def resultsConfig( - target: PromotedAccountsFlowRequest - ): RecommendationResultsConfig = { - RecommendationResultsConfig( - target.params(PromotedAccountsFlowParams.ResultSizeParam), - target.params(PromotedAccountsFlowParams.BatchSizeParam) - ) - } - - override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("promoted_accounts_flow") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.docx new file mode 100644 index 000000000..68df67449 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.scala deleted file mode 100644 index 010aea509..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -abstract class PromotedAccountsFlowParams[A](default: A) extends Param[A](default) { - override val statName: String = "ads/" + this.getClass.getSimpleName -} - -object PromotedAccountsFlowParams { - - // number of total slots returned to the end user, available to put ads - case object TargetEligibility extends PromotedAccountsFlowParams[Boolean](true) - case object ResultSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue) - case object BatchSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue) - case object FetchCandidateSourceBudget - extends PromotedAccountsFlowParams[Duration](1000.millisecond) - -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.docx new file mode 100644 index 000000000..a8d71c689 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.scala deleted file mode 100644 index 61afb3049..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads -import com.twitter.follow_recommendations.common.clients.adserver.AdRequest -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasExcludedUserIds -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class PromotedAccountsFlowRequest( - override val clientContext: ClientContext, - override val params: Params, - displayLocation: DisplayLocation, - profileId: Option[Long], - // note we also add userId and profileId to excludeUserIds - excludeIds: Seq[Long]) - extends HasParams - with HasClientContext - with HasExcludedUserIds - with HasDisplayLocation { - def toAdsRequest(fetchProductionPromotedAccounts: Boolean): AdRequest = { - AdRequest( - clientContext = clientContext, - displayLocation = displayLocation, - isTest = Some(!fetchProductionPromotedAccounts), - profileUserId = profileId - ) - } - override val excludedUserIds: Seq[Long] = { - excludeIds ++ clientContext.userId.toSeq ++ profileId.toSeq - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.docx new file mode 100644 index 000000000..52db28e12 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.scala deleted file mode 100644 index 5327911c2..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads -import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedCandidateUser -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.AdMetadata -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails - -object PromotedAccountsUtil { - def toCandidateUser(promotedCandidateUser: PromotedCandidateUser): CandidateUser = { - CandidateUser( - id = promotedCandidateUser.id, - score = None, - adMetadata = - Some(AdMetadata(promotedCandidateUser.position, promotedCandidateUser.adImpression)), - reason = Some( - Reason( - accountProof = Some(AccountProof(followProof = Some(promotedCandidateUser.followProof)))) - ), - userCandidateSourceDetails = Some( - UserCandidateSourceDetails( - promotedCandidateUser.primaryCandidateSource, - Map.empty, - Map.empty, - None)) - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD deleted file mode 100644 index 886c0fe5a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD.docx new file mode 100644 index 000000000..8344c5049 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.docx new file mode 100644 index 000000000..056b33c88 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.scala deleted file mode 100644 index 30dfa0d42..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.scala +++ /dev/null @@ -1,202 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource -import com.twitter.follow_recommendations.common.base.GatedPredicateBase -import com.twitter.follow_recommendations.common.base.ParamPredicate -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.RecommendationFlow -import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate -import com.twitter.follow_recommendations.common.predicates.InactivePredicate -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker -import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform -import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform -import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil -import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderFlow @Inject() ( - contentRecommenderFlowCandidateSourceRegistry: ContentRecommenderFlowCandidateSourceRegistry, - recentFollowingPredicate: RecentFollowingPredicate, - gizmoduckPredicate: GizmoduckPredicate, - inactivePredicate: InactivePredicate, - sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, - invalidRelationshipPredicate: InvalidRelationshipPredicate, - trackingTokenTransform: TrackingTokenTransform, - baseStatsReceiver: StatsReceiver) - extends RecommendationFlow[ContentRecommenderRequest, CandidateUser] - with RecommendationFlowBaseSideEffectsUtil[ContentRecommenderRequest, CandidateUser] - with CandidateSourceHoldbackUtil { - - override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("content_recommender_flow") - - override val qualityFactorObserver: Option[QualityFactorObserver] = { - val config = LinearLatencyQualityFactorConfig( - qualityFactorBounds = - BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0), - initialDelay = 60.seconds, - targetLatency = 100.milliseconds, - targetLatencyPercentile = 95.0, - delta = 0.001 - ) - val qualityFactor = LinearLatencyQualityFactor(config) - val observer = LinearLatencyQualityFactorObserver(qualityFactor) - statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat) - Some(observer) - } - - protected override def targetEligibility: Predicate[ContentRecommenderRequest] = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.TargetEligibility - ) - - protected override def candidateSources( - target: ContentRecommenderRequest - ): Seq[CandidateSource[ContentRecommenderRequest, CandidateUser]] = { - import EnrichedCandidateSource._ - val identifiers = ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params).keySet - val selected = contentRecommenderFlowCandidateSourceRegistry.select(identifiers) - val budget = - target.params(ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond).millisecond - filterCandidateSources(target, selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq) - } - - protected override val preRankerCandidateFilter: Predicate[ - (ContentRecommenderRequest, CandidateUser) - ] = { - val preRankerFilterStats = statsReceiver.scope("pre_ranker") - val recentFollowingPredicateStats = preRankerFilterStats.scope("recent_following_predicate") - val invalidRelationshipPredicateStats = - preRankerFilterStats.scope("invalid_relationship_predicate") - - object recentFollowingGatedPredicate - extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( - recentFollowingPredicate, - recentFollowingPredicateStats - ) { - override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = - item._1.params(ContentRecommenderParams.EnableRecentFollowingPredicate) - } - - object invalidRelationshipGatedPredicate - extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( - invalidRelationshipPredicate, - invalidRelationshipPredicateStats - ) { - override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = - item._1.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate) - } - - ExcludedUserIdPredicate - .observe(preRankerFilterStats.scope("exclude_user_id_predicate")) - .andThen(recentFollowingGatedPredicate.observe(recentFollowingPredicateStats)) - .andThen(invalidRelationshipGatedPredicate.observe(invalidRelationshipPredicateStats)) - } - - /** - * rank the candidates - */ - protected override def selectRanker( - target: ContentRecommenderRequest - ): Ranker[ContentRecommenderRequest, CandidateUser] = { - val rankersStatsReceiver = statsReceiver.scope("rankers") - WeightedCandidateSourceRanker - .build[ContentRecommenderRequest]( - ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params), - randomSeed = target.getRandomizationSeed - ).observe(rankersStatsReceiver.scope("weighted_candidate_source_ranker")) - } - - /** - * transform the candidates after ranking - */ - protected override def postRankerTransform: Transform[ - ContentRecommenderRequest, - CandidateUser - ] = { - new DedupTransform[ContentRecommenderRequest, CandidateUser] - .observe(statsReceiver.scope("dedupping")) - } - - protected override def validateCandidates: Predicate[ - (ContentRecommenderRequest, CandidateUser) - ] = { - val stats = statsReceiver.scope("validate_candidates") - val gizmoduckPredicateStats = stats.scope("gizmoduck_predicate") - val inactivePredicateStats = stats.scope("inactive_predicate") - val sgsPredicateStats = stats.scope("sgs_predicate") - - val includeGizmoduckPredicate = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.EnableGizmoduckPredicate) - .map[(ContentRecommenderRequest, CandidateUser)] { - case (request: ContentRecommenderRequest, _) => - request - } - - val includeInactivePredicate = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.EnableInactivePredicate) - .map[(ContentRecommenderRequest, CandidateUser)] { - case (request: ContentRecommenderRequest, _) => - request - } - - val includeInvalidTargetCandidateRelationshipTypesPredicate = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate) - .map[(ContentRecommenderRequest, CandidateUser)] { - case (request: ContentRecommenderRequest, _) => - request - } - - Predicate - .andConcurrently[(ContentRecommenderRequest, CandidateUser)]( - Seq( - gizmoduckPredicate.observe(gizmoduckPredicateStats).gate(includeGizmoduckPredicate), - inactivePredicate.observe(inactivePredicateStats).gate(includeInactivePredicate), - sgsPredicate - .observe(sgsPredicateStats).gate( - includeInvalidTargetCandidateRelationshipTypesPredicate), - ) - ) - } - - /** - * transform the candidates into results and return - */ - protected override def transformResults: Transform[ContentRecommenderRequest, CandidateUser] = { - trackingTokenTransform - } - - /** - * configuration for recommendation results - */ - protected override def resultsConfig( - target: ContentRecommenderRequest - ): RecommendationResultsConfig = { - RecommendationResultsConfig( - target.maxResults.getOrElse(target.params(ContentRecommenderParams.ResultSizeParam)), - target.params(ContentRecommenderParams.BatchSizeParam) - ) - } - -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.docx new file mode 100644 index 000000000..4bdf1cacf Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.scala deleted file mode 100644 index 4a6c61042..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderFlowCandidateSourceRegistry @Inject() ( - // social based - forwardPhoneBookSource: ForwardPhoneBookSource, - forwardEmailBookSource: ForwardEmailBookSource, - reversePhoneBookSource: ReversePhoneBookSource, - reverseEmailBookSource: ReverseEmailBookSource, - offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, - triangularLoopsSource: TriangularLoopsSource, - userUserGraphCandidateSource: UserUserGraphCandidateSource, - realGraphOonSource: RealGraphOonV2Source, - recentFollowingRecentFollowingExpansionSource: RecentFollowingRecentFollowingExpansionSource, - // activity based - recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, - recentEngagementSimilarUsersSource: RecentEngagementSimilarUsersSource, - repeatedProfileVisitsSource: RepeatedProfileVisitsSource, - // geo based - popCountrySource: PopCountrySource, - popGeohashSource: PopGeohashSource, - popCountryBackFillSource: PopCountryBackFillSource, - crowdSearchAccountsSource: CrowdSearchAccountsSource, - topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, - ppmiLocaleFollowSource: PPMILocaleFollowSource, - baseStatsReceiver: StatsReceiver) - extends CandidateSourceRegistry[ContentRecommenderRequest, CandidateUser] { - - override val statsReceiver = baseStatsReceiver - .scope("content_recommender_flow", "candidate_sources") - - override val sources: Set[CandidateSource[ContentRecommenderRequest, CandidateUser]] = Seq( - forwardPhoneBookSource, - forwardEmailBookSource, - reversePhoneBookSource, - reverseEmailBookSource, - offlineStrongTiePredictionSource, - triangularLoopsSource, - userUserGraphCandidateSource, - realGraphOonSource, - recentFollowingRecentFollowingExpansionSource, - recentFollowingSimilarUsersSource, - recentEngagementSimilarUsersSource, - repeatedProfileVisitsSource, - popCountrySource, - popGeohashSource, - popCountryBackFillSource, - crowdSearchAccountsSource, - topOrganicFollowsAccountsSource, - ppmiLocaleFollowSource, - ).toSet -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.docx new file mode 100644 index 000000000..2691a1f8e Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.scala deleted file mode 100644 index 845a6ec0a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.timelines.configapi.Params - -object ContentRecommenderFlowCandidateSourceWeights { - - def getWeights( - params: Params - ): Map[CandidateSourceIdentifier, Double] = { - Map[CandidateSourceIdentifier, Double]( - // Social based - UserUserGraphCandidateSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight), - ForwardPhoneBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight), - ReversePhoneBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight), - ForwardEmailBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight), - ReverseEmailBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight), - TriangularLoopsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight), - OfflineStrongTiePredictionSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight), - RecentFollowingRecentFollowingExpansionSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight), - RecentFollowingSimilarUsersSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight), - // Activity based - RealGraphOonV2Source.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight), - RecentEngagementSimilarUsersSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight), - RepeatedProfileVisitsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight), - // Geo based - PopCountrySource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight), - PopGeohashSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight), - PopCountryBackFillSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight), - PPMILocaleFollowSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight), - CrowdSearchAccountsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight), - TopOrganicFollowsAccountsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight), - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.docx new file mode 100644 index 000000000..96094876c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.scala deleted file mode 100644 index 462de260b..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.timelines.configapi.FSBoundedParam - -object ContentRecommenderFlowCandidateSourceWeightsParams { - // Social based - case object ForwardPhoneBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ForwardPhoneBookSourceWeight, - 1d, - 0d, - 1000d) - case object ForwardEmailBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ForwardEmailBookSourceWeight, - 1d, - 0d, - 1000d) - case object ReversePhoneBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ReversePhoneBookSourceWeight, - 1d, - 0d, - 1000d) - case object ReverseEmailBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ReverseEmailBookSourceWeight, - 1d, - 0d, - 1000d) - case object OfflineStrongTiePredictionSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.OfflineStrongTiePredictionSourceWeight, - 1d, - 0d, - 1000d) - case object TriangularLoopsSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.TriangularLoopsSourceWeight, - 1d, - 0d, - 1000d) - case object UserUserGraphSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.UserUserGraphSourceWeight, - 1d, - 0d, - 1000d) - case object NewFollowingNewFollowingExpansionSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.NewFollowingNewFollowingExpansionSourceWeight, - 1d, - 0d, - 1000d) - // Activity based - case object NewFollowingSimilarUserSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.NewFollowingSimilarUserSourceWeight, - 1d, - 0d, - 1000d) - case object RecentEngagementSimilarUserSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.RecentEngagementSimilarUserSourceWeight, - 1d, - 0d, - 1000d) - case object RepeatedProfileVisitsSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.RepeatedProfileVisitsSourceWeight, - 1d, - 0d, - 1000d) - case object RealGraphOonSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.RealGraphOonSourceWeight, - 1d, - 0d, - 1000d) - // Geo based - case object PopCountrySourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PopCountrySourceWeight, - 1d, - 0d, - 1000d) - case object PopGeohashSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PopGeohashSourceWeight, - 1d, - 0d, - 1000d) - case object PopCountryBackfillSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PopCountryBackfillSourceWeight, - 1d, - 0d, - 1000d) - case object PPMILocaleFollowSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PPMILocaleFollowSourceWeight, - 1d, - 0d, - 1000d) - case object TopOrganicFollowsAccountsSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.TopOrganicFollowsAccountsSourceWeight, - 1d, - 0d, - 1000d) - case object CrowdSearchAccountSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.CrowdSearchAccountSourceWeight, - 1d, - 0d, - 1000d) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.docx new file mode 100644 index 000000000..250b1c260 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.scala deleted file mode 100644 index a24032c84..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderFlowFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq( - ContentRecommenderParams.IncludeActivityBasedCandidateSource, - ContentRecommenderParams.IncludeSocialBasedCandidateSource, - ContentRecommenderParams.IncludeGeoBasedCandidateSource, - ContentRecommenderParams.IncludeHomeTimelineTweetRecsCandidateSource, - ContentRecommenderParams.IncludeSocialProofEnforcedCandidateSource, - ContentRecommenderParams.EnableRecentFollowingPredicate, - ContentRecommenderParams.EnableGizmoduckPredicate, - ContentRecommenderParams.EnableInactivePredicate, - ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate, - ContentRecommenderParams.IncludeNewFollowingNewFollowingExpansionCandidateSource, - ContentRecommenderParams.IncludeMoreGeoBasedCandidateSource, - ContentRecommenderParams.TargetEligibility, - ContentRecommenderParams.GetFollowersFromSgs, - ContentRecommenderParams.EnableInvalidRelationshipPredicate, - ) - - override val intFSParams: Seq[FSBoundedParam[Int]] = - Seq( - ContentRecommenderParams.ResultSizeParam, - ContentRecommenderParams.BatchSizeParam, - ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond, - ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond, - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = - Seq( - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight, - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.docx new file mode 100644 index 000000000..6bca9feaa Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.scala deleted file mode 100644 index ff51dc9f6..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -object ContentRecommenderFlowFeatureSwitchKeys { - val TargetUserEligible = "content_recommender_flow_target_eligible" - val ResultSize = "content_recommender_flow_result_size" - val BatchSize = "content_recommender_flow_batch_size" - val RecentFollowingPredicateBudgetInMillisecond = - "content_recommender_flow_recent_following_predicate_budget_in_ms" - val CandidateGenerationBudgetInMillisecond = - "content_recommender_flow_candidate_generation_budget_in_ms" - val EnableRecentFollowingPredicate = "content_recommender_flow_enable_recent_following_predicate" - val EnableGizmoduckPredicate = "content_recommender_flow_enable_gizmoduck_predicate" - val EnableInactivePredicate = "content_recommender_flow_enable_inactive_predicate" - val EnableInvalidTargetCandidateRelationshipPredicate = - "content_recommender_flow_enable_invalid_target_candidate_relationship_predicate" - val IncludeActivityBasedCandidateSource = - "content_recommender_flow_include_activity_based_candidate_source" - val IncludeSocialBasedCandidateSource = - "content_recommender_flow_include_social_based_candidate_source" - val IncludeGeoBasedCandidateSource = - "content_recommender_flow_include_geo_based_candidate_source" - val IncludeHomeTimelineTweetRecsCandidateSource = - "content_recommender_flow_include_home_timeline_tweet_recs_candidate_source" - val IncludeSocialProofEnforcedCandidateSource = - "content_recommender_flow_include_social_proof_enforced_candidate_source" - val IncludeNewFollowingNewFollowingExpansionCandidateSource = - "content_recommender_flow_include_new_following_new_following_expansion_candidate_source" - val IncludeMoreGeoBasedCandidateSource = - "content_recommender_flow_include_more_geo_based_candidate_source" - val GetFollowersFromSgs = "content_recommender_flow_get_followers_from_sgs" - val EnableInvalidRelationshipPredicate = - "content_recommender_flow_enable_invalid_relationship_predicate" - - // Candidate source weight param keys - // Social based - val ForwardPhoneBookSourceWeight = - "content_recommender_flow_candidate_source_weight_forward_phone_book" - val ForwardEmailBookSourceWeight = - "content_recommender_flow_candidate_source_weight_forward_email_book" - val ReversePhoneBookSourceWeight = - "content_recommender_flow_candidate_source_weight_reverse_phone_book" - val ReverseEmailBookSourceWeight = - "content_recommender_flow_candidate_source_weight_reverse_email_book" - val OfflineStrongTiePredictionSourceWeight = - "content_recommender_flow_candidate_source_weight_offline_stp" - val TriangularLoopsSourceWeight = - "content_recommender_flow_candidate_source_weight_triangular_loops" - val UserUserGraphSourceWeight = "content_recommender_flow_candidate_source_weight_user_user_graph" - val NewFollowingNewFollowingExpansionSourceWeight = - "content_recommender_flow_candidate_source_weight_new_following_new_following_expansion" - // Activity based - val NewFollowingSimilarUserSourceWeight = - "content_recommender_flow_candidate_source_weight_new_following_similar_user" - val RecentEngagementSimilarUserSourceWeight = - "content_recommender_flow_candidate_source_weight_recent_engagement_similar_user" - val RepeatedProfileVisitsSourceWeight = - "content_recommender_flow_candidate_source_weight_repeated_profile_visits" - val RealGraphOonSourceWeight = "content_recommender_flow_candidate_source_weight_real_graph_oon" - // Geo based - val PopCountrySourceWeight = "content_recommender_flow_candidate_source_weight_pop_country" - val PopGeohashSourceWeight = "content_recommender_flow_candidate_source_weight_pop_geohash" - val PopCountryBackfillSourceWeight = - "content_recommender_flow_candidate_source_weight_pop_country_backfill" - val PPMILocaleFollowSourceWeight = - "content_recommender_flow_candidate_source_weight_ppmi_locale_follow" - val TopOrganicFollowsAccountsSourceWeight = - "content_recommender_flow_candidate_source_weight_top_organic_follow_account" - val CrowdSearchAccountSourceWeight = - "content_recommender_flow_candidate_source_weight_crowd_search_account" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.docx new file mode 100644 index 000000000..75cf79f9c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.scala deleted file mode 100644 index 6b43325af..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -abstract class ContentRecommenderParams[A](default: A) extends Param[A](default) { - override val statName: String = "content_recommender/" + this.getClass.getSimpleName -} - -object ContentRecommenderParams { - - case object TargetEligibility - extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.TargetUserEligible, true) - - case object ResultSizeParam - extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.ResultSize, 15, 1, 500) - case object BatchSizeParam - extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.BatchSize, 15, 1, 500) - case object RecentFollowingPredicateBudgetInMillisecond - extends FSBoundedParam[Int]( - ContentRecommenderFlowFeatureSwitchKeys.RecentFollowingPredicateBudgetInMillisecond, - 8, - 1, - 50) - case object FetchCandidateSourceBudgetInMillisecond - extends FSBoundedParam[Int]( - ContentRecommenderFlowFeatureSwitchKeys.CandidateGenerationBudgetInMillisecond, - 60, - 1, - 80) - case object EnableRecentFollowingPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableRecentFollowingPredicate, - true) - case object EnableGizmoduckPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableGizmoduckPredicate, - false) - case object EnableInactivePredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableInactivePredicate, - false) - case object EnableInvalidTargetCandidateRelationshipPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidTargetCandidateRelationshipPredicate, - false) - case object IncludeActivityBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeActivityBasedCandidateSource, - true) - case object IncludeSocialBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialBasedCandidateSource, - true) - case object IncludeGeoBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeGeoBasedCandidateSource, - true) - case object IncludeHomeTimelineTweetRecsCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeHomeTimelineTweetRecsCandidateSource, - false) - case object IncludeSocialProofEnforcedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialProofEnforcedCandidateSource, - false) - case object IncludeNewFollowingNewFollowingExpansionCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeNewFollowingNewFollowingExpansionCandidateSource, - false) - - case object IncludeMoreGeoBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeMoreGeoBasedCandidateSource, - false) - - case object GetFollowersFromSgs - extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.GetFollowersFromSgs, false) - - case object EnableInvalidRelationshipPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate, - false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.docx new file mode 100644 index 000000000..de221d3aa Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.scala deleted file mode 100644 index 5952314e5..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasExcludedUserIds -import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode -import com.twitter.follow_recommendations.common.models.HasInvalidRelationshipUserIds -import com.twitter.follow_recommendations.common.models.HasRecentFollowedByUserIds -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.HasUserState -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class ContentRecommenderRequest( - override val params: Params, - override val clientContext: ClientContext, - inputExcludeUserIds: Seq[Long], - override val recentFollowedUserIds: Option[Seq[Long]], - override val recentFollowedByUserIds: Option[Seq[Long]], - override val invalidRelationshipUserIds: Option[Set[Long]], - override val displayLocation: DisplayLocation, - maxResults: Option[Int] = None, - override val debugOptions: Option[DebugOptions] = None, - override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, - override val userState: Option[UserState] = None) - extends HasParams - with HasClientContext - with HasDisplayLocation - with HasDebugOptions - with HasRecentFollowedUserIds - with HasRecentFollowedByUserIds - with HasInvalidRelationshipUserIds - with HasExcludedUserIds - with HasUserState - with HasGeohashAndCountryCode { - override val excludedUserIds: Seq[Long] = { - inputExcludeUserIds ++ clientContext.userId.toSeq - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.docx new file mode 100644 index 000000000..61601f7dd Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.scala deleted file mode 100644 index 769f9ce51..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueOptionalWithStats -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStatsWithin -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.stitch.Stitch - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderRequestBuilder @Inject() ( - socialGraph: SocialGraphClient, - userLocationFetcher: UserLocationFetcher, - userStateClient: UserStateClient, - statsReceiver: StatsReceiver) { - - val stats: StatsReceiver = statsReceiver.scope("content_recommender_request_builder") - val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds") - private val invalidRelationshipUsersMaxSizeCounter = - invalidRelationshipUsersStats.counter("maxSize") - private val invalidRelationshipUsersNotMaxSizeCounter = - invalidRelationshipUsersStats.counter("notMaxSize") - - def build(req: ProductRequest): Stitch[ContentRecommenderRequest] = { - val userStateStitch = Stitch - .collect(req.recommendationRequest.clientContext.userId.map(userId => - userStateClient.getUserState(userId))).map(_.flatten) - val recentFollowedUserIdsStitch = - Stitch - .collect(req.recommendationRequest.clientContext.userId.map { userId => - rescueWithStatsWithin( - socialGraph.getRecentFollowedUserIds(userId), - stats, - "recentFollowedUserIds", - req - .params( - ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond).millisecond - ) - }) - val recentFollowedByUserIdsStitch = - if (req.params(ContentRecommenderParams.GetFollowersFromSgs)) { - Stitch - .collect( - req.recommendationRequest.clientContext.userId.map(userId => - rescueWithStatsWithin( - socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId), - stats, - "recentFollowedByUserIds", - req - .params(ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond) - .millisecond - ))) - } else Stitch.None - val invalidRelationshipUserIdsStitch: Stitch[Option[Seq[Long]]] = - if (req.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate)) { - Stitch - .collect( - req.recommendationRequest.clientContext.userId.map { userId => - rescueWithStats( - socialGraph - .getInvalidRelationshipUserIdsFromCachedColumn(userId) - .onSuccess(ids => - if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) { - invalidRelationshipUsersMaxSizeCounter.incr() - } else { - invalidRelationshipUsersNotMaxSizeCounter.incr() - }), - stats, - "invalidRelationshipUserIds" - ) - } - ) - } else { - Stitch.None - } - val locationStitch = - rescueOptionalWithStats( - userLocationFetcher.getGeohashAndCountryCode( - req.recommendationRequest.clientContext.userId, - req.recommendationRequest.clientContext.ipAddress - ), - stats, - "userLocation" - ) - Stitch - .join( - recentFollowedUserIdsStitch, - recentFollowedByUserIdsStitch, - invalidRelationshipUserIdsStitch, - locationStitch, - userStateStitch) - .map { - case ( - recentFollowedUserIds, - recentFollowedByUserIds, - invalidRelationshipUserIds, - location, - userState) => - ContentRecommenderRequest( - req.params, - req.recommendationRequest.clientContext, - req.recommendationRequest.excludedIds.getOrElse(Nil), - recentFollowedUserIds, - recentFollowedByUserIds, - invalidRelationshipUserIds.map(_.toSet), - req.recommendationRequest.displayLocation, - req.recommendationRequest.maxResults, - req.recommendationRequest.debugParams.flatMap(_.debugOptions), - location, - userState - ) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD deleted file mode 100644 index 9129e17b8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD +++ /dev/null @@ -1,58 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD.docx new file mode 100644 index 000000000..716972123 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.docx new file mode 100644 index 000000000..0ea45de46 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.scala deleted file mode 100644 index ed15d566c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.sims.LinearRegressionFollow2vecNearestNeighborsStore -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceScorer -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PostNuxMlCandidateSourceRegistry @Inject() ( - crowdSearchAccountsCandidateSource: CrowdSearchAccountsSource, - topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, - linearRegressionfollow2vecNearestNeighborsStore: LinearRegressionFollow2vecNearestNeighborsStore, - forwardEmailBookSource: ForwardEmailBookSource, - forwardPhoneBookSource: ForwardPhoneBookSource, - offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, - onlineSTPSource: OnlineSTPSourceScorer, - popCountrySource: PopCountrySource, - popCountryBackFillSource: PopCountryBackFillSource, - popGeohashSource: PopGeohashSource, - recentEngagementDirectFollowSimilarUsersSource: RecentEngagementSimilarUsersSource, - recentEngagementNonDirectFollowSource: RecentEngagementNonDirectFollowSource, - recentEngagementDirectFollowSalsaExpansionSource: RecentEngagementDirectFollowSalsaExpansionSource, - recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, - realGraphOonV2Source: RealGraphOonV2Source, - repeatedProfileVisitSource: RepeatedProfileVisitsSource, - reverseEmailBookSource: ReverseEmailBookSource, - reversePhoneBookSource: ReversePhoneBookSource, - triangularLoopsSource: TriangularLoopsSource, - userUserGraphCandidateSource: UserUserGraphCandidateSource, - ppmiLocaleFollowSource: PPMILocaleFollowSource, - popGeohashQualityFollowSource: PopGeohashQualityFollowSource, - baseStatsReceiver: StatsReceiver, -) extends CandidateSourceRegistry[PostNuxMlRequest, CandidateUser] { - import EnrichedCandidateSource._ - - override val statsReceiver = baseStatsReceiver - .scope("post_nux_ml_flow", "candidate_sources") - - // sources primarily based on social graph signals - private[this] val socialSources = Seq( - linearRegressionfollow2vecNearestNeighborsStore.mapKeys[PostNuxMlRequest]( - _.getOptionalUserId.toSeq), - forwardEmailBookSource, - forwardPhoneBookSource, - offlineStrongTiePredictionSource, - onlineSTPSource, - reverseEmailBookSource, - reversePhoneBookSource, - triangularLoopsSource, - ) - - // sources primarily based on geo signals - private[this] val geoSources = Seq( - popCountrySource, - popCountryBackFillSource, - popGeohashSource, - popGeohashQualityFollowSource, - topOrganicFollowsAccountsSource, - crowdSearchAccountsCandidateSource, - ppmiLocaleFollowSource, - ) - - // sources primarily based on recent activity signals - private[this] val activitySources = Seq( - repeatedProfileVisitSource, - recentEngagementDirectFollowSalsaExpansionSource.mapKeys[PostNuxMlRequest]( - _.getOptionalUserId.toSeq), - recentEngagementDirectFollowSimilarUsersSource, - recentEngagementNonDirectFollowSource.mapKeys[PostNuxMlRequest](_.getOptionalUserId.toSeq), - recentFollowingSimilarUsersSource, - realGraphOonV2Source, - userUserGraphCandidateSource, - ) - - override val sources: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] = ( - geoSources ++ socialSources ++ activitySources - ).toSet -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.docx new file mode 100644 index 000000000..bb24b4fe5 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.scala deleted file mode 100644 index 9492747a4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.scala +++ /dev/null @@ -1,177 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.Param - -abstract class PostNuxMlCandidateSourceWeightParams[A](default: A) extends Param[A](default) { - override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName -} - -object PostNuxMlCandidateSourceWeightParams { - - case object CandidateWeightCrowdSearch - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightCrowdSearch, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightTopOrganicFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTopOrganicFollow, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPPMILocaleFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPPMILocaleFollow, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightForwardEmailBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardEmailBook, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightForwardPhoneBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardPhoneBook, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightOfflineStrongTiePrediction - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOfflineStrongTiePrediction, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightOnlineStp - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOnlineStp, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopCountry - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopCountry, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopGeohash - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohash, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopGeohashQualityFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohashQualityFollow, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopGeoBackfill - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeoBackfill, - 1, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentFollowingSimilarUsers - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentFollowingSimilarUsers, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentEngagementDirectFollowSalsaExpansion - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementDirectFollowSalsaExpansion, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentEngagementNonDirectFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementNonDirectFollow, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentEngagementSimilarUsers - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementSimilarUsers, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRepeatedProfileVisits - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRepeatedProfileVisits, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightFollow2vecNearestNeighbors - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightFollow2vecNearestNeighbors, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightReverseEmailBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReverseEmailBook, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightReversePhoneBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReversePhoneBook, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightTriangularLoops - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTriangularLoops, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightTwoHopRandomWalk - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTwoHopRandomWalk, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightUserUserGraph - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightUserUserGraph, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightRealGraphOonV2 - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRealGraphOonV2, - 1.0, - 0.0, - 2000.0 - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.docx new file mode 100644 index 000000000..cff9cde9c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala deleted file mode 100644 index 14e982a41..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala +++ /dev/null @@ -1,193 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.IdentityRanker -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models._ -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.fatigue_ranker.ImpressionBasedFatigueRanker -import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRanker -import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRankerParams -import com.twitter.follow_recommendations.common.rankers.interleave_ranker.InterleaveRanker -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.HydrateFeaturesTransform -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRanker -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker -import com.twitter.follow_recommendations.configapi.candidates.HydrateCandidateParamsTransform -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams - -/** - * Used to build the combined ranker comprising 4 stages of ranking: - * - weighted sampler - * - truncating to the top N merged results for ranking - * - ML ranker - * - Interleaving ranker for producer-side experiments - * - impression-based fatigueing - */ -@Singleton -class PostNuxMlCombinedRankerBuilder[ - T <: HasParams with HasSimilarToContext with HasClientContext with HasExcludedUserIds with HasDisplayLocation with HasDebugOptions with HasPreFetchedFeature with HasDismissedUserIds with HasQualityFactor] @Inject() ( - firstNRanker: FirstNRanker[T], - hydrateFeaturesTransform: HydrateFeaturesTransform[T], - hydrateCandidateParamsTransform: HydrateCandidateParamsTransform[T], - mlRanker: MlRanker[T], - statsReceiver: StatsReceiver) { - private[this] val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_ranker") - - // we construct each ranker independently and chain them together - def build( - request: T, - candidateSourceWeights: Map[CandidateSourceIdentifier, Double] - ): Ranker[T, CandidateUser] = { - val displayLocationStats = stats.scope(request.displayLocation.toString) - val weightedRankerStats: StatsReceiver = - displayLocationStats.scope("weighted_candidate_source_ranker") - val firstNRankerStats: StatsReceiver = - displayLocationStats.scope("first_n_ranker") - val hydrateCandidateParamsStats = - displayLocationStats.scope("hydrate_candidate_params") - val fatigueRankerStats = displayLocationStats.scope("fatigue_ranker") - val interleaveRankerStats = - displayLocationStats.scope("interleave_ranker") - val allRankersStats = displayLocationStats.scope("all_rankers") - - // Checking if the heavy-ranker is an experimental model. - // If it is, InterleaveRanker and candidate parameter hydration are disabled. - // *NOTE* that consumer-side experiments should at any time take a small % of traffic, less - // than 20% for instance, to leave enough room for producer experiments. Increasing bucket - // size for producer experiments lead to other issues and is not a viable option for faster - // experiments. - val requestRankerId = request.params(MlRankerParams.RequestScorerIdParam) - if (requestRankerId != RankerId.PostNuxProdRanker) { - hydrateCandidateParamsStats.counter(s"disabled_by_${requestRankerId.toString}").incr() - interleaveRankerStats.counter(s"disabled_by_${requestRankerId.toString}").incr() - } - - // weighted ranker that samples from the candidate sources - val weightedRanker = WeightedCandidateSourceRanker - .build[T]( - candidateSourceWeights, - request.params(PostNuxMlParams.CandidateShuffler).shuffle(request.getRandomizationSeed), - randomSeed = request.getRandomizationSeed - ).observe(weightedRankerStats) - - // ranker that takes the first n results (ie truncates output) while merging duplicates - val firstNRankerObs = firstNRanker.observe(firstNRankerStats) - // either ML ranker that uses deepbirdv2 to score or no ranking - val mainRanker: Ranker[T, CandidateUser] = - buildMainRanker(request, requestRankerId == RankerId.PostNuxProdRanker, displayLocationStats) - // fatigue ranker that uses wtf impressions to fatigue - val fatigueRanker = buildFatigueRanker(request, fatigueRankerStats).observe(fatigueRankerStats) - - // interleaveRanker combines rankings from several rankers and enforces candidates' ranks in - // experiment buckets according to their assigned ranker model. - val interleaveRanker = - buildInterleaveRanker( - request, - requestRankerId == RankerId.PostNuxProdRanker, - interleaveRankerStats) - .observe(interleaveRankerStats) - - weightedRanker - .andThen(firstNRankerObs) - .andThen(mainRanker) - .andThen(fatigueRanker) - .andThen(interleaveRanker) - .observe(allRankersStats) - } - - def buildMainRanker( - request: T, - isMainRankerPostNuxProd: Boolean, - displayLocationStats: StatsReceiver - ): Ranker[T, CandidateUser] = { - - // note that we may be disabling heavy ranker for users not bucketed - // (due to empty results from the new candidate source) - // need a better solution in the future - val mlRankerStats = displayLocationStats.scope("ml_ranker") - val noMlRankerStats = displayLocationStats.scope("no_ml_ranker") - val hydrateFeaturesStats = - displayLocationStats.scope("hydrate_features") - val hydrateCandidateParamsStats = - displayLocationStats.scope("hydrate_candidate_params") - val notHydrateCandidateParamsStats = - displayLocationStats.scope("not_hydrate_candidate_params") - val rankerStats = displayLocationStats.scope("ranker") - val mlRankerDisabledByExperimentsCounter = - mlRankerStats.counter("disabled_by_experiments") - val mlRankerDisabledByQualityFactorCounter = - mlRankerStats.counter("disabled_by_quality_factor") - - val disabledByQualityFactor = request.qualityFactor - .exists(_ <= request.params(PostNuxMlParams.TurnoffMLScorerQFThreshold)) - - if (disabledByQualityFactor) - mlRankerDisabledByQualityFactorCounter.incr() - - if (request.params(PostNuxMlParams.UseMlRanker) && !disabledByQualityFactor) { - - val hydrateFeatures = hydrateFeaturesTransform - .observe(hydrateFeaturesStats) - - val optionalHydratedParamsTransform: Transform[T, CandidateUser] = { - // We disable candidate parameter hydration for experimental heavy-ranker models. - if (isMainRankerPostNuxProd && - request.params(PostNuxMlParams.EnableCandidateParamHydration)) { - hydrateCandidateParamsTransform - .observe(hydrateCandidateParamsStats) - } else { - new IdentityTransform[T, CandidateUser]() - .observe(notHydrateCandidateParamsStats) - } - } - val candidateSize = request.params(FirstNRankerParams.CandidatesToRank) - Ranker - .chain( - hydrateFeatures.andThen(optionalHydratedParamsTransform), - mlRanker.observe(mlRankerStats), - ) - .within( - request.params(PostNuxMlParams.MlRankerBudget), - rankerStats.scope(s"n$candidateSize")) - } else { - new IdentityRanker[T, CandidateUser].observe(noMlRankerStats) - } - } - - def buildInterleaveRanker( - request: T, - isMainRankerPostNuxProd: Boolean, - interleaveRankerStats: StatsReceiver - ): Ranker[T, CandidateUser] = { - // InterleaveRanker is enabled only for display locations powered by the PostNux heavy-ranker. - if (request.params(PostNuxMlParams.EnableInterleaveRanker) && - // InterleaveRanker is disabled for requests with experimental heavy-rankers. - isMainRankerPostNuxProd) { - new InterleaveRanker[T](interleaveRankerStats) - } else { - new IdentityRanker[T, CandidateUser]() - } - } - - def buildFatigueRanker( - request: T, - fatigueRankerStats: StatsReceiver - ): Ranker[T, CandidateUser] = { - if (request.params(PostNuxMlParams.EnableFatigueRanker)) { - ImpressionBasedFatigueRanker - .build[T]( - fatigueRankerStats - ).within(request.params(PostNuxMlParams.FatigueRankerBudget), fatigueRankerStats) - } else { - new IdentityRanker[T, CandidateUser]() - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.docx new file mode 100644 index 000000000..2442ef67d Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.scala deleted file mode 100644 index 092f07100..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.scala +++ /dev/null @@ -1,304 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource._ -import com.twitter.follow_recommendations.common.base._ -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicate -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate -import com.twitter.follow_recommendations.common.transforms.ranker_id.RandomRankerIdTransform -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate -import com.twitter.follow_recommendations.common.predicates.CandidateParamPredicate -import com.twitter.follow_recommendations.common.predicates.CandidateSourceParamPredicate -import com.twitter.follow_recommendations.common.predicates.CuratedCompetitorListPredicate -import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate -import com.twitter.follow_recommendations.common.predicates.InactivePredicate -import com.twitter.follow_recommendations.common.predicates.PreviouslyRecommendedUserIdsPredicate -import com.twitter.follow_recommendations.common.predicates.user_activity.NonNearZeroUserActivityPredicate -import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform -import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProofTransform -import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform -import com.twitter.follow_recommendations.common.transforms.weighted_sampling.SamplingTransform -import com.twitter.follow_recommendations.configapi.candidates.CandidateUserParamsFactory -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform -import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.timelines.configapi.Params -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.predicates.hss.HssPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate -import com.twitter.follow_recommendations.common.transforms.modify_social_proof.RemoveAccountProofTransform -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.RecommendationFlowData -import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.stitch.Stitch - -/** - * We use this flow for all post-nux display locations that would use a machine-learning-based-ranker - * eg HTL, Sidebar, etc - * Note that the RankedPostNuxFlow is used primarily for scribing/data collection, and doesn't - * incorporate all of the other components in a flow (candidate source generation, predicates etc) - */ -@Singleton -class PostNuxMlFlow @Inject() ( - postNuxMlCandidateSourceRegistry: PostNuxMlCandidateSourceRegistry, - postNuxMlCombinedRankerBuilder: PostNuxMlCombinedRankerBuilder[PostNuxMlRequest], - curatedCompetitorListPredicate: CuratedCompetitorListPredicate, - gizmoduckPredicate: GizmoduckPredicate, - sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, - hssPredicate: HssPredicate, - invalidRelationshipPredicate: InvalidRelationshipPredicate, - recentFollowingPredicate: RecentFollowingPredicate, - nonNearZeroUserActivityPredicate: NonNearZeroUserActivityPredicate, - inactivePredicate: InactivePredicate, - dismissedCandidatePredicate: DismissedCandidatePredicate, - previouslyRecommendedUserIdsPredicate: PreviouslyRecommendedUserIdsPredicate, - modifySocialProofTransform: ModifySocialProofTransform, - removeAccountProofTransform: RemoveAccountProofTransform, - trackingTokenTransform: TrackingTokenTransform, - randomRankerIdTransform: RandomRankerIdTransform, - candidateParamsFactory: CandidateUserParamsFactory[PostNuxMlRequest], - samplingTransform: SamplingTransform, - frsLogger: FrsLogger, - baseStatsReceiver: StatsReceiver) - extends RecommendationFlow[PostNuxMlRequest, CandidateUser] - with RecommendationFlowBaseSideEffectsUtil[PostNuxMlRequest, CandidateUser] - with CandidateSourceHoldbackUtil { - override protected val targetEligibility: Predicate[PostNuxMlRequest] = - new ParamPredicate[PostNuxMlRequest](PostNuxMlParams.TargetEligibility) - - override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("post_nux_ml_flow") - - override val qualityFactorObserver: Option[QualityFactorObserver] = { - val config = LinearLatencyQualityFactorConfig( - qualityFactorBounds = - BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0), - initialDelay = 60.seconds, - targetLatency = 700.milliseconds, - targetLatencyPercentile = 95.0, - delta = 0.001 - ) - val qualityFactor = LinearLatencyQualityFactor(config) - val observer = LinearLatencyQualityFactorObserver(qualityFactor) - statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat) - Some(observer) - } - - override protected def updateTarget(request: PostNuxMlRequest): Stitch[PostNuxMlRequest] = { - Stitch.value( - request.copy(qualityFactor = qualityFactorObserver.map(_.qualityFactor.currentValue)) - ) - } - - private[post_nux_ml] def getCandidateSourceIdentifiers( - params: Params - ): Set[CandidateSourceIdentifier] = { - PostNuxMlFlowCandidateSourceWeights.getWeights(params).keySet - } - - override protected def candidateSources( - request: PostNuxMlRequest - ): Seq[CandidateSource[PostNuxMlRequest, CandidateUser]] = { - val identifiers = getCandidateSourceIdentifiers(request.params) - val selected: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] = - postNuxMlCandidateSourceRegistry.select(identifiers) - val budget: Duration = request.params(PostNuxMlParams.FetchCandidateSourceBudget) - filterCandidateSources( - request, - selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq) - } - - override protected val preRankerCandidateFilter: Predicate[(PostNuxMlRequest, CandidateUser)] = { - val stats = statsReceiver.scope("pre_ranker") - - object excludeNearZeroUserPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - nonNearZeroUserActivityPredicate, - stats.scope("exclude_near_zero_predicate") - ) { - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.ExcludeNearZeroCandidates) - } - - object invalidRelationshipGatedPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - invalidRelationshipPredicate, - stats.scope("invalid_relationship_predicate") - ) { - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.EnableInvalidRelationshipPredicate) - } - - ExcludedUserIdPredicate - .observe(stats.scope("exclude_user_id_predicate")) - .andThen( - recentFollowingPredicate.observe(stats.scope("recent_following_predicate")) - ) - .andThen( - dismissedCandidatePredicate.observe(stats.scope("dismissed_candidate_predicate")) - ) - .andThen( - previouslyRecommendedUserIdsPredicate.observe( - stats.scope("previously_recommended_user_ids_predicate")) - ) - .andThen( - invalidRelationshipGatedPredicate.observe(stats.scope("invalid_relationship_predicate")) - ) - .andThen( - excludeNearZeroUserPredicate.observe(stats.scope("exclude_near_zero_user_state")) - ) - .observe(stats.scope("overall_pre_ranker_candidate_filter")) - } - - override protected def selectRanker( - request: PostNuxMlRequest - ): Ranker[PostNuxMlRequest, CandidateUser] = { - postNuxMlCombinedRankerBuilder.build( - request, - PostNuxMlFlowCandidateSourceWeights.getWeights(request.params)) - } - - override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = { - new DedupTransform[PostNuxMlRequest, CandidateUser] - .observe(statsReceiver.scope("dedupping")) - .andThen( - samplingTransform - .gated(PostNuxMlParams.SamplingTransformEnabled) - .observe(statsReceiver.scope("samplingtransform"))) - } - - override protected val validateCandidates: Predicate[(PostNuxMlRequest, CandidateUser)] = { - val stats = statsReceiver.scope("validate_candidates") - val competitorPredicate = - curatedCompetitorListPredicate.map[(PostNuxMlRequest, CandidateUser)](_._2) - - val producerHoldbackPredicate = new CandidateParamPredicate[CandidateUser]( - GlobalParams.KeepUserCandidate, - FilterReason.CandidateSideHoldback - ).map[(PostNuxMlRequest, CandidateUser)] { - case (request, user) => candidateParamsFactory(user, request) - } - val pymkProducerHoldbackPredicate = new CandidateSourceParamPredicate( - GlobalParams.KeepSocialUserCandidate, - FilterReason.CandidateSideHoldback, - CandidateSourceHoldbackUtil.SocialCandidateSourceIds - ).map[(PostNuxMlRequest, CandidateUser)] { - case (request, user) => candidateParamsFactory(user, request) - } - val sgsPredicateStats = stats.scope("sgs_predicate") - object sgsGatedPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - sgsPredicate.observe(sgsPredicateStats), - sgsPredicateStats - ) { - - /** - * When SGS predicate is turned off, only query SGS exists API for (user, candidate, relationship) - * when the user's number of invalid relationships exceeds the threshold during request - * building step. This is to minimize load to SGS and underlying Flock DB. - */ - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.EnableSGSPredicate) || - SocialGraphClient.enablePostRankerSgsPredicate( - item._1.invalidRelationshipUserIds.getOrElse(Set.empty).size) - } - - val hssPredicateStats = stats.scope("hss_predicate") - object hssGatedPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - hssPredicate.observe(hssPredicateStats), - hssPredicateStats - ) { - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.EnableHssPredicate) - } - - Predicate - .andConcurrently[(PostNuxMlRequest, CandidateUser)]( - Seq( - competitorPredicate.observe(stats.scope("curated_competitor_predicate")), - gizmoduckPredicate.observe(stats.scope("gizmoduck_predicate")), - sgsGatedPredicate, - hssGatedPredicate, - inactivePredicate.observe(stats.scope("inactive_predicate")), - ) - ) - // to avoid dilutions, we need to apply the receiver holdback predicates at the very last step - .andThen(pymkProducerHoldbackPredicate.observe(stats.scope("pymk_receiver_side_holdback"))) - .andThen(producerHoldbackPredicate.observe(stats.scope("receiver_side_holdback"))) - .observe(stats.scope("overall_validate_candidates")) - } - - override protected val transformResults: Transform[PostNuxMlRequest, CandidateUser] = { - modifySocialProofTransform - .gated(EnableGFSSocialProofTransform) - .andThen(trackingTokenTransform) - .andThen(randomRankerIdTransform.gated(PostNuxMlParams.LogRandomRankerId)) - .andThen(removeAccountProofTransform.gated(PostNuxMlParams.EnableRemoveAccountProofTransform)) - } - - override protected def resultsConfig(request: PostNuxMlRequest): RecommendationResultsConfig = { - RecommendationResultsConfig( - request.maxResults.getOrElse(request.params(PostNuxMlParams.ResultSizeParam)), - request.params(PostNuxMlParams.BatchSizeParam) - ) - } - - override def applySideEffects( - target: PostNuxMlRequest, - candidateSources: Seq[CandidateSource[PostNuxMlRequest, CandidateUser]], - candidatesFromCandidateSources: Seq[CandidateUser], - mergedCandidates: Seq[CandidateUser], - filteredCandidates: Seq[CandidateUser], - rankedCandidates: Seq[CandidateUser], - transformedCandidates: Seq[CandidateUser], - truncatedCandidates: Seq[CandidateUser], - results: Seq[CandidateUser] - ): Stitch[Unit] = { - frsLogger.logRecommendationFlowData[PostNuxMlRequest]( - target, - RecommendationFlowData[PostNuxMlRequest]( - target, - PostNuxMlFlow.identifier, - candidateSources, - candidatesFromCandidateSources, - mergedCandidates, - filteredCandidates, - rankedCandidates, - transformedCandidates, - truncatedCandidates, - results - ) - ) - super.applySideEffects( - target, - candidateSources, - candidatesFromCandidateSources, - mergedCandidates, - filteredCandidates, - rankedCandidates, - transformedCandidates, - truncatedCandidates, - results - ) - } -} - -object PostNuxMlFlow { - val identifier = RecommendationPipelineIdentifier("PostNuxMlFlow") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.docx new file mode 100644 index 000000000..9358ba2cb Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.scala deleted file mode 100644 index edb447cba..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims.Follow2vecNearestNeighborsStore -import com.twitter.follow_recommendations.common.candidate_sources.stp.BaseOnlineSTPSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk.TwoHopRandomWalkSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlCandidateSourceWeightParams._ -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.timelines.configapi.Params - -object PostNuxMlFlowCandidateSourceWeights { - - def getWeights(params: Params): Map[CandidateSourceIdentifier, Double] = { - Map[CandidateSourceIdentifier, Double]( - // Social based - PPMILocaleFollowSource.Identifier -> params(CandidateWeightPPMILocaleFollow), - Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression -> params( - CandidateWeightFollow2vecNearestNeighbors), - RecentFollowingSimilarUsersSource.Identifier -> params( - CandidateWeightRecentFollowingSimilarUsers), - BaseOnlineSTPSource.Identifier -> params(CandidateWeightOnlineStp), - OfflineStrongTiePredictionSource.Identifier -> params( - CandidateWeightOfflineStrongTiePrediction), - ForwardEmailBookSource.Identifier -> params(CandidateWeightForwardEmailBook), - ForwardPhoneBookSource.Identifier -> params(CandidateWeightForwardPhoneBook), - ReverseEmailBookSource.Identifier -> params(CandidateWeightReverseEmailBook), - ReversePhoneBookSource.Identifier -> params(CandidateWeightReversePhoneBook), - TriangularLoopsSource.Identifier -> params(CandidateWeightTriangularLoops), - TwoHopRandomWalkSource.Identifier -> params(CandidateWeightTwoHopRandomWalk), - UserUserGraphCandidateSource.Identifier -> params(CandidateWeightUserUserGraph), - // Geo based - PopCountrySource.Identifier -> params(CandidateWeightPopCountry), - PopCountryBackFillSource.Identifier -> params(CandidateWeightPopGeoBackfill), - PopGeohashSource.Identifier -> params(CandidateWeightPopGeohash), - PopGeohashQualityFollowSource.Identifier -> params(CandidateWeightPopGeohashQualityFollow), - CrowdSearchAccountsSource.Identifier -> params(CandidateWeightCrowdSearch), - TopOrganicFollowsAccountsSource.Identifier -> params(CandidateWeightTopOrganicFollow), - // Engagement based - RealGraphOonV2Source.Identifier -> params(CandidateWeightRealGraphOonV2), - RecentEngagementNonDirectFollowSource.Identifier -> params( - CandidateWeightRecentEngagementNonDirectFollow), - RecentEngagementSimilarUsersSource.Identifier -> params( - CandidateWeightRecentEngagementSimilarUsers), - RepeatedProfileVisitsSource.Identifier -> params(CandidateWeightRepeatedProfileVisits), - RecentEngagementDirectFollowSalsaExpansionSource.Identifier -> params( - CandidateWeightRecentEngagementDirectFollowSalsaExpansion), - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.docx new file mode 100644 index 000000000..71f5af28a Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala deleted file mode 100644 index f329cbd13..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -object PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys { - val CandidateWeightCrowdSearch = "post_nux_ml_flow_candidate_source_weights_user_crowd_search" - val CandidateWeightTopOrganicFollow = - "post_nux_ml_flow_candidate_source_weights_top_organic_follow" - val CandidateWeightPPMILocaleFollow = - "post_nux_ml_flow_candidate_source_weights_user_ppmi_locale_follow" - val CandidateWeightForwardEmailBook = - "post_nux_ml_flow_candidate_source_weights_user_forward_email_book" - val CandidateWeightForwardPhoneBook = - "post_nux_ml_flow_candidate_source_weights_user_forward_phone_book" - val CandidateWeightOfflineStrongTiePrediction = - "post_nux_ml_flow_candidate_source_weights_user_offline_strong_tie_prediction" - val CandidateWeightOnlineStp = "post_nux_ml_flow_candidate_source_weights_user_online_stp" - val CandidateWeightPopCountry = "post_nux_ml_flow_candidate_source_weights_user_pop_country" - val CandidateWeightPopGeohash = "post_nux_ml_flow_candidate_source_weights_user_pop_geohash" - val CandidateWeightPopGeohashQualityFollow = - "post_nux_ml_flow_candidate_source_weights_user_pop_geohash_quality_follow" - val CandidateWeightPopGeoBackfill = - "post_nux_ml_flow_candidate_source_weights_user_pop_geo_backfill" - val CandidateWeightRecentFollowingSimilarUsers = - "post_nux_ml_flow_candidate_source_weights_user_recent_following_similar_users" - val CandidateWeightRecentEngagementDirectFollowSalsaExpansion = - "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_direct_follow_salsa_expansion" - val CandidateWeightRecentEngagementNonDirectFollow = - "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_non_direct_follow" - val CandidateWeightRecentEngagementSimilarUsers = - "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_similar_users" - val CandidateWeightRepeatedProfileVisits = - "post_nux_ml_flow_candidate_source_weights_user_repeated_profile_visits" - val CandidateWeightFollow2vecNearestNeighbors = - "post_nux_ml_flow_candidate_source_weights_user_follow2vec_nearest_neighbors" - val CandidateWeightReverseEmailBook = - "post_nux_ml_flow_candidate_source_weights_user_reverse_email_book" - val CandidateWeightReversePhoneBook = - "post_nux_ml_flow_candidate_source_weights_user_reverse_phone_book" - val CandidateWeightTriangularLoops = - "post_nux_ml_flow_candidate_source_weights_user_triangular_loops" - val CandidateWeightTwoHopRandomWalk = - "post_nux_ml_flow_candidate_source_weights_user_two_hop_random_walk" - val CandidateWeightUserUserGraph = - "post_nux_ml_flow_candidate_source_weights_user_user_user_graph" - val CandidateWeightRealGraphOonV2 = - "post_nux_ml_flow_candidate_source_weights_user_real_graph_oon_v2" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.docx new file mode 100644 index 000000000..4880777b6 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.scala deleted file mode 100644 index 0dd059dad..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.NoShuffle -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.RandomShuffler -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PostNuxMlFlowFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - PostNuxMlParams.OnlineSTPEnabled, - PostNuxMlParams.SamplingTransformEnabled, - PostNuxMlParams.Follow2VecLinearRegressionEnabled, - PostNuxMlParams.UseMlRanker, - PostNuxMlParams.EnableCandidateParamHydration, - PostNuxMlParams.EnableInterleaveRanker, - PostNuxMlParams.EnableAdhocRanker, - PostNuxMlParams.ExcludeNearZeroCandidates, - PostNuxMlParams.IncludeRepeatedProfileVisitsCandidateSource, - PostNuxMlParams.EnableInterestsOptOutPredicate, - PostNuxMlParams.EnableSGSPredicate, - PostNuxMlParams.EnableInvalidRelationshipPredicate, - PostNuxMlParams.EnableRemoveAccountProofTransform, - PostNuxMlParams.EnablePPMILocaleFollowSourceInPostNux, - PostNuxMlParams.EnableRealGraphOonV2, - PostNuxMlParams.GetFollowersFromSgs, - PostNuxMlRequestBuilderParams.EnableInvalidRelationshipPredicate - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - PostNuxMlCandidateSourceWeightParams.CandidateWeightCrowdSearch, - PostNuxMlCandidateSourceWeightParams.CandidateWeightTopOrganicFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPPMILocaleFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardEmailBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardPhoneBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightOfflineStrongTiePrediction, - PostNuxMlCandidateSourceWeightParams.CandidateWeightOnlineStp, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopCountry, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohash, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohashQualityFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeoBackfill, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentFollowingSimilarUsers, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementDirectFollowSalsaExpansion, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementNonDirectFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementSimilarUsers, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRepeatedProfileVisits, - PostNuxMlCandidateSourceWeightParams.CandidateWeightFollow2vecNearestNeighbors, - PostNuxMlCandidateSourceWeightParams.CandidateWeightReverseEmailBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightReversePhoneBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightTriangularLoops, - PostNuxMlCandidateSourceWeightParams.CandidateWeightTwoHopRandomWalk, - PostNuxMlCandidateSourceWeightParams.CandidateWeightUserUserGraph, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRealGraphOonV2, - PostNuxMlParams.TurnoffMLScorerQFThreshold - ) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - PostNuxMlParams.MlRankerBudget, - PostNuxMlRequestBuilderParams.TopicIdFetchBudget, - PostNuxMlRequestBuilderParams.DismissedIdScanBudget, - PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget - ) - - override val gatedOverridesMap = Map( - PostNuxMlFlowFeatureSwitchKeys.EnableRandomDataCollection -> Seq( - PostNuxMlParams.CandidateShuffler := new RandomShuffler[CandidateUser], - PostNuxMlParams.LogRandomRankerId := true - ), - PostNuxMlFlowFeatureSwitchKeys.EnableNoShuffler -> Seq( - PostNuxMlParams.CandidateShuffler := new NoShuffle[CandidateUser] - ), - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.docx new file mode 100644 index 000000000..c08bd1a8d Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.scala deleted file mode 100644 index 6a44c4bbb..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -object PostNuxMlFlowFeatureSwitchKeys { - val UseMlRanker = "post_nux_ml_flow_use_ml_ranker" - val EnableCandidateParamHydration = "post_nux_ml_flow_enable_candidate_param_hydration" - val OnlineSTPEnabled = "post_nux_ml_flow_online_stp_source_enabled" - val Follow2VecLinearRegressionEnabled = "post_nux_ml_flow_follow_to_vec_lr_source_enabled" - val EnableRandomDataCollection = "post_nux_ml_flow_random_data_collection_enabled" - val EnableAdhocRanker = "post_nux_ml_flow_adhoc_ranker_enabled" - val EnableFatigueRanker = "post_nux_ml_flow_fatigue_ranker_enabled" - val EnableInterleaveRanker = "post_nux_ml_flow_interleave_ranker_enabled" - val IncludeRepeatedProfileVisitsCandidateSource = - "post_nux_ml_flow_include_repeated_profile_visits_candidate_source" - val MLRankerBudget = "post_nux_ml_flow_ml_ranker_budget_millis" - val EnableNoShuffler = "post_nux_ml_flow_no_shuffler" - val SamplingTransformEnabled = "post_nux_ml_flow_sampling_transform_enabled" - val ExcludeNearZeroCandidates = "post_nux_ml_flow_exclude_near_zero_candidates" - val EnableInterestsOptOutPredicate = "post_nux_ml_flow_enable_interests_opt_out_predicate" - val EnableRemoveAccountProofTransform = "post_nux_ml_flow_enable_remove_account_proof_transform" - val EnablePPMILocaleFollowSourceInPostNux = "post_nux_ml_flow_enable_ppmilocale_follow_source" - val EnableInvalidRelationshipPredicate = "post_nux_ml_flow_enable_invalid_relationship_predicate" - val EnableRealGraphOonV2 = "post_nux_ml_flow_enable_real_graph_oon_v2" - val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate" - val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate" - val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs" - val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threhsold" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.docx new file mode 100644 index 000000000..21c08282c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.scala deleted file mode 100644 index cb5cf3648..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.CandidateShuffler -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.ExponentialShuffler -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -abstract class PostNuxMlParams[A](default: A) extends Param[A](default) { - override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName -} - -object PostNuxMlParams { - - // infra params: - case object FetchCandidateSourceBudget extends PostNuxMlParams[Duration](90.millisecond) - - // WTF Impression Store has very high tail latency (p9990 or p9999), but p99 latency is pretty good (~100ms) - // set the time budget for this step to be 200ms to make the performance of service more predictable - case object FatigueRankerBudget extends PostNuxMlParams[Duration](200.millisecond) - - case object MlRankerBudget - extends FSBoundedParam[Duration]( - name = PostNuxMlFlowFeatureSwitchKeys.MLRankerBudget, - default = 400.millisecond, - min = 100.millisecond, - max = 800.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - // product params: - case object TargetEligibility extends PostNuxMlParams[Boolean](true) - - case object ResultSizeParam extends PostNuxMlParams[Int](3) - case object BatchSizeParam extends PostNuxMlParams[Int](12) - - case object CandidateShuffler - extends PostNuxMlParams[CandidateShuffler[CandidateUser]]( - new ExponentialShuffler[CandidateUser]) - case object LogRandomRankerId extends PostNuxMlParams[Boolean](false) - - // whether or not to use the ml ranker at all (feature hydration + ranker) - case object UseMlRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.UseMlRanker, false) - - // whether or not to enable candidate param hydration in postnux_ml_flow - case object EnableCandidateParamHydration - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableCandidateParamHydration, false) - - // Whether or not OnlineSTP candidates are considered in the final pool of candidates. - // If set to `false`, the candidate source will be removed *after* all other considerations. - case object OnlineSTPEnabled - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.OnlineSTPEnabled, false) - - // Whether or not the candidates are sampled from a Plackett-Luce model - case object SamplingTransformEnabled - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.SamplingTransformEnabled, false) - - // Whether or not Follow2Vec candidates are considered in the final pool of candidates. - // If set to `false`, the candidate source will be removed *after* all other considerations. - case object Follow2VecLinearRegressionEnabled - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.Follow2VecLinearRegressionEnabled, - false) - - // Whether or not to enable AdhocRanker to allow adhoc, non-ML, score modifications. - case object EnableAdhocRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableAdhocRanker, false) - - // Whether the impression-based fatigue ranker is enabled or not. - case object EnableFatigueRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableFatigueRanker, true) - - // whether or not to enable InterleaveRanker for producer-side experiments. - case object EnableInterleaveRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterleaveRanker, false) - - // whether to exclude users in near zero user state - case object ExcludeNearZeroCandidates - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.ExcludeNearZeroCandidates, false) - - case object EnablePPMILocaleFollowSourceInPostNux - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.EnablePPMILocaleFollowSourceInPostNux, - false) - - case object EnableInterestsOptOutPredicate - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterestsOptOutPredicate, false) - - case object EnableInvalidRelationshipPredicate - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate, - false) - - // Totally disabling SGS predicate need to disable EnableInvalidRelationshipPredicate as well - case object EnableSGSPredicate - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableSGSPredicate, true) - - case object EnableHssPredicate - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableHssPredicate, true) - - // Whether or not to include RepeatedProfileVisits as one of the candidate sources in the PostNuxMlFlow. If false, - // RepeatedProfileVisitsSource would not be run for the users in candidate_generation. - case object IncludeRepeatedProfileVisitsCandidateSource - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.IncludeRepeatedProfileVisitsCandidateSource, - false) - - case object EnableRealGraphOonV2 - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableRealGraphOonV2, false) - - case object GetFollowersFromSgs - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.GetFollowersFromSgs, false) - - case object EnableRemoveAccountProofTransform - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.EnableRemoveAccountProofTransform, - false) - - // quality factor threshold to turn off ML ranker completely - object TurnoffMLScorerQFThreshold - extends FSBoundedParam[Double]( - name = PostNuxMlFlowFeatureSwitchKeys.TurnOffMLScorerQFThreshold, - default = 0.3, - min = 0.1, - max = 1.0) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.docx new file mode 100644 index 000000000..6f9c6cf1e Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.scala deleted file mode 100644 index 2cb112638..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models._ -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class PostNuxMlRequest( - override val params: Params, - override val clientContext: ClientContext, - override val similarToUserIds: Seq[Long], - inputExcludeUserIds: Seq[Long], - override val recentFollowedUserIds: Option[Seq[Long]], - override val invalidRelationshipUserIds: Option[Set[Long]], - override val recentFollowedByUserIds: Option[Seq[Long]], - override val dismissedUserIds: Option[Seq[Long]], - override val displayLocation: DisplayLocation, - maxResults: Option[Int] = None, - override val debugOptions: Option[DebugOptions] = None, - override val wtfImpressions: Option[Seq[WtfImpression]], - override val uttInterestIds: Option[Seq[Long]] = None, - override val customInterests: Option[Seq[String]] = None, - override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, - inputPreviouslyRecommendedUserIds: Option[Set[Long]] = None, - inputPreviouslyFollowedUserIds: Option[Set[Long]] = None, - override val isSoftUser: Boolean = false, - override val userState: Option[UserState] = None, - override val qualityFactor: Option[Double] = None) - extends HasParams - with HasSimilarToContext - with HasClientContext - with HasExcludedUserIds - with HasDisplayLocation - with HasDebugOptions - with HasGeohashAndCountryCode - with HasPreFetchedFeature - with HasDismissedUserIds - with HasInterestIds - with HasPreviousRecommendationsContext - with HasIsSoftUser - with HasUserState - with HasInvalidRelationshipUserIds - with HasQualityFactor { - override val excludedUserIds: Seq[Long] = { - inputExcludeUserIds ++ clientContext.userId.toSeq ++ similarToUserIds - } - override val previouslyRecommendedUserIDs: Set[Long] = - inputPreviouslyRecommendedUserIds.getOrElse(Set.empty) - override val previouslyFollowedUserIds: Set[Long] = - inputPreviouslyFollowedUserIds.getOrElse(Set.empty) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.docx new file mode 100644 index 000000000..0e6ed16f8 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.scala deleted file mode 100644 index aeb248b7f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.scala +++ /dev/null @@ -1,173 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.dismiss_store.DismissStore -import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher -import com.twitter.follow_recommendations.common.clients.impression_store.WtfImpressionStore -import com.twitter.follow_recommendations.common.clients.interests_service.InterestServiceClient -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient -import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicateParams -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils._ -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.DismissedIdScanBudget -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.TopicIdFetchBudget -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.inject.Logging -import com.twitter.stitch.Stitch -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PostNuxMlRequestBuilder @Inject() ( - socialGraph: SocialGraphClient, - wtfImpressionStore: WtfImpressionStore, - dismissStore: DismissStore, - userLocationFetcher: UserLocationFetcher, - interestServiceClient: InterestServiceClient, - userStateClient: UserStateClient, - statsReceiver: StatsReceiver) - extends Logging { - - val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_request_builder") - val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds") - private val invalidRelationshipUsersMaxSizeCounter = - invalidRelationshipUsersStats.counter("maxSize") - private val invalidRelationshipUsersNotMaxSizeCounter = - invalidRelationshipUsersStats.counter("notMaxSize") - - def build( - req: ProductRequest, - previouslyRecommendedUserIds: Option[Set[Long]] = None, - previouslyFollowedUserIds: Option[Set[Long]] = None - ): Stitch[PostNuxMlRequest] = { - val dl = req.recommendationRequest.displayLocation - val resultsStitch = Stitch.collect( - req.recommendationRequest.clientContext.userId - .map { userId => - val lookBackDuration = req.params(DismissedCandidatePredicateParams.LookBackDuration) - val negativeStartTs = -(Time.now - lookBackDuration).inMillis - val recentFollowedUserIdsStitch = - rescueWithStats( - socialGraph.getRecentFollowedUserIds(userId), - stats, - "recentFollowedUserIds") - val invalidRelationshipUserIdsStitch = - if (req.params(PostNuxMlParams.EnableInvalidRelationshipPredicate)) { - rescueWithStats( - socialGraph - .getInvalidRelationshipUserIds(userId) - .onSuccess(ids => - if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) { - invalidRelationshipUsersMaxSizeCounter.incr() - } else { - invalidRelationshipUsersNotMaxSizeCounter.incr() - }), - stats, - "invalidRelationshipUserIds" - ) - } else { - Stitch.value(Seq.empty) - } - // recentFollowedByUserIds are only used in experiment candidate sources - val recentFollowedByUserIdsStitch = if (req.params(PostNuxMlParams.GetFollowersFromSgs)) { - rescueWithStats( - socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId), - stats, - "recentFollowedByUserIds") - } else Stitch.value(Seq.empty) - val wtfImpressionsStitch = - rescueWithStatsWithin( - wtfImpressionStore.get(userId, dl), - stats, - "wtfImpressions", - req.params(WTFImpressionsScanBudget)) - val dismissedUserIdsStitch = - rescueWithStatsWithin( - dismissStore.get(userId, negativeStartTs, None), - stats, - "dismissedUserIds", - req.params(DismissedIdScanBudget)) - val locationStitch = - rescueOptionalWithStats( - userLocationFetcher.getGeohashAndCountryCode( - Some(userId), - req.recommendationRequest.clientContext.ipAddress), - stats, - "userLocation" - ) - val topicIdsStitch = - rescueWithStatsWithin( - interestServiceClient.fetchUttInterestIds(userId), - stats, - "topicIds", - req.params(TopicIdFetchBudget)) - val userStateStitch = - rescueOptionalWithStats(userStateClient.getUserState(userId), stats, "userState") - Stitch.join( - recentFollowedUserIdsStitch, - invalidRelationshipUserIdsStitch, - recentFollowedByUserIdsStitch, - dismissedUserIdsStitch, - wtfImpressionsStitch, - locationStitch, - topicIdsStitch, - userStateStitch - ) - }) - - resultsStitch.map { - case Some( - ( - recentFollowedUserIds, - invalidRelationshipUserIds, - recentFollowedByUserIds, - dismissedUserIds, - wtfImpressions, - locationInfo, - topicIds, - userState)) => - PostNuxMlRequest( - params = req.params, - clientContext = req.recommendationRequest.clientContext, - similarToUserIds = Nil, - inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil), - recentFollowedUserIds = Some(recentFollowedUserIds), - invalidRelationshipUserIds = Some(invalidRelationshipUserIds.toSet), - recentFollowedByUserIds = Some(recentFollowedByUserIds), - dismissedUserIds = Some(dismissedUserIds), - displayLocation = dl, - maxResults = req.recommendationRequest.maxResults, - debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions), - wtfImpressions = Some(wtfImpressions), - geohashAndCountryCode = locationInfo, - uttInterestIds = Some(topicIds), - inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds, - inputPreviouslyFollowedUserIds = previouslyFollowedUserIds, - isSoftUser = req.recommendationRequest.isSoftUser, - userState = userState - ) - case _ => - PostNuxMlRequest( - params = req.params, - clientContext = req.recommendationRequest.clientContext, - similarToUserIds = Nil, - inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil), - recentFollowedUserIds = None, - invalidRelationshipUserIds = None, - recentFollowedByUserIds = None, - dismissedUserIds = None, - displayLocation = dl, - maxResults = req.recommendationRequest.maxResults, - debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions), - wtfImpressions = None, - geohashAndCountryCode = None, - inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds, - inputPreviouslyFollowedUserIds = previouslyFollowedUserIds, - isSoftUser = req.recommendationRequest.isSoftUser, - userState = None - ) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.docx new file mode 100644 index 000000000..4e9377345 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.scala deleted file mode 100644 index da60f0382..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion - -object PostNuxMlRequestBuilderParams { - case object TopicIdFetchBudget - extends FSBoundedParam[Duration]( - name = "post_nux_ml_request_builder_topic_id_fetch_budget_millis", - default = 200.millisecond, - min = 80.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - case object DismissedIdScanBudget - extends FSBoundedParam[Duration]( - name = "post_nux_ml_request_builder_dismissed_id_scan_budget_millis", - default = 200.millisecond, - min = 80.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - case object WTFImpressionsScanBudget - extends FSBoundedParam[Duration]( - name = "post_nux_ml_request_builder_wtf_impressions_scan_budget_millis", - default = 200.millisecond, - min = 80.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - case object EnableInvalidRelationshipPredicate - extends FSParam[Boolean]( - name = "post_nux_ml_request_builder_enable_invalid_relationship_predicate", - false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD deleted file mode 100644 index a35992e93..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD.docx new file mode 100644 index 000000000..0aad13613 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.docx new file mode 100644 index 000000000..d7bd33518 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala deleted file mode 100644 index 8b920c556..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala +++ /dev/null @@ -1,164 +0,0 @@ -package com.twitter.follow_recommendations.logging - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.HasIsSoftUser -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.follow_recommendations.logging.thriftscala.RecommendationLog -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.RecommendationFlowData -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.follow_recommendations.models.ScoringUserRequest -import com.twitter.follow_recommendations.models.ScoringUserResponse -import com.twitter.inject.annotations.Flag -import com.twitter.logging.LoggerFactory -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.scribelib.marshallers.ClientDataProvider -import com.twitter.scribelib.marshallers.ExternalRefererDataProvider -import com.twitter.scribelib.marshallers.ScribeSerialization -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/** - * This is the standard logging class we use to log data into: - * 1) logs.follow_recommendations_logs - * - * This logger logs data for 2 endpoints: getRecommendations, scoreUserCandidates - * All data scribed via this logger have to be converted into the same thrift type: RecommendationLog - * - * 2) logs.frs_recommendation_flow_logs - * - * This logger logs recommendation flow data for getRecommendations requests - * All data scribed via this logger have to be converted into the same thrift type: FrsRecommendationFlowLog - */ -@Singleton -class FrsLogger @Inject() ( - @Named(GuiceNamedConstants.REQUEST_LOGGER) loggerFactory: LoggerFactory, - @Named(GuiceNamedConstants.FLOW_LOGGER) flowLoggerFactory: LoggerFactory, - stats: StatsReceiver, - @Flag("log_results") serviceShouldLogResults: Boolean) - extends ScribeSerialization { - private val logger = loggerFactory.apply() - private val flowLogger = flowLoggerFactory.apply() - private val logRecommendationCounter = stats.counter("scribe_recommendation") - private val logScoringCounter = stats.counter("scribe_scoring") - private val logRecommendationFlowCounter = stats.counter("scribe_recommendation_flow") - - def logRecommendationResult( - request: RecommendationRequest, - response: RecommendationResponse - ): Unit = { - if (!request.isSoftUser) { - val log = - RecommendationLog(request.toOfflineThrift, response.toOfflineThrift, Time.now.inMillis) - logRecommendationCounter.incr() - logger.info( - serializeThrift( - log, - FrsLogger.LogCategory, - FrsLogger.mkProvider(request.clientContext) - )) - } - } - - def logScoringResult(request: ScoringUserRequest, response: ScoringUserResponse): Unit = { - if (!request.isSoftUser) { - val log = - RecommendationLog( - request.toRecommendationRequest.toOfflineThrift, - response.toRecommendationResponse.toOfflineThrift, - Time.now.inMillis) - logScoringCounter.incr() - logger.info( - serializeThrift( - log, - FrsLogger.LogCategory, - FrsLogger.mkProvider(request.toRecommendationRequest.clientContext) - )) - } - } - - def logRecommendationFlowData[Target <: HasClientContext with HasIsSoftUser with HasParams]( - request: Target, - flowData: RecommendationFlowData[Target] - ): Unit = { - if (!request.isSoftUser && request.params(GlobalParams.EnableRecommendationFlowLogs)) { - val log = flowData.toRecommendationFlowLogOfflineThrift - logRecommendationFlowCounter.incr() - flowLogger.info( - serializeThrift( - log, - FrsLogger.FlowLogCategory, - FrsLogger.mkProvider(request.clientContext) - )) - } - } - - // We prefer the settings given in the user request, and if none provided we default to the - // aurora service configuration. - def shouldLog(debugParamsOpt: Option[DebugParams]): Boolean = - debugParamsOpt match { - case Some(debugParams) => - debugParams.debugOptions match { - case Some(debugOptions) => - !debugOptions.doNotLog - case None => - serviceShouldLogResults - } - case None => - serviceShouldLogResults - } - -} - -object FrsLogger { - val LogCategory = "follow_recommendations_logs" - val FlowLogCategory = "frs_recommendation_flow_logs" - - def mkProvider(clientContext: ClientContext) = new ClientDataProvider { - - /** The id of the current user. When the user is logged out, this method should return None. */ - override val userId: Option[Long] = clientContext.userId - - /** The id of the guest, which is present in logged-in or loged-out states */ - override val guestId: Option[Long] = clientContext.guestId - - /** The personalization id (pid) of the user, used to personalize Twitter services */ - override val personalizationId: Option[String] = None - - /** The id of the individual device the user is currently using. This id will be unique for different users' devices. */ - override val deviceId: Option[String] = clientContext.deviceId - - /** The OAuth application id of the application the user is currently using */ - override val clientApplicationId: Option[Long] = clientContext.appId - - /** The OAuth parent application id of the application the user is currently using */ - override val parentApplicationId: Option[Long] = None - - /** The two-letter, upper-case country code used to designate the country from which the scribe event occurred */ - override val countryCode: Option[String] = clientContext.countryCode - - /** The two-letter, lower-case language code used to designate the probably language spoken by the scribe event initiator */ - override val languageCode: Option[String] = clientContext.languageCode - - /** The user-agent header used to identify the client browser or device that the user is currently active on */ - override val userAgent: Option[String] = clientContext.userAgent - - /** Whether the user is accessing Twitter via a secured connection */ - override val isSsl: Option[Boolean] = Some(true) - - /** The referring URL to the current page for web-based clients, if applicable */ - override val referer: Option[String] = None - - /** - * The external site, partner, or email that lead to the current Twitter application. Returned value consists of a - * tuple including the encrypted referral data and the type of referral - */ - override val externalReferer: Option[ExternalRefererDataProvider] = None - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD deleted file mode 100644 index 597ab76c4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD.docx new file mode 100644 index 000000000..a1794e82b Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.docx new file mode 100644 index 000000000..a1cc37273 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.scala deleted file mode 100644 index 38215c44b..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.models - -object CandidateSourceType extends Enumeration { - type CandidateSourceType = Value - val Social = Value("social") - val GeoAndInterests = Value("geo_and_interests") - val ActivityContextual = Value("activity_contextual") - val None = Value("none") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.docx new file mode 100644 index 000000000..19c568628 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.scala deleted file mode 100644 index a5702b2b3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.timelines.configapi.Params - -case class CandidateUserDebugParams(paramsMap: Map[Long, Params]) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.docx new file mode 100644 index 000000000..c7419d524 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.scala deleted file mode 100644 index dee7f9b6a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.common.models.DebugOptions.fromDebugParamsThrift -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.timelines.configapi.{FeatureValue => ConfigApiFeatureValue} - -case class DebugParams( - featureOverrides: Option[Map[String, ConfigApiFeatureValue]], - debugOptions: Option[DebugOptions]) - -object DebugParams { - def fromThrift(thrift: t.DebugParams): DebugParams = DebugParams( - featureOverrides = thrift.featureOverrides.map { map => - map.mapValues(FeatureValue.fromThrift).toMap - }, - debugOptions = Some( - fromDebugParamsThrift(thrift) - ) - ) - def toOfflineThrift(model: DebugParams): offline.OfflineDebugParams = - offline.OfflineDebugParams(randomizationSeed = model.debugOptions.flatMap(_.randomizationSeed)) -} - -trait HasFrsDebugParams { - def frsDebugParams: Option[DebugParams] -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.docx new file mode 100644 index 000000000..a63f8009d Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.scala deleted file mode 100644 index 59f0adfd7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.FlowContext -import com.twitter.follow_recommendations.common.models.RecentlyEngagedUserId -import com.twitter.follow_recommendations.logging.thriftscala.OfflineDisplayContext -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import scala.reflect.ClassTag -import scala.reflect.classTag - -trait DisplayContext { - def toOfflineThrift: offline.OfflineDisplayContext -} - -object DisplayContext { - case class Profile(profileId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Profile(offline.OfflineProfile(profileId)) - } - case class Search(searchQuery: String) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Search(offline.OfflineSearch(searchQuery)) - } - case class Rux(focalAuthorId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Rux(offline.OfflineRux(focalAuthorId)) - } - - case class Topic(topicId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Topic(offline.OfflineTopic(topicId)) - } - - case class ReactiveFollow(followedUserIds: Seq[Long]) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.ReactiveFollow(offline.OfflineReactiveFollow(followedUserIds)) - } - - case class NuxInterests(flowContext: Option[FlowContext], uttInterestIds: Option[Seq[Long]]) - extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.NuxInterests( - offline.OfflineNuxInterests(flowContext.map(_.toOfflineThrift))) - } - - case class PostNuxFollowTask(flowContext: Option[FlowContext]) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.PostNuxFollowTask( - offline.OfflinePostNuxFollowTask(flowContext.map(_.toOfflineThrift))) - } - - case class AdCampaignTarget(similarToUserIds: Seq[Long]) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.AdCampaignTarget( - offline.OfflineAdCampaignTarget(similarToUserIds)) - } - - case class ConnectTab( - byfSeedUserIds: Seq[Long], - similarToUserIds: Seq[Long], - engagedUserIds: Seq[RecentlyEngagedUserId]) - extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.ConnectTab( - offline.OfflineConnectTab( - byfSeedUserIds, - similarToUserIds, - engagedUserIds.map(user => user.toOfflineThrift))) - } - - case class SimilarToUser(similarToUserId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.SimilarToUser(offline.OfflineSimilarToUser(similarToUserId)) - } - - def fromThrift(tDisplayContext: t.DisplayContext): DisplayContext = tDisplayContext match { - case t.DisplayContext.Profile(p) => Profile(p.profileId) - case t.DisplayContext.Search(s) => Search(s.searchQuery) - case t.DisplayContext.Rux(r) => Rux(r.focalAuthorId) - case t.DisplayContext.Topic(t) => Topic(t.topicId) - case t.DisplayContext.ReactiveFollow(f) => ReactiveFollow(f.followedUserIds) - case t.DisplayContext.NuxInterests(n) => - NuxInterests(n.flowContext.map(FlowContext.fromThrift), n.uttInterestIds) - case t.DisplayContext.AdCampaignTarget(a) => - AdCampaignTarget(a.similarToUserIds) - case t.DisplayContext.ConnectTab(connect) => - ConnectTab( - connect.byfSeedUserIds, - connect.similarToUserIds, - connect.recentlyEngagedUserIds.map(RecentlyEngagedUserId.fromThrift)) - case t.DisplayContext.SimilarToUser(r) => - SimilarToUser(r.similarToUserId) - case t.DisplayContext.PostNuxFollowTask(p) => - PostNuxFollowTask(p.flowContext.map(FlowContext.fromThrift)) - case t.DisplayContext.UnknownUnionField(t) => - throw new UnknownDisplayContextException(t.field.name) - } - - def getDisplayContextAs[T <: DisplayContext: ClassTag](displayContext: DisplayContext): T = - displayContext match { - case context: T => context - case _ => - throw new UnexpectedDisplayContextTypeException( - displayContext, - classTag[T].getClass.getSimpleName) - } -} - -class UnknownDisplayContextException(name: String) - extends Exception(s"Unknown DisplayContext in Thrift: ${name}") - -class UnexpectedDisplayContextTypeException(displayContext: DisplayContext, expectedType: String) - extends Exception(s"DisplayContext ${displayContext} not of expected type ${expectedType}") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.docx new file mode 100644 index 000000000..9271998d0 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.scala deleted file mode 100644 index 66f0afafa..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.timelines.configapi._ - -object FeatureValue { - def fromThrift(thriftFeatureValue: t.FeatureValue): FeatureValue = thriftFeatureValue match { - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.BoolValue(bool)) => - BooleanFeatureValue(bool) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.StrValue(string)) => - StringFeatureValue(string) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.IntValue(int)) => - NumberFeatureValue(int) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.LongValue(long)) => - NumberFeatureValue(long) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.UnknownUnionField(field)) => - throw new UnknownFeatureValueException(s"Primitive: ${field.field.name}") - case t.FeatureValue.UnknownUnionField(field) => - throw new UnknownFeatureValueException(field.field.name) - } -} - -class UnknownFeatureValueException(fieldName: String) - extends Exception(s"Unknown FeatureValue name in thrift: ${fieldName}") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.docx new file mode 100644 index 000000000..ef0837141 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.scala deleted file mode 100644 index 06b19ac46..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.HasUserState -import com.twitter.follow_recommendations.common.utils.UserSignupUtil -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.util.Time - -case class RecommendationFlowData[Target <: HasClientContext]( - request: Target, - recommendationFlowIdentifier: RecommendationPipelineIdentifier, - candidateSources: Seq[CandidateSource[Target, CandidateUser]], - candidatesFromCandidateSources: Seq[CandidateUser], - mergedCandidates: Seq[CandidateUser], - filteredCandidates: Seq[CandidateUser], - rankedCandidates: Seq[CandidateUser], - transformedCandidates: Seq[CandidateUser], - truncatedCandidates: Seq[CandidateUser], - results: Seq[CandidateUser]) - extends HasMarshalling { - - import RecommendationFlowData._ - - lazy val toRecommendationFlowLogOfflineThrift: offline.RecommendationFlowLog = { - val userMetadata = userToOfflineRecommendationFlowUserMetadata(request) - val signals = userToOfflineRecommendationFlowSignals(request) - val filteredCandidateSourceCandidates = - candidatesToOfflineRecommendationFlowCandidateSourceCandidates( - candidateSources, - filteredCandidates - ) - val rankedCandidateSourceCandidates = - candidatesToOfflineRecommendationFlowCandidateSourceCandidates( - candidateSources, - rankedCandidates - ) - val truncatedCandidateSourceCandidates = - candidatesToOfflineRecommendationFlowCandidateSourceCandidates( - candidateSources, - truncatedCandidates - ) - - offline.RecommendationFlowLog( - ClientContextConverter.toFRSOfflineClientContextThrift(request.clientContext), - userMetadata, - signals, - Time.now.inMillis, - recommendationFlowIdentifier.name, - Some(filteredCandidateSourceCandidates), - Some(rankedCandidateSourceCandidates), - Some(truncatedCandidateSourceCandidates) - ) - } -} - -object RecommendationFlowData { - def userToOfflineRecommendationFlowUserMetadata[Target <: HasClientContext]( - request: Target - ): Option[offline.OfflineRecommendationFlowUserMetadata] = { - val userSignupAge = UserSignupUtil.userSignupAge(request).map(_.inDays) - val userState = request match { - case req: HasUserState => req.userState.map(_.name) - case _ => None - } - Some(offline.OfflineRecommendationFlowUserMetadata(userSignupAge, userState)) - } - - def userToOfflineRecommendationFlowSignals[Target <: HasClientContext]( - request: Target - ): Option[offline.OfflineRecommendationFlowSignals] = { - val countryCode = request.getCountryCode - Some(offline.OfflineRecommendationFlowSignals(countryCode)) - } - - def candidatesToOfflineRecommendationFlowCandidateSourceCandidates[Target <: HasClientContext]( - candidateSources: Seq[CandidateSource[Target, CandidateUser]], - candidates: Seq[CandidateUser], - ): Seq[offline.OfflineRecommendationFlowCandidateSourceCandidates] = { - val candidatesGroupedByCandidateSources = - candidates.groupBy( - _.getPrimaryCandidateSource.getOrElse(CandidateSourceIdentifier("NoCandidateSource"))) - - candidateSources.map(candidateSource => { - val candidates = - candidatesGroupedByCandidateSources.get(candidateSource.identifier).toSeq.flatten - val candidateUserIds = candidates.map(_.id) - val candidateUserScores = candidates.map(_.score).exists(_.nonEmpty) match { - case true => Some(candidates.map(_.score.getOrElse(-1.0))) - case false => None - } - offline.OfflineRecommendationFlowCandidateSourceCandidates( - candidateSource.identifier.name, - candidateUserIds, - candidateUserScores - ) - }) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.docx new file mode 100644 index 000000000..5f2624729 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.scala deleted file mode 100644 index fa768b536..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext - -case class RecommendationRequest( - clientContext: ClientContext, - displayLocation: DisplayLocation, - displayContext: Option[DisplayContext], - maxResults: Option[Int], - cursor: Option[String], - excludedIds: Option[Seq[Long]], - fetchPromotedContent: Option[Boolean], - debugParams: Option[DebugParams] = None, - userLocationState: Option[String] = None, - isSoftUser: Boolean = false) { - def toOfflineThrift: offline.OfflineRecommendationRequest = offline.OfflineRecommendationRequest( - ClientContextConverter.toFRSOfflineClientContextThrift(clientContext), - displayLocation.toOfflineThrift, - displayContext.map(_.toOfflineThrift), - maxResults, - cursor, - excludedIds, - fetchPromotedContent, - debugParams.map(DebugParams.toOfflineThrift) - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.docx new file mode 100644 index 000000000..460a2fc1b Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.scala deleted file mode 100644 index fadff377b..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling - -case class RecommendationResponse(recommendations: Seq[Recommendation]) extends HasMarshalling { - lazy val toThrift: t.RecommendationResponse = - t.RecommendationResponse(recommendations.map(_.toThrift)) - - lazy val toOfflineThrift: offline.OfflineRecommendationResponse = - offline.OfflineRecommendationResponse(recommendations.map(_.toOfflineThrift)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.docx new file mode 100644 index 000000000..7a1d63cf2 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.scala deleted file mode 100644 index a8798bda2..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext -import com.twitter.product_mixer.core.model.marshalling.request.{Request => ProductMixerRequest} - -case class Request( - override val maxResults: Option[Int], - override val debugParams: Option[request.DebugParams], - override val productContext: Option[ProductContext], - override val product: request.Product, - override val clientContext: ClientContext, - override val serializedRequestCursor: Option[String], - override val frsDebugParams: Option[DebugParams], - displayLocation: DisplayLocation, - excludedIds: Option[Seq[Long]], - fetchPromotedContent: Option[Boolean], - userLocationState: Option[String] = None) - extends ProductMixerRequest - with HasFrsDebugParams {} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.docx new file mode 100644 index 000000000..19bc13a2d Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.scala deleted file mode 100644 index 84d9d3ee3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models._ -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class ScoringUserRequest( - override val clientContext: ClientContext, - override val displayLocation: DisplayLocation, - override val params: Params, - override val debugOptions: Option[DebugOptions] = None, - override val recentFollowedUserIds: Option[Seq[Long]], - override val recentFollowedByUserIds: Option[Seq[Long]], - override val wtfImpressions: Option[Seq[WtfImpression]], - override val similarToUserIds: Seq[Long], - candidates: Seq[CandidateUser], - debugParams: Option[DebugParams] = None, - isSoftUser: Boolean = false) - extends HasClientContext - with HasDisplayLocation - with HasParams - with HasDebugOptions - with HasPreFetchedFeature - with HasSimilarToContext { - def toOfflineThrift: offline.OfflineScoringUserRequest = offline.OfflineScoringUserRequest( - ClientContextConverter.toFRSOfflineClientContextThrift(clientContext), - displayLocation.toOfflineThrift, - candidates.map(_.toOfflineUserThrift) - ) - def toRecommendationRequest: RecommendationRequest = RecommendationRequest( - clientContext = clientContext, - displayLocation = displayLocation, - displayContext = None, - maxResults = None, - cursor = None, - excludedIds = None, - fetchPromotedContent = None, - debugParams = debugParams, - isSoftUser = isSoftUser - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.docx new file mode 100644 index 000000000..a2c3886cd Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.scala deleted file mode 100644 index 4611386d3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class ScoringUserResponse(candidates: Seq[CandidateUser]) { - lazy val toThrift: t.ScoringUserResponse = - t.ScoringUserResponse(candidates.map(_.toUserThrift)) - - lazy val toRecommendationResponse: RecommendationResponse = RecommendationResponse(candidates) - - lazy val toOfflineThrift: offline.OfflineScoringUserResponse = - offline.OfflineScoringUserResponse(candidates.map(_.toOfflineUserThrift)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD deleted file mode 100644 index 4874d636e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD.docx new file mode 100644 index 000000000..f4e1bb65b Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.docx new file mode 100644 index 000000000..930b9d607 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.scala deleted file mode 100644 index 023b4c63e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.models.failures - -import com.twitter.product_mixer.core.pipeline.pipeline_failure.CandidateSourceTimeout -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure - -object TimeoutPipelineFailure { - def apply(candidateSourceName: String): PipelineFailure = { - PipelineFailure( - CandidateSourceTimeout, - s"Candidate Source $candidateSourceName timed out before returning candidates") - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.docx new file mode 100644 index 000000000..2ed698fab Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.scala deleted file mode 100644 index b75b6753e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.abdecider.ABDeciderFactory -import com.twitter.abdecider.LoggingABDecider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.inject.TwitterModule -import com.twitter.logging.LoggerFactory -import javax.inject.Singleton - -object ABDeciderModule extends TwitterModule { - @Provides - @Singleton - def provideABDecider( - stats: StatsReceiver, - @Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) factory: LoggerFactory - ): LoggingABDecider = { - - val ymlPath = "/usr/local/config/abdecider/abdecider.yml" - - val abDeciderFactory = ABDeciderFactory( - abDeciderYmlPath = ymlPath, - scribeLogger = Some(factory()), - environment = Some("production") - ) - - abDeciderFactory.buildWithLogging() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD deleted file mode 100644 index e7fb68380..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD.docx new file mode 100644 index 000000000..1e4ab940c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.docx new file mode 100644 index 000000000..4613c18f8 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.scala deleted file mode 100644 index ef3865bf2..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.follow_recommendations.configapi.ConfigBuilder -import com.twitter.inject.TwitterModule -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.Config -import javax.inject.Singleton - -object ConfigApiModule extends TwitterModule { - @Provides - @Singleton - def providesDeciderGateBuilder(decider: Decider): DeciderGateBuilder = - new DeciderGateBuilder(decider) - - @Provides - @Singleton - def providesConfig(configBuilder: ConfigBuilder): Config = configBuilder.build() -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.docx new file mode 100644 index 000000000..7447ee123 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.scala deleted file mode 100644 index 4ab0e4eba..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.inject.annotations.Flag -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.annotations.DarkTrafficService -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.inject.TwitterModule -import com.twitter.inject.thrift.filters.DarkTrafficFilter -import com.twitter.servo.decider.DeciderGateBuilder - -object DiffyModule extends TwitterModule { - // diffy.dest is defined in the Follow Recommendations Service aurora file - // and points to the Dark Traffic Proxy server - private val destFlag = - flag[String]("diffy.dest", "/$/nil", "Resolvable name of diffy-service or proxy") - - @Provides - @Singleton - @DarkTrafficService - def provideDarkTrafficService( - serviceIdentifier: ServiceIdentifier - ): FollowRecommendationsThriftService.ReqRepServicePerEndpoint = { - ThriftMux.client - .withClientId(ClientId("follow_recos_service_darktraffic_proxy_client")) - .withMutualTls(serviceIdentifier) - .servicePerEndpoint[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]( - dest = destFlag(), - label = "darktrafficproxy" - ) - } - - @Provides - @Singleton - def provideDarkTrafficFilter( - @DarkTrafficService darkService: FollowRecommendationsThriftService.ReqRepServicePerEndpoint, - deciderGateBuilder: DeciderGateBuilder, - statsReceiver: StatsReceiver, - @Flag("environment") env: String - ): DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint] = { - // sampleFunction is used to determine which requests should get replicated - // to the dark traffic proxy server - val sampleFunction: Any => Boolean = { _ => - // check whether the current FRS instance is deployed in production - env match { - case "prod" => - statsReceiver.scope("provideDarkTrafficFilter").counter("prod").incr() - destFlag.isDefined && deciderGateBuilder - .keyToFeature(DeciderKey.EnableTrafficDarkReading).isAvailable(RandomRecipient) - case _ => - statsReceiver.scope("provideDarkTrafficFilter").counter("devel").incr() - // replicate zero requests if in non-production environment - false - } - } - new DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]( - darkService, - sampleFunction, - forwardAfterService = true, - statsReceiver.scope("DarkTrafficFilter"), - lookupByMethod = true - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.docx new file mode 100644 index 000000000..653080641 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.scala deleted file mode 100644 index 1600344b6..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.twitter.abdecider.LoggingABDecider -import com.twitter.featureswitches.v2.Feature -import com.twitter.featureswitches.v2.FeatureFilter -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.v2.builder.FeatureSwitchesBuilder -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES -import com.twitter.inject.TwitterModule -import javax.inject.Named -import javax.inject.Singleton - -object FeaturesSwitchesModule extends TwitterModule { - private val DefaultConfigRepoPath = "/usr/local/config" - private val FeaturesPath = "/features/onboarding/follow-recommendations-service/main" - val isLocal = flag("configrepo.local", false, "Is the server running locally or in a DC") - val localConfigRepoPath = flag( - "local.configrepo", - System.getProperty("user.home") + "/workspace/config", - "Path to your local config repo" - ) - - @Provides - @Singleton - def providesFeatureSwitches( - abDecider: LoggingABDecider, - statsReceiver: StatsReceiver - ): FeatureSwitches = { - val configRepoPath = if (isLocal()) { - localConfigRepoPath() - } else { - DefaultConfigRepoPath - } - - FeatureSwitchesBuilder - .createDefault(FeaturesPath, abDecider, Some(statsReceiver)) - .configRepoAbsPath(configRepoPath) - .serviceDetailsFromAurora() - .build() - } - - @Provides - @Singleton - @Named(PRODUCER_SIDE_FEATURE_SWITCHES) - def providesProducerFeatureSwitches( - abDecider: LoggingABDecider, - statsReceiver: StatsReceiver - ): FeatureSwitches = { - val configRepoPath = if (isLocal()) { - localConfigRepoPath() - } else { - DefaultConfigRepoPath - } - - /** - * Feature Switches evaluate all tied FS Keys on Params construction time, which is very inefficient - * for producer/candidate side holdbacks because we have 100s of candidates, and 100s of FS which result - * in 10,000 FS evaluations when we want 1 per candidate (100 total), so we create a new FS Client - * which has a [[ProducerFeatureFilter]] set for feature filter to reduce the FS Keys we evaluate. - */ - FeatureSwitchesBuilder - .createDefault(FeaturesPath, abDecider, Some(statsReceiver.scope("producer_side_fs"))) - .configRepoAbsPath(configRepoPath) - .serviceDetailsFromAurora() - .addFeatureFilter(ProducerFeatureFilter) - .build() - } -} - -case object ProducerFeatureFilter extends FeatureFilter { - private val AllowedKeys = Set( - "post_nux_ml_flow_candidate_user_scorer_id", - "frs_receiver_holdback_keep_social_user_candidate", - "frs_receiver_holdback_keep_user_candidate") - - override def filter(feature: Feature): Option[Feature] = { - if (AllowedKeys.exists(feature.parameters.contains)) { - Some(feature) - } else { - None - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.docx new file mode 100644 index 000000000..304831296 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala deleted file mode 100644 index f8ff5ae94..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.follow_recommendations.modules -import com.twitter.inject.TwitterModule - -object FlagsModule extends TwitterModule { - flag[Boolean]( - name = "fetch_prod_promoted_accounts", - help = "Whether or not to fetch production promoted accounts (true / false)" - ) - flag[Boolean]( - name = "interests_opt_out_prod_enabled", - help = "Whether to fetch intersts opt out data from the prod strato column or not" - ) - flag[Boolean]( - name = "log_results", - default = false, - help = "Whether to log results such that we use them for scoring or metrics" - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.docx new file mode 100644 index 000000000..32a3f4729 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.scala deleted file mode 100644 index 218f3b973..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.twitter.follow_recommendations.products.ProdProductRegistry -import com.twitter.follow_recommendations.products.common.ProductRegistry -import com.twitter.inject.TwitterModule -import javax.inject.Singleton - -object ProductRegistryModule extends TwitterModule { - override protected def configure(): Unit = { - bind[ProductRegistry].to[ProdProductRegistry].in[Singleton] - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.docx new file mode 100644 index 000000000..ff1663698 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.scala deleted file mode 100644 index 035cc04bf..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.inject.TwitterModule -import com.twitter.relevance.ep_model.common.CommonConstants -import com.twitter.relevance.ep_model.scorer.EPScorer -import com.twitter.relevance.ep_model.scorer.EPScorerBuilder -import java.io.File -import java.io.FileOutputStream -import scala.language.postfixOps - -object ScorerModule extends TwitterModule { - private val STPScorerPath = "/quality/stp_models/20141223" - - private def fileFromResource(resource: String): File = { - val inputStream = getClass.getResourceAsStream(resource) - val file = File.createTempFile(resource, "temp") - val fos = new FileOutputStream(file) - Iterator - .continually(inputStream.read) - .takeWhile(-1 !=) - .foreach(fos.write) - file - } - - @Provides - @Singleton - def provideEpScorer: EPScorer = { - val modelPath = - fileFromResource(STPScorerPath + "/" + CommonConstants.EP_MODEL_FILE_NAME).getAbsolutePath - val trainingConfigPath = - fileFromResource(STPScorerPath + "/" + CommonConstants.TRAINING_CONFIG).getAbsolutePath - val epScorer = new EPScorerBuilder - epScorer - .withModelPath(modelPath) - .withTrainingConfig(trainingConfigPath) - .build() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.docx new file mode 100644 index 000000000..ebbfe782f Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.scala deleted file mode 100644 index 35af77c1a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.inject.TwitterModule -import com.twitter.logging.BareFormatter -import com.twitter.logging.HandlerFactory -import com.twitter.logging.Level -import com.twitter.logging.LoggerFactory -import com.twitter.logging.NullHandler -import com.twitter.logging.QueueingHandler -import com.twitter.logging.ScribeHandler - -object ScribeModule extends TwitterModule { - val useProdLogger = flag( - name = "scribe.use_prod_loggers", - default = false, - help = "whether to use production logging for service" - ) - - @Provides - @Singleton - @Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) - def provideClientEventsLoggerFactory(stats: StatsReceiver): LoggerFactory = { - val loggerCategory = "client_event" - val clientEventsHandler: HandlerFactory = if (useProdLogger()) { - QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = loggerCategory, - formatter = BareFormatter, - level = Some(Level.INFO), - statsReceiver = stats.scope("client_event_scribe") - ) - ) - } else { () => NullHandler } - LoggerFactory( - node = "abdecider", - level = Some(Level.INFO), - useParents = false, - handlers = clientEventsHandler :: Nil - ) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.REQUEST_LOGGER) - def provideFollowRecommendationsLoggerFactory(stats: StatsReceiver): LoggerFactory = { - val loggerCategory = "follow_recommendations_logs" - val handlerFactory: HandlerFactory = if (useProdLogger()) { - QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = loggerCategory, - formatter = BareFormatter, - level = Some(Level.INFO), - statsReceiver = stats.scope("follow_recommendations_logs_scribe") - ) - ) - } else { () => NullHandler } - LoggerFactory( - node = loggerCategory, - level = Some(Level.INFO), - useParents = false, - handlers = handlerFactory :: Nil - ) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.FLOW_LOGGER) - def provideFrsRecommendationFlowLoggerFactory(stats: StatsReceiver): LoggerFactory = { - val loggerCategory = "frs_recommendation_flow_logs" - val handlerFactory: HandlerFactory = if (useProdLogger()) { - QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = loggerCategory, - formatter = BareFormatter, - level = Some(Level.INFO), - statsReceiver = stats.scope("frs_recommendation_flow_logs_scribe") - ) - ) - } else { () => NullHandler } - LoggerFactory( - node = loggerCategory, - level = Some(Level.INFO), - useParents = false, - handlers = handlerFactory :: Nil - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.docx new file mode 100644 index 000000000..175952713 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.scala deleted file mode 100644 index 0572e43bf..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.memcached.ZookeeperStateMonitor.DefaultTimer -import com.twitter.inject.TwitterModule -import com.twitter.util.Timer - -object TimerModule extends TwitterModule { - @Provides - @Singleton - def providesTimer: Timer = DefaultTimer -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD deleted file mode 100644 index 5840c0f2f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD.docx new file mode 100644 index 000000000..022e05d12 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.docx new file mode 100644 index 000000000..f7b9472af Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.scala deleted file mode 100644 index 9a0dbb995..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.follow_recommendations.products - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.products.common.ProductRegistry -import com.twitter.follow_recommendations.products.explore_tab.ExploreTabProduct -import com.twitter.follow_recommendations.products.home_timeline.HomeTimelineProduct -import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.HomeTimelineTweetRecsProduct -import com.twitter.follow_recommendations.products.sidebar.SidebarProduct - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ProdProductRegistry @Inject() ( - exploreTabProduct: ExploreTabProduct, - homeTimelineProduct: HomeTimelineProduct, - homeTimelineTweetRecsProduct: HomeTimelineTweetRecsProduct, - sidebarProduct: SidebarProduct, -) extends ProductRegistry { - - override val products: Seq[common.Product] = - Seq( - exploreTabProduct, - homeTimelineProduct, - homeTimelineTweetRecsProduct, - sidebarProduct - ) - - override val displayLocationProductMap: Map[DisplayLocation, common.Product] = - products.groupBy(_.displayLocation).flatMap { - case (loc, products) => - assert(products.size == 1, s"Found more than 1 Product for ${loc}") - products.headOption.map { product => loc -> product } - } - - override def getProductByDisplayLocation(displayLocation: DisplayLocation): common.Product = { - displayLocationProductMap.getOrElse( - displayLocation, - throw new MissingProductException(displayLocation)) - } -} - -class MissingProductException(displayLocation: DisplayLocation) - extends Exception(s"No Product found for ${displayLocation}") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD deleted file mode 100644 index 4b32816e4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD.docx new file mode 100644 index 000000000..1b84a56d1 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.docx new file mode 100644 index 000000000..3967ba8ac Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.scala deleted file mode 100644 index c00d8c407..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.products.common - -abstract class ProductException(message: String) extends Exception(message) - -class MissingFieldException(productRequest: ProductRequest, fieldName: String) - extends ProductException( - s"Missing ${fieldName} field for ${productRequest.recommendationRequest.displayLocation} request") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.docx new file mode 100644 index 000000000..6d0e55d51 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.scala deleted file mode 100644 index 28c348204..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.follow_recommendations.products.common - -import com.twitter.follow_recommendations.assembler.models.Layout -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.product_mixer.core.model.marshalling.request.{Product => ProductMixerProduct} -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params - -trait Product { - - /** Each product also requires a human-readable name. - * You can change this at any time - */ - def name: String - - /** - * Every product needs a machine-friendly identifier for internal use. - * You should use the same name as the product package name. - * Except dashes are better than underscore - * - * Avoid changing this once it's in production. - */ - def identifier: String - - def displayLocation: DisplayLocation - - def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] - - /** - * Blender is responsible for blending together the candidates generated by different flows used - * in a product. For example, if a product uses two flows, it is blender's responsibility to - * interleave their generated candidates together and make a unified sequence of candidates. - */ - def blender: Transform[ProductRequest, Recommendation] - - /** - * It is resultsTransformer job to do any final transformations needed on the final list of - * candidates generated by a product. For example, if a final quality check on candidates needed, - * resultsTransformer will handle it. - */ - def resultsTransformer(request: ProductRequest): Stitch[Transform[ProductRequest, Recommendation]] - - def enabled(request: ProductRequest): Stitch[Boolean] - - def layout: Option[Layout] = None - - def productMixerProduct: Option[ProductMixerProduct] = None -} - -case class ProductRequest(recommendationRequest: RecommendationRequest, params: Params) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.docx new file mode 100644 index 000000000..2cdc5c310 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.scala deleted file mode 100644 index fbe486536..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.products.common - -import com.twitter.follow_recommendations.common.models.DisplayLocation - -trait ProductRegistry { - def products: Seq[Product] - def displayLocationProductMap: Map[DisplayLocation, Product] - def getProductByDisplayLocation(displayLocation: DisplayLocation): Product -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD deleted file mode 100644 index 2f9412612..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD.docx new file mode 100644 index 000000000..ad87d91a0 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.docx new file mode 100644 index 000000000..f290ae967 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.scala deleted file mode 100644 index a49fccb45..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.products.explore_tab - -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.explore_tab.configapi.ExploreTabParams -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ExploreTabProduct @Inject() ( - postNuxMlFlow: PostNuxMlFlow, - postNuxMlRequestBuilder: PostNuxMlRequestBuilder) - extends Product { - override val name: String = "Explore Tab" - - override val identifier: String = "explore-tab" - - override val displayLocation: DisplayLocation = DisplayLocation.ExploreTab - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => - Seq(postNuxMlFlow.mapKey({ _: ProductRequest => postNuxMlRequest })) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = - new IdentityTransform[ProductRequest, Recommendation] - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = - Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) - - override def enabled(request: ProductRequest): Stitch[Boolean] = { - // Ideally we should hook up is_soft_user as custom FS field and disable the product through FS - val enabledForUserType = !request.recommendationRequest.isSoftUser || request.params( - ExploreTabParams.EnableProductForSoftUser) - Stitch.value(request.params(ExploreTabParams.EnableProduct) && enabledForUserType) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD deleted file mode 100644 index 3bb732e35..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD.docx new file mode 100644 index 000000000..8d97abeec Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.docx new file mode 100644 index 000000000..ff3ea33d3 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.scala deleted file mode 100644 index 092252aca..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.products.explore_tab.configapi - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.products.explore_tab.configapi.ExploreTabParams._ -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ExploreTabFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq(EnableProductForSoftUser) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.docx new file mode 100644 index 000000000..325278fcb Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.scala deleted file mode 100644 index b9d9d3b87..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.products.explore_tab.configapi - -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.configapi.FSParam - -object ExploreTabParams { - object EnableProduct extends Param[Boolean](false) - object EnableProductForSoftUser - extends FSParam[Boolean]("explore_tab_enable_product_for_soft_user", false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD deleted file mode 100644 index 4b0586ff7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD.docx new file mode 100644 index 000000000..fa1303832 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.docx new file mode 100644 index 000000000..b0667ecb9 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.scala deleted file mode 100644 index a2051c150..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline - -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product - -case object HTLProductMixer extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("HomeTimeline") - override val stringCenterProject: Option[String] = Some("people-discovery") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.docx new file mode 100644 index 000000000..1efdd696c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.scala deleted file mode 100644 index 590aab182..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline - -import com.twitter.follow_recommendations.assembler.models.ActionConfig -import com.twitter.follow_recommendations.assembler.models.FollowedByUsersProof -import com.twitter.follow_recommendations.assembler.models.FooterConfig -import com.twitter.follow_recommendations.assembler.models.GeoContextProof -import com.twitter.follow_recommendations.assembler.models.HeaderConfig -import com.twitter.follow_recommendations.assembler.models.Layout -import com.twitter.follow_recommendations.assembler.models.TitleConfig -import com.twitter.follow_recommendations.assembler.models.UserListLayout -import com.twitter.follow_recommendations.assembler.models.UserListOptions -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlow -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlowRequest -import com.twitter.follow_recommendations.blenders.PromotedAccountsBlender -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.home_timeline.configapi.HomeTimelineParams._ -import com.twitter.inject.Injector -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.product.guice.ProductScope -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeTimelineProduct @Inject() ( - postNuxMlFlow: PostNuxMlFlow, - postNuxMlRequestBuilder: PostNuxMlRequestBuilder, - promotedAccountsFlow: PromotedAccountsFlow, - promotedAccountsBlender: PromotedAccountsBlender, - productScope: ProductScope, - injector: Injector, -) extends Product { - - override val name: String = "Home Timeline" - - override val identifier: String = "home-timeline" - - override val displayLocation: DisplayLocation = DisplayLocation.HomeTimeline - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => - Seq( - postNuxMlFlow.mapKey({ request: ProductRequest => postNuxMlRequest }), - promotedAccountsFlow.mapKey(mkPromotedAccountsRequest)) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = { - promotedAccountsBlender.mapTarget[ProductRequest](getMaxResults) - } - - private val identityTransform = new IdentityTransform[ProductRequest, Recommendation] - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(identityTransform) - - override def enabled(request: ProductRequest): Stitch[Boolean] = - Stitch.value(request.params(EnableProduct)) - - override def layout: Option[Layout] = { - productMixerProduct.map { product => - val homeTimelineStrings = productScope.let(product) { - injector.instance[HomeTimelineStrings] - } - UserListLayout( - header = Some(HeaderConfig(TitleConfig(homeTimelineStrings.whoToFollowModuleTitle))), - userListOptions = UserListOptions(userBioEnabled = true, userBioTruncated = true, None), - socialProofs = Some( - Seq( - FollowedByUsersProof( - homeTimelineStrings.whoToFollowFollowedByManyUserSingleString, - homeTimelineStrings.whoToFollowFollowedByManyUserDoubleString, - homeTimelineStrings.whoToFollowFollowedByManyUserMultipleString - ), - GeoContextProof(homeTimelineStrings.whoToFollowPopularInCountryKey) - )), - footer = Some( - FooterConfig( - Some(ActionConfig(homeTimelineStrings.whoToFollowModuleFooter, "http://twitter.com")))) - ) - } - } - - override def productMixerProduct: Option[request.Product] = Some(HTLProductMixer) - - private[home_timeline] def mkPromotedAccountsRequest( - req: ProductRequest - ): PromotedAccountsFlowRequest = { - PromotedAccountsFlowRequest( - req.recommendationRequest.clientContext, - req.params, - req.recommendationRequest.displayLocation, - None, - req.recommendationRequest.excludedIds.getOrElse(Nil) - ) - } - - private[home_timeline] def getMaxResults(req: ProductRequest): Int = { - req.recommendationRequest.maxResults.getOrElse( - req.params(DefaultMaxResults) - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.docx new file mode 100644 index 000000000..cd0ec0620 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.scala deleted file mode 100644 index 75819555e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline - -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.ExternalStringRegistry -import com.twitter.stringcenter.client.core.ExternalString -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class HomeTimelineStrings @Inject() ( - @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { - private val externalStringRegistry = externalStringRegistryProvider.get() - val whoToFollowFollowedByManyUserSingleString: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserSingle") - val whoToFollowFollowedByManyUserDoubleString: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserDouble") - val whoToFollowFollowedByManyUserMultipleString: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserMultiple") - val whoToFollowPopularInCountryKey: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.popularInCountry") - val whoToFollowModuleTitle: ExternalString = - externalStringRegistry.createProdString("WhoToFollowModule.title") - val whoToFollowModuleFooter: ExternalString = - externalStringRegistry.createProdString("WhoToFollowModule.pivot") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD deleted file mode 100644 index 3bb732e35..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD.docx new file mode 100644 index 000000000..8d97abeec Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.docx new file mode 100644 index 000000000..e81008a65 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.scala deleted file mode 100644 index 15e97b3a5..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline.configapi - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.products.home_timeline.configapi.HomeTimelineParams._ -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeTimelineFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq(EnableWritingServingHistory) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - DurationGuardrailToForceSuggest, - SuggestBasedFatigueDuration - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.docx new file mode 100644 index 000000000..94406df61 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.scala deleted file mode 100644 index 65ab5ae23..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline.configapi - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object HomeTimelineParams { - object EnableProduct extends Param[Boolean](false) - - object DefaultMaxResults extends Param[Int](20) - - object EnableWritingServingHistory - extends FSParam[Boolean]("home_timeline_enable_writing_serving_history", false) - - object DurationGuardrailToForceSuggest - extends FSBoundedParam[Duration]( - name = "home_timeline_duration_guardrail_to_force_suggest_in_hours", - default = 0.hours, - min = 0.hours, - max = 1000.hours) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromHours - } - - object SuggestBasedFatigueDuration - extends FSBoundedParam[Duration]( - name = "home_timeline_suggest_based_fatigue_duration_in_hours", - default = 0.hours, - min = 0.hours, - max = 1000.hours) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromHours - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD deleted file mode 100644 index 140cc928d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD.docx new file mode 100644 index 000000000..30a36644c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.docx new file mode 100644 index 000000000..ef7e3cdaa Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.scala deleted file mode 100644 index a5586f296..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline_tweet_recs - -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlow -import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams._ -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -/* - * This "DisplayLocation" is used to generate user recommendations using the ContentRecommenderFlow. These recommendations are later used downstream - * to generate recommended tweets on Home Timeline. - */ -@Singleton -class HomeTimelineTweetRecsProduct @Inject() ( - contentRecommenderFlow: ContentRecommenderFlow, - contentRecommenderRequestBuilder: ContentRecommenderRequestBuilder) - extends Product { - override val name: String = "Home Timeline Tweet Recs" - - override val identifier: String = "home-timeline-tweet-recs" - - override val displayLocation: DisplayLocation = DisplayLocation.HomeTimelineTweetRecs - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - contentRecommenderRequestBuilder.build(request).map { contentRecommenderRequest => - Seq(contentRecommenderFlow.mapKey({ request: ProductRequest => contentRecommenderRequest })) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = - new IdentityTransform[ProductRequest, Recommendation] - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = - Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) - - override def enabled(request: ProductRequest): Stitch[Boolean] = - Stitch.value(request.params(EnableProduct)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD deleted file mode 100644 index 9be8d9647..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "configapi/configapi-decider", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD.docx new file mode 100644 index 000000000..fb688c491 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.docx new file mode 100644 index 000000000..69bf0254c Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.scala deleted file mode 100644 index 319a51847..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi - -import com.twitter.timelines.configapi.Param - -object HomeTimelineTweetRecsParams { - object EnableProduct extends Param[Boolean](false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD deleted file mode 100644 index f469a4748..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD.docx new file mode 100644 index 000000000..153b829a7 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.docx new file mode 100644 index 000000000..ad195c8b3 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.scala deleted file mode 100644 index 29f788011..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.follow_recommendations.products.sidebar - -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlow -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlowRequest -import com.twitter.follow_recommendations.blenders.PromotedAccountsBlender -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.sidebar.configapi.SidebarParams -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SidebarProduct @Inject() ( - postNuxMlFlow: PostNuxMlFlow, - postNuxMlRequestBuilder: PostNuxMlRequestBuilder, - promotedAccountsFlow: PromotedAccountsFlow, - promotedAccountsBlender: PromotedAccountsBlender) - extends Product { - override val name: String = "Sidebar" - - override val identifier: String = "sidebar" - - override val displayLocation: DisplayLocation = DisplayLocation.Sidebar - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => - Seq( - postNuxMlFlow.mapKey({ _: ProductRequest => postNuxMlRequest }), - promotedAccountsFlow.mapKey(mkPromotedAccountsRequest) - ) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = { - promotedAccountsBlender.mapTarget[ProductRequest](getMaxResults) - } - - private[sidebar] def mkPromotedAccountsRequest( - req: ProductRequest - ): PromotedAccountsFlowRequest = { - PromotedAccountsFlowRequest( - req.recommendationRequest.clientContext, - req.params, - req.recommendationRequest.displayLocation, - None, - req.recommendationRequest.excludedIds.getOrElse(Nil) - ) - } - - private[sidebar] def getMaxResults(req: ProductRequest): Int = { - req.recommendationRequest.maxResults.getOrElse( - req.params(SidebarParams.DefaultMaxResults) - ) - } - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = - Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) - - override def enabled(request: ProductRequest): Stitch[Boolean] = - Stitch.value(request.params(SidebarParams.EnableProduct)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD deleted file mode 100644 index 6fee24f89..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD.docx new file mode 100644 index 000000000..e1cfd4086 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.docx new file mode 100644 index 000000000..d7415525f Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.scala deleted file mode 100644 index bbd026495..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.products.sidebar.configapi - -import com.twitter.timelines.configapi.Param - -object SidebarParams { - object EnableProduct extends Param[Boolean](false) - - object DefaultMaxResults extends Param[Int](20) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD deleted file mode 100644 index fe29c22de..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD.docx new file mode 100644 index 000000000..c271d08dc Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.docx new file mode 100644 index 000000000..ffcfa8cc0 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.scala deleted file mode 100644 index 7567fe9ce..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService.GetRecommendations -import com.twitter.follow_recommendations.thriftscala.ClientContext -import com.twitter.follow_recommendations.thriftscala.DebugParams -import com.twitter.follow_recommendations.thriftscala.DisplayContext -import com.twitter.follow_recommendations.thriftscala.DisplayLocation -import com.twitter.follow_recommendations.thriftscala.Profile -import com.twitter.follow_recommendations.thriftscala.RecommendationRequest -import com.twitter.inject.Logging -import com.twitter.inject.utils.Handler -import com.twitter.scrooge.Request -import com.twitter.scrooge.Response -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowRecommendationsServiceWarmupHandler @Inject() (warmup: ThriftWarmup) - extends Handler - with Logging { - - private val clientId = ClientId("thrift-warmup-client") - - override def handle(): Unit = { - val testIds = Seq(1L) - def warmupQuery(userId: Long, displayLocation: DisplayLocation): RecommendationRequest = { - val clientContext = ClientContext( - userId = Some(userId), - guestId = None, - appId = Some(258901L), - ipAddress = Some("0.0.0.0"), - userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), - countryCode = Some("US"), - languageCode = Some("en"), - isTwoffice = None, - userRoles = None, - deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") - ) - RecommendationRequest( - clientContext = clientContext, - displayLocation = displayLocation, - displayContext = None, - maxResults = Some(3), - fetchPromotedContent = Some(false), - debugParams = Some(DebugParams(doNotLog = Some(true))) - ) - } - - // Add FRS display locations here if they should be targeted for warm-up - // when FRS is starting from a fresh state after a deploy - val displayLocationsToWarmUp: Seq[DisplayLocation] = Seq( - DisplayLocation.HomeTimeline, - DisplayLocation.HomeTimelineReverseChron, - DisplayLocation.ProfileSidebar, - DisplayLocation.NuxInterests, - DisplayLocation.NuxPymk - ) - - try { - clientId.asCurrent { - // Iterate over each user ID created for testing - testIds foreach { id => - // Iterate over each display location targeted for warm-up - displayLocationsToWarmUp foreach { displayLocation => - val warmupReq = warmupQuery(id, displayLocation) - info(s"Sending warm-up request to service with query: $warmupReq") - warmup.sendRequest( - method = GetRecommendations, - req = Request(GetRecommendations.Args(warmupReq)))(assertWarmupResponse) - // send the request one more time so that it goes through cache hits - warmup.sendRequest( - method = GetRecommendations, - req = Request(GetRecommendations.Args(warmupReq)))(assertWarmupResponse) - } - } - } - } catch { - case e: Throwable => - // we don't want a warmup failure to prevent start-up - error(e.getMessage, e) - } - info("Warm-up done.") - } - - /* Private */ - - private def assertWarmupResponse(result: Try[Response[GetRecommendations.SuccessType]]): Unit = { - // we collect and log any exceptions from the result. - result match { - case Return(_) => // ok - case Throw(exception) => - warn() - error(s"Error performing warm-up request: ${exception.getMessage}", exception) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.docx new file mode 100644 index 000000000..8b4c8f1db Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.scala deleted file mode 100644 index daff9040d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.StatsReceiver -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.timelines.configapi.Params -import com.twitter.follow_recommendations.common.utils.DisplayLocationProductConverterUtil -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.{DebugParams => FrsDebugParams} -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.follow_recommendations.models.Request -import com.twitter.product_mixer.core.model.marshalling.request.{ - DebugParams => ProductMixerDebugParams -} -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.stitch.Stitch - -@Singleton -class ProductMixerRecommendationService @Inject() ( - productPipelineRegistry: ProductPipelineRegistry, - resultLogger: FrsLogger, - baseStats: StatsReceiver) { - - private val stats = baseStats.scope("product_mixer_recos_service_stats") - private val loggingStats = stats.scope("logged") - - def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] = { - if (params(DeciderParams.EnableRecommendations)) { - val productMixerRequest = convertToProductMixerRequest(request) - - productPipelineRegistry - .getProductPipeline[Request, RecommendationResponse](productMixerRequest.product) - .process(ProductPipelineRequest(productMixerRequest, params)).onSuccess { response => - if (resultLogger.shouldLog(request.debugParams)) { - loggingStats.counter().incr() - resultLogger.logRecommendationResult(request, response) - } - } - } else { - Stitch.value(RecommendationResponse(Nil)) - } - - } - - def convertToProductMixerRequest(frsRequest: RecommendationRequest): Request = { - Request( - maxResults = frsRequest.maxResults, - debugParams = convertToProductMixerDebugParams(frsRequest.debugParams), - productContext = None, - product = - DisplayLocationProductConverterUtil.displayLocationToProduct(frsRequest.displayLocation), - clientContext = frsRequest.clientContext, - serializedRequestCursor = frsRequest.cursor, - frsDebugParams = frsRequest.debugParams, - displayLocation = frsRequest.displayLocation, - excludedIds = frsRequest.excludedIds, - fetchPromotedContent = frsRequest.fetchPromotedContent, - userLocationState = frsRequest.userLocationState - ) - } - - private def convertToProductMixerDebugParams( - frsDebugParams: Option[FrsDebugParams] - ): Option[ProductMixerDebugParams] = { - frsDebugParams.map { debugParams => - ProductMixerDebugParams(debugParams.featureOverrides, None) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.docx new file mode 100644 index 000000000..66593a4fa Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.scala deleted file mode 100644 index 1c949f03d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton -import scala.util.Random - -@Singleton -class ProductPipelineSelector @Inject() ( - recommendationsService: RecommendationsService, - productMixerRecommendationService: ProductMixerRecommendationService, - productPipelineSelectorConfig: ProductPipelineSelectorConfig, - baseStats: StatsReceiver) { - - private val frsStats = baseStats.scope("follow_recommendations_service") - private val stats = frsStats.scope("product_pipeline_selector_parity") - - private val readFromProductMixerCounter = stats.counter("select_product_mixer") - private val readFromOldFRSCounter = stats.counter("select_old_frs") - - def selectPipeline( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - productPipelineSelectorConfig - .getDarkReadAndExpParams(request.displayLocation).map { darkReadAndExpParam => - if (params(darkReadAndExpParam.expParam)) { - readFromProductMixerPipeline(request, params) - } else if (params(darkReadAndExpParam.darkReadParam)) { - darkReadAndReturnResult(request, params) - } else { - readFromOldFrsPipeline(request, params) - } - }.getOrElse(readFromOldFrsPipeline(request, params)) - } - - private def readFromProductMixerPipeline( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - readFromProductMixerCounter.incr() - productMixerRecommendationService.get(request, params) - } - - private def readFromOldFrsPipeline( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - readFromOldFRSCounter.incr() - recommendationsService.get(request, params) - } - - private def darkReadAndReturnResult( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - val darkReadStats = stats.scope("select_dark_read", request.displayLocation.toFsName) - darkReadStats.counter("count").incr() - - // If no seed is set, create a random one that both requests will use to remove differences - // in randomness for the WeightedCandidateSourceRanker - val randomizationSeed = new Random().nextLong() - - val oldFRSPiplelineRequest = request.copy( - debugParams = Some( - request.debugParams.getOrElse( - DebugParams(None, Some(DebugOptions(randomizationSeed = Some(randomizationSeed)))))) - ) - val productMixerPipelineRequest = request.copy( - debugParams = Some( - request.debugParams.getOrElse( - DebugParams( - None, - Some(DebugOptions(doNotLog = true, randomizationSeed = Some(randomizationSeed)))))) - ) - - StatsUtil - .profileStitch( - readFromOldFrsPipeline(oldFRSPiplelineRequest, params), - darkReadStats.scope("frs_timing")).applyEffect { frsOldPipelineResponse => - Stitch.async( - StatsUtil - .profileStitch( - readFromProductMixerPipeline(productMixerPipelineRequest, params), - darkReadStats.scope("product_mixer_timing")).liftToOption().map { - case Some(frsProductMixerResponse) => - darkReadStats.counter("product_mixer_pipeline_success").incr() - compare(request, frsOldPipelineResponse, frsProductMixerResponse) - case None => - darkReadStats.counter("product_mixer_pipeline_failure").incr() - } - ) - } - } - - def compare( - request: RecommendationRequest, - frsOldPipelineResponse: RecommendationResponse, - frsProductMixerResponse: RecommendationResponse - ): Unit = { - val compareStats = stats.scope("pipeline_comparison", request.displayLocation.toFsName) - compareStats.counter("total-comparisons").incr() - - val oldFrsMap = frsOldPipelineResponse.recommendations.map { user => user.id -> user }.toMap - val productMixerMap = frsProductMixerResponse.recommendations.map { user => - user.id -> user - }.toMap - - compareTopNResults(3, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(5, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(25, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(50, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(75, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - - // Compare individual matching candidates - oldFrsMap.keys.foreach(userId => { - if (productMixerMap.contains(userId)) { - (oldFrsMap(userId), productMixerMap(userId)) match { - case (oldFrsUser: CandidateUser, productMixerUser: CandidateUser) => - compareStats.counter("matching-user-count").incr() - compareUser(oldFrsUser, productMixerUser, compareStats) - case _ => - compareStats.counter("unknown-user-type-count").incr() - } - } else { - compareStats.counter("missing-user-count").incr() - } - }) - } - - private def compareTopNResults( - n: Int, - frsOldPipelineResponse: RecommendationResponse, - frsProductMixerResponse: RecommendationResponse, - compareStats: StatsReceiver - ): Unit = { - if (frsOldPipelineResponse.recommendations.size >= n && frsProductMixerResponse.recommendations.size >= n) { - val oldFrsPipelineFirstN = frsOldPipelineResponse.recommendations.take(n).map(_.id) - val productMixerPipelineFirstN = frsProductMixerResponse.recommendations.take(n).map(_.id) - - if (oldFrsPipelineFirstN.sorted == productMixerPipelineFirstN.sorted) - compareStats.counter(s"first-$n-sorted-equal-ids").incr() - if (oldFrsPipelineFirstN == productMixerPipelineFirstN) - compareStats.counter(s"first-$n-unsorted-ids-equal").incr() - else - compareStats.counter(s"first-$n-unsorted-ids-unequal").incr() - } - } - - private def compareUser( - oldFrsUser: CandidateUser, - productMixerUser: CandidateUser, - stats: StatsReceiver - ): Unit = { - val userStats = stats.scope("matching-user") - - if (oldFrsUser.score != productMixerUser.score) - userStats.counter("mismatch-score").incr() - if (oldFrsUser.reason != productMixerUser.reason) - userStats.counter("mismatch-reason").incr() - if (oldFrsUser.userCandidateSourceDetails != productMixerUser.userCandidateSourceDetails) - userStats.counter("mismatch-userCandidateSourceDetails").incr() - if (oldFrsUser.adMetadata != productMixerUser.adMetadata) - userStats.counter("mismatch-adMetadata").incr() - if (oldFrsUser.trackingToken != productMixerUser.trackingToken) - userStats.counter("mismatch-trackingToken").incr() - if (oldFrsUser.dataRecord != productMixerUser.dataRecord) - userStats.counter("mismatch-dataRecord").incr() - if (oldFrsUser.scores != productMixerUser.scores) - userStats.counter("mismatch-scores").incr() - if (oldFrsUser.infoPerRankingStage != productMixerUser.infoPerRankingStage) - userStats.counter("mismatch-infoPerRankingStage").incr() - if (oldFrsUser.params != productMixerUser.params) - userStats.counter("mismatch-params").incr() - if (oldFrsUser.engagements != productMixerUser.engagements) - userStats.counter("mismatch-engagements").incr() - if (oldFrsUser.recommendationFlowIdentifier != productMixerUser.recommendationFlowIdentifier) - userStats.counter("mismatch-recommendationFlowIdentifier").incr() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.docx new file mode 100644 index 000000000..b0b2d06c8 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.scala deleted file mode 100644 index a1cac3316..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param -import javax.inject.Singleton - -@Singleton -class ProductPipelineSelectorConfig { - private val paramsMap: Map[DisplayLocation, DarkReadAndExpParams] = Map.empty - - def getDarkReadAndExpParams( - displayLocation: DisplayLocation - ): Option[DarkReadAndExpParams] = { - paramsMap.get(displayLocation) - } -} - -case class DarkReadAndExpParams(darkReadParam: Param[Boolean], expParam: FSParam[Boolean]) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.docx new file mode 100644 index 000000000..631d45cf8 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.scala deleted file mode 100644 index 967790a08..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.products.common.ProductRegistry -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.stitch.Stitch -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ProductRecommenderService @Inject() ( - productRegistry: ProductRegistry, - statsReceiver: StatsReceiver) { - - private val stats = statsReceiver.scope("ProductRecommenderService") - - def getRecommendations( - request: RecommendationRequest, - params: Params - ): Stitch[Seq[Recommendation]] = { - val displayLocation = request.displayLocation - val displayLocationStatName = displayLocation.toString - val locationStats = stats.scope(displayLocationStatName) - val loggedInOrOutStats = if (request.clientContext.userId.isDefined) { - stats.scope("logged_in").scope(displayLocationStatName) - } else { - stats.scope("logged_out").scope(displayLocationStatName) - } - - loggedInOrOutStats.counter("requests").incr() - val product = productRegistry.getProductByDisplayLocation(displayLocation) - val productRequest = ProductRequest(request, params) - val productEnabledStitch = - StatsUtil.profileStitch(product.enabled(productRequest), locationStats.scope("enabled")) - productEnabledStitch.flatMap { productEnabled => - if (productEnabled && params(EnableWhoToFollowProducts)) { - loggedInOrOutStats.counter("enabled").incr() - val stitch = for { - workflows <- StatsUtil.profileStitch( - product.selectWorkflows(productRequest), - locationStats.scope("select_workflows")) - workflowRecos <- StatsUtil.profileStitch( - Stitch.collect( - workflows.map(_.process(productRequest).map(_.result.getOrElse(Seq.empty)))), - locationStats.scope("execute_workflows") - ) - blendedCandidates <- StatsUtil.profileStitch( - product.blender.transform(productRequest, workflowRecos.flatten), - locationStats.scope("blend_results")) - resultsTransformer <- StatsUtil.profileStitch( - product.resultsTransformer(productRequest), - locationStats.scope("results_transformer")) - transformedCandidates <- StatsUtil.profileStitch( - resultsTransformer.transform(productRequest, blendedCandidates), - locationStats.scope("execute_results_transformer")) - } yield { - transformedCandidates - } - StatsUtil.profileStitchResults[Seq[Recommendation]](stitch, locationStats, _.size) - } else { - loggedInOrOutStats.counter("disabled").incr() - locationStats.counter("disabled_product").incr() - Stitch.Nil - } - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.docx new file mode 100644 index 000000000..da68cf7df Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.scala deleted file mode 100644 index e4bc1e3c0..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecommendationsService @Inject() ( - productRecommenderService: ProductRecommenderService, - resultLogger: FrsLogger) { - def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] = { - if (params(DeciderParams.EnableRecommendations)) { - productRecommenderService - .getRecommendations(request, params).map(RecommendationResponse).onSuccess { response => - if (resultLogger.shouldLog(request.debugParams)) { - resultLogger.logRecommendationResult(request, response) - } - } - } else { - Stitch.value(RecommendationResponse(Nil)) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.docx new file mode 100644 index 000000000..a06a2ad1d Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.scala deleted file mode 100644 index b3a8c6664..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil.profileStitchSeqResults -import com.twitter.follow_recommendations.common.clients.impression_store.WtfImpressionStore -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.HydrateFeaturesTransform -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRanker -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.ScoringUserRequest -import com.twitter.follow_recommendations.models.ScoringUserResponse -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class UserScoringService @Inject() ( - socialGraph: SocialGraphClient, - wtfImpressionStore: WtfImpressionStore, - hydrateFeaturesTransform: HydrateFeaturesTransform[ScoringUserRequest], - mlRanker: MlRanker[ScoringUserRequest], - resultLogger: FrsLogger, - stats: StatsReceiver) { - - private val scopedStats: StatsReceiver = stats.scope(this.getClass.getSimpleName) - private val disabledCounter: Counter = scopedStats.counter("disabled") - - def get(request: ScoringUserRequest): Stitch[ScoringUserResponse] = { - if (request.params(DeciderParams.EnableScoreUserCandidates)) { - val hydratedRequest = hydrate(request) - val candidatesStitch = hydratedRequest.flatMap { req => - hydrateFeaturesTransform.transform(req, request.candidates).flatMap { - candidateWithFeatures => - mlRanker.rank(req, candidateWithFeatures) - } - } - profileStitchSeqResults(candidatesStitch, scopedStats) - .map(ScoringUserResponse) - .onSuccess { response => - if (resultLogger.shouldLog(request.debugParams)) { - resultLogger.logScoringResult(request, response) - } - } - } else { - disabledCounter.incr() - Stitch.value(ScoringUserResponse(Nil)) - } - } - - private def hydrate(request: ScoringUserRequest): Stitch[ScoringUserRequest] = { - val allStitches = Stitch.collect(request.clientContext.userId.map { userId => - val recentFollowedUserIdsStitch = - rescueWithStats( - socialGraph.getRecentFollowedUserIds(userId), - stats, - "recentFollowedUserIds") - val recentFollowedByUserIdsStitch = - rescueWithStats( - socialGraph.getRecentFollowedByUserIds(userId), - stats, - "recentFollowedByUserIds") - val wtfImpressionsStitch = - rescueWithStats( - wtfImpressionStore.get(userId, request.displayLocation), - stats, - "wtfImpressions") - Stitch.join(recentFollowedUserIdsStitch, recentFollowedByUserIdsStitch, wtfImpressionsStitch) - }) - allStitches.map { - case Some((recentFollowedUserIds, recentFollowedByUserIds, wtfImpressions)) => - request.copy( - recentFollowedUserIds = - if (recentFollowedUserIds.isEmpty) None else Some(recentFollowedUserIds), - recentFollowedByUserIds = - if (recentFollowedByUserIds.isEmpty) None else Some(recentFollowedByUserIds), - wtfImpressions = if (wtfImpressions.isEmpty) None else Some(wtfImpressions) - ) - case _ => request - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD deleted file mode 100644 index 8b2c2d041..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/exceptions", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/modules", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/response", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD.docx new file mode 100644 index 000000000..5c9f1b5e5 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.docx new file mode 100644 index 000000000..559d680f7 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.scala deleted file mode 100644 index f3a09a6d7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.follow_recommendations.service.exceptions - -import com.twitter.finatra.thrift.exceptions.ExceptionMapper -import com.twitter.inject.Logging -import com.twitter.util.Future -import javax.inject.Singleton - -@Singleton -class UnknownLoggingExceptionMapper extends ExceptionMapper[Exception, Throwable] with Logging { - def handleException(throwable: Exception): Future[Throwable] = { - error( - s"Unmapped Exception: ${throwable.getMessage} - ${throwable.getStackTrace.mkString(", \n\t")}", - throwable - ) - - Future.exception(throwable) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD deleted file mode 100644 index e92976ad8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD.docx new file mode 100644 index 000000000..3327875ea Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.docx new file mode 100644 index 000000000..1e569ae22 Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.scala deleted file mode 100644 index 60a28f1a8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.follow_recommendations.utils - -import com.twitter.follow_recommendations.common.candidate_sources.addressbook._ -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentStrongEngagementDirectFollowSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.MutualFollowStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.BaseOnlineSTPSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.SocialProofEnforcedOfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk.TwoHopRandomWalkSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.follow_recommendations.models.CandidateSourceType -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.timelines.configapi.HasParams - -trait CandidateSourceHoldbackUtil { - import CandidateSourceHoldbackUtil._ - def filterCandidateSources[T <: HasParams]( - request: T, - sources: Seq[CandidateSource[T, CandidateUser]] - ): Seq[CandidateSource[T, CandidateUser]] = { - val typeToFilter = request.params(GlobalParams.CandidateSourcesToFilter) - val sourcesToFilter = CandidateSourceTypeToMap.get(typeToFilter).getOrElse(Set.empty) - sources.filterNot { source => sourcesToFilter.contains(source.identifier) } - } -} - -object CandidateSourceHoldbackUtil { - final val ContextualActivityCandidateSourceIds: Set[CandidateSourceIdentifier] = - Set( - RecentFollowingSimilarUsersSource.Identifier, - RecentEngagementNonDirectFollowSource.Identifier, - RecentEngagementSimilarUsersSource.Identifier, - RecentStrongEngagementDirectFollowSimilarUsersSource.Identifier, - SwitchingSimsSource.Identifier, - ) - - final val SocialCandidateSourceIds: Set[CandidateSourceIdentifier] = - Set( - ForwardEmailBookSource.Identifier, - ForwardPhoneBookSource.Identifier, - ReverseEmailBookSource.Identifier, - ReversePhoneBookSource.Identifier, - RecentFollowingRecentFollowingExpansionSource.Identifier, - BaseOnlineSTPSource.Identifier, - MutualFollowStrongTiePredictionSource.Identifier, - OfflineStrongTiePredictionSource.Identifier, - SocialProofEnforcedOfflineStrongTiePredictionSource.Identifier, - TriangularLoopsSource.Identifier, - TwoHopRandomWalkSource.Identifier - ) - - final val GeoCandidateSourceIds: Set[CandidateSourceIdentifier] = - Set( - PPMILocaleFollowSource.Identifier, - PopCountrySource.Identifier, - PopGeohashSource.Identifier, - PopCountryBackFillSource.Identifier, - PopGeoSource.Identifier, - ) - - final val CandidateSourceTypeToMap: Map[CandidateSourceType.Value, Set[ - CandidateSourceIdentifier - ]] = - Map( - CandidateSourceType.Social -> SocialCandidateSourceIds, - CandidateSourceType.ActivityContextual -> ContextualActivityCandidateSourceIds, - CandidateSourceType.GeoAndInterests -> GeoCandidateSourceIds - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.docx b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.docx new file mode 100644 index 000000000..967eb48da Binary files /dev/null and b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.docx differ diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.scala deleted file mode 100644 index 9304fb398..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.follow_recommendations.utils - -import com.twitter.follow_recommendations.common.base.RecommendationFlow -import com.twitter.follow_recommendations.common.base.SideEffectsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch - -trait RecommendationFlowBaseSideEffectsUtil[Target <: HasClientContext, Candidate <: CandidateUser] - extends SideEffectsUtil[Target, Candidate] { - recommendationFlow: RecommendationFlow[Target, Candidate] => - - override def applySideEffects( - target: Target, - candidateSources: Seq[CandidateSource[Target, Candidate]], - candidatesFromCandidateSources: Seq[Candidate], - mergedCandidates: Seq[Candidate], - filteredCandidates: Seq[Candidate], - rankedCandidates: Seq[Candidate], - transformedCandidates: Seq[Candidate], - truncatedCandidates: Seq[Candidate], - results: Seq[Candidate] - ): Stitch[Unit] = { - Stitch.async( - Stitch.collect( - Seq( - applySideEffectsCandidateSourceCandidates( - target, - candidateSources, - candidatesFromCandidateSources), - applySideEffectsMergedCandidates(target, mergedCandidates), - applySideEffectsFilteredCandidates(target, filteredCandidates), - applySideEffectsRankedCandidates(target, rankedCandidates), - applySideEffectsTransformedCandidates(target, transformedCandidates), - applySideEffectsTruncatedCandidates(target, truncatedCandidates), - applySideEffectsResults(target, results) - ) - )) - } - - /* - In subclasses, override functions below to apply custom side effects at each step in pipeline. - Call super.applySideEffectsXYZ to scribe basic scribes implemented in this parent class - */ - def applySideEffectsCandidateSourceCandidates( - target: Target, - candidateSources: Seq[CandidateSource[Target, Candidate]], - candidatesFromCandidateSources: Seq[Candidate] - ): Stitch[Unit] = { - val candidatesGroupedByCandidateSources = - candidatesFromCandidateSources.groupBy( - _.getPrimaryCandidateSource.getOrElse(CandidateSourceIdentifier("NoCandidateSource"))) - - target.getOptionalUserId match { - case Some(userId) => - val userAgeOpt = SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays) - userAgeOpt match { - case Some(userAge) if userAge <= 30 => - candidateSources.map { candidateSource => - { - val candidateSourceStats = statsReceiver.scope(candidateSource.identifier.name) - - val isEmpty = - !candidatesGroupedByCandidateSources.keySet.contains(candidateSource.identifier) - - if (userAge <= 1) - candidateSourceStats - .scope("user_age", "1", "empty").counter(isEmpty.toString).incr() - if (userAge <= 7) - candidateSourceStats - .scope("user_age", "7", "empty").counter(isEmpty.toString).incr() - if (userAge <= 30) - candidateSourceStats - .scope("user_age", "30", "empty").counter(isEmpty.toString).incr() - } - } - case _ => Nil - } - case None => Nil - } - Stitch.Unit - } - - def applySideEffectsBaseCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = Stitch.Unit - - def applySideEffectsMergedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsFilteredCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsRankedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsTransformedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsTruncatedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsResults( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/BUILD b/follow-recommendations-service/thrift/src/main/thrift/BUILD deleted file mode 100644 index e5cbd19cf..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -create_thrift_libraries( - base_name = "thrift", - sources = ["*.thrift"], - platform = "java8", - tags = ["bazel-compatible"], - dependency_roots = [ - "finatra-internal/thrift/src/main/thrift", - "follow-recommendations-service/thrift/src/main/thrift/logging:thrift", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift", - "src/thrift/com/twitter/ads/adserver:adserver_common", - "src/thrift/com/twitter/ml/api:data", - "src/thrift/com/twitter/suggests/controller_data", - ], - generate_languages = [ - "java", - "scala", - "strato", - ], - provides_java_name = "follow-recommendations-java", - provides_scala_name = "follow-recommendations-scala", -) diff --git a/follow-recommendations-service/thrift/src/main/thrift/BUILD.docx b/follow-recommendations-service/thrift/src/main/thrift/BUILD.docx new file mode 100644 index 000000000..1b516de6b Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/BUILD.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/assembler.docx b/follow-recommendations-service/thrift/src/main/thrift/assembler.docx new file mode 100644 index 000000000..e84f632a6 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/assembler.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/assembler.thrift b/follow-recommendations-service/thrift/src/main/thrift/assembler.thrift deleted file mode 100644 index eb782d0fe..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/assembler.thrift +++ /dev/null @@ -1,42 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -struct Header { - 1: required Title title -} - -struct Title { - 1: required string text -} - -struct Footer { - 1: optional Action action -} - -struct Action { - 1: required string text - 2: required string actionURL -} - -struct UserList { - 1: required bool userBioEnabled - 2: required bool userBioTruncated - 3: optional i64 userBioMaxLines - 4: optional FeedbackAction feedbackAction -} - -struct Carousel { - 1: optional FeedbackAction feedbackAction -} - -union WTFPresentation { - 1: UserList userBioList - 2: Carousel carousel -} - -struct DismissUserId {} - -union FeedbackAction { - 1: DismissUserId dismissUserId -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/client_context.docx b/follow-recommendations-service/thrift/src/main/thrift/client_context.docx new file mode 100644 index 000000000..ed0d55918 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/client_context.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/client_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/client_context.thrift deleted file mode 100644 index adbdc407a..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/client_context.thrift +++ /dev/null @@ -1,19 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -// Caller/Client level specific context (e.g, user id/guest id/app id). -struct ClientContext { - 1: optional i64 userId(personalDataType='UserId') - 2: optional i64 guestId(personalDataType='GuestId') - 3: optional i64 appId(personalDataType='AppId') - 4: optional string ipAddress(personalDataType='IpAddress') - 5: optional string userAgent(personalDataType='UserAgent') - 6: optional string countryCode(personalDataType='InferredCountry') - 7: optional string languageCode(personalDataType='InferredLanguage') - 9: optional bool isTwoffice(personalDataType='InferredLocation') - 10: optional set userRoles - 11: optional string deviceId(personalDataType='DeviceId') - 12: optional i64 guestIdAds(personalDataType='GuestId') - 13: optional i64 guestIdMarketing(personalDataType='GuestId') -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/debug.docx b/follow-recommendations-service/thrift/src/main/thrift/debug.docx new file mode 100644 index 000000000..b4354c30a Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/debug.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/debug.thrift b/follow-recommendations-service/thrift/src/main/thrift/debug.thrift deleted file mode 100644 index a41c59114..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/debug.thrift +++ /dev/null @@ -1,73 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendation - -// These are broken into their own union -// because we can have features that are -// complex flavors of these (such as Seq) -union PrimitiveFeatureValue { - 1: i32 intValue - 2: i64 longValue - 3: string strValue - 4: bool boolValue -} - -union FeatureValue { - 1: PrimitiveFeatureValue primitiveValue -} - -struct DebugParams { - 1: optional map featureOverrides - 2: optional i64 randomizationSeed - 3: optional bool includeDebugInfoInResults - 4: optional bool doNotLog -} - -enum DebugCandidateSourceIdentifier { - UTT_INTERESTS_RELATED_USERS_SOURCE = 0 - UTT_PRODUCER_EXPANSION_SOURCE = 1 - UTT_SEED_ACCOUNT_SOURCE = 2 - BYF_USER_FOLLOW_CLUSTER_SIMS_SOURCE = 3 - BYF_USER_FOLLOW_CLUSTER_SOURCE = 4 - USER_FOLLOW_CLUSTER_SOURCE = 5 - RECENT_SEARCH_BASED_SOURCE = 6 - PEOPLE_ACTIVITY_RECENT_ENGAGEMENT_SOURCE = 7 - PEOPLE_ACTIVITY_RECENT_ENGAGEMENT_SIMS_SOURCE = 8, - REVERSE_PHONE_BOOK_SOURCE = 9, - REVERSE_EMAIL_BOOK_SOURCE = 10, - SIMS_DEBUG_STORE = 11, - UTT_PRODUCER_ONLINE_MBCG_SOURCE = 12, - BONUS_FOLLOW_CONDITIONAL_ENGAGEMENT_STORE = 13, - // 14 (BONUS_FOLLOW_PMI_STORE) was deleted as it's not used anymore - FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 15, - OFFLINE_STP = 16, - OFFLINE_STP_BIG = 17, - OFFLINE_MUTUAL_FOLLOW_EXPANSION = 18, - REPEATED_PROFILE_VISITS = 19, - TIME_DECAY_FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 20, - LINEAR_REGRESSION_FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 21, - REAL_GRAPH_EXPANSION_SOURCE = 22, - RELATABLE_ACCOUNTS_BY_INTEREST = 23, - EMAIL_TWEET_CLICK = 24, - GOOD_TWEET_CLICK_ENGAGEMENTS = 25, - ENGAGED_FOLLOWER_RATIO = 26, - TWEET_SHARE_ENGAGEMENTS = 27, - BULK_FRIEND_FOLLOWS = 28, - REAL_GRAPH_OON_V2_SOURCE = 30, - CROWD_SEARCH_ACCOUNTS = 31, - POP_GEOHASH = 32, - POP_COUNTRY = 33, - POP_COUNTRY_BACKFILL = 34, - TWEET_SHARER_TO_SHARE_RECIPIENT_ENGAGEMENTS = 35, - TWEET_AUTHOR_TO_SHARE_RECIPIENT_ENGAGEMENTS = 36, - BULK_FRIEND_FOLLOWS_NEW_USER = 37, - ONLINE_STP_EPSCORER = 38, - ORGANIC_FOLLOW_ACCOUNTS = 39, - NUX_LO_HISTORY = 40, - TRAFFIC_ATTRIBUTION_ACCOUNTS = 41, - ONLINE_STP_RAW_ADDRESS_BOOK = 42, - POP_GEOHASH_QUALITY_FOLLOW = 43, - NOTIFICATION_ENGAGEMENT = 44, - EFR_BY_WORLDWIDE_PICTURE_PRODUCER = 45, - POP_GEOHASH_REAL_GRAPH = 46, -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/display_context.docx b/follow-recommendations-service/thrift/src/main/thrift/display_context.docx new file mode 100644 index 000000000..1657bf2c9 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/display_context.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/display_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/display_context.thrift deleted file mode 100644 index cfd613b71..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/display_context.thrift +++ /dev/null @@ -1,62 +0,0 @@ -include "flows.thrift" -include "recently_engaged_user_id.thrift" - -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -struct Profile { - 1: required i64 profileId(personalDataType='UserId') -}(hasPersonalData='true') - -struct Search { - 1: required string searchQuery(personalDataType='SearchQuery') -}(hasPersonalData='true') - -struct Rux { - 1: required i64 focalAuthorId(personalDataType='UserId') -}(hasPersonalData='true') - -struct Topic { - 1: required i64 topicId(personalDataType = 'TopicFollow') -}(hasPersonalData='true') - -struct ReactiveFollow { - 1: required list followedUserIds(personalDataType='UserId') -}(hasPersonalData='true') - -struct NuxInterests { - 1: optional flows.FlowContext flowContext // set for recommendation inside an interactive flow - 2: optional list uttInterestIds // if provided, we use these interestIds for generating candidates instead of for example fetching user selected interests -}(hasPersonalData='true') - -struct AdCampaignTarget { - 1: required list similarToUserIds(personalDataType='UserId') -}(hasPersonalData='true') - -struct ConnectTab { - 1: required list byfSeedUserIds(personalDataType='UserId') - 2: required list similarToUserIds(personalDataType='UserId') - 3: required list recentlyEngagedUserIds -}(hasPersonalData='true') - -struct SimilarToUser { - 1: required i64 similarToUserId(personalDataType='UserId') -}(hasPersonalData='true') - -struct PostNuxFollowTask { - 1: optional flows.FlowContext flowContext // set for recommendation inside an interactive flow -}(hasPersonalData='true') - -union DisplayContext { - 1: Profile profile - 2: Search search - 3: Rux rux - 4: Topic topic - 5: ReactiveFollow reactiveFollow - 6: NuxInterests nuxInterests - 7: AdCampaignTarget adCampaignTarget - 8: ConnectTab connectTab - 9: SimilarToUser similarToUser - 10: PostNuxFollowTask postNuxFollowTask -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/display_location.docx b/follow-recommendations-service/thrift/src/main/thrift/display_location.docx new file mode 100644 index 000000000..618845c1d Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/display_location.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/display_location.thrift b/follow-recommendations-service/thrift/src/main/thrift/display_location.thrift deleted file mode 100644 index d94b9842e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/display_location.thrift +++ /dev/null @@ -1,55 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -enum DisplayLocation { - SIDEBAR = 0 - PROFILE_SIDEBAR = 2 - CLUSTER_FOLLOW = 7 - NEW_USER_SARUS_BACKFILL = 12 - PROFILE_DEVICE_FOLLOW = 23 - RECOS_BACKFILL = 32 - HOME_TIMELINE = 39 # HOME_TIMELINE_WTF in Hermit - PROFILE_TOP_FOLLOWING = 42 - PROFILE_TOP_FOLLOWERS = 43 - PEOPLE_PLUS_PLUS = 47 - EXPLORE_TAB = 57 - MagicRecs = 59 # Account recommendation in notification - AB_UPLOAD_INJECTION = 60 - /** - * To prevent setting 2 display locations with the same index in FRS. - * - * The display location should be added to the following files: - * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift - * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift - * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala - */ - CAMPAIGN_FORM = 61 - RUX_LANDING_PAGE = 62 - PROFILE_BONUS_FOLLOW = 63 - ELECTION_EXPLORE_WTF = 64 - HTL_BONUS_FOLLOW = 65 - TOPIC_LANDING_PAGE_HEADER = 66 - NUX_PYMK = 67 - NUX_INTERESTS = 68 - REACTIVE_FOLLOW = 69 - RUX_PYMK = 70 - INDIA_COVID19_CURATED_ACCOUNTS_WTF = 71 - NUX_TOPIC_BONUS_FOLLOW = 72 - TWEET_NOTIFICATION_RECS = 73 - HTL_SPACE_HOSTS = 74 - POST_NUX_FOLLOW_TASK = 75 - TOPIC_LANDING_PAGE = 76 - USER_TYPEAHEAD_PREFETCH = 77 - HOME_TIMELINE_RELATABLE_ACCOUNTS = 78 - NUX_GEO_CATEGORY = 79 - NUX_INTERESTS_CATEGORY = 80 - NUX_PYMK_CATEGORY = 81 - TOP_ARTICLES = 82 - HOME_TIMELINE_TWEET_RECS = 83 - HTL_BULK_FRIEND_FOLLOWS = 84 - NUX_AUTO_FOLLOW = 85 - SEARCH_BONUS_FOLLOW = 86 - CONTENT_RECOMMENDER = 87 - HOME_TIMELINE_REVERSE_CHRON = 88 -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/engagementType.docx b/follow-recommendations-service/thrift/src/main/thrift/engagementType.docx new file mode 100644 index 000000000..4de496df0 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/engagementType.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/engagementType.thrift b/follow-recommendations-service/thrift/src/main/thrift/engagementType.thrift deleted file mode 100644 index ef028d008..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/engagementType.thrift +++ /dev/null @@ -1,11 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -enum EngagementType { - Click = 0 - Like = 1 - Mention = 2 - Retweet = 3 - ProfileView = 4 -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/flows.docx b/follow-recommendations-service/thrift/src/main/thrift/flows.docx new file mode 100644 index 000000000..cc4112ffc Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/flows.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/flows.thrift b/follow-recommendations-service/thrift/src/main/thrift/flows.thrift deleted file mode 100644 index 894ebf81e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/flows.thrift +++ /dev/null @@ -1,20 +0,0 @@ -/* - * This file defines additional thrift objects that should be specified in FRS request for context of recommendation, specifically the previous recommendations / new interactions in an interactive flow (series of follow steps). These typically are sent from OCF - */ - -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -struct FlowRecommendation { - 1: required i64 userId(personalDataType='UserId') -}(hasPersonalData='true') - -struct RecommendationStep { - 1: required list recommendations - 2: required set followedUserIds(personalDataType='UserId') -}(hasPersonalData='true') - -struct FlowContext { - 1: required list steps -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.docx b/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.docx new file mode 100644 index 000000000..21a6cd79e Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.thrift b/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.thrift deleted file mode 100644 index 40aadc0b6..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.thrift +++ /dev/null @@ -1,100 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "assembler.thrift" -include "client_context.thrift" -include "debug.thrift" -include "display_context.thrift" -include "display_location.thrift" -include "recommendations.thrift" -include "recently_engaged_user_id.thrift" - -include "finatra-thrift/finatra_thrift_exceptions.thrift" -include "com/twitter/product_mixer/core/pipeline_execution_result.thrift" - -struct RecommendationRequest { - 1: required client_context.ClientContext clientContext - 2: required display_location.DisplayLocation displayLocation - 3: optional display_context.DisplayContext displayContext - // Max results to return - 4: optional i32 maxResults - // Cursor to continue returning results if any - 5: optional string cursor - // IDs of Content to exclude from recommendations - 6: optional list excludedIds(personalDataType='UserId') - // Whether to also get promoted content - 7: optional bool fetchPromotedContent - 8: optional debug.DebugParams debugParams - 9: optional string userLocationState(personalDataType='InferredLocation') -}(hasPersonalData='true') - - -struct RecommendationResponse { - 1: required list recommendations -}(hasPersonalData='true') - -// for scoring a list of candidates, while logging hydrated features -struct ScoringUserRequest { - 1: required client_context.ClientContext clientContext - 2: required display_location.DisplayLocation displayLocation - 3: required list candidates - 4: optional debug.DebugParams debugParams -}(hasPersonalData='true') - -struct ScoringUserResponse { - 1: required list candidates // empty for now -}(hasPersonalData='true') - -// for getting the list of candidates generated by a single candidate source -struct DebugCandidateSourceRequest { - 1: required client_context.ClientContext clientContext - 2: required debug.DebugCandidateSourceIdentifier candidateSource - 3: optional list uttInterestIds - 4: optional debug.DebugParams debugParams - 5: optional list recentlyFollowedUserIds - 6: optional list recentlyEngagedUserIds - 7: optional list byfSeedUserIds - 8: optional list similarToUserIds - 9: required bool applySgsPredicate - 10: optional i32 maxResults -}(hasPersonalData='true') - -service FollowRecommendationsThriftService { - RecommendationResponse getRecommendations(1: RecommendationRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - RecommendationDisplayResponse getRecommendationDisplayResponse(1: RecommendationRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - // temporary endpoint for feature hydration and logging for data collection. - ScoringUserResponse scoreUserCandidates(1: ScoringUserRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - // Debug endpoint for getting recommendations of a single candidate source. We can remove this endpoint when ProMix provide this functionality and we integrate with it. - RecommendationResponse debugCandidateSource(1: DebugCandidateSourceRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - - // Get the full execution log for a pipeline (used by our debugging tools) - pipeline_execution_result.PipelineExecutionResult executePipeline(1: RecommendationRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) -} - -struct RecommendationDisplayResponse { - 1: required list hydratedRecommendation - 2: optional assembler.Header header - 3: optional assembler.Footer footer - 4: optional assembler.WTFPresentation wtfPresentation -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.docx b/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.docx new file mode 100644 index 000000000..f34099e41 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.thrift b/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.thrift deleted file mode 100644 index 404b0ae29..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.thrift +++ /dev/null @@ -1,9 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -// struct used for storing the history of computing and serving of recommendations to a user -struct FollowRecommendationsServingHistory { - 1: required i64 lastComputationTimeMs (personalDataType = 'PrivateTimestamp') - 2: required i64 lastServingTimeMs (personalDataType = 'PrivateTimestamp') -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD b/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD deleted file mode 100644 index 2f769d498..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -create_thrift_libraries( - base_name = "thrift", - sources = ["*.thrift"], - platform = "java8", - tags = ["bazel-compatible"], - dependency_roots = [ - "src/thrift/com/twitter/ads/adserver:adserver_common", - "src/thrift/com/twitter/ml/api:data", - "src/thrift/com/twitter/suggests/controller_data", - ], - generate_languages = [ - "java", - "scala", - "strato", - ], - provides_java_name = "follow-recommendations-logging-java", - provides_scala_name = "follow-recommendations-logging-scala", -) diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD.docx new file mode 100644 index 000000000..b7c090f45 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.docx new file mode 100644 index 000000000..5e0038eab Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.thrift deleted file mode 100644 index 2b6e454b2..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.thrift +++ /dev/null @@ -1,14 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -// Offline equal of ClientContext -struct OfflineClientContext { - 1: optional i64 userId(personalDataType='UserId') - 2: optional i64 guestId(personalDataType='GuestId') - 3: optional i64 appId(personalDataType='AppId') - 4: optional string countryCode(personalDataType='InferredCountry') - 5: optional string languageCode(personalDataType='InferredLanguage') - 6: optional i64 guestIdAds(personalDataType='GuestId') - 7: optional i64 guestIdMarketing(personalDataType='GuestId') -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/debug.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/debug.docx new file mode 100644 index 000000000..dcd23c153 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/debug.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/debug.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/debug.thrift deleted file mode 100644 index 882dca005..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/debug.thrift +++ /dev/null @@ -1,8 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendation.logging - -// subset of DebugParams -struct OfflineDebugParams { - 1: optional i64 randomizationSeed // track if the request was randomly ranked or not -}(persisted='true', hasPersonalData='false') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.docx new file mode 100644 index 000000000..55a076bc4 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.thrift deleted file mode 100644 index c38850011..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.thrift +++ /dev/null @@ -1,66 +0,0 @@ -include "logging/flows.thrift" -include "logging/recently_engaged_user_id.thrift" - -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -// Offline equal of Profile DisplayContext -struct OfflineProfile { - 1: required i64 profileId(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -// Offline equal of Search DisplayContext -struct OfflineSearch { - 1: required string searchQuery(personalDataType='SearchQuery') -}(persisted='true', hasPersonalData='true') - -// Offline equal of Rux Landing Page DisplayContext -struct OfflineRux { - 1: required i64 focalAuthorId(personalDataType="UserId") -}(persisted='true', hasPersonalData='true') - -// Offline equal of Topic DisplayContext -struct OfflineTopic { - 1: required i64 topicId(personalDataType = 'TopicFollow') -}(persisted='true', hasPersonalData='true') - -struct OfflineReactiveFollow { - 1: required list followedUserIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineNuxInterests { - 1: optional flows.OfflineFlowContext flowContext // set for recommendation inside an interactive flow -}(persisted='true', hasPersonalData='true') - -struct OfflineAdCampaignTarget { - 1: required list similarToUserIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineConnectTab { - 1: required list byfSeedUserIds(personalDataType='UserId') - 2: required list similarToUserIds(personalDataType='UserId') - 3: required list recentlyEngagedUserIds -}(persisted='true', hasPersonalData='true') - -struct OfflineSimilarToUser { - 1: required i64 similarToUserId(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflinePostNuxFollowTask { - 1: optional flows.OfflineFlowContext flowContext // set for recommendation inside an interactive flow -}(persisted='true', hasPersonalData='true') - -// Offline equal of DisplayContext -union OfflineDisplayContext { - 1: OfflineProfile profile - 2: OfflineSearch search - 3: OfflineRux rux - 4: OfflineTopic topic - 5: OfflineReactiveFollow reactiveFollow - 6: OfflineNuxInterests nuxInterests - 7: OfflineAdCampaignTarget adCampaignTarget - 8: OfflineConnectTab connectTab - 9: OfflineSimilarToUser similarToUser - 10: OfflinePostNuxFollowTask postNuxFollowTask -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.docx new file mode 100644 index 000000000..9629d7185 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift deleted file mode 100644 index a4dbbecd4..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift +++ /dev/null @@ -1,55 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -/** - * Make sure you add the new DL to the following files and redeploy our attribution jobs - * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift - * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift - * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala - */ - -// Offline equal of DisplayLocation -enum OfflineDisplayLocation { - SIDEBAR = 0 - PROFILE_SIDEBAR = 2 - CLUSTER_FOLLOW = 7 - NEW_USER_SARUS_BACKFILL = 12 - PROFILE_DEVICE_FOLLOW = 23 - RECOS_BACKFILL = 32 - HOME_TIMELINE = 39 - PROFILE_TOP_FOLLOWING = 42 - PROFILE_TOP_FOLLOWERS = 43 - PEOPLE_PLUS_PLUS = 47 - EXPLORE_TAB = 57 - MagicRecs = 59 - AB_UPLOAD_INJECTION = 60 - CAMPAIGN_FORM = 61 - RUX_LANDING_PAGE = 62 - PROFILE_BONUS_FOLLOW = 63 - ELECTION_EXPLORE_WTF = 64 - HTL_BONUS_FOLLOW = 65 - TOPIC_LANDING_PAGE_HEADER = 66 - NUX_PYMK = 67 - NUX_INTERESTS = 68 - REACTIVE_FOLLOW = 69 - RUX_PYMK = 70 - INDIA_COVID19_CURATED_ACCOUNTS_WTF=71 - NUX_TOPIC_BONUS_FOLLOW = 72 - TWEET_NOTIFICATION_RECS = 73 - HTL_SPACE_HOSTS = 74 - POST_NUX_FOLLOW_TASK = 75 - TOPIC_LANDING_PAGE = 76 - USER_TYPEAHEAD_PREFETCH = 77 - HOME_TIMELINE_RELATABLE_ACCOUNTS = 78 - NUX_GEO_CATEGORY = 79 - NUX_INTERESTS_CATEGORY = 80 - NUX_PYMK_CATEGORY = 81 - TOP_ARTICLES = 82 - HOME_TIMELINE_TWEET_RECS = 83 - HTL_BULK_FRIEND_FOLLOWS = 84 - NUX_AUTO_FOLLOW = 85 - SEARCH_BONUS_FOLLOW = 86 - CONTENT_RECOMMENDER = 87 - HOME_TIMELINE_REVERSE_CHRON = 88 -}(persisted='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.docx new file mode 100644 index 000000000..4f5ed301f Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.thrift deleted file mode 100644 index 75191f16f..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.thrift +++ /dev/null @@ -1,11 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -enum EngagementType { - Click = 0 - Like = 1 - Mention = 2 - Retweet = 3 - ProfileView = 4 -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/flows.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/flows.docx new file mode 100644 index 000000000..86dbd8e76 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/flows.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/flows.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/flows.thrift deleted file mode 100644 index 98551c08e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/flows.thrift +++ /dev/null @@ -1,16 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -struct OfflineFlowRecommendation { - 1: required i64 userId(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationStep { - 1: required list recommendations - 2: required set followedUserIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineFlowContext { - 1: required list steps -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/logs.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/logs.docx new file mode 100644 index 000000000..9b39481f5 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/logs.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/logs.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/logs.thrift deleted file mode 100644 index 33f09cfb9..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/logs.thrift +++ /dev/null @@ -1,72 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "client_context.thrift" -include "debug.thrift" -include "display_context.thrift" -include "display_location.thrift" -include "recommendations.thrift" - -struct OfflineRecommendationRequest { - 1: required client_context.OfflineClientContext clientContext - 2: required display_location.OfflineDisplayLocation displayLocation - 3: optional display_context.OfflineDisplayContext displayContext - 4: optional i32 maxResults - 5: optional string cursor - 6: optional list excludedIds(personalDataType='UserId') - 7: optional bool fetchPromotedContent - 8: optional debug.OfflineDebugParams debugParams -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationResponse { - 1: required list recommendations -}(persisted='true', hasPersonalData='true') - -struct RecommendationLog { - 1: required OfflineRecommendationRequest request - 2: required OfflineRecommendationResponse response - 3: required i64 timestampMs -}(persisted='true', hasPersonalData='true') - -struct OfflineScoringUserRequest { - 1: required client_context.OfflineClientContext clientContext - 2: required display_location.OfflineDisplayLocation displayLocation - 3: required list candidates -}(persisted='true', hasPersonalData='true') - -struct OfflineScoringUserResponse { - 1: required list candidates -}(persisted='true', hasPersonalData='true') - -struct ScoredUsersLog { - 1: required OfflineScoringUserRequest request - 2: required OfflineScoringUserResponse response - 3: required i64 timestampMs -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationFlowUserMetadata { - 1: optional i32 userSignupAge(personalDataType = 'AgeOfAccount') - 2: optional string userState(personalDataType = 'UserState') -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationFlowSignals { - 1: optional string countryCode(personalDataType='InferredCountry') -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationFlowCandidateSourceCandidates { - 1: required string candidateSourceName - 2: required list candidateUserIds(personalDataType='UserId') - 3: optional list candidateUserScores -}(persisted='true', hasPersonalData='true') - -struct RecommendationFlowLog { - 1: required client_context.OfflineClientContext clientContext - 2: optional OfflineRecommendationFlowUserMetadata userMetadata - 3: optional OfflineRecommendationFlowSignals signals - 4: required i64 timestampMs - 5: required string recommendationFlowIdentifier - 6: optional list filteredCandidates - 7: optional list rankedCandidates - 8: optional list truncatedCandidates -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.docx new file mode 100644 index 000000000..93bd9e157 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.thrift deleted file mode 100644 index 6fc24d919..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.thrift +++ /dev/null @@ -1,62 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -// Proof based on Follow relationship -struct FollowProof { - 1: required list userIds(personalDataType='UserId') - 2: required i32 numIds(personalDataType='CountOfFollowersAndFollowees') -}(persisted='true', hasPersonalData='true') - -// Similar to userIds in the context (e.g. profileId) -struct SimilarToProof { - 1: required list userIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -// Proof based on geo location -struct PopularInGeoProof { - 1: required string location(personalDataType='InferredLocation') -}(persisted='true', hasPersonalData='true') - -// Proof based on ttt interest -struct TttInterestProof { - 1: required i64 interestId(personalDataType='ProvidedInterests') - 2: required string interestDisplayName(personalDataType='ProvidedInterests') -}(persisted='true', hasPersonalData='true') - -// Proof based on topics -struct TopicProof { - 1: required i64 topicId(personalDataType='ProvidedInterests') -}(persisted='true', hasPersonalData='true') - -// Proof based on custom interest / search queries -struct CustomInterestProof { - 1: required string customerInterest(personalDataType='SearchQuery') -}(persisted='true', hasPersonalData='true') - -// Proof based on tweet authors -struct TweetsAuthorProof { - 1: required list tweetIds(personalDataType='TweetId') -}(persisted='true', hasPersonalData='true') - -// Proof candidate is of device follow type -struct DeviceFollowProof { - 1: required bool isDeviceFollow(personalDataType='OtherDeviceInfo') -}(persisted='true', hasPersonalData='true') - -// Account level proof that should be attached to each candidate -struct AccountProof { - 1: optional FollowProof followProof - 2: optional SimilarToProof similarToProof - 3: optional PopularInGeoProof popularInGeoProof - 4: optional TttInterestProof tttInterestProof - 5: optional TopicProof topicProof - 6: optional CustomInterestProof customInterestProof - 7: optional TweetsAuthorProof tweetsAuthorProof - 8: optional DeviceFollowProof deviceFollowProof - -}(persisted='true', hasPersonalData='true') - -struct Reason { - 1: optional AccountProof accountProof -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.docx new file mode 100644 index 000000000..d768ba197 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.thrift deleted file mode 100644 index f0af960b9..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.thrift +++ /dev/null @@ -1,10 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "engagementType.thrift" - -struct RecentlyEngagedUserId { - 1: required i64 id(personalDataType='UserId') - 2: required engagementType.EngagementType engagementType -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.docx new file mode 100644 index 000000000..108d89f84 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.thrift deleted file mode 100644 index bf94e41b8..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.thrift +++ /dev/null @@ -1,26 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "com/twitter/ads/adserver/adserver_common.thrift" -include "reasons.thrift" -include "tracking.thrift" -include "scoring.thrift" - -// Offline equal of UserRecommendation -struct OfflineUserRecommendation { - 1: required i64 userId(personalDataType='UserId') - // reason for this suggestions, eg: social context - 2: optional reasons.Reason reason - // present if it is a promoted account - 3: optional adserver_common.AdImpression adImpression - // tracking token (unserialized) for attribution - 4: optional tracking.TrackingToken trackingToken - // scoring details - 5: optional scoring.ScoringDetails scoringDetails -}(persisted='true', hasPersonalData='true') - -// Offline equal of Recommendation -union OfflineRecommendation { - 1: OfflineUserRecommendation user -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.docx new file mode 100644 index 000000000..50861872b Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.thrift deleted file mode 100644 index e1524662d..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.thrift +++ /dev/null @@ -1,38 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "com/twitter/ml/api/data.thrift" - -struct CandidateSourceDetails { - 1: optional map candidateSourceScores - 2: optional i32 primarySource -}(persisted='true', hasPersonalData='false') - -struct Score { - 1: required double value - 2: optional string rankerId - 3: optional string scoreType -}(persisted='true', hasPersonalData='false') // scoring and ranking info per ranking stage - -// Contains (1) the ML-based heavy ranker and score (2) scores and rankers in producer experiment framework -struct Scores { - 1: required list scores - 2: optional string selectedRankerId - 3: required bool isInProducerScoringExperiment -}(persisted='true', hasPersonalData='false') - -struct RankingInfo { - 1: optional Scores scores - 2: optional i32 rank -}(persisted='true', hasPersonalData='false') - -// this encapsulates all information related to the ranking process from generation to scoring -struct ScoringDetails { - 1: optional CandidateSourceDetails candidateSourceDetails - 2: optional double score // The ML-based heavy ranker score - 3: optional data.DataRecord dataRecord - 4: optional list rankerIds // all ranker ids, including (1) ML-based heavy ranker (2) non-ML adhoc rankers - 5: optional map infoPerRankingStage // scoring and ranking info per ranking stage -}(persisted='true', hasPersonalData='true') - diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.docx b/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.docx new file mode 100644 index 000000000..4fc06b78f Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.thrift deleted file mode 100644 index 067ba1a46..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.thrift +++ /dev/null @@ -1,16 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "com/twitter/suggests/controller_data/controller_data.thrift" -include "display_location.thrift" - -struct TrackingToken { - // trace-id of the request - 1: required i64 sessionId (personalDataType='SessionId') - 2: optional display_location.OfflineDisplayLocation displayLocation - // 64-bit encoded binary attributes of our recommendation - 3: optional controller_data.ControllerData controllerData - // WTF Algorithm Id (backward compatibility) - 4: optional i32 algoId -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/reasons.docx b/follow-recommendations-service/thrift/src/main/thrift/reasons.docx new file mode 100644 index 000000000..018438fb6 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/reasons.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/reasons.thrift b/follow-recommendations-service/thrift/src/main/thrift/reasons.thrift deleted file mode 100644 index 299e88885..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/reasons.thrift +++ /dev/null @@ -1,61 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -// Proof based on Follow relationship -struct FollowProof { - 1: required list userIds(personalDataType='UserId') - 2: required i32 numIds(personalDataType='CountOfFollowersAndFollowees') -}(hasPersonalData='true') - -// Similar to userIds in the context (e.g. profileId) -struct SimilarToProof { - 1: required list userIds(personalDataType='UserId') -}(hasPersonalData='true') - -// Proof based on geo location -struct PopularInGeoProof { - 1: required string location(personalDataType='InferredLocation') -}(hasPersonalData='true') - -// Proof based on ttt interest -struct TttInterestProof { - 1: required i64 interestId(personalDataType='ProvidedInterests') - 2: required string interestDisplayName(personalDataType='ProvidedInterests') -}(hasPersonalData='true') - -// Proof based on topics -struct TopicProof { - 1: required i64 topicId(personalDataType='ProvidedInterests') -}(hasPersonalData='true') - -// Proof based on custom interest / search queries -struct CustomInterestProof { - 1: required string query(personalDataType='SearchQuery') -}(hasPersonalData='true') - -// Proof based on tweet authors -struct TweetsAuthorProof { - 1: required list tweetIds(personalDataType='TweetId') -}(hasPersonalData='true') - -// Proof candidate is of device follow type -struct DeviceFollowProof { - 1: required bool isDeviceFollow(personalDataType='OtherDeviceInfo') -}(hasPersonalData='true') - -// Account level proof that should be attached to each candidate -struct AccountProof { - 1: optional FollowProof followProof - 2: optional SimilarToProof similarToProof - 3: optional PopularInGeoProof popularInGeoProof - 4: optional TttInterestProof tttInterestProof - 5: optional TopicProof topicProof - 6: optional CustomInterestProof customInterestProof - 7: optional TweetsAuthorProof tweetsAuthorProof - 8: optional DeviceFollowProof deviceFollowProof -}(hasPersonalData='true') - -struct Reason { - 1: optional AccountProof accountProof -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.docx b/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.docx new file mode 100644 index 000000000..d0fa13f80 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.thrift b/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.thrift deleted file mode 100644 index 6a13bd31e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.thrift +++ /dev/null @@ -1,10 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "engagementType.thrift" - -struct RecentlyEngagedUserId { - 1: required i64 id(personalDataType='UserId') - 2: required engagementType.EngagementType engagementType -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/recommendations.docx b/follow-recommendations-service/thrift/src/main/thrift/recommendations.docx new file mode 100644 index 000000000..9eedb040e Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/recommendations.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/recommendations.thrift b/follow-recommendations-service/thrift/src/main/thrift/recommendations.thrift deleted file mode 100644 index 1070bb11c..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/recommendations.thrift +++ /dev/null @@ -1,40 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "com/twitter/ads/adserver/adserver_common.thrift" -include "debug.thrift" -include "reasons.thrift" -include "scoring.thrift" - -struct UserRecommendation { - 1: required i64 userId(personalDataType='UserId') - // reason for this suggestions, eg: social context - 2: optional reasons.Reason reason - // present if it is a promoted account - 3: optional adserver_common.AdImpression adImpression - // tracking token for attribution - 4: optional string trackingInfo - // scoring details - 5: optional scoring.ScoringDetails scoringDetails - 6: optional string recommendationFlowIdentifier - // FeatureSwitch overrides for candidates: - 7: optional map featureOverrides -}(hasPersonalData='true') - -union Recommendation { - 1: UserRecommendation user -}(hasPersonalData='true') - -struct HydratedUserRecommendation { - 1: required i64 userId(personalDataType='UserId') - 2: optional string socialProof - // present if it is a promoted account, used by clients for determining ad impression - 3: optional adserver_common.AdImpression adImpression - // tracking token for attribution - 4: optional string trackingInfo -}(hasPersonalData='true') - -union HydratedRecommendation { - 1: HydratedUserRecommendation hydratedUserRecommendation -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/scoring.docx b/follow-recommendations-service/thrift/src/main/thrift/scoring.docx new file mode 100644 index 000000000..264136037 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/scoring.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/scoring.thrift b/follow-recommendations-service/thrift/src/main/thrift/scoring.thrift deleted file mode 100644 index 33111baf8..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/scoring.thrift +++ /dev/null @@ -1,49 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "com/twitter/ml/api/data.thrift" - -struct CandidateSourceDetails { - 1: optional map candidateSourceScores - 2: optional i32 primarySource - 3: optional map candidateSourceRanks -}(hasPersonalData='false') - -struct Score { - 1: required double value - 2: optional string rankerId - 3: optional string scoreType -}(hasPersonalData='false') - -// Contains (1) the ML-based heavy ranker and score (2) scores and rankers in producer experiment framework -struct Scores { - 1: required list scores - 2: optional string selectedRankerId - 3: required bool isInProducerScoringExperiment -}(hasPersonalData='false') - -struct RankingInfo { - 1: optional Scores scores - 2: optional i32 rank -}(hasPersonalData='false') - -// this encapsulates all information related to the ranking process from generation to scoring -struct ScoringDetails { - 1: optional CandidateSourceDetails candidateSourceDetails - 2: optional double score - 3: optional data.DataRecord dataRecord - 4: optional list rankerIds - 5: optional DebugDataRecord debugDataRecord // this field is not logged as it's only used for debugging - 6: optional map infoPerRankingStage // scoring and ranking info per ranking stage -}(hasPersonalData='true') - -// exactly the same as a data record, except that we store the feature name instead of the id -struct DebugDataRecord { - 1: optional set binaryFeatures; // stores BINARY features - 2: optional map continuousFeatures; // stores CONTINUOUS features - 3: optional map discreteFeatures; // stores DISCRETE features - 4: optional map stringFeatures; // stores STRING features - 5: optional map> sparseBinaryFeatures; // stores sparse BINARY features - 6: optional map> sparseContinuousFeatures; // sparse CONTINUOUS features -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/tracking.docx b/follow-recommendations-service/thrift/src/main/thrift/tracking.docx new file mode 100644 index 000000000..cb8719fa4 Binary files /dev/null and b/follow-recommendations-service/thrift/src/main/thrift/tracking.docx differ diff --git a/follow-recommendations-service/thrift/src/main/thrift/tracking.thrift b/follow-recommendations-service/thrift/src/main/thrift/tracking.thrift deleted file mode 100644 index 81111ead8..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/tracking.thrift +++ /dev/null @@ -1,17 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "com/twitter/suggests/controller_data/controller_data.thrift" -include "display_location.thrift" - -// struct used for tracking/attribution purposes in our offline pipelines -struct TrackingToken { - // trace-id of the request - 1: required i64 sessionId (personalDataType='SessionId') - 2: optional display_location.DisplayLocation displayLocation - // 64-bit encoded binary attributes of our recommendation - 3: optional controller_data.ControllerData controllerData - // WTF Algorithm Id (backward compatibility) - 4: optional i32 algoId -}(hasPersonalData='true') diff --git a/graph-feature-service/BUILD.bazel b/graph-feature-service/BUILD.bazel deleted file mode 100644 index afad5ce5d..000000000 --- a/graph-feature-service/BUILD.bazel +++ /dev/null @@ -1,67 +0,0 @@ -alias( - name = "graph_feature_service-server", - target = ":graph_feature_service-server_lib", -) - -target( - name = "graph_feature_service-server_lib", - dependencies = [ - "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server", - ], -) - -alias( - name = "graph_feature_service-worker", - target = ":graph_feature_service-worker_lib", -) - -target( - name = "graph_feature_service-worker_lib", - dependencies = [ - "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/worker", - ], -) - -jvm_binary( - name = "server-bin", - basename = "graph_feature_service-server", - main = "com.twitter.graph_feature_service.server.Main", - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":graph_feature_service-server", - "3rdparty/jvm/ch/qos/logback:logback-classic", - "finagle/finagle-zipkin-scribe/src/main/scala", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server/logback-classic/src/main/scala", - ], -) - -jvm_binary( - name = "worker-bin", - basename = "graph_feature_service-worker", - main = "com.twitter.graph_feature_service.worker.Main", - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":graph_feature_service-worker", - "3rdparty/jvm/ch/qos/logback:logback-classic", - "finagle/finagle-zipkin-scribe/src/main/scala", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server/logback-classic/src/main/scala", - ], -) - -jvm_app( - name = "server-bundle", - basename = "graph_feature_service-server-dist", - binary = ":server-bin", - tags = ["bazel-compatible"], -) - -jvm_app( - name = "worker-bundle", - basename = "graph_feature_service-worker-dist", - binary = ":worker-bin", - tags = ["bazel-compatible"], -) diff --git a/graph-feature-service/BUILD.docx b/graph-feature-service/BUILD.docx new file mode 100644 index 000000000..e15ed83a8 Binary files /dev/null and b/graph-feature-service/BUILD.docx differ diff --git a/graph-feature-service/README.docx b/graph-feature-service/README.docx new file mode 100644 index 000000000..cf99c4c1f Binary files /dev/null and b/graph-feature-service/README.docx differ diff --git a/graph-feature-service/README.md b/graph-feature-service/README.md deleted file mode 100644 index 8305988b6..000000000 --- a/graph-feature-service/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Graph Feature Service - -Graph Feature Service (GFS) is a distributed system that can provide various graph features for given pairs of users. For instance, given source user A and candidate user C, GFS can answer questions like “how many of A’s followings have favorited C”, “how many of A’s followings are following C”, and “how much C is similar to the users that A has favorited“. \ No newline at end of file diff --git a/graph-feature-service/doc/common.docx b/graph-feature-service/doc/common.docx new file mode 100644 index 000000000..c8ca12321 Binary files /dev/null and b/graph-feature-service/doc/common.docx differ diff --git a/graph-feature-service/doc/common.md b/graph-feature-service/doc/common.md deleted file mode 100644 index 70d293c84..000000000 --- a/graph-feature-service/doc/common.md +++ /dev/null @@ -1,62 +0,0 @@ -# Common thrift types - -GFS uses several thrift datastructures which are common to multiple queries. They are listed below. - -## EdgeType - -`EdgeType` is a thrift enum which specifies which edge types to query for the graph. - -```thrift -enum EdgeType { - FOLLOWING, - FOLLOWED_BY, - FAVORITE, - FAVORITED_BY, - RETWEET, - RETWEETED_BY, - REPLY, - REPLYED_BY, - MENTION, - MENTIONED_BY, - MUTUAL_FOLLOW, - SIMILAR_TO, // more edge types (like block, report, etc.) can be supported later. - RESERVED_12, - RESERVED_13, - RESERVED_14, - RESERVED_15, - RESERVED_16, - RESERVED_17, - RESERVED_18, - RESERVED_19, - RESERVED_20 -} -``` - -For an example of how this is used, consider the `GetNeighbors` query. If we set the `edgeType` field -of the `GfsNeighborsRequest`, the response will contain all the users that the specified user follows. -If, on the other hand, we set `edgeType` to be `FollowedBy` it will return all the users who are -followed by the specified user. - -## FeatureType - -`FeatureType` is a thrift struct which is used in queries which require two edge types. - -```thrift -struct FeatureType { - 1: required EdgeType leftEdgeType // edge type from source user - 2: required EdgeType rightEdgeType // edge type from candidate user -}(persisted="true") -``` - -## UserWithScore - -The candidate generation queries return lists of candidates together with a computed score for the -relevant feature. `UserWithScore` is a thrift struct which bundles together a candidate's ID with -the score. - -```thrift -struct UserWithScore { - 1: required i64 userId - 2: required double score -} -``` diff --git a/graph-feature-service/doc/getintersection.docx b/graph-feature-service/doc/getintersection.docx new file mode 100644 index 000000000..6528c9284 Binary files /dev/null and b/graph-feature-service/doc/getintersection.docx differ diff --git a/graph-feature-service/doc/getintersection.md b/graph-feature-service/doc/getintersection.md deleted file mode 100644 index 6053729bb..000000000 --- a/graph-feature-service/doc/getintersection.md +++ /dev/null @@ -1,43 +0,0 @@ -# GetIntersection - -## Request and response syntax - -A `GetIntersection` call takes as input a `GfsIntersectionRequest` thrift struct. - -```thrift -struct GfsIntersectionRequest { - 1: required i64 userId - 2: required list candidateUserIds - 3: required list featureTypes -} -``` - -The response is returned in a `GfsIntersectionResponse` thrift struct. - -```thrift -struct GfsIntersectionResponse { - 1: required i64 userId - 2: required list results -} - -struct GfsIntersectionResult { - 1: required i64 candidateUserId - 2: required list intersectionValues -} - -struct IntersectionValue { - 1: required FeatureType featureType - 2: optional i32 count - 3: optional list intersectionIds - 4: optional i32 leftNodeDegree - 5: optional i32 rightNodeDegree -}(persisted="true") -``` - -## Behavior - -The `GfsIntersectionResponse` contains in its `results` field a `GfsIntersectionResult` for every candidate in `candidateIds` which contains an `IntersectionValue` for every `FeatureType` in the request's `featureTypes` field. - -The `IntersectionValue` contains the size of the intersection between the `leftEdgeType` edges from `userId` and the `rightEdgeType` edges from `candidateId` in the `count` field, as well as their respective degrees in the graphs in `leftNodeDegree` and `rightNodeDegree` respectively. - -**Note:** the `intersectionIds` field currently only contains `Nil`. diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/BUILD.bazel b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/BUILD.bazel deleted file mode 100644 index e6b2e92be..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - platform = "java8", - tags = [ - "bazel-compatible", - "bazel-only", - ], - dependencies = ["src/scala/com/twitter/storehaus_internal/util"], -) diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/BUILD.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/BUILD.docx new file mode 100644 index 000000000..385921109 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/BUILD.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/Configs.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/Configs.docx new file mode 100644 index 000000000..a3a03cd95 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/Configs.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/Configs.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/Configs.scala deleted file mode 100644 index 20647d68c..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common/Configs.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.graph_feature_service.common - -import com.twitter.conversions.DurationOps._ -import com.twitter.util.Duration -import com.twitter.util.Time -import java.nio.ByteBuffer -import scala.util.hashing.MurmurHash3 - -object Configs { - - // NOTE: notify #recos-platform slack room, if you want to change this. - // This SHOULD be updated together with NUM_SHARDS in worker.aurora - final val NumGraphShards: Int = 40 - - final val TopKRealGraph: Int = 512 - - final val BaseHdfsPath: String = "/user/cassowary/processed/gfs/constant_db/" - - // whether or not to write in_value and out_value graphs. Used in the scalding job. - final val EnableValueGraphs: Boolean = true - // whether or not to write in_key and out_key graphs. Used in the scalding job. - final val EnableKeyGraphs: Boolean = false - - final val FollowOutValPath: String = "follow_out_val/" - final val FollowOutKeyPath: String = "follow_out_key/" - final val FollowInValPath: String = "follow_in_val/" - final val FollowInKeyPath: String = "follow_in_key/" - - final val MutualFollowValPath: String = "mutual_follow_val/" - final val MutualFollowKeyPath: String = "mutual_follow_key/" - - final val FavoriteOutValPath: String = "favorite_out_val/" - final val FavoriteInValPath: String = "favorite_in_val/" - final val FavoriteOutKeyPath: String = "favorite_out_key/" - final val FavoriteInKeyPath: String = "favorite_in_key/" - - final val RetweetOutValPath: String = "retweet_out_val/" - final val RetweetInValPath: String = "retweet_in_val/" - final val RetweetOutKeyPath: String = "retweet_out_key/" - final val RetweetInKeyPath: String = "retweet_in_key/" - - final val MentionOutValPath: String = "mention_out_val/" - final val MentionInValPath: String = "mention_in_val/" - final val MentionOutKeyPath: String = "mention_out_key/" - final val MentionInKeyPath: String = "mention_in_key/" - - final val MemCacheTTL: Duration = 8.hours - - final val RandomSeed: Int = 39582942 - - def getTimedHdfsShardPath(shardId: Int, path: String, time: Time): String = { - val timeStr = time.format("yyyy/MM/dd") - s"$path/$timeStr/shard_$shardId" - } - - def getHdfsPath(path: String, overrideBaseHdfsPath: Option[String] = None): String = { - val basePath = overrideBaseHdfsPath.getOrElse(BaseHdfsPath) - s"$basePath$path" - } - - private def hash(kArr: Array[Byte], seed: Int): Int = { - MurmurHash3.bytesHash(kArr, seed) & 0x7fffffff // keep positive - } - - private def hashLong(l: Long, seed: Int): Int = { - hash(ByteBuffer.allocate(8).putLong(l).array(), seed) - } - - def shardForUser(userId: Long): Int = { - hashLong(userId, RandomSeed) % NumGraphShards - } - -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/BUILD.bazel b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/BUILD.bazel deleted file mode 100644 index c20a5e04c..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -scala_library( - sources = ["**/*.scala"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/lz4:lz4-java", - "3rdparty/jvm/org/slf4j:slf4j-api", - "discovery-common/src/main/scala/com/twitter/discovery/common/stats", - "finagle/finagle-http/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "finatra/inject/inject-utils/src/main/scala", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", - "graph-feature-service/src/main/resources", - "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/common", - "graph-feature-service/src/main/scala/com/twitter/graph_feature_service/util", - "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "servo/request/src/main/scala", - "src/scala/com/twitter/storehaus_internal/memcache", - "util/util-app/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/BUILD.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/BUILD.docx new file mode 100644 index 000000000..ccbe4146a Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/BUILD.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/Main.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/Main.docx new file mode 100644 index 000000000..ed3cc4927 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/Main.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/Main.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/Main.scala deleted file mode 100644 index 5980afdf2..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/Main.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.graph_feature_service.server - -import com.google.inject.Module -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.mtls.thriftmux.Mtls -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters.{ - AccessLoggingFilter, - LoggingMDCFilter, - StatsFilter, - ThriftMDCFilter, - TraceIdMDCFilter -} -import com.twitter.finatra.mtls.thriftmux.modules.MtlsThriftWebFormsModule -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.graph_feature_service.server.controllers.ServerController -import com.twitter.graph_feature_service.server.handlers.ServerWarmupHandler -import com.twitter.graph_feature_service.server.modules.{ - GetIntersectionStoreModule, - GraphFeatureServiceWorkerClientsModule, - ServerFlagsModule -} -import com.twitter.graph_feature_service.thriftscala -import com.twitter.inject.thrift.modules.ThriftClientIdModule - -object Main extends ServerMain - -class ServerMain extends ThriftServer with Mtls { - - override val name = "graph_feature_service-server" - - override val modules: Seq[Module] = { - Seq( - ServerFlagsModule, - DeciderModule, - ThriftClientIdModule, - GraphFeatureServiceWorkerClientsModule, - GetIntersectionStoreModule, - new MtlsThriftWebFormsModule[thriftscala.Server.MethodPerEndpoint](this) - ) - } - - override def configureThrift(router: ThriftRouter): Unit = { - router - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[AccessLoggingFilter] - .filter[StatsFilter] - .add[ServerController] - } - - override protected def warmup(): Unit = { - handle[ServerWarmupHandler]() - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/controllers/ServerController.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/controllers/ServerController.docx new file mode 100644 index 000000000..8535dc53f Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/controllers/ServerController.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/controllers/ServerController.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/controllers/ServerController.scala deleted file mode 100644 index ca8973f05..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/controllers/ServerController.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.graph_feature_service.server.controllers - -import com.twitter.discovery.common.stats.DiscoveryStatsFilter -import com.twitter.finagle.Service -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.thrift.Controller -import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler.GetIntersectionRequest -import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler -import com.twitter.graph_feature_service.thriftscala -import com.twitter.graph_feature_service.thriftscala.Server.GetIntersection -import com.twitter.graph_feature_service.thriftscala.Server.GetPresetIntersection -import com.twitter.graph_feature_service.thriftscala._ -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ServerController @Inject() ( - serverGetIntersectionHandler: ServerGetIntersectionHandler -)( - implicit statsReceiver: StatsReceiver) - extends Controller(thriftscala.Server) { - - private val getIntersectionService: Service[GetIntersectionRequest, GfsIntersectionResponse] = - new DiscoveryStatsFilter(statsReceiver.scope("srv").scope("get_intersection")) - .andThen(Service.mk(serverGetIntersectionHandler)) - - val getIntersection: Service[GetIntersection.Args, GfsIntersectionResponse] = { args => - // TODO: Disable updateCache after HTL switch to use PresetIntersection endpoint. - getIntersectionService( - GetIntersectionRequest.fromGfsIntersectionRequest(args.request, cacheable = true)) - } - handle(GetIntersection) { getIntersection } - - def getPresetIntersection: Service[ - GetPresetIntersection.Args, - GfsIntersectionResponse - ] = { args => - // TODO: Refactor after HTL switch to PresetIntersection - val cacheable = args.request.presetFeatureTypes == PresetFeatureTypes.HtlTwoHop - getIntersectionService( - GetIntersectionRequest.fromGfsPresetIntersectionRequest(args.request, cacheable)) - } - - handle(GetPresetIntersection) { getPresetIntersection } - -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerGetIntersectionHandler.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerGetIntersectionHandler.docx new file mode 100644 index 000000000..dbae23a45 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerGetIntersectionHandler.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerGetIntersectionHandler.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerGetIntersectionHandler.scala deleted file mode 100644 index 2c77c1f54..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerGetIntersectionHandler.scala +++ /dev/null @@ -1,198 +0,0 @@ -package com.twitter.graph_feature_service.server.handlers - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.graph_feature_service.server.handlers.ServerGetIntersectionHandler.GetIntersectionRequest -import com.twitter.graph_feature_service.server.stores.FeatureTypesEncoder -import com.twitter.graph_feature_service.server.stores.GetIntersectionStore.GetIntersectionQuery -import com.twitter.graph_feature_service.thriftscala.PresetFeatureTypes -import com.twitter.graph_feature_service.thriftscala._ -import com.twitter.graph_feature_service.util.FeatureTypesCalculator -import com.twitter.servo.request.RequestHandler -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Memoize -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class ServerGetIntersectionHandler @Inject() ( - @Named("ReadThroughGetIntersectionStore") - readThroughStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult], - @Named("BypassCacheGetIntersectionStore") - readOnlyStore: ReadableStore[GetIntersectionQuery, CachedIntersectionResult] -)( - implicit statsReceiver: StatsReceiver) - extends RequestHandler[GetIntersectionRequest, GfsIntersectionResponse] { - - import ServerGetIntersectionHandler._ - - // TODO: Track all the stats based on PresetFeatureType and update the dashboard - private val stats: StatsReceiver = statsReceiver.scope("srv").scope("get_intersection") - private val numCandidatesCount = stats.counter("total_num_candidates") - private val numCandidatesStat = stats.stat("num_candidates") - private val numFeaturesStat = stats.stat("num_features") - private val userEmptyCount = stats.counter("user_empty_count") - private val candidateEmptyRateStat = stats.stat("candidate_empty_rate") - private val candidateNumEmptyStat = stats.stat("candidate_num_empty") - private val missedRateStat = stats.stat("miss_rate") - private val numMissedStat = stats.stat("num_missed") - - // Assume the order from HTL doesn't change. Only log the HTL query now. - private val featureStatMap = FeatureTypesCalculator.presetFeatureTypes.map { feature => - val featureString = s"${feature.leftEdgeType.name}_${feature.rightEdgeType.name}" - feature -> Array( - stats.counter(s"feature_type_${featureString}_total"), - stats.counter(s"feature_type_${featureString}_count_zero"), - stats.counter(s"feature_type_${featureString}_left_zero"), - stats.counter(s"feature_type_${featureString}_right_zero") - ) - }.toMap - - private val sourceCandidateNumStats = Memoize[PresetFeatureTypes, Stat] { presetFeature => - stats.stat(s"source_candidate_num_${presetFeature.name}") - } - - override def apply(request: GetIntersectionRequest): Future[GfsIntersectionResponse] = { - val featureTypes = request.calculatedFeatureTypes - val numCandidates = request.candidateUserIds.length - val numFeatures = featureTypes.length - - numCandidatesCount.incr(numCandidates) - numCandidatesStat.add(numCandidates) - numFeaturesStat.add(numFeatures) - sourceCandidateNumStats(request.presetFeatureTypes).add(numCandidates) - - // Note: do not change the orders of features and candidates. - val candidateIds = request.candidateUserIds - - if (featureTypes.isEmpty || candidateIds.isEmpty) { - Future.value(DefaultGfsIntersectionResponse) - } else { - Future - .collect { - val getIntersectionStore = if (request.cacheable) readThroughStore else readOnlyStore - getIntersectionStore.multiGet(GetIntersectionQuery.buildQueries(request)) - }.map { responses => - val results = responses.collect { - case (query, Some(result)) => - query.candidateId -> GfsIntersectionResult( - query.candidateId, - query.calculatedFeatureTypes.zip(result.values).map { - case (featureType, value) => - IntersectionValue( - featureType, - Some(value.count), - if (value.intersectionIds.isEmpty) None else Some(value.intersectionIds), - Some(value.leftNodeDegree), - Some(value.rightNodeDegree) - ) - } - ) - } - - // Keep the response order same as input - val processedResults = candidateIds.map { candidateId => - results.getOrElse(candidateId, GfsIntersectionResult(candidateId, List.empty)) - } - - val candidateEmptyNum = - processedResults.count( - _.intersectionValues.exists(value => isZero(value.rightNodeDegree))) - - val numMissed = processedResults.count(_.intersectionValues.size != numFeatures) - - if (processedResults.exists( - _.intersectionValues.forall(value => isZero(value.leftNodeDegree)))) { - userEmptyCount.incr() - } - - candidateNumEmptyStat.add(candidateEmptyNum) - candidateEmptyRateStat.add(candidateEmptyNum.toFloat / numCandidates) - numMissedStat.add(numMissed) - missedRateStat.add(numMissed.toFloat / numCandidates) - - processedResults.foreach { result => - result.intersectionValues.zip(featureTypes).foreach { - case (value, featureType) => - featureStatMap.get(featureType).foreach { statsArray => - statsArray(TotalIndex).incr() - if (isZero(value.count)) { - statsArray(CountIndex).incr() - } - if (isZero(value.leftNodeDegree)) { - statsArray(LeftIndex).incr() - } - if (isZero(value.rightNodeDegree)) { - statsArray(RightIndex).incr() - } - } - } - } - - GfsIntersectionResponse(processedResults) - } - } - - } - -} - -private[graph_feature_service] object ServerGetIntersectionHandler { - - case class GetIntersectionRequest( - userId: Long, - candidateUserIds: Seq[Long], - featureTypes: Seq[FeatureType], - presetFeatureTypes: PresetFeatureTypes, - intersectionIdLimit: Option[Int], - cacheable: Boolean) { - - lazy val calculatedFeatureTypes: Seq[FeatureType] = - FeatureTypesCalculator.getFeatureTypes(presetFeatureTypes, featureTypes) - - lazy val calculatedFeatureTypesString: String = - FeatureTypesEncoder(calculatedFeatureTypes) - } - - object GetIntersectionRequest { - - def fromGfsIntersectionRequest( - request: GfsIntersectionRequest, - cacheable: Boolean - ): GetIntersectionRequest = { - GetIntersectionRequest( - request.userId, - request.candidateUserIds, - request.featureTypes, - PresetFeatureTypes.Empty, - request.intersectionIdLimit, - cacheable) - } - - def fromGfsPresetIntersectionRequest( - request: GfsPresetIntersectionRequest, - cacheable: Boolean - ): GetIntersectionRequest = { - GetIntersectionRequest( - request.userId, - request.candidateUserIds, - List.empty, - request.presetFeatureTypes, - request.intersectionIdLimit, - cacheable) - } - } - - private val DefaultGfsIntersectionResponse = GfsIntersectionResponse() - - private val TotalIndex = 0 - private val CountIndex = 1 - private val LeftIndex = 2 - private val RightIndex = 3 - - def isZero(opt: Option[Int]): Boolean = { - !opt.exists(_ != 0) - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerWarmupHandler.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerWarmupHandler.docx new file mode 100644 index 000000000..9c27b656d Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerWarmupHandler.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerWarmupHandler.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerWarmupHandler.scala deleted file mode 100644 index 3e31f2c8f..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/handlers/ServerWarmupHandler.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.graph_feature_service.server.handlers - -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.graph_feature_service.thriftscala.EdgeType.FavoritedBy -import com.twitter.graph_feature_service.thriftscala.EdgeType.FollowedBy -import com.twitter.graph_feature_service.thriftscala.EdgeType.Following -import com.twitter.graph_feature_service.thriftscala.Server.GetIntersection -import com.twitter.graph_feature_service.thriftscala.FeatureType -import com.twitter.graph_feature_service.thriftscala.GfsIntersectionRequest -import com.twitter.inject.utils.Handler -import com.twitter.scrooge.Request -import com.twitter.util.logging.Logger -import javax.inject.Inject -import javax.inject.Singleton -import scala.util.Random - -@Singleton -class ServerWarmupHandler @Inject() (warmup: ThriftWarmup) extends Handler { - - val logger: Logger = Logger("WarmupHandler") - - // TODO: Add the testing accounts to warm-up the service. - private val testingAccounts: Array[Long] = Seq.empty.toArray - - private def getRandomRequest: GfsIntersectionRequest = { - GfsIntersectionRequest( - testingAccounts(Random.nextInt(testingAccounts.length)), - testingAccounts, - Seq(FeatureType(Following, FollowedBy), FeatureType(Following, FavoritedBy)) - ) - } - - override def handle(): Unit = { - warmup.sendRequest( - GetIntersection, - Request( - GetIntersection.Args( - getRandomRequest - )), - 10 - )() - - logger.info("Warmup Done!") - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GetIntersectionStoreModule.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GetIntersectionStoreModule.docx new file mode 100644 index 000000000..4bde9304c Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GetIntersectionStoreModule.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GetIntersectionStoreModule.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GetIntersectionStoreModule.scala deleted file mode 100644 index cae99b3a6..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GetIntersectionStoreModule.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.graph_feature_service.server.modules - -import com.google.inject.Provides -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.graph_feature_service.common.Configs._ -import com.twitter.graph_feature_service.server.stores.GetIntersectionStore -import com.twitter.graph_feature_service.server.stores.GetIntersectionStore.GetIntersectionQuery -import com.twitter.graph_feature_service.thriftscala.CachedIntersectionResult -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.memcache.MemcacheStore -import com.twitter.storehaus_internal.util.{ClientName, ZkEndPoint} -import com.twitter.util.Duration -import javax.inject.{Named, Singleton} - -/** - * Initialize the MemCache based GetIntersectionStore. - * The Key of MemCache is UserId~CandidateId~FeatureTypes~IntersectionIdLimit. - */ -object GetIntersectionStoreModule extends TwitterModule { - - private[this] val requestTimeout: Duration = 25.millis - private[this] val retries: Int = 0 - - @Provides - @Named("ReadThroughGetIntersectionStore") - @Singleton - def provideReadThroughGetIntersectionStore( - graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, - serviceIdentifier: ServiceIdentifier, - @Flag(ServerFlagNames.MemCacheClientName) memCacheName: String, - @Flag(ServerFlagNames.MemCachePath) memCachePath: String - )( - implicit statsReceiver: StatsReceiver - ): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] = { - buildMemcacheStore( - graphFeatureServiceWorkerClients, - memCacheName, - memCachePath, - serviceIdentifier) - } - - @Provides - @Named("BypassCacheGetIntersectionStore") - @Singleton - def provideReadOnlyGetIntersectionStore( - graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, - )( - implicit statsReceiver: StatsReceiver - ): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] = { - // Bypass the Memcache. - GetIntersectionStore(graphFeatureServiceWorkerClients, statsReceiver) - } - - private[this] def buildMemcacheStore( - graphFeatureServiceWorkerClients: GraphFeatureServiceWorkerClients, - memCacheName: String, - memCachePath: String, - serviceIdentifier: ServiceIdentifier, - )( - implicit statsReceiver: StatsReceiver - ): ReadableStore[GetIntersectionQuery, CachedIntersectionResult] = { - val backingStore = GetIntersectionStore(graphFeatureServiceWorkerClients, statsReceiver) - - val cacheClient = MemcacheStore.memcachedClient( - name = ClientName(memCacheName), - dest = ZkEndPoint(memCachePath), - timeout = requestTimeout, - retries = retries, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - - ObservedMemcachedReadableStore.fromCacheClient[GetIntersectionQuery, CachedIntersectionResult]( - backingStore = backingStore, - cacheClient = cacheClient, - ttl = MemCacheTTL - )( - valueInjection = LZ4Injection.compose(CompactScalaCodec(CachedIntersectionResult)), - statsReceiver = statsReceiver.scope("mem_cache"), - keyToString = { key => - s"L~${key.userId}~${key.candidateId}~${key.featureTypesString}~${key.intersectionIdLimit}" - } - ) - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GraphFeatureServiceWorkerClientsModule.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GraphFeatureServiceWorkerClientsModule.docx new file mode 100644 index 000000000..152ee14e3 Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GraphFeatureServiceWorkerClientsModule.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GraphFeatureServiceWorkerClientsModule.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GraphFeatureServiceWorkerClientsModule.scala deleted file mode 100644 index a6f56827e..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/GraphFeatureServiceWorkerClientsModule.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.graph_feature_service.server.modules - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.service.RetryBudget -import com.twitter.graph_feature_service.thriftscala -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.util.{Await, Duration} -import javax.inject.Singleton - -case class GraphFeatureServiceWorkerClients( - workers: Seq[thriftscala.Worker.MethodPerEndpoint]) - -object GraphFeatureServiceWorkerClientsModule extends TwitterModule { - private[this] val closeableGracePeriod: Duration = 1.second - private[this] val requestTimeout: Duration = 25.millis - - @Provides - @Singleton - def provideGraphFeatureServiceWorkerClient( - @Flag(ServerFlagNames.NumWorkers) numWorkers: Int, - @Flag(ServerFlagNames.ServiceRole) serviceRole: String, - @Flag(ServerFlagNames.ServiceEnv) serviceEnv: String, - serviceIdentifier: ServiceIdentifier - ): GraphFeatureServiceWorkerClients = { - - val workers: Seq[thriftscala.Worker.MethodPerEndpoint] = - (0 until numWorkers).map { id => - val dest = s"/srv#/$serviceEnv/local/$serviceRole/graph_feature_service-worker-$id" - - val client = ThriftMux.client - .withRequestTimeout(requestTimeout) - .withRetryBudget(RetryBudget.Empty) - .withMutualTls(serviceIdentifier) - .build[thriftscala.Worker.MethodPerEndpoint](dest, s"worker-$id") - - onExit { - val closeable = client.asClosable - Await.result(closeable.close(closeableGracePeriod), closeableGracePeriod) - } - - client - } - - GraphFeatureServiceWorkerClients(workers) - } -} diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/LZ4Injection.docx b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/LZ4Injection.docx new file mode 100644 index 000000000..95d0ed2ef Binary files /dev/null and b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/LZ4Injection.docx differ diff --git a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/LZ4Injection.scala b/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/LZ4Injection.scala deleted file mode 100644 index 3018dcd9c..000000000 --- a/graph-feature-service/src/main/scala/com/twitter/graph_feature_service/server/modules/LZ4Injection.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.graph_feature_service.server.modules - -import com.twitter.bijection.Injection -import scala.util.Try -import net.jpountz.lz4.{LZ4CompressorWithLength, LZ4DecompressorWithLength, LZ4Factory} - -object LZ4Injection extends Injection[Array[Byte], Array[Byte]] { - private val lz4Factory = LZ4Factory.fastestInstance() - private val fastCompressor = new LZ4CompressorWithLength(lz4Factory.fastCompressor()) - private val decompressor = new LZ4DecompressorWithLength(lz4Factory.fastDecompressor()) - - override def apply(a: Array[Byte]): Array[Byte] = LZ4Injection.fastCompressor.compress(a) - - override def invert(b: Array[Byte]): Try[Array[Byte]] = Try { - LZ4Injection.decompressor.decompress(b) - } -}