diff --git a/follow-recommendations-service/BUILD b/follow-recommendations-service/BUILD deleted file mode 100644 index 5210cbd88..000000000 --- a/follow-recommendations-service/BUILD +++ /dev/null @@ -1,48 +0,0 @@ -# Without this alias, library :follow-recommendations-service_lib would conflict with :bin -alias( - name = "follow-recommendations-service", - target = ":follow-recommendations-service_lib", -) - -target( - name = "follow-recommendations-service_lib", - dependencies = [ - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - ], -) - -jvm_binary( - name = "bin", - basename = "follow-recommendations-service", - main = "com.twitter.follow_recommendations.FollowRecommendationsServiceThriftServerMain", - runtime_platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - ":follow-recommendations-service", - "3rdparty/jvm/ch/qos/logback:logback-classic", - "finagle/finagle-zipkin-scribe/src/main/scala", - "finatra/inject/inject-logback/src/main/scala", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "twitter-server-internal/src/main/scala", - "twitter-server/logback-classic/src/main/scala", - ], -) - -# Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app -jvm_app( - name = "follow-recommendations-service-app", - archive = "zip", - binary = ":bin", - bundles = [ - bundle( - fileset = [ - "server/src/main/resources/*", - "server/src/main/resources/**/*", - ], - owning_target = "follow-recommendations-service/server/src/main/resources:frs_resources", - relative_to = "server/src/main/resources", - ), - ], - tags = ["bazel-compatible"], -) diff --git a/follow-recommendations-service/CONFIG.ini b/follow-recommendations-service/CONFIG.ini deleted file mode 100644 index c8173efa8..000000000 --- a/follow-recommendations-service/CONFIG.ini +++ /dev/null @@ -1,24 +0,0 @@ -[code-coverage] -package = com.twitter.follow_recommendations - -[docbird] -project_name = follow-recommendations-service -project_type = service -; example settings: -; -; project_name = fluffybird -; description = fluffybird is a service for fluffing up feathers. -; tags = python,documentation,fluffybird -; project_type = service -; - allowed options: essay, library, service, hub, cookbook, styleguide, policy -; owner_links = roster -; - allowed options: roster, find, email -; scrolling_tocs = yes -; comments = yes -; verifications = yes -; support_widget = yes -; health_score = yes -; sticky_sidebar = no - -[jira] -project = CJREL diff --git a/follow-recommendations-service/FRS_architecture.png b/follow-recommendations-service/FRS_architecture.png deleted file mode 100644 index eda105e45..000000000 Binary files a/follow-recommendations-service/FRS_architecture.png and /dev/null differ diff --git a/follow-recommendations-service/README.md b/follow-recommendations-service/README.md deleted file mode 100644 index 1640184a5..000000000 --- a/follow-recommendations-service/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Follow Recommendations Service - -## Introduction to the Follow Recommendations Service (FRS) -The Follow Recommendations Service (FRS) is a robust recommendation engine designed to provide users with personalized suggestions for accounts to follow. At present, FRS supports Who-To-Follow (WTF) module recommendations across a variety of Twitter product interfaces. Additionally, by suggesting tweet authors, FRS also delivers FutureGraph tweet recommendations, which consist of tweets from accounts that users may be interested in following in the future. - -## Design -The system is tailored to accommodate diverse use cases, such as Post New-User-Experience (NUX), advertisements, FutureGraph tweets, and more. Each use case features a unique display location identifier. To view all display locations, refer to the following path: `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala`. - -Recommendation steps are customized according to each display location. Common and high-level steps are encapsulated within the "RecommendationFlow," which includes operations like candidate generation, ranker selection, filtering, transformation, and beyond. To explore all flows, refer to this path: `follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows`. - -For each product (corresponding to a display location), one or multiple flows can be selected to generate candidates based on code and configurations. To view all products, refer to the following path: `follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs`. - -The FRS overview diagram is depicted below: - -![FRS_architecture.png](FRS_architecture.png) - - -### Candidate Generation -During this step, FRS utilizes various user signals and algorithms to identify candidates from all Twitter accounts. The candidate source folder is located at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/`, with a README file provided within each candidate source folder. - -### Filtering -In this phase, FRS applies different filtering logic after generating account candidates to improve quality and health. Filtering may occur before and/or after the ranking step, with heavier filtering logic (e.g., higher latency) typically applied after the ranking step. The filters' folder is located at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates`. - -### Ranking -During this step, FRS employs both Machine Learning (ML) and heuristic rule-based candidate ranking. For the ML ranker, ML features are fetched beforehand (i.e., feature hydration), -and a DataRecord (the Twitter-standard Machine Learning data format used to represent feature data, labels, and predictions when training or serving) is constructed for each pair. -These pairs are then sent to a separate ML prediction service, which houses the ML model trained offline. -The ML prediction service returns a prediction score, representing the probability that a user will follow and engage with the candidate. -This score is a weighted sum of p(follow|recommendation) and p(positive engagement|follow), and FRS uses this score to rank the candidates. - -The rankers' folder is located at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers`. - -### Transform -In this phase, the sequence of candidates undergoes necessary transformations, such as deduplication, attaching social proof (i.e., "followed by XX user"), adding tracking tokens, and more. -The transformers' folder can be found at `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms`. - -### Truncation -During this final step, FRS trims the candidate pool to a specified size. This process ensures that only the most relevant and engaging candidates are presented to users while maintaining an optimal user experience. - -By implementing these comprehensive steps and adapting to various use cases, the Follow Recommendations Service (FRS) effectively curates tailored suggestions for Twitter users, enhancing their overall experience and promoting meaningful connections within the platform. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/BUILD deleted file mode 100644 index d02506a85..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/guava", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "finagle/finagle-core/src/main", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "stitch/stitch-core", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/CandidateSourceRegistry.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/CandidateSourceRegistry.scala deleted file mode 100644 index ee9cfbbe5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/CandidateSourceRegistry.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource.toEnriched -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -// a helper structure to register and select candidate sources based on identifiers -trait CandidateSourceRegistry[Target, Candidate] { - - val statsReceiver: StatsReceiver - - def sources: Set[CandidateSource[Target, Candidate]] - - final lazy val candidateSources: Map[ - CandidateSourceIdentifier, - CandidateSource[Target, Candidate] - ] = { - val map = sources.map { c => - c.identifier -> c.observe(statsReceiver) - }.toMap - - if (map.size != sources.size) { - throw new IllegalArgumentException("Duplicate Candidate Source Identifiers") - } - - map - } - - def select( - identifiers: Set[CandidateSourceIdentifier] - ): Set[CandidateSource[Target, Candidate]] = { - // fails loud if the candidate source is not registered - identifiers.map(candidateSources(_)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/EnrichedCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/EnrichedCandidateSource.scala deleted file mode 100644 index 9d8507528..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/EnrichedCandidateSource.scala +++ /dev/null @@ -1,164 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.util.TimeoutException -import scala.language.implicitConversions - -class EnrichedCandidateSource[Target, Candidate](original: CandidateSource[Target, Candidate]) { - - /** - * Gate the candidate source based on the Predicate of target. - * It returns results only if the predicate returns Valid. - * - * @param predicate - * @return - */ - def gate(predicate: Predicate[Target]): CandidateSource[Target, Candidate] = { - throw new UnsupportedOperationException() - } - - def observe(statsReceiver: StatsReceiver): CandidateSource[Target, Candidate] = { - val originalIdentifier = original.identifier - val stats = statsReceiver.scope(originalIdentifier.name) - new CandidateSource[Target, Candidate] { - val identifier = originalIdentifier - override def apply(target: Target): Stitch[Seq[Candidate]] = { - StatsUtil.profileStitchSeqResults[Candidate](original(target), stats) - } - } - } - - /** - * Map target type into new target type (1 to optional mapping) - */ - def stitchMapKey[Target2]( - targetMapper: Target2 => Stitch[Option[Target]] - ): CandidateSource[Target2, Candidate] = { - val targetsMapper: Target2 => Stitch[Seq[Target]] = { target => - targetMapper(target).map(_.toSeq) - } - stitchMapKeys(targetsMapper) - } - - /** - * Map target type into new target type (1 to many mapping) - */ - def stitchMapKeys[Target2]( - targetMapper: Target2 => Stitch[Seq[Target]] - ): CandidateSource[Target2, Candidate] = { - new CandidateSource[Target2, Candidate] { - val identifier = original.identifier - override def apply(target: Target2): Stitch[Seq[Candidate]] = { - for { - mappedTargets <- targetMapper(target) - results <- Stitch.traverse(mappedTargets)(original(_)) - } yield results.flatten - } - } - } - - /** - * Map target type into new target type (1 to many mapping) - */ - def mapKeys[Target2]( - targetMapper: Target2 => Seq[Target] - ): CandidateSource[Target2, Candidate] = { - val stitchMapper: Target2 => Stitch[Seq[Target]] = { target => - Stitch.value(targetMapper(target)) - } - stitchMapKeys(stitchMapper) - } - - /** - * Map candidate types to new type based on candidateMapper - */ - def mapValues[Candidate2]( - candidateMapper: Candidate => Stitch[Option[Candidate2]] - ): CandidateSource[Target, Candidate2] = { - - new CandidateSource[Target, Candidate2] { - val identifier = original.identifier - override def apply(target: Target): Stitch[Seq[Candidate2]] = { - original(target).flatMap { candidates => - val results = Stitch.traverse(candidates)(candidateMapper(_)) - results.map(_.flatten) - } - } - } - } - - /** - * Map candidate types to new type based on candidateMapper - */ - def mapValue[Candidate2]( - candidateMapper: Candidate => Candidate2 - ): CandidateSource[Target, Candidate2] = { - val stitchMapper: Candidate => Stitch[Option[Candidate2]] = { c => - Stitch.value(Some(candidateMapper(c))) - } - mapValues(stitchMapper) - } - - /** - * This method wraps the candidate source in a designated timeout so that a single candidate - * source does not result in a timeout for the entire flow - */ - def within( - candidateTimeout: Duration, - statsReceiver: StatsReceiver - ): CandidateSource[Target, Candidate] = { - val originalIdentifier = original.identifier - val timeoutCounter = - statsReceiver.counter(originalIdentifier.name, "timeout") - - new CandidateSource[Target, Candidate] { - val identifier = originalIdentifier - override def apply(target: Target): Stitch[Seq[Candidate]] = { - original - .apply(target) - .within(candidateTimeout)(com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - timeoutCounter.incr() - Stitch.Nil - } - } - } - } - - def failOpenWithin( - candidateTimeout: Duration, - statsReceiver: StatsReceiver - ): CandidateSource[Target, Candidate] = { - val originalIdentifier = original.identifier - val timeoutCounter = - statsReceiver.counter(originalIdentifier.name, "timeout") - - new CandidateSource[Target, Candidate] { - val identifier = originalIdentifier - override def apply(target: Target): Stitch[Seq[Candidate]] = { - original - .apply(target) - .within(candidateTimeout)(com.twitter.finagle.util.DefaultTimer) - .handle { - case _: TimeoutException => - timeoutCounter.incr() - Seq.empty - case e: Exception => - statsReceiver - .scope("candidate_source_error").scope(originalIdentifier.name).counter( - e.getClass.getSimpleName).incr - Seq.empty - } - } - } - } -} - -object EnrichedCandidateSource { - implicit def toEnriched[K, V](original: CandidateSource[K, V]): EnrichedCandidateSource[K, V] = - new EnrichedCandidateSource(original) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/ParamPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/ParamPredicate.scala deleted file mode 100644 index f457527bc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/ParamPredicate.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.follow_recommendations.common.models.FilterReason.ParamReason -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Param - -case class ParamPredicate[Request <: HasParams](param: Param[Boolean]) extends Predicate[Request] { - - def apply(request: Request): Stitch[PredicateResult] = { - if (request.params(param)) { - Stitch.value(PredicateResult.Valid) - } else { - Stitch.value(PredicateResult.Invalid(Set(ParamReason(param.statName)))) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Predicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Predicate.scala deleted file mode 100644 index e5b40ed82..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Predicate.scala +++ /dev/null @@ -1,282 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch - -trait Predicate[-Q] { - - def apply(item: Q): Stitch[PredicateResult] - def arrow: Arrow[Q, PredicateResult] = Arrow.apply(apply) - - def map[K](mapper: K => Q): Predicate[K] = Predicate(arrow.contramap(mapper)) - - /** - * check the predicate results for a batch of items for convenience. - * - * mark it as final to avoid potential abuse usage - */ - final def batch(items: Seq[Q]): Stitch[Seq[PredicateResult]] = { - this.arrow.traverse(items) - } - - /** - * Syntax sugar for functions which take in 2 inputs as a tuple. - */ - def apply[Q1, Q2](item1: Q1, item2: Q2)(implicit ev: ((Q1, Q2)) => Q): Stitch[PredicateResult] = { - apply((item1, item2)) - } - - /** - * Runs the predicates in sequence. The returned predicate will return true iff both the predicates return true. - * ie. it is an AND operation - * - * We short-circuit the evaluation, ie we don't evaluate the 2nd predicate if the 1st is false - * - * @param p predicate to run in sequence - * - * @return a new predicate object that represents the logical AND of both predicates - */ - def andThen[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1] = { - Predicate({ query: Q1 => - apply(query).flatMap { - case PredicateResult.Valid => p(query) - case PredicateResult.Invalid(reasons) => Stitch.value(PredicateResult.Invalid(reasons)) - } - }) - } - - /** - * Creates a predicate which runs the current & given predicate in sequence. - * The returned predicate will return true if either current or given predicate returns true. - * That is, given predicate will be only run if current predicate returns false. - * - * @param p predicate to run in sequence - * - * @return new predicate object that represents the logical OR of both predicates. - * if both are invalid, the reason would be the set of all invalid reasons. - */ - def or[Q1 <: Q](p: Predicate[Q1]): Predicate[Q1] = { - Predicate({ query: Q1 => - apply(query).flatMap { - case PredicateResult.Valid => Stitch.value(PredicateResult.Valid) - case PredicateResult.Invalid(reasons) => - p(query).flatMap { - case PredicateResult.Valid => Stitch.value(PredicateResult.Valid) - case PredicateResult.Invalid(newReasons) => - Stitch.value(PredicateResult.Invalid(reasons ++ newReasons)) - } - } - }) - } - - /* - * Runs the predicate only if the provided predicate is valid, otherwise returns valid. - * */ - def gate[Q1 <: Q](gatingPredicate: Predicate[Q1]): Predicate[Q1] = { - Predicate { query: Q1 => - gatingPredicate(query).flatMap { result => - if (result == PredicateResult.Valid) { - apply(query) - } else { - Stitch.value(PredicateResult.Valid) - } - } - } - } - - def observe(statsReceiver: StatsReceiver): Predicate[Q] = Predicate( - StatsUtil.profilePredicateResult(this.arrow, statsReceiver)) - - def convertToFailOpenWithResultType(resultType: PredicateResult): Predicate[Q] = { - Predicate { query: Q => - apply(query).handle { - case _: Exception => - resultType - } - - } - } - -} - -class TruePredicate[Q] extends Predicate[Q] { - override def apply(item: Q): Stitch[PredicateResult] = Predicate.AlwaysTrueStitch -} - -class FalsePredicate[Q](reason: FilterReason) extends Predicate[Q] { - val InvalidResult = Stitch.value(PredicateResult.Invalid(Set(reason))) - override def apply(item: Q): Stitch[PredicateResult] = InvalidResult -} - -object Predicate { - - val AlwaysTrueStitch = Stitch.value(PredicateResult.Valid) - - val NumBatchesStat = "num_batches_stats" - val NumBatchesCount = "num_batches" - - def apply[Q](func: Q => Stitch[PredicateResult]): Predicate[Q] = new Predicate[Q] { - override def apply(item: Q): Stitch[PredicateResult] = func(item) - - override val arrow: Arrow[Q, PredicateResult] = Arrow(func) - } - - def apply[Q](outerArrow: Arrow[Q, PredicateResult]): Predicate[Q] = new Predicate[Q] { - override def apply(item: Q): Stitch[PredicateResult] = arrow(item) - - override val arrow: Arrow[Q, PredicateResult] = outerArrow - } - - /** - * Given some items, this function - * 1. chunks them up in groups - * 2. lazily applies a predicate on each group - * 3. filters based on the predicate - * 4. takes first numToTake items. - * - * If numToTake is satisfied, then any later predicates are not called. - * - * @param items items of type Q - * @param predicate predicate that determines whether an item is acceptable - * @param batchSize batch size to call the predicate with - * @param numToTake max number of items to return - * @param stats stats receiver - * @tparam Q type of item - * - * @return a future of K items - */ - def batchFilterTake[Q]( - items: Seq[Q], - predicate: Predicate[Q], - batchSize: Int, - numToTake: Int, - stats: StatsReceiver - ): Stitch[Seq[Q]] = { - - def take( - input: Iterator[Stitch[Seq[Q]]], - prev: Seq[Q], - takeSize: Int, - numOfBatch: Int - ): Stitch[(Seq[Q], Int)] = { - if (input.hasNext) { - val currFut = input.next() - currFut.flatMap { curr => - val taken = curr.take(takeSize) - val combined = prev ++ taken - if (taken.size < takeSize) - take(input, combined, takeSize - taken.size, numOfBatch + 1) - else Stitch.value((combined, numOfBatch + 1)) - } - } else { - Stitch.value((prev, numOfBatch)) - } - } - - val batchedItems = items.view.grouped(batchSize) - val batchedFutures = batchedItems.map { batch => - Stitch.traverse(batch)(predicate.apply).map { conds => - (batch.zip(conds)).withFilter(_._2.value).map(_._1) - } - } - take(batchedFutures, Nil, numToTake, 0).map { - case (filtered: Seq[Q], numOfBatch: Int) => - stats.stat(NumBatchesStat).add(numOfBatch) - stats.counter(NumBatchesCount).incr(numOfBatch) - filtered - } - } - - /** - * filter a list of items based on the predicate - * - * @param items a list of items - * @param predicate predicate of the item - * @tparam Q item type - * @return the list of items that satisfy the predicate - */ - def filter[Q](items: Seq[Q], predicate: Predicate[Q]): Stitch[Seq[Q]] = { - predicate.batch(items).map { results => - items.zip(results).collect { - case (item, PredicateResult.Valid) => item - } - } - } - - /** - * filter a list of items based on the predicate given the target - * - * @param target target item - * @param items a list of items - * @param predicate predicate of the (target, item) pair - * @tparam Q item type - * @return the list of items that satisfy the predicate given the target - */ - def filter[T, Q](target: T, items: Seq[Q], predicate: Predicate[(T, Q)]): Stitch[Seq[Q]] = { - predicate.batch(items.map(i => (target, i))).map { results => - items.zip(results).collect { - case (item, PredicateResult.Valid) => item - } - } - } - - /** - * Returns a predicate, where an element is true iff it that element is true for all input predicates. - * ie. it is an AND operation - * - * This is done concurrently. - * - * @param predicates list of predicates - * @tparam Q Type parameter - * - * @return new predicate object that is the logical "and" of the input predicates - */ - def andConcurrently[Q](predicates: Seq[Predicate[Q]]): Predicate[Q] = { - Predicate { query: Q => - Stitch.traverse(predicates)(p => p(query)).map { predicateResults => - val allInvalid = predicateResults - .collect { - case PredicateResult.Invalid(reason) => - reason - } - if (allInvalid.isEmpty) { - PredicateResult.Valid - } else { - val allInvalidReasons = allInvalid.reduce(_ ++ _) - PredicateResult.Invalid(allInvalidReasons) - } - } - } - } -} - -/** - * applies the underlying predicate when the param is on. - */ -abstract class GatedPredicateBase[Q]( - underlyingPredicate: Predicate[Q], - stats: StatsReceiver = NullStatsReceiver) - extends Predicate[Q] { - def gate(item: Q): Boolean - - val underlyingPredicateTotal = stats.counter("underlying_total") - val underlyingPredicateValid = stats.counter("underlying_valid") - val underlyingPredicateInvalid = stats.counter("underlying_invalid") - val notGatedCounter = stats.counter("not_gated") - - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - - override def apply(item: Q): Stitch[PredicateResult] = { - if (gate(item)) { - underlyingPredicateTotal.incr() - underlyingPredicate(item) - } else { - notGatedCounter.incr() - ValidStitch - } - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/PredicateResult.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/PredicateResult.scala deleted file mode 100644 index 002e90275..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/PredicateResult.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.follow_recommendations.common.models.FilterReason - -sealed trait PredicateResult { - def value: Boolean -} - -object PredicateResult { - - case object Valid extends PredicateResult { - override val value = true - } - - case class Invalid(reasons: Set[FilterReason] = Set.empty[FilterReason]) extends PredicateResult { - override val value = false - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Ranker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Ranker.scala deleted file mode 100644 index 27eb50457..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Ranker.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.util.TimeoutException - -/** - * Ranker is a special kind of transform that would only change the order of a list of items. - * If a single item is given, it "may" attach additional scoring information to the item. - * - * @tparam Target target to recommend the candidates - * @tparam Candidate candidate type to rank - */ -trait Ranker[Target, Candidate] extends Transform[Target, Candidate] { ranker => - - def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] - - override def transform(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { - rank(target, candidates) - } - - override def observe(statsReceiver: StatsReceiver): Ranker[Target, Candidate] = { - val originalRanker = this - new Ranker[Target, Candidate] { - override def rank(target: Target, items: Seq[Candidate]): Stitch[Seq[Candidate]] = { - statsReceiver.counter(Transform.InputCandidatesCount).incr(items.size) - statsReceiver.stat(Transform.InputCandidatesStat).add(items.size) - StatsUtil.profileStitchSeqResults(originalRanker.rank(target, items), statsReceiver) - } - } - } - - def reverse: Ranker[Target, Candidate] = new Ranker[Target, Candidate] { - def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = - ranker.rank(target, candidates).map(_.reverse) - } - - def andThen(other: Ranker[Target, Candidate]): Ranker[Target, Candidate] = { - val original = this - new Ranker[Target, Candidate] { - def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { - original.rank(target, candidates).flatMap { results => other.rank(target, results) } - } - } - } - - /** - * This method wraps the Ranker in a designated timeout. - * If the ranker timeouts, it would return the original candidates directly, - * instead of failing the whole recommendation flow - */ - def within(timeout: Duration, statsReceiver: StatsReceiver): Ranker[Target, Candidate] = { - val timeoutCounter = statsReceiver.counter("timeout") - val original = this - new Ranker[Target, Candidate] { - override def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { - original - .rank(target, candidates) - .within(timeout)(com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - timeoutCounter.incr() - Stitch.value(candidates) - } - } - } - } -} - -object Ranker { - - def chain[Target, Candidate]( - transformer: Transform[Target, Candidate], - ranker: Ranker[Target, Candidate] - ): Ranker[Target, Candidate] = { - new Ranker[Target, Candidate] { - def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { - transformer - .transform(target, candidates) - .flatMap { results => ranker.rank(target, results) } - } - } - } -} - -class IdentityRanker[Target, Candidate] extends Ranker[Target, Candidate] { - def rank(target: Target, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = - Stitch.value(candidates) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/RecommendationFlow.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/RecommendationFlow.scala deleted file mode 100644 index 6bddc9751..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/RecommendationFlow.scala +++ /dev/null @@ -1,250 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineResult -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.stitch.Stitch - -/** - * configs for results generated from the recommendation flow - * - * @param desiredCandidateCount num of desired candidates to return - * @param batchForCandidatesCheck batch size for candidates check - */ -case class RecommendationResultsConfig(desiredCandidateCount: Int, batchForCandidatesCheck: Int) - -trait BaseRecommendationFlow[Target, Candidate <: UniversalNoun[Long]] { - val identifier = RecommendationPipelineIdentifier("RecommendationFlow") - - def process( - pipelineRequest: Target - ): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] - - def mapKey[Target2](fn: Target2 => Target): BaseRecommendationFlow[Target2, Candidate] = { - val original = this - new BaseRecommendationFlow[Target2, Candidate] { - override def process( - pipelineRequest: Target2 - ): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] = - original.process(fn(pipelineRequest)) - } - } -} - -/** - * Defines a typical recommendation flow to fetch, filter, rank and transform candidates. - * - * 1. targetEligibility: determine the eligibility of target request - * 2. candidateSources: fetch candidates from candidate sources based on target type - * 3. preRankerCandidateFilter: light filtering of candidates - * 4. ranker: ranking of candidates (could be composed of multiple stages, light ranking, heavy ranking and etc) - * 5. postRankerTransform: deduping, grouping, rule based promotion / demotions and etc - * 6. validateCandidates: heavy filters to determine the eligibility of the candidates. - * will only be applied to candidates that we expect to return. - * 7. transformResults: transform the individual candidates into desired format (e.g. hydrate social proof) - * - * Note that the actual implementations may not need to implement all the steps if not needed - * (could just leave to IdentityRanker if ranking is not needed). - * - * Theoretically, the actual implementation could override the above flow to add - * more steps (e.g. add a transform step before ranking). - * But it is recommended to add the additional steps into this base flow if the step proves - * to have significant justification, or merge it into an existing step if it is a minor change. - * - * @tparam Target type of target request - * @tparam Candidate type of candidate to return - */ -trait RecommendationFlow[Target, Candidate <: UniversalNoun[Long]] - extends BaseRecommendationFlow[Target, Candidate] - with SideEffectsUtil[Target, Candidate] { - - /** - * optionally update or enrich the request before executing the flows - */ - protected def updateTarget(target: Target): Stitch[Target] = Stitch.value(target) - - /** - * check if the target is eligible for the flow - */ - protected def targetEligibility: Predicate[Target] - - /** - * define the candidate sources that should be used for the given target - */ - protected def candidateSources(target: Target): Seq[CandidateSource[Target, Candidate]] - - /** - * filter invalid candidates before the ranking phase. - */ - protected def preRankerCandidateFilter: Predicate[(Target, Candidate)] - - /** - * rank the candidates - */ - protected def selectRanker(target: Target): Ranker[Target, Candidate] - - /** - * transform the candidates after ranking (e.g. dedupping, grouping and etc) - */ - protected def postRankerTransform: Transform[Target, Candidate] - - /** - * filter invalid candidates before returning the results. - * - * Some heavy filters e.g. SGS filter could be applied in this step - */ - protected def validateCandidates: Predicate[(Target, Candidate)] - - /** - * transform the candidates into results and return - */ - protected def transformResults: Transform[Target, Candidate] - - /** - * configuration for recommendation results - */ - protected def resultsConfig(target: Target): RecommendationResultsConfig - - /** - * track the quality factor the recommendation pipeline - */ - protected def qualityFactorObserver: Option[QualityFactorObserver] = None - - def statsReceiver: StatsReceiver - - /** - * high level monitoring for the whole flow - * (make sure to add monitoring for each individual component by yourself) - * - * additional candidates: count, stats, non_empty_count - * target eligibility: latency, success, failures, request, count, valid_count, invalid_count, invalid_reasons - * candidate generation: latency, success, failures, request, count, non_empty_count, results_stat - * pre ranker filter: latency, success, failures, request, count, non_empty_count, results_stat - * ranker: latency, success, failures, request, count, non_empty_count, results_stat - * post ranker: latency, success, failures, request, count, non_empty_count, results_stat - * filter and take: latency, success, failures, request, count, non_empty_count, results_stat, batch count - * transform results: latency, success, failures, request, count, non_empty_count, results_stat - */ - import RecommendationFlow._ - lazy val additionalCandidatesStats = statsReceiver.scope(AdditionalCandidatesStats) - lazy val targetEligibilityStats = statsReceiver.scope(TargetEligibilityStats) - lazy val candidateGenerationStats = statsReceiver.scope(CandidateGenerationStats) - lazy val preRankerFilterStats = statsReceiver.scope(PreRankerFilterStats) - lazy val rankerStats = statsReceiver.scope(RankerStats) - lazy val postRankerTransformStats = statsReceiver.scope(PostRankerTransformStats) - lazy val filterAndTakeStats = statsReceiver.scope(FilterAndTakeStats) - lazy val transformResultsStats = statsReceiver.scope(TransformResultsStats) - - lazy val overallStats = statsReceiver.scope(OverallStats) - - import StatsUtil._ - - override def process( - pipelineRequest: Target - ): Stitch[RecommendationPipelineResult[Candidate, Seq[Candidate]]] = { - - observeStitchQualityFactor( - profileStitchSeqResults( - updateTarget(pipelineRequest).flatMap { target => - profilePredicateResult(targetEligibility(target), targetEligibilityStats).flatMap { - case PredicateResult.Valid => processValidTarget(target, Seq.empty) - case PredicateResult.Invalid(_) => Stitch.Nil - } - }, - overallStats - ).map { candidates => - RecommendationPipelineResult.empty.withResult(candidates) - }, - qualityFactorObserver, - overallStats - ) - } - - protected def processValidTarget( - target: Target, - additionalCandidates: Seq[Candidate] - ): Stitch[Seq[Candidate]] = { - - /** - * A basic recommendation flow looks like this: - * - * 1. fetch candidates from candidate sources - * 2. blend candidates with existing candidates - * 3. filter the candidates (light filters) before ranking - * 4. ranking - * 5. filter and truncate the candidates using postRankerCandidateFilter - * 6. transform the candidates based on product requirement - */ - val candidateSourcesToFetch = candidateSources(target) - for { - candidates <- profileStitchSeqResults( - Stitch.traverse(candidateSourcesToFetch)(_(target)).map(_.flatten), - candidateGenerationStats - ) - mergedCandidates = - profileSeqResults(additionalCandidates, additionalCandidatesStats) ++ - candidates - filteredCandidates <- profileStitchSeqResults( - Predicate.filter(target, mergedCandidates, preRankerCandidateFilter), - preRankerFilterStats - ) - rankedCandidates <- profileStitchSeqResults( - selectRanker(target).rank(target, filteredCandidates), - rankerStats - ) - transformed <- profileStitchSeqResults( - postRankerTransform.transform(target, rankedCandidates), - postRankerTransformStats - ) - truncated <- profileStitchSeqResults( - take(target, transformed, resultsConfig(target)), - filterAndTakeStats - ) - results <- profileStitchSeqResults( - transformResults.transform(target, truncated), - transformResultsStats - ) - _ <- applySideEffects( - target, - candidateSourcesToFetch, - candidates, - mergedCandidates, - filteredCandidates, - rankedCandidates, - transformed, - truncated, - results) - } yield results - } - - private[this] def take( - target: Target, - candidates: Seq[Candidate], - config: RecommendationResultsConfig - ): Stitch[Seq[Candidate]] = { - Predicate - .batchFilterTake( - candidates.map(c => (target, c)), - validateCandidates, - config.batchForCandidatesCheck, - config.desiredCandidateCount, - statsReceiver - ).map(_.map(_._2)) - } -} - -object RecommendationFlow { - - val AdditionalCandidatesStats = "additional_candidates" - val TargetEligibilityStats = "target_eligibility" - val CandidateGenerationStats = "candidate_generation" - val PreRankerFilterStats = "pre_ranker_filter" - val RankerStats = "ranker" - val PostRankerTransformStats = "post_ranker_transform" - val FilterAndTakeStats = "filter_and_take" - val TransformResultsStats = "transform_results" - val OverallStats = "overall" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/SideEffectsUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/SideEffectsUtil.scala deleted file mode 100644 index 2c922f580..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/SideEffectsUtil.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch - -/** - * SideEffectsUtil applies side effects to the intermediate candidate results from a recommendation flow pipeline. - * - * @tparam Target target to recommend the candidates - * @tparam Candidate candidate type to rank - */ -trait SideEffectsUtil[Target, Candidate] { - def applySideEffects( - target: Target, - candidateSources: Seq[CandidateSource[Target, Candidate]], - candidatesFromCandidateSources: Seq[Candidate], - mergedCandidates: Seq[Candidate], - filteredCandidates: Seq[Candidate], - rankedCandidates: Seq[Candidate], - transformedCandidates: Seq[Candidate], - truncatedCandidates: Seq[Candidate], - results: Seq[Candidate] - ): Stitch[Unit] = Stitch.Unit -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/StatsUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/StatsUtil.scala deleted file mode 100644 index eb99a909e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/StatsUtil.scala +++ /dev/null @@ -1,272 +0,0 @@ -package com.twitter.follow_recommendations.common.base -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.stitch.Arrow -import com.twitter.stitch.Stitch -import com.twitter.util.Stopwatch -import java.util.concurrent.TimeUnit -import scala.util.control.NonFatal - -object StatsUtil { - val LatencyName = "latency_ms" - val RequestName = "requests" - val SuccessName = "success" - val FailureName = "failures" - val ResultsName = "results" - val ResultsStat = "results_stat" - val EmptyResultsName = "empty" - val NonEmptyResultsName = "non_empty" - val ValidCount = "valid" - val InvalidCount = "invalid" - val InvalidHasReasons = "has_reasons" - val Reasons = "reasons" - val QualityFactorStat = "quality_factor_stat" - val QualityFactorCounts = "quality_factor_counts" - - /** - * Helper function for timing a stitch, returning the original stitch. - */ - def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T] = { - - Stitch - .time(stitch) - .map { - case (response, stitchRunDuration) => - stat.counter(RequestName).incr() - stat.stat(LatencyName).add(stitchRunDuration.inMilliseconds) - response - .onSuccess { _ => stat.counter(SuccessName).incr() } - .onFailure { e => - stat.counter(FailureName).incr() - stat.scope(FailureName).counter(getCleanClassName(e)).incr() - } - } - .lowerFromTry - } - - /** - * Helper function for timing an arrow, returning the original arrow. - */ - def profileArrow[T, U](arrow: Arrow[T, U], stat: StatsReceiver): Arrow[T, U] = { - - Arrow - .time(arrow) - .map { - case (response, stitchRunDuration) => - stat.counter(RequestName).incr() - stat.stat(LatencyName).add(stitchRunDuration.inMilliseconds) - response - .onSuccess { _ => stat.counter(SuccessName).incr() } - .onFailure { e => - stat.counter(FailureName).incr() - stat.scope(FailureName).counter(getCleanClassName(e)).incr() - } - } - .lowerFromTry - } - - /** - * Helper function to count and track the distribution of results - */ - def profileResults[T](results: T, stat: StatsReceiver, size: T => Int): T = { - val numResults = size(results) - stat.counter(ResultsName).incr(numResults) - if (numResults == 0) { - stat.counter(EmptyResultsName).incr() - results - } else { - stat.stat(ResultsStat).add(numResults) - stat.counter(NonEmptyResultsName).incr() - results - } - } - - /** - * Helper function to count and track the distribution of a list of results - */ - def profileSeqResults[T](results: Seq[T], stat: StatsReceiver): Seq[T] = { - profileResults[Seq[T]](results, stat, _.size) - } - - /** - * Helper function for timing a stitch and count the number of results, returning the original stitch. - */ - def profileStitchResults[T](stitch: Stitch[T], stat: StatsReceiver, size: T => Int): Stitch[T] = { - profileStitch(stitch, stat).onSuccess { results => profileResults(results, stat, size) } - } - - /** - * Helper function for timing an arrow and count the number of results, returning the original arrow. - */ - def profileArrowResults[T, U]( - arrow: Arrow[T, U], - stat: StatsReceiver, - size: U => Int - ): Arrow[T, U] = { - profileArrow(arrow, stat).onSuccess { results => profileResults(results, stat, size) } - } - - /** - * Helper function for timing a stitch and count a seq of results, returning the original stitch. - */ - def profileStitchSeqResults[T](stitch: Stitch[Seq[T]], stat: StatsReceiver): Stitch[Seq[T]] = { - profileStitchResults[Seq[T]](stitch, stat, _.size) - } - - /** - * Helper function for timing a stitch and count optional results, returning the original stitch. - */ - def profileStitchOptionalResults[T]( - stitch: Stitch[Option[T]], - stat: StatsReceiver - ): Stitch[Option[T]] = { - profileStitchResults[Option[T]](stitch, stat, _.size) - } - - /** - * Helper function for timing a stitch and count a map of results, returning the original stitch. - */ - def profileStitchMapResults[K, V]( - stitch: Stitch[Map[K, V]], - stat: StatsReceiver - ): Stitch[Map[K, V]] = { - profileStitchResults[Map[K, V]](stitch, stat, _.size) - } - - def getCleanClassName(obj: Object): String = - obj.getClass.getSimpleName.stripSuffix("$") - - /** - * Helper function for timing a stitch and count a list of PredicateResult - */ - def profilePredicateResults( - predicateResult: Stitch[Seq[PredicateResult]], - statsReceiver: StatsReceiver - ): Stitch[Seq[PredicateResult]] = { - profileStitch[Seq[PredicateResult]]( - predicateResult, - statsReceiver - ).onSuccess { - _.map { - case PredicateResult.Valid => - statsReceiver.counter(ValidCount).incr() - case PredicateResult.Invalid(reasons) => - statsReceiver.counter(InvalidCount).incr() - reasons.map { filterReason => - statsReceiver.counter(InvalidHasReasons).incr() - statsReceiver.scope(Reasons).counter(filterReason.reason).incr() - } - } - } - } - - /** - * Helper function for timing a stitch and count individual PredicateResult - */ - def profilePredicateResult( - predicateResult: Stitch[PredicateResult], - statsReceiver: StatsReceiver - ): Stitch[PredicateResult] = { - profilePredicateResults( - predicateResult.map(Seq(_)), - statsReceiver - ).map(_.head) - } - - /** - * Helper function for timing an arrow and count a list of PredicateResult - */ - def profilePredicateResults[Q]( - predicateResult: Arrow[Q, Seq[PredicateResult]], - statsReceiver: StatsReceiver - ): Arrow[Q, Seq[PredicateResult]] = { - profileArrow[Q, Seq[PredicateResult]]( - predicateResult, - statsReceiver - ).onSuccess { - _.map { - case PredicateResult.Valid => - statsReceiver.counter(ValidCount).incr() - case PredicateResult.Invalid(reasons) => - statsReceiver.counter(InvalidCount).incr() - reasons.map { filterReason => - statsReceiver.counter(InvalidHasReasons).incr() - statsReceiver.scope(Reasons).counter(filterReason.reason).incr() - } - } - } - } - - /** - * Helper function for timing an arrow and count individual PredicateResult - */ - def profilePredicateResult[Q]( - predicateResult: Arrow[Q, PredicateResult], - statsReceiver: StatsReceiver - ): Arrow[Q, PredicateResult] = { - profilePredicateResults( - predicateResult.map(Seq(_)), - statsReceiver - ).map(_.head) - } - - /** - * Helper function for timing a stitch code block - */ - def profileStitchSeqResults[T]( - stats: StatsReceiver - )( - block: => Stitch[Seq[T]] - ): Stitch[Seq[T]] = { - stats.counter(RequestName).incr() - profileStitch(stats.stat(LatencyName), TimeUnit.MILLISECONDS) { - block onSuccess { r => - if (r.isEmpty) stats.counter(EmptyResultsName).incr() - stats.stat(ResultsStat).add(r.size) - } onFailure { e => - { - stats.counter(FailureName).incr() - stats.scope(FailureName).counter(e.getClass.getName).incr() - } - } - } - } - - /** - * Time a given asynchronous `f` using the given `unit`. - */ - def profileStitch[A](stat: Stat, unit: TimeUnit)(f: => Stitch[A]): Stitch[A] = { - val start = Stopwatch.timeNanos() - try { - f.respond { _ => stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) } - } catch { - case NonFatal(e) => - stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) - Stitch.exception(e) - } - } - - def observeStitchQualityFactor[T]( - stitch: Stitch[T], - qualityFactorObserverOption: Option[QualityFactorObserver], - statsReceiver: StatsReceiver - ): Stitch[T] = { - qualityFactorObserverOption - .map { observer => - Stitch - .time(stitch) - .map { - case (response, stitchRunDuration) => - observer(response, stitchRunDuration) - val qfVal = observer.qualityFactor.currentValue.floatValue() * 10000 - statsReceiver.counter(QualityFactorCounts).incr() - statsReceiver - .stat(QualityFactorStat) - .add(qfVal) - response - } - .lowerFromTry - }.getOrElse(stitch) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Transform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Transform.scala deleted file mode 100644 index c870db2f6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base/Transform.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.follow_recommendations.common.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Param - -/** - * transform a or a list of candidate for target T - * - * @tparam T target type - * @tparam C candidate type - */ -trait Transform[-T, C] { - - // you need to implement at least one of the two methods here. - def transformItem(target: T, item: C): Stitch[C] = { - transform(target, Seq(item)).map(_.head) - } - - def transform(target: T, items: Seq[C]): Stitch[Seq[C]] - - def mapTarget[T2](mapper: T2 => T): Transform[T2, C] = { - val original = this - new Transform[T2, C] { - override def transformItem(target: T2, item: C): Stitch[C] = { - original.transformItem(mapper(target), item) - } - override def transform(target: T2, items: Seq[C]): Stitch[Seq[C]] = { - original.transform(mapper(target), items) - } - } - } - - /** - * sequential composition. we execute this' transform first, followed by the other's transform - */ - def andThen[T1 <: T](other: Transform[T1, C]): Transform[T1, C] = { - val original = this - new Transform[T1, C] { - override def transformItem(target: T1, item: C): Stitch[C] = - original.transformItem(target, item).flatMap(other.transformItem(target, _)) - override def transform(target: T1, items: Seq[C]): Stitch[Seq[C]] = - original.transform(target, items).flatMap(other.transform(target, _)) - } - } - - def observe(statsReceiver: StatsReceiver): Transform[T, C] = { - val originalTransform = this - new Transform[T, C] { - override def transform(target: T, items: Seq[C]): Stitch[Seq[C]] = { - statsReceiver.counter(Transform.InputCandidatesCount).incr(items.size) - statsReceiver.stat(Transform.InputCandidatesStat).add(items.size) - StatsUtil.profileStitchSeqResults(originalTransform.transform(target, items), statsReceiver) - } - - override def transformItem(target: T, item: C): Stitch[C] = { - statsReceiver.counter(Transform.InputCandidatesCount).incr() - StatsUtil.profileStitch(originalTransform.transformItem(target, item), statsReceiver) - } - } - } -} - -trait GatedTransform[T <: HasParams, C] extends Transform[T, C] { - def gated(param: Param[Boolean]): Transform[T, C] = { - val original = this - (target: T, items: Seq[C]) => { - if (target.params(param)) { - original.transform(target, items) - } else { - Stitch.value(items) - } - } - } -} - -object Transform { - val InputCandidatesCount = "input_candidates" - val InputCandidatesStat = "input_candidates_stat" -} - -class IdentityTransform[T, C] extends Transform[T, C] { - override def transform(target: T, items: Seq[C]): Stitch[Seq[C]] = Stitch.value(items) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/AddressBookParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/AddressBookParams.scala deleted file mode 100644 index f93a60d1b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/AddressBookParams.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.addressbook - -import com.twitter.timelines.configapi.FSParam - -object AddressBookParams { - // Used by display locations that want only to read from the ABV2 Client and ignore Manhattan - // Currently the only display location that does this is the ABUploadInjection DisplayLocation - object ReadFromABV2Only extends FSParam[Boolean]("addressbook_read_only_from_abv2", false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/BUILD deleted file mode 100644 index ddbabad19..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/hermit/usercontacts:hermit-usercontacts-scala", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardEmailBookSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardEmailBookSource.scala deleted file mode 100644 index d291459ce..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardEmailBookSource.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.addressbook - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.AddressBookParams.ReadFromABV2Only -import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient -import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType -import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.ForwardEmailBookClientColumn -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForwardEmailBookSource @Inject() ( - forwardEmailBookClientColumn: ForwardEmailBookClientColumn, - addressBookClient: AddressbookClient, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = - ForwardEmailBookSource.Identifier - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - /** - * Generate a list of candidates for the target - */ - override def apply( - target: HasParams with HasClientContext - ): Stitch[Seq[CandidateUser]] = { - val candidateUsers: Stitch[Seq[Long]] = target.getOptionalUserId - .map { userId => - rescueWithStats( - addressBookClient.getUsers( - userId = userId, - identifiers = - Seq(RecordIdentifier(userId = Some(userId), email = None, phoneNumber = None)), - batchSize = AddressbookClient.AddressBook2BatchSize, - edgeType = ForwardEmailBookSource.DefaultEdgeType, - fetcherOption = - if (target.params.apply(ReadFromABV2Only)) None - else Some(forwardEmailBookClientColumn.fetcher), - queryOption = AddressbookClient - .createQueryOption( - edgeType = ForwardEmailBookSource.DefaultEdgeType, - isPhone = ForwardEmailBookSource.IsPhone) - ), - stats, - "AddressBookClient" - ) - }.getOrElse(Stitch.Nil) - - candidateUsers - .map( - _.take(ForwardEmailBookSource.NumEmailBookEntries) - .map(CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) - .withCandidateSource(identifier))) - } -} - -object ForwardEmailBookSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.ForwardEmailBook.toString) - val NumEmailBookEntries: Int = 1000 - val IsPhone = false - val DefaultEdgeType: EdgeType = EdgeType.Forward -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardPhoneBookSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardPhoneBookSource.scala deleted file mode 100644 index bb1f61f05..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ForwardPhoneBookSource.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.addressbook - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.AddressBookParams.ReadFromABV2Only -import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient -import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType -import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.ForwardPhoneContactsClientColumn -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForwardPhoneBookSource @Inject() ( - forwardPhoneContactsClientColumn: ForwardPhoneContactsClientColumn, - addressBookClient: AddressbookClient, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = - ForwardPhoneBookSource.Identifier - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - /** - * Generate a list of candidates for the target - */ - override def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - val candidateUsers: Stitch[Seq[Long]] = target.getOptionalUserId - .map { userId => - rescueWithStats( - addressBookClient.getUsers( - userId, - identifiers = - Seq(RecordIdentifier(userId = Some(userId), email = None, phoneNumber = None)), - batchSize = AddressbookClient.AddressBook2BatchSize, - edgeType = ForwardPhoneBookSource.DefaultEdgeType, - fetcherOption = - if (target.params.apply(ReadFromABV2Only)) None - else Some(forwardPhoneContactsClientColumn.fetcher), - queryOption = AddressbookClient - .createQueryOption( - edgeType = ForwardPhoneBookSource.DefaultEdgeType, - isPhone = ForwardPhoneBookSource.IsPhone) - ), - stats, - "AddressBookClient" - ) - }.getOrElse(Stitch.Nil) - - candidateUsers - .map( - _.take(ForwardPhoneBookSource.NumPhoneBookEntries) - .map(CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) - .withCandidateSource(identifier))) - } -} - -object ForwardPhoneBookSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.ForwardPhoneBook.toString) - val NumPhoneBookEntries: Int = 1000 - val IsPhone = true - val DefaultEdgeType: EdgeType = EdgeType.Forward -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/README.md deleted file mode 100644 index 37b04a638..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Address Book Candidate Source -Provides the accounts of a given user's forward and reverse phone and email book contacts. -It is only available when the user has synced their address book with the service. - diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReverseEmailBookSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReverseEmailBookSource.scala deleted file mode 100644 index 6e89f8c24..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReverseEmailBookSource.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.addressbook - -import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient -import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType -import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier -import com.twitter.follow_recommendations.common.clients.email_storage_service.EmailStorageServiceClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueOptionalWithStats -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.ReverseEmailContactsClientColumn -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ReverseEmailBookSource @Inject() ( - reverseEmailContactsClientColumn: ReverseEmailContactsClientColumn, - essClient: EmailStorageServiceClient, - addressBookClient: AddressbookClient, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] { - override val identifier: CandidateSourceIdentifier = ReverseEmailBookSource.Identifier - private val rescueStats = statsReceiver.scope("ReverseEmailBookSource") - - /** - * Generate a list of candidates for the target - */ - override def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - val reverseCandidatesFromEmail = target.getOptionalUserId - .map { userId => - val verifiedEmailStitchOpt = - rescueOptionalWithStats( - essClient.getVerifiedEmail(userId, PurposeOfProcessing.ContentRecommendations), - rescueStats, - "getVerifiedEmail") - verifiedEmailStitchOpt.flatMap { emailOpt => - rescueWithStats( - addressBookClient.getUsers( - userId = userId, - identifiers = emailOpt - .map(email => - RecordIdentifier(userId = None, email = Some(email), phoneNumber = None)).toSeq, - batchSize = ReverseEmailBookSource.NumEmailBookEntries, - edgeType = ReverseEmailBookSource.DefaultEdgeType, - fetcherOption = - if (target.params(AddressBookParams.ReadFromABV2Only)) None - else Some(reverseEmailContactsClientColumn.fetcher) - ), - rescueStats, - "AddressBookClient" - ) - } - }.getOrElse(Stitch.Nil) - - reverseCandidatesFromEmail.map( - _.take(ReverseEmailBookSource.NumEmailBookEntries) - .map( - CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) - .withCandidateSource(identifier)) - ) - } -} - -object ReverseEmailBookSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.ReverseEmailBookIbis.toString) - val NumEmailBookEntries: Int = 500 - val IsPhone = false - val DefaultEdgeType: EdgeType = EdgeType.Reverse -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReversePhoneBookSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReversePhoneBookSource.scala deleted file mode 100644 index 4dbe5a617..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook/ReversePhoneBookSource.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.addressbook - -import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookClient -import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType -import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier -import com.twitter.follow_recommendations.common.clients.phone_storage_service.PhoneStorageServiceClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.ReversePhoneContactsClientColumn -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ReversePhoneBookSource @Inject() ( - reversePhoneContactsClientColumn: ReversePhoneContactsClientColumn, - pssClient: PhoneStorageServiceClient, - addressBookClient: AddressbookClient, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = ReversePhoneBookSource.Identifier - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - /** - * Generate a list of candidates for the target - */ - override def apply(target: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - val reverseCandidatesFromPhones: Stitch[Seq[Long]] = target.getOptionalUserId - .map { userId => - pssClient - .getPhoneNumbers(userId, PurposeOfProcessing.ContentRecommendations) - .flatMap { phoneNumbers => - rescueWithStats( - addressBookClient.getUsers( - userId = userId, - identifiers = phoneNumbers.map(phoneNumber => - RecordIdentifier(userId = None, email = None, phoneNumber = Some(phoneNumber))), - batchSize = ReversePhoneBookSource.NumPhoneBookEntries, - edgeType = ReversePhoneBookSource.DefaultEdgeType, - fetcherOption = - if (target.params(AddressBookParams.ReadFromABV2Only)) None - else Some(reversePhoneContactsClientColumn.fetcher), - queryOption = AddressbookClient.createQueryOption( - edgeType = ReversePhoneBookSource.DefaultEdgeType, - isPhone = ReversePhoneBookSource.IsPhone) - ), - stats, - "AddressBookClient" - ) - } - }.getOrElse(Stitch.Nil) - - reverseCandidatesFromPhones.map( - _.take(ReversePhoneBookSource.NumPhoneBookEntries) - .map( - CandidateUser(_, score = Some(CandidateUser.DefaultCandidateScore)) - .withCandidateSource(identifier)) - ) - } -} - -object ReversePhoneBookSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.ReversePhoneBook.toString) - val NumPhoneBookEntries: Int = 500 - val IsPhone = true - val DefaultEdgeType: EdgeType = EdgeType.Reverse -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/BUILD deleted file mode 100644 index 275137bf9..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "content-recommender/thrift/src/main/thrift:thrift-scala", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/scala/com/twitter/onboarding/relevance/features/ymbii", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/CachedCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/CachedCandidateSource.scala deleted file mode 100644 index c0196c304..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/CachedCandidateSource.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.util.Duration - -class CachedCandidateSource[K <: Object, V <: Object]( - candidateSource: CandidateSource[K, V], - maxCacheSize: Int, - cacheTTL: Duration, - statsReceiver: StatsReceiver, - override val identifier: CandidateSourceIdentifier) - extends CandidateSource[K, V] { - - private val cache = StitchCache[K, Seq[V]]( - maxCacheSize = maxCacheSize, - ttl = cacheTTL, - statsReceiver = statsReceiver.scope(identifier.name, "cache"), - underlyingCall = (k: K) => candidateSource(k) - ) - - override def apply(target: K): Stitch[Seq[V]] = cache.readThrough(target) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/ExperimentalCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/ExperimentalCandidateSource.scala deleted file mode 100644 index 9e2b0e6e9..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/ExperimentalCandidateSource.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Param -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -/** - * A wrapper of CandidateSource to make it easier to do experimentation - * on new candidate generation algorithms - * - * @param baseSource base candidate source - * @param darkreadAlgorithmParam controls whether or not to darkread candidates (fetch them even if they will not be included) - * @param keepCandidatesParam controls whether or not to keep candidates from the base source - * @param resultCountThresholdParam controls how many results the source must return to bucket the user and return results (greater-than-or-equal-to) - * @tparam T request type. it must extend HasParams - * @tparam V value type - */ -class ExperimentalCandidateSource[T <: HasParams, V]( - baseSource: CandidateSource[T, V], - darkreadAlgorithmParam: Param[Boolean], - keepCandidatesParam: Param[Boolean], - resultCountThresholdParam: Param[Int], - baseStatsReceiver: StatsReceiver) - extends CandidateSource[T, V] { - - override val identifier: CandidateSourceIdentifier = baseSource.identifier - private[base] val statsReceiver = - baseStatsReceiver.scope(s"Experimental/${identifier.name}") - private[base] val requestsCounter = statsReceiver.counter("requests") - private[base] val resultCountGreaterThanThresholdCounter = - statsReceiver.counter("with_results_at_or_above_count_threshold") - private[base] val keepResultsCounter = statsReceiver.counter("keep_results") - private[base] val discardResultsCounter = statsReceiver.counter("discard_results") - - override def apply(request: T): Stitch[Seq[V]] = { - if (request.params(darkreadAlgorithmParam)) { - requestsCounter.incr() - fetchFromCandidateSourceAndProcessResults(request) - } else { - Stitch.Nil - } - } - - private def fetchFromCandidateSourceAndProcessResults(request: T): Stitch[Seq[V]] = { - baseSource(request).map { results => - if (results.length >= request.params(resultCountThresholdParam)) { - processResults(results, request.params(keepCandidatesParam)) - } else { - Nil - } - } - } - - private def processResults(results: Seq[V], keepResults: Boolean): Seq[V] = { - resultCountGreaterThanThresholdCounter.incr() - if (keepResults) { - keepResultsCounter.incr() - results - } else { - discardResultsCounter.incr() - Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/RealGraphExpansionRepository.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/RealGraphExpansionRepository.scala deleted file mode 100644 index 8add11fa6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/RealGraphExpansionRepository.scala +++ /dev/null @@ -1,208 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.candidate_sources.base.RealGraphExpansionRepository.DefaultScore -import com.twitter.follow_recommendations.common.candidate_sources.base.RealGraphExpansionRepository.MaxNumIntermediateNodesToKeep -import com.twitter.follow_recommendations.common.candidate_sources.base.RealGraphExpansionRepository.FirstDegreeCandidatesTimeout -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models._ -import com.twitter.onboarding.relevance.features.ymbii.ExpansionCandidateScores -import com.twitter.onboarding.relevance.features.ymbii.RawYMBIICandidateFeatures -import com.twitter.onboarding.relevance.store.thriftscala.CandidatesFollowedV1 -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.util.Duration -import scala.collection.immutable -import scala.util.control.NonFatal - -private final case class InterestExpansionCandidate( - userID: Long, - score: Double, - features: RawYMBIICandidateFeatures) - -abstract class RealGraphExpansionRepository[Request]( - realgraphExpansionStore: Fetcher[ - Long, - Unit, - CandidatesFollowedV1 - ], - override val identifier: CandidateSourceIdentifier, - statsReceiver: StatsReceiver = NullStatsReceiver, - maxUnderlyingCandidatesToQuery: Int = 50, - maxCandidatesToReturn: Int = 40, - overrideUnderlyingTimeout: Option[Duration] = None, - appendSocialProof: Boolean = false) - extends CandidateSource[ - Request, - CandidateUser - ] { - - val underlyingCandidateSource: Seq[ - CandidateSource[ - Request, - CandidateUser - ] - ] - - private val stats = statsReceiver.scope(this.getClass.getSimpleName).scope(identifier.name) - private val underlyingCandidateSourceFailureStats = - stats.scope("underlying_candidate_source_failure") - - def apply( - request: Request, - ): Stitch[Seq[CandidateUser]] = { - - val candidatesFromUnderlyingSourcesStitch: Seq[Stitch[Seq[CandidateUser]]] = - underlyingCandidateSource.map { candidateSource => - candidateSource - .apply(request) - .within(overrideUnderlyingTimeout.getOrElse(FirstDegreeCandidatesTimeout))( - DefaultTimer - ) - .handle { - case NonFatal(e) => - underlyingCandidateSourceFailureStats - .counter(candidateSource.identifier.name, e.getClass.getSimpleName).incr() - Seq.empty - } - } - - for { - underlyingCandidatesFromEachAlgo <- Stitch.collect(candidatesFromUnderlyingSourcesStitch) - // The first algorithm in the list has the highest priority. Depending on if its not - // populated, fall back to other algorithms. Once a particular algorithm is chosen, only - // take the top few candidates from the underlying store for expansion. - underlyingCandidatesTuple = - underlyingCandidatesFromEachAlgo - .zip(underlyingCandidateSource) - .find(_._1.nonEmpty) - - underlyingAlgorithmUsed: Option[CandidateSourceIdentifier] = underlyingCandidatesTuple.map { - case (_, candidateSource) => candidateSource.identifier - } - - // Take maxUnderlyingCandidatesToQuery to query realgraphExpansionStore - underlyingCandidates = - underlyingCandidatesTuple - .map { - case (candidates, candidateSource) => - stats - .scope("underlyingAlgorithmUsedScope").counter( - candidateSource.identifier.name).incr() - candidates - } - .getOrElse(Seq.empty) - .sortBy(_.score.getOrElse(DefaultScore))(Ordering.Double.reverse) - .take(maxUnderlyingCandidatesToQuery) - - underlyingCandidateMap: Map[Long, Double] = underlyingCandidates.map { candidate => - (candidate.id, candidate.score.getOrElse(DefaultScore)) - }.toMap - - expansionCandidates <- - Stitch - .traverse(underlyingCandidateMap.keySet.toSeq) { candidateId => - Stitch.join( - Stitch.value(candidateId), - realgraphExpansionStore.fetch(candidateId).map(_.v)) - - }.map(_.toMap) - - rerankedCandidates: Seq[InterestExpansionCandidate] = - rerankCandidateExpansions(underlyingCandidateMap, expansionCandidates) - - rerankedCandidatesFiltered = rerankedCandidates.take(maxCandidatesToReturn) - - } yield { - rerankedCandidatesFiltered.map { candidate => - val socialProofReason = if (appendSocialProof) { - val socialProofIds = candidate.features.expansionCandidateScores - .map(_.intermediateCandidateId) - Some( - Reason(Some( - AccountProof(followProof = Some(FollowProof(socialProofIds, socialProofIds.size)))))) - } else { - None - } - CandidateUser( - id = candidate.userID, - score = Some(candidate.score), - reason = socialProofReason, - userCandidateSourceDetails = Some( - UserCandidateSourceDetails( - primaryCandidateSource = Some(identifier), - candidateSourceFeatures = Map(identifier -> Seq(candidate.features)) - )) - ).addAddressBookMetadataIfAvailable(underlyingAlgorithmUsed.toSeq) - } - } - } - - /** - * Expands underlying candidates, returning them in sorted order. - * - * @param underlyingCandidatesMap A map from underlying candidate id to score - * @param expansionCandidateMap A map from underlying candidate id to optional expansion candidates - * @return A sorted sequence of expansion candidates and associated scores - */ - private def rerankCandidateExpansions( - underlyingCandidatesMap: Map[Long, Double], - expansionCandidateMap: Map[Long, Option[CandidatesFollowedV1]] - ): Seq[InterestExpansionCandidate] = { - - // extract features - val candidates: Seq[(Long, ExpansionCandidateScores)] = for { - (underlyingCandidateId, underlyingCandidateScore) <- underlyingCandidatesMap.toSeq - expansionCandidates = - expansionCandidateMap - .get(underlyingCandidateId) - .flatten - .map(_.candidatesFollowed) - .getOrElse(Seq.empty) - expansionCandidate <- expansionCandidates - } yield expansionCandidate.candidateID -> ExpansionCandidateScores( - underlyingCandidateId, - Some(underlyingCandidateScore), - Some(expansionCandidate.score) - ) - - // merge intermediate nodes for the same candidate - val dedupedCandidates: Seq[(Long, Seq[ExpansionCandidateScores])] = - candidates.groupBy(_._1).mapValues(_.map(_._2).sortBy(_.intermediateCandidateId)).toSeq - - // score the candidate - val candidatesWithTotalScore: Seq[((Long, Seq[ExpansionCandidateScores]), Double)] = - dedupedCandidates.map { candidate: (Long, Seq[ExpansionCandidateScores]) => - ( - candidate, - candidate._2.map { ieScore: ExpansionCandidateScores => - ieScore.scoreFromUserToIntermediateCandidate.getOrElse(DefaultScore) * - ieScore.scoreFromIntermediateToExpansionCandidate.getOrElse(DefaultScore) - }.sum) - } - - // sort candidate by score - for { - ((candidate, edges), score) <- candidatesWithTotalScore.sortBy(_._2)(Ordering[Double].reverse) - } yield InterestExpansionCandidate( - candidate, - score, - RawYMBIICandidateFeatures( - edges.size, - edges.take(MaxNumIntermediateNodesToKeep).to[immutable.Seq]) - ) - } - -} - -object RealGraphExpansionRepository { - private val FirstDegreeCandidatesTimeout: Duration = 250.milliseconds - private val MaxNumIntermediateNodesToKeep = 20 - private val DefaultScore = 0.0d - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderParams.scala deleted file mode 100644 index a4a0c9784..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderParams.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object SimilarUserExpanderParams { - - case object EnableNonDirectFollowExpansion - extends FSParam[Boolean]("similar_user_enable_non_direct_follow_expansion", true) - - case object EnableSimsExpandSeedAccountsSort - extends FSParam[Boolean]("similar_user_enable_sims_expander_seed_account_sort", false) - - case object DefaultExpansionInputCount - extends FSBoundedParam[Int]( - name = "similar_user_default_expansion_input_count", - default = Integer.MAX_VALUE, - min = 0, - max = Integer.MAX_VALUE) - - case object DefaultFinalCandidatesReturnedCount - extends FSBoundedParam[Int]( - name = "similar_user_default_final_candidates_returned_count", - default = Integer.MAX_VALUE, - min = 0, - max = Integer.MAX_VALUE) - - case object DefaultEnableImplicitEngagedExpansion - extends FSParam[Boolean]("similar_user_enable_implicit_engaged_expansion", true) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderRepository.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderRepository.scala deleted file mode 100644 index 336902b34..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SimilarUserExpanderRepository.scala +++ /dev/null @@ -1,313 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.DefaultEnableImplicitEngagedExpansion -import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.DefaultExpansionInputCount -import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.DefaultFinalCandidatesReturnedCount -import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.EnableNonDirectFollowExpansion -import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderParams.EnableSimsExpandSeedAccountsSort -import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderRepository.DefaultCandidateBuilder -import com.twitter.follow_recommendations.common.candidate_sources.base.SimilarUserExpanderRepository.DefaultScore -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.EngagementType -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.follow_recommendations.common.models.SimilarToProof -import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails -import com.twitter.hermit.candidate.thriftscala.Candidates -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class SecondDegreeCandidate(userId: Long, score: Double, socialProof: Option[Seq[Long]]) - -abstract class SimilarUserExpanderRepository[-Request <: HasParams]( - override val identifier: CandidateSourceIdentifier, - similarToCandidatesFetcher: Fetcher[ - Long, - Unit, - Candidates - ], - expansionInputSizeParam: FSBoundedParam[Int] = DefaultExpansionInputCount, - candidatesReturnedSizeParam: FSBoundedParam[Int] = DefaultFinalCandidatesReturnedCount, - enableImplicitEngagedExpansion: FSParam[Boolean] = DefaultEnableImplicitEngagedExpansion, - thresholdToAvoidExpansion: Int = 30, - maxExpansionPerCandidate: Option[Int] = None, - includingOriginalCandidates: Boolean = false, - scorer: (Double, Double) => Double = SimilarUserExpanderRepository.DefaultScorer, - aggregator: (Seq[Double]) => Double = ScoreAggregator.Max, - candidateBuilder: (Long, CandidateSourceIdentifier, Double, CandidateUser) => CandidateUser = - DefaultCandidateBuilder) - extends TwoHopExpansionCandidateSource[ - Request, - CandidateUser, - SecondDegreeCandidate, - CandidateUser - ] { - - val originalCandidateSource: CandidateSource[Request, CandidateUser] - val backupOriginalCandidateSource: Option[CandidateSource[Request, CandidateUser]] = None - - override def firstDegreeNodes(request: Request): Stitch[Seq[CandidateUser]] = { - - val originalCandidatesStitch: Stitch[Seq[CandidateUser]] = - originalCandidateSource(request) - - val backupCandidatesStitch: Stitch[Seq[CandidateUser]] = - if (request.params(EnableNonDirectFollowExpansion)) { - backupOriginalCandidateSource.map(_.apply(request)).getOrElse(Stitch.Nil) - } else { - Stitch.Nil - } - - val firstDegreeCandidatesCombinedStitch: Stitch[Seq[CandidateUser]] = - Stitch - .join(originalCandidatesStitch, backupCandidatesStitch).map { - case (firstDegreeOrigCandidates, backupFirstDegreeCandidates) => - if (request.params(EnableSimsExpandSeedAccountsSort)) { - firstDegreeOrigCandidates ++ backupFirstDegreeCandidates sortBy { - -_.score.getOrElse(DefaultScore) - } - } else { - firstDegreeOrigCandidates ++ backupFirstDegreeCandidates - } - } - - val candidatesAfterImplicitEngagementsRemovalStitch: Stitch[Seq[CandidateUser]] = - getCandidatesAfterImplicitEngagementFiltering( - request.params, - firstDegreeCandidatesCombinedStitch) - - val firstDegreeCandidatesCombinedTrimmed = candidatesAfterImplicitEngagementsRemovalStitch.map { - candidates: Seq[CandidateUser] => - candidates.take(request.params(expansionInputSizeParam)) - } - - firstDegreeCandidatesCombinedTrimmed.map { firstDegreeResults: Seq[CandidateUser] => - if (firstDegreeResults.nonEmpty && firstDegreeResults.size < thresholdToAvoidExpansion) { - firstDegreeResults - .groupBy(_.id).mapValues( - _.maxBy(_.score) - ).values.toSeq - } else { - Nil - } - } - - } - - override def secondaryDegreeNodes( - request: Request, - firstDegreeCandidate: CandidateUser - ): Stitch[Seq[SecondDegreeCandidate]] = { - similarToCandidatesFetcher.fetch(firstDegreeCandidate.id).map(_.v).map { candidateListOption => - candidateListOption - .map { candidatesList => - candidatesList.candidates.map(candidate => - SecondDegreeCandidate(candidate.userId, candidate.score, candidate.socialProof)) - }.getOrElse(Nil) - } - - } - - override def aggregateAndScore( - req: Request, - firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SecondDegreeCandidate]] - ): Stitch[Seq[CandidateUser]] = { - - val similarExpanderResults = firstDegreeToSecondDegreeNodesMap.flatMap { - case (firstDegreeCandidate, seqOfSecondDegreeCandidates) => - val sourceScore = firstDegreeCandidate.score.getOrElse(DefaultScore) - val results: Seq[CandidateUser] = seqOfSecondDegreeCandidates.map { secondDegreeCandidate => - val score = scorer(sourceScore, secondDegreeCandidate.score) - candidateBuilder(secondDegreeCandidate.userId, identifier, score, firstDegreeCandidate) - } - maxExpansionPerCandidate match { - case None => results - case Some(limit) => results.sortBy(-_.score.getOrElse(DefaultScore)).take(limit) - } - }.toSeq - - val allCandidates = { - if (includingOriginalCandidates) - firstDegreeToSecondDegreeNodesMap.keySet.toSeq - else - Nil - } ++ similarExpanderResults - - val groupedCandidates: Seq[CandidateUser] = allCandidates - .groupBy(_.id) - .flatMap { - case (_, candidates) => - val finalScore = aggregator(candidates.map(_.score.getOrElse(DefaultScore))) - val candidateSourceDetailsCombined = aggregateCandidateSourceDetails(candidates) - val accountSocialProofcombined = aggregateAccountSocialProof(candidates) - - candidates.headOption.map( - _.copy( - score = Some(finalScore), - reason = accountSocialProofcombined, - userCandidateSourceDetails = candidateSourceDetailsCombined) - .withCandidateSource(identifier)) - } - .toSeq - - Stitch.value( - groupedCandidates - .sortBy { -_.score.getOrElse(DefaultScore) }.take(req.params(candidatesReturnedSizeParam)) - ) - } - - def aggregateCandidateSourceDetails( - candidates: Seq[CandidateUser] - ): Option[UserCandidateSourceDetails] = { - candidates - .map { candidate => - candidate.userCandidateSourceDetails.map(_.candidateSourceScores).getOrElse(Map.empty) - }.reduceLeftOption { (scoreMap1, scoreMap2) => - scoreMap1 ++ scoreMap2 - }.map { - UserCandidateSourceDetails(primaryCandidateSource = None, _) - } - - } - - def aggregateAccountSocialProof(candidates: Seq[CandidateUser]): Option[Reason] = { - candidates - .map { candidate => - ( - candidate.reason - .flatMap(_.accountProof.flatMap(_.similarToProof.map(_.similarTo))).getOrElse(Nil), - candidate.reason - .flatMap(_.accountProof.flatMap(_.followProof.map(_.followedBy))).getOrElse(Nil), - candidate.reason - .flatMap(_.accountProof.flatMap(_.followProof.map(_.numIds))).getOrElse(0) - ) - }.reduceLeftOption { (accountProofOne, accountProofTwo) => - ( - // merge similarToIds - accountProofOne._1 ++ accountProofTwo._1, - // merge followedByIds - accountProofOne._2 ++ accountProofTwo._2, - // add numIds - accountProofOne._3 + accountProofTwo._3) - }.map { proofs => - Reason(accountProof = Some( - AccountProof( - similarToProof = Some(SimilarToProof(proofs._1)), - followProof = if (proofs._2.nonEmpty) Some(FollowProof(proofs._2, proofs._3)) else None - ))) - } - } - - def getCandidatesAfterImplicitEngagementFiltering( - params: Params, - firstDegreeCandidatesStitch: Stitch[Seq[CandidateUser]] - ): Stitch[Seq[CandidateUser]] = { - - if (!params(enableImplicitEngagedExpansion)) { - - /** - * Remove candidates whose engagement types only contain implicit engagements - * (e.g. Profile View, Tweet Click) and only expand those candidates who contain explicit - * engagements. - */ - firstDegreeCandidatesStitch.map { candidates => - candidates.filter { cand => - cand.engagements.exists(engage => - engage == EngagementType.Like || engage == EngagementType.Retweet || engage == EngagementType.Mention) - } - } - } else { - firstDegreeCandidatesStitch - } - } - -} - -object SimilarUserExpanderRepository { - val DefaultScorer: (Double, Double) => Double = (sourceScore: Double, similarScore: Double) => - similarScore - val MultiplyScorer: (Double, Double) => Double = (sourceScore: Double, similarScore: Double) => - sourceScore * similarScore - val SourceScorer: (Double, Double) => Double = (sourceScore: Double, similarScore: Double) => - sourceScore - - val DefaultScore = 0.0d - - val DefaultCandidateBuilder: ( - Long, - CandidateSourceIdentifier, - Double, - CandidateUser - ) => CandidateUser = - ( - userId: Long, - _: CandidateSourceIdentifier, - score: Double, - candidate: CandidateUser - ) => { - val originalCandidateSourceDetails = - candidate.userCandidateSourceDetails.flatMap { candSourceDetails => - candSourceDetails.primaryCandidateSource.map { primaryCandidateSource => - UserCandidateSourceDetails( - primaryCandidateSource = None, - candidateSourceScores = Map(primaryCandidateSource -> candidate.score)) - } - } - CandidateUser( - id = userId, - score = Some(score), - userCandidateSourceDetails = originalCandidateSourceDetails, - reason = - Some(Reason(Some(AccountProof(similarToProof = Some(SimilarToProof(Seq(candidate.id))))))) - ) - } - - val FollowClusterCandidateBuilder: ( - Long, - CandidateSourceIdentifier, - Double, - CandidateUser - ) => CandidateUser = - (userId: Long, _: CandidateSourceIdentifier, score: Double, candidate: CandidateUser) => { - val originalCandidateSourceDetails = - candidate.userCandidateSourceDetails.flatMap { candSourceDetails => - candSourceDetails.primaryCandidateSource.map { primaryCandidateSource => - UserCandidateSourceDetails( - primaryCandidateSource = None, - candidateSourceScores = Map(primaryCandidateSource -> candidate.score)) - } - } - - val originalFollowCluster = candidate.reason - .flatMap(_.accountProof.flatMap(_.followProof.map(_.followedBy))) - - CandidateUser( - id = userId, - score = Some(score), - userCandidateSourceDetails = originalCandidateSourceDetails, - reason = Some( - Reason( - Some( - AccountProof( - similarToProof = Some(SimilarToProof(Seq(candidate.id))), - followProof = originalFollowCluster.map(follows => - FollowProof(follows, follows.size))))) - ) - ) - } -} - -object ScoreAggregator { - // aggregate the same candidates with same id by taking the one with largest score - val Max: Seq[Double] => Double = (candidateScores: Seq[Double]) => { candidateScores.max } - - // aggregate the same candidates with same id by taking the sum of the scores - val Sum: Seq[Double] => Double = (candidateScores: Seq[Double]) => { candidateScores.sum } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSource.scala deleted file mode 100644 index f650c9f6d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSource.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProof -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration - -abstract class SocialProofEnforcedCandidateSource( - candidateSource: CandidateSource[HasClientContext with HasParams, CandidateUser], - modifySocialProof: ModifySocialProof, - minNumSocialProofsRequired: Int, - override val identifier: CandidateSourceIdentifier, - baseStatsReceiver: StatsReceiver) - extends CandidateSource[HasClientContext with HasParams, CandidateUser] { - - val statsReceiver = baseStatsReceiver.scope(identifier.name) - - override def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] = { - val mustCallSgs: Boolean = target.params(SocialProofEnforcedCandidateSourceParams.MustCallSgs) - val callSgsCachedColumn: Boolean = - target.params(SocialProofEnforcedCandidateSourceParams.CallSgsCachedColumn) - val QueryIntersectionIdsNum: Int = - target.params(SocialProofEnforcedCandidateSourceParams.QueryIntersectionIdsNum) - val MaxNumCandidatesToAnnotate: Int = - target.params(SocialProofEnforcedCandidateSourceParams.MaxNumCandidatesToAnnotate) - val gfsIntersectionIdsNum: Int = - target.params(SocialProofEnforcedCandidateSourceParams.GfsIntersectionIdsNum) - val sgsIntersectionIdsNum: Int = - target.params(SocialProofEnforcedCandidateSourceParams.SgsIntersectionIdsNum) - val gfsLagDuration: Duration = - target.params(SocialProofEnforcedCandidateSourceParams.GfsLagDurationInDays) - - candidateSource(target) - .flatMap { candidates => - val candidatesWithoutEnoughSocialProof = candidates - .collect { - case candidate if !candidate.followedBy.exists(_.size >= minNumSocialProofsRequired) => - candidate - } - statsReceiver - .stat("candidates_with_no_social_proofs").add(candidatesWithoutEnoughSocialProof.size) - val candidatesToAnnotate = - candidatesWithoutEnoughSocialProof.take(MaxNumCandidatesToAnnotate) - statsReceiver.stat("candidates_to_annotate").add(candidatesToAnnotate.size) - - val annotatedCandidatesMapStitch = target.getOptionalUserId - .map { userId => - modifySocialProof - .hydrateSocialProof( - userId, - candidatesToAnnotate, - Some(QueryIntersectionIdsNum), - mustCallSgs, - callSgsCachedColumn, - gfsLagDuration = gfsLagDuration, - gfsIntersectionIds = gfsIntersectionIdsNum, - sgsIntersectionIds = sgsIntersectionIdsNum - ).map { annotatedCandidates => - annotatedCandidates - .map(annotatedCandidate => (annotatedCandidate.id, annotatedCandidate)).toMap - } - }.getOrElse(Stitch.value(Map.empty[Long, CandidateUser])) - - annotatedCandidatesMapStitch.map { annotatedCandidatesMap => - candidates - .flatMap { candidate => - if (candidate.followedBy.exists(_.size >= minNumSocialProofsRequired)) { - Some(candidate) - } else { - annotatedCandidatesMap.get(candidate.id).collect { - case annotatedCandidate - if annotatedCandidate.followedBy.exists( - _.size >= minNumSocialProofsRequired) => - annotatedCandidate - } - } - }.map(_.withCandidateSource(identifier)) - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceFSConfig.scala deleted file mode 100644 index a74164f28..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceFSConfig.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SocialProofEnforcedCandidateSourceFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq( - SocialProofEnforcedCandidateSourceParams.MustCallSgs, - SocialProofEnforcedCandidateSourceParams.CallSgsCachedColumn, - ) - override val intFSParams: Seq[FSBoundedParam[Int]] = - Seq( - SocialProofEnforcedCandidateSourceParams.QueryIntersectionIdsNum, - SocialProofEnforcedCandidateSourceParams.MaxNumCandidatesToAnnotate, - SocialProofEnforcedCandidateSourceParams.GfsIntersectionIdsNum, - SocialProofEnforcedCandidateSourceParams.SgsIntersectionIdsNum, - ) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - SocialProofEnforcedCandidateSourceParams.GfsLagDurationInDays - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceParams.scala deleted file mode 100644 index 36e50e59f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/SocialProofEnforcedCandidateSourceParams.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -object SocialProofEnforcedCandidateSourceParams { - case object MustCallSgs - extends FSParam[Boolean]("social_proof_enforced_candidate_source_must_call_sgs", true) - - case object CallSgsCachedColumn - extends FSParam[Boolean]( - "social_proof_enforced_candidate_source_call_sgs_cached_column", - false) - - case object QueryIntersectionIdsNum - extends FSBoundedParam[Int]( - name = "social_proof_enforced_candidate_source_query_intersection_ids_num", - default = 3, - min = 0, - max = Integer.MAX_VALUE) - - case object MaxNumCandidatesToAnnotate - extends FSBoundedParam[Int]( - name = "social_proof_enforced_candidate_source_max_num_candidates_to_annotate", - default = 50, - min = 0, - max = Integer.MAX_VALUE) - - case object GfsIntersectionIdsNum - extends FSBoundedParam[Int]( - name = "social_proof_enforced_candidate_source_gfs_intersection_ids_num", - default = 3, - min = 0, - max = Integer.MAX_VALUE) - - case object SgsIntersectionIdsNum - extends FSBoundedParam[Int]( - name = "social_proof_enforced_candidate_source_sgs_intersection_ids_num", - default = 10, - min = 0, - max = Integer.MAX_VALUE) - - case object GfsLagDurationInDays - extends FSBoundedParam[Duration]( - name = "social_proof_enforced_candidate_source_gfs_lag_duration_in_days", - default = 14.days, - min = 1.days, - max = 60.days) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromDays - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherSource.scala deleted file mode 100644 index ea6fa57fa..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherSource.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher - -abstract class StratoFetcherSource[K, U, V]( - fetcher: Fetcher[K, U, V], - view: U, - override val identifier: CandidateSourceIdentifier) - extends CandidateSource[K, CandidateUser] { - - def map(user: K, v: V): Seq[CandidateUser] - - override def apply(target: K): Stitch[Seq[CandidateUser]] = { - fetcher - .fetch(target, view) - .map { result => - result.v - .map { candidates => map(target, candidates) } - .getOrElse(Nil) - .map(_.withCandidateSource(identifier)) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherWithUnitViewSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherWithUnitViewSource.scala deleted file mode 100644 index 1f1572ee6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/StratoFetcherWithUnitViewSource.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher - -abstract class StratoFetcherWithUnitViewSource[K, V]( - fetcher: Fetcher[K, Unit, V], - override val identifier: CandidateSourceIdentifier) - extends StratoFetcherSource[K, Unit, V](fetcher, Unit, identifier) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TweetAuthorsCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TweetAuthorsCandidateSource.scala deleted file mode 100644 index 541b24837..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TweetAuthorsCandidateSource.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.follow_recommendations.common.models.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch - -/** - * base trait for tweet authors based algorithms, e.g. topical tweet authors, twistly, ... - * - * @tparam Target target type - * @tparam Candidate output candidate types - */ -trait TweetAuthorsCandidateSource[-Target, +Candidate] extends CandidateSource[Target, Candidate] { - - /** - * fetch Tweet candidates - */ - def getTweetCandidates(target: Target): Stitch[Seq[TweetCandidate]] - - /** - * fetch authorId - */ - def getTweetAuthorId(tweetCandidate: TweetCandidate): Stitch[Option[Long]] - - /** - * wrap candidate ID and TweetAuthorProof in Candidate - */ - def toCandidate(authorId: Long, tweetIds: Seq[Long], score: Option[Double]): Candidate - - /** - * aggregate scores, default to the first score - */ - def aggregator(scores: Seq[Double]): Double = - scores.headOption.getOrElse(TweetAuthorsCandidateSource.DefaultScore) - - /** - * aggregation method for a group of tweet candidates - */ - def aggregateAndScore( - target: Target, - tweetCandidates: Seq[TweetCandidate] - ): Seq[Candidate] - - /** - * generate a list of candidates for the target - */ - def build( - target: Target - ): Stitch[Seq[Candidate]] = { - // Fetch Tweet candidates and hydrate author IDs - val tweetCandidatesStitch = for { - tweetCandidates <- getTweetCandidates(target) - authorIds <- Stitch.collect(tweetCandidates.map(getTweetAuthorId(_))) - } yield { - for { - (authorIdOpt, tweetCandidate) <- authorIds.zip(tweetCandidates) - authorId <- authorIdOpt - } yield tweetCandidate.copy(authorId = authorId) - } - - // Aggregate and score, convert to candidate - tweetCandidatesStitch.map(aggregateAndScore(target, _)) - } - - def apply(target: Target): Stitch[Seq[Candidate]] = - build(target) -} - -object TweetAuthorsCandidateSource { - final val DefaultScore: Double = 0.0 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TwoHopExpansionCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TwoHopExpansionCandidateSource.scala deleted file mode 100644 index 40c699d22..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base/TwoHopExpansionCandidateSource.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.base - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch - -/** - * base trait for two-hop expansion based algorithms, e.g. online_stp, phonebook_prediction, - * recent following sims, recent engagement sims, ... - * - * @tparam Target target type - * @tparam FirstDegree type of first degree nodes - * @tparam SecondaryDegree type of secondary degree nodes - * @tparam Candidate output candidate types - */ -trait TwoHopExpansionCandidateSource[-Target, FirstDegree, SecondaryDegree, +Candidate] - extends CandidateSource[Target, Candidate] { - - /** - * fetch first degree nodes given request - */ - def firstDegreeNodes(req: Target): Stitch[Seq[FirstDegree]] - - /** - * fetch secondary degree nodes given request and first degree nodes - */ - def secondaryDegreeNodes(req: Target, node: FirstDegree): Stitch[Seq[SecondaryDegree]] - - /** - * aggregate and score the candidates to generate final results - */ - def aggregateAndScore( - req: Target, - firstDegreeToSecondDegreeNodesMap: Map[FirstDegree, Seq[SecondaryDegree]] - ): Stitch[Seq[Candidate]] - - /** - * Generate a list of candidates for the target - */ - def apply(target: Target): Stitch[Seq[Candidate]] = { - for { - firstDegreeNodes <- firstDegreeNodes(target) - secondaryDegreeNodes <- Stitch.traverse(firstDegreeNodes)(secondaryDegreeNodes(target, _)) - aggregated <- aggregateAndScore(target, firstDegreeNodes.zip(secondaryDegreeNodes).toMap) - } yield aggregated - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/BUILD deleted file mode 100644 index de3bc0a6b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/onboarding/relevance/crowd_search_accounts:crowd_search_accounts-scala", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsFSConfig.scala deleted file mode 100644 index 520983b60..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsFSConfig.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CrowdSearchAccountsFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - CrowdSearchAccountsParams.CandidateSourceEnabled, - ) - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - CrowdSearchAccountsParams.CandidateSourceWeight, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsParams.scala deleted file mode 100644 index a167b7768..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsParams.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumSeqParam -import com.twitter.timelines.configapi.FSParam - -object CrowdSearchAccountsParams { - // whether or not to fetch CrowdSearchAccounts candidate sources - case object CandidateSourceEnabled - extends FSParam[Boolean]("crowd_search_accounts_candidate_source_enabled", false) - - /** - * Contains the logic key for account filtering and ranking. Currently we have 3 main logic keys - * - new_daily: filtering top searched accounts with max daily searches based on new users - * - new_weekly: filtering top searched accounts with max weekly searches based on new users - * - daily: filtering top searched accounts with max daily searches - * - weekly: filtering top searched accounts with max weekly searches - * Mapping of the Logic Id to Logic key is done via @enum AccountsFilteringAndRankingLogic - */ - case object AccountsFilteringAndRankingLogics - extends FSEnumSeqParam[AccountsFilteringAndRankingLogicId.type]( - name = "crowd_search_accounts_filtering_and_ranking_logic_ids", - default = Seq(AccountsFilteringAndRankingLogicId.SearchesWeekly), - enum = AccountsFilteringAndRankingLogicId) - - case object CandidateSourceWeight - extends FSBoundedParam[Double]( - "crowd_search_accounts_candidate_source_weight", - default = 1200, - min = 0.001, - max = 2000) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsSource.scala deleted file mode 100644 index 6d3e903a1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/CrowdSearchAccountsSource.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts - -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.AccountsFilteringAndRankingLogics -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.CandidateSourceEnabled -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode -import com.twitter.hermit.model.Algorithm -import com.twitter.onboarding.relevance.crowd_search_accounts.thriftscala.CrowdSearchAccounts -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.CrowdSearchAccountsClientColumn -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -object AccountsFilteringAndRankingLogicId extends Enumeration { - type AccountsFilteringAndRankingLogicId = Value - - val NewSearchesDaily: AccountsFilteringAndRankingLogicId = Value("new_searches_daily") - val NewSearchesWeekly: AccountsFilteringAndRankingLogicId = Value("new_searches_weekly") - val SearchesDaily: AccountsFilteringAndRankingLogicId = Value("searches_daily") - val SearchesWeekly: AccountsFilteringAndRankingLogicId = Value("searches_weekly") -} - -object CrowdSearchAccountsSource { - val MaxCacheSize = 500 - val CacheTTL: Duration = Duration.fromHours(24) - - type Target = HasParams with HasClientContext with HasGeohashAndCountryCode - - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.CrowdSearchAccounts.toString) -} - -@Singleton -class CrowdSearchAccountsSource @Inject() ( - crowdSearchAccountsClientColumn: CrowdSearchAccountsClientColumn, - statsReceiver: StatsReceiver, -) extends CandidateSource[CrowdSearchAccountsSource.Target, CandidateUser] - with Logging { - - /** @see [[CandidateSourceIdentifier]] */ - override val identifier: CandidateSourceIdentifier = - CrowdSearchAccountsSource.Identifier - - private val stats = statsReceiver.scope(identifier.name) - private val requestsStats = stats.counter("requests") - private val noCountryCodeStats = stats.counter("no_country_code") - private val successStats = stats.counter("success") - private val errorStats = stats.counter("error") - - private val cache = StitchCache[String, Option[CrowdSearchAccounts]]( - maxCacheSize = CrowdSearchAccountsSource.MaxCacheSize, - ttl = CrowdSearchAccountsSource.CacheTTL, - statsReceiver = statsReceiver.scope(identifier.name, "cache"), - underlyingCall = (k: String) => { - crowdSearchAccountsClientColumn.fetcher - .fetch(k) - .map { result => result.v } - } - ) - - /** returns a Seq of ''potential'' content */ - override def apply( - target: CrowdSearchAccountsSource.Target - ): Stitch[Seq[CandidateUser]] = { - if (!target.params(CandidateSourceEnabled)) { - return Stitch.value(Seq[CandidateUser]()) - } - requestsStats.incr() - target.getCountryCode - .orElse(target.geohashAndCountryCode.flatMap(_.countryCode)).map { countryCode => - Stitch - .collect(target - .params(AccountsFilteringAndRankingLogics).map(logic => - cache.readThrough(countryCode.toUpperCase() + "-" + logic))) - .onSuccess(_ => { - successStats.incr() - }) - .onFailure(t => { - debug("candidate source failed identifier = %s".format(identifier), t) - errorStats.incr() - }) - .map(transformCrowdSearchAccountsToCandidateSource) - }.getOrElse { - noCountryCodeStats.incr() - Stitch.value(Seq[CandidateUser]()) - } - } - - private def transformCrowdSearchAccountsToCandidateSource( - crowdSearchAccounts: Seq[Option[CrowdSearchAccounts]] - ): Seq[CandidateUser] = { - crowdSearchAccounts - .flatMap(opt => - opt - .map(accounts => - accounts.accounts.map(account => - CandidateUser( - id = account.accountId, - score = Some(account.searchActivityScore), - ).withCandidateSource(identifier))) - .getOrElse(Seq[CandidateUser]())) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/README.md deleted file mode 100644 index 043279b44..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Crowd Search Candidate Source -Provides the most searched accounts within a specific country over the past 1 and 7 days. -* When we refer to "most searched accounts", we are referring to accounts that have been clicked on the most frequently by users after they see search results in both the typeahead and search results page. -* The results returned by the service have undergone health filters. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BUILD deleted file mode 100644 index 2493a4a93..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BasePopGeoHashSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BasePopGeoHashSource.scala deleted file mode 100644 index 862046bbc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/BasePopGeoHashSource.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.google.inject.Singleton -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject - -@Singleton -class BasePopGeohashSource @Inject() ( - popGeoSource: CandidateSource[String, CandidateUser], - statsReceiver: StatsReceiver) - extends CandidateSource[ - HasParams with HasClientContext with HasGeohashAndCountryCode, - CandidateUser - ] - with BasePopGeohashSourceConfig { - - val stats: StatsReceiver = statsReceiver - - // counter to check if we found a geohash value in the request - val foundGeohashCounter: Counter = stats.counter("found_geohash_value") - // counter to check if we are missing a geohash value in the request - val missingGeohashCounter: Counter = stats.counter("missing_geohash_value") - - /** @see [[CandidateSourceIdentifier]] */ - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - "BasePopGeohashSource") - - override def apply( - target: HasParams with HasClientContext with HasGeohashAndCountryCode - ): Stitch[Seq[CandidateUser]] = { - if (!candidateSourceEnabled(target)) { - return Stitch.Nil - } - target.geohashAndCountryCode - .flatMap(_.geohash).map { geohash => - foundGeohashCounter.incr() - val keys = (minGeohashLength(target) to math.min(maxGeohashLength(target), geohash.length)) - .map("geohash_" + geohash.take(_)).reverse - if (returnResultFromAllPrecision(target)) { - Stitch - .collect(keys.map(popGeoSource.apply)).map( - _.flatten.map(_.withCandidateSource(identifier)) - ) - } else { - Stitch - .collect(keys.map(popGeoSource.apply)).map( - _.find(_.nonEmpty) - .getOrElse(Nil) - .take(maxResults(target)).map(_.withCandidateSource(identifier)) - ) - } - }.getOrElse { - missingGeohashCounter.incr() - Stitch.Nil - } - } -} - -trait BasePopGeohashSourceConfig { - type Target = HasParams with HasClientContext - def maxResults(target: Target): Int = 200 - def minGeohashLength(target: Target): Int = 2 - def maxGeohashLength(target: Target): Int = 4 - def returnResultFromAllPrecision(target: Target): Boolean = false - def candidateSourceEnabled(target: Target): Boolean = false -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountryBackFillSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountryBackFillSource.scala deleted file mode 100644 index e2a5f9baf..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountryBackFillSource.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject - -@Singleton -class PopCountryBackFillSource @Inject() (popGeoSource: PopGeoSource) - extends CandidateSource[HasClientContext with HasParams, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = PopCountryBackFillSource.Identifier - - override def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] = { - target.getOptionalUserId - .map(_ => - popGeoSource(PopCountryBackFillSource.DefaultKey) - .map(_.take(PopCountryBackFillSource.MaxResults).map(_.withCandidateSource(identifier)))) - .getOrElse(Stitch.Nil) - } -} - -object PopCountryBackFillSource { - val Identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(Algorithm.PopCountryBackFill.toString) - val MaxResults = 40 - val DefaultKey = "country_US" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountrySource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountrySource.scala deleted file mode 100644 index de6377df6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopCountrySource.scala +++ /dev/null @@ -1,63 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.google.inject.Singleton -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode -import com.twitter.follow_recommendations.common.models.HasUserState -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject - -@Singleton -class PopCountrySource @Inject() ( - popGeoSource: PopGeoSource, - statsReceiver: StatsReceiver) - extends CandidateSource[ - HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode, - CandidateUser - ] { - - override val identifier: CandidateSourceIdentifier = PopCountrySource.Identifier - val stats: StatsReceiver = statsReceiver.scope("PopCountrySource") - - // counter to check if we found a country code value in the request - val foundCountryCodeCounter: Counter = stats.counter("found_country_code_value") - // counter to check if we are missing a country code value in the request - val missingCountryCodeCounter: Counter = stats.counter("missing_country_code_value") - - override def apply( - target: HasClientContext with HasParams with HasUserState with HasGeohashAndCountryCode - ): Stitch[Seq[CandidateUser]] = { - target.geohashAndCountryCode - .flatMap(_.countryCode).map { countryCode => - foundCountryCodeCounter.incr() - if (target.userState.exists(PopCountrySource.BlacklistedTargetUserStates.contains)) { - Stitch.Nil - } else { - popGeoSource("country_" + countryCode) - .map(_.take(PopCountrySource.MaxResults).map(_.withCandidateSource(identifier))) - } - }.getOrElse { - missingCountryCodeCounter.incr() - Stitch.Nil - } - } -} - -object PopCountrySource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.PopCountry.toString) - val MaxResults = 40 - val BlacklistedTargetUserStates: Set[UserState] = Set( - UserState.HeavyTweeter, - UserState.HeavyNonTweeter, - UserState.MediumTweeter, - UserState.MediumNonTweeter) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSource.scala deleted file mode 100644 index ad473a282..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSource.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo -import com.google.inject.Singleton -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.PopularInGeoProof -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.hermit.model.Algorithm -import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.UniquePopQualityFollowUsersInPlaceClientColumn -import com.twitter.util.Duration -import javax.inject.Inject - -@Singleton -class PopGeohashQualityFollowSource @Inject() ( - popGeoSource: PopGeoQualityFollowSource, - statsReceiver: StatsReceiver) - extends BasePopGeohashSource( - popGeoSource = popGeoSource, - statsReceiver = statsReceiver.scope("PopGeohashQualityFollowSource"), - ) { - override val identifier: CandidateSourceIdentifier = PopGeohashQualityFollowSource.Identifier - override def maxResults(target: Target): Int = { - target.params(PopGeoQualityFollowSourceParams.PopGeoSourceMaxResultsPerPrecision) - } - override def minGeohashLength(target: Target): Int = { - target.params(PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMinPrecision) - } - override def maxGeohashLength(target: Target): Int = { - target.params(PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMaxPrecision) - } - override def returnResultFromAllPrecision(target: Target): Boolean = { - target.params(PopGeoQualityFollowSourceParams.PopGeoSourceReturnFromAllPrecisions) - } - override def candidateSourceEnabled(target: Target): Boolean = { - target.params(PopGeoQualityFollowSourceParams.CandidateSourceEnabled) - } -} - -object PopGeohashQualityFollowSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.PopGeohashQualityFollow.toString) -} - -object PopGeoQualityFollowSource { - val MaxCacheSize = 20000 - val CacheTTL: Duration = Duration.fromHours(24) - val MaxResults = 200 -} - -@Singleton -class PopGeoQualityFollowSource @Inject() ( - popGeoQualityFollowClientColumn: UniquePopQualityFollowUsersInPlaceClientColumn, - statsReceiver: StatsReceiver, -) extends CandidateSource[String, CandidateUser] { - - /** @see [[CandidateSourceIdentifier]] */ - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - "PopGeoQualityFollowSource") - - private val cache = StitchCache[String, Option[PopUsersInPlace]]( - maxCacheSize = PopGeoQualityFollowSource.MaxCacheSize, - ttl = PopGeoQualityFollowSource.CacheTTL, - statsReceiver = statsReceiver.scope(identifier.name, "cache"), - underlyingCall = (k: String) => { - popGeoQualityFollowClientColumn.fetcher - .fetch(k) - .map { result => result.v } - } - ) - - override def apply(target: String): Stitch[Seq[CandidateUser]] = { - val result: Stitch[Option[PopUsersInPlace]] = cache.readThrough(target) - result.map { pu => - pu.map { candidates => - candidates.popUsers.sortBy(-_.score).take(PopGeoQualityFollowSource.MaxResults).map { - candidate => - CandidateUser( - id = candidate.userId, - score = Some(candidate.score), - reason = Some( - Reason( - Some( - AccountProof( - popularInGeoProof = Some(PopularInGeoProof(location = candidates.place)) - ) - ) - ) - ) - ) - } - }.getOrElse(Nil) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceFSConfig.scala deleted file mode 100644 index 4d8577522..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceFSConfig.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PopGeoQualityFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig { - override val intFSParams: Seq[FSBoundedParam[Int] with FSName] = Seq( - PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMaxPrecision, - PopGeoQualityFollowSourceParams.PopGeoSourceGeoHashMinPrecision, - PopGeoQualityFollowSourceParams.PopGeoSourceMaxResultsPerPrecision - ) - override val doubleFSParams: Seq[FSBoundedParam[Double] with FSName] = Seq( - PopGeoQualityFollowSourceParams.CandidateSourceWeight - ) - override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( - PopGeoQualityFollowSourceParams.CandidateSourceEnabled, - PopGeoQualityFollowSourceParams.PopGeoSourceReturnFromAllPrecisions - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceParams.scala deleted file mode 100644 index dac92df52..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoQualityFollowSourceParams.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object PopGeoQualityFollowSourceParams { - case object CandidateSourceEnabled - extends FSParam[Boolean]("pop_geo_quality_follow_source_enabled", false) - - case object PopGeoSourceGeoHashMinPrecision - extends FSBoundedParam[Int]( - "pop_geo_quality_follow_source_geo_hash_min_precision", - default = 2, - min = 0, - max = 10) - - case object PopGeoSourceGeoHashMaxPrecision - extends FSBoundedParam[Int]( - "pop_geo_quality_follow_source_geo_hash_max_precision", - default = 3, - min = 0, - max = 10) - - case object PopGeoSourceReturnFromAllPrecisions - extends FSParam[Boolean]( - "pop_geo_quality_follow_source_return_from_all_precisions", - default = false) - - case object PopGeoSourceMaxResultsPerPrecision - extends FSBoundedParam[Int]( - "pop_geo_quality_follow_source_max_results_per_precision", - default = 200, - min = 0, - max = 1000) - - case object CandidateSourceWeight - extends FSBoundedParam[Double]( - "pop_geo_quality_follow_source_weight", - default = 200, - min = 0.001, - max = 2000) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSource.scala deleted file mode 100644 index 2db13ed69..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSource.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.base.CachedCandidateSource -import com.twitter.follow_recommendations.common.candidate_sources.base.StratoFetcherWithUnitViewSource -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.PopularInGeoProof -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.util.Duration -import javax.inject.Inject - -@Singleton -class BasePopGeoSource @Inject() ( - @Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) fetcher: Fetcher[ - String, - Unit, - PopUsersInPlace - ]) extends StratoFetcherWithUnitViewSource[String, PopUsersInPlace]( - fetcher, - BasePopGeoSource.Identifier) { - - override def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] = - BasePopGeoSource.map(target, candidates) -} - -object BasePopGeoSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("BasePopGeoSource") - val MaxResults = 200 - - def map(target: String, candidates: PopUsersInPlace): Seq[CandidateUser] = - candidates.popUsers.sortBy(-_.score).take(BasePopGeoSource.MaxResults).view.map { candidate => - CandidateUser( - id = candidate.userId, - score = Some(candidate.score), - reason = Some( - Reason( - Some( - AccountProof( - popularInGeoProof = Some(PopularInGeoProof(location = candidates.place)) - ) - ) - ) - ) - ) - } -} - -@Singleton -class PopGeoSource @Inject() (basePopGeoSource: BasePopGeoSource, statsReceiver: StatsReceiver) - extends CachedCandidateSource[String, CandidateUser]( - basePopGeoSource, - PopGeoSource.MaxCacheSize, - PopGeoSource.CacheTTL, - statsReceiver, - PopGeoSource.Identifier) - -object PopGeoSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("PopGeoSource") - val MaxCacheSize = 20000 - val CacheTTL: Duration = 1.hours -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceFSConfig.scala deleted file mode 100644 index ea3f6ce38..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceFSConfig.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PopGeoSourceFSConfig @Inject() () extends FeatureSwitchConfig { - override val intFSParams: Seq[FSBoundedParam[Int] with FSName] = Seq( - PopGeoSourceParams.PopGeoSourceGeoHashMaxPrecision, - PopGeoSourceParams.PopGeoSourceMaxResultsPerPrecision, - PopGeoSourceParams.PopGeoSourceGeoHashMinPrecision, - ) - override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( - PopGeoSourceParams.PopGeoSourceReturnFromAllPrecisions, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceParams.scala deleted file mode 100644 index a63e320b4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeoSourceParams.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object PopGeoSourceParams { - case object PopGeoSourceGeoHashMinPrecision - extends FSBoundedParam[Int]( - "pop_geo_source_geo_hash_min_precision", - default = 2, - min = 0, - max = 10) - - case object PopGeoSourceGeoHashMaxPrecision - extends FSBoundedParam[Int]( - "pop_geo_source_geo_hash_max_precision", - default = 4, - min = 0, - max = 10) - - case object PopGeoSourceReturnFromAllPrecisions - extends FSParam[Boolean]("pop_geo_source_return_from_all_precisions", default = false) - - case object PopGeoSourceMaxResultsPerPrecision - extends FSBoundedParam[Int]( - "pop_geo_source_max_results_per_precision", - default = 200, - min = 0, - max = 1000) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeohashSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeohashSource.scala deleted file mode 100644 index 9447b4867..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/PopGeohashSource.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.geo - -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import javax.inject.Inject - -@Singleton -class PopGeohashSource @Inject() ( - popGeoSource: PopGeoSource, - statsReceiver: StatsReceiver) - extends BasePopGeohashSource( - popGeoSource = popGeoSource, - statsReceiver = statsReceiver.scope("PopGeohashSource"), - ) { - override def candidateSourceEnabled(target: Target): Boolean = true - override val identifier: CandidateSourceIdentifier = PopGeohashSource.Identifier - override def minGeohashLength(target: Target): Int = { - target.params(PopGeoSourceParams.PopGeoSourceGeoHashMinPrecision) - } - override def maxResults(target: Target): Int = { - target.params(PopGeoSourceParams.PopGeoSourceMaxResultsPerPrecision) - } - override def maxGeohashLength(target: Target): Int = { - target.params(PopGeoSourceParams.PopGeoSourceGeoHashMaxPrecision) - } - override def returnResultFromAllPrecision(target: Target): Boolean = { - target.params(PopGeoSourceParams.PopGeoSourceReturnFromAllPrecisions) - } -} - -object PopGeohashSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.PopGeohash.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/README.md deleted file mode 100644 index 13c2f245f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Pop Geo Candidate Source -Provides the most followed / quality followed accounts in a specific country and a geolocation within past 2 weeks. -* A "quality follow" refers to any follow that leads to visible engagement, such as favorites, mentions, retweets, direct messages, replies, and quote tweets. The engagement must be allowed in either direction, and must occur on the day of the follow or within one subsequent day. Additionally, there must be no unfollowing, blocking, muting, or reporting of the account in the same time period. -* The minimum geolocation precision used is ±20 km (12 mi), and precise user geolocation is not utilized. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/BUILD deleted file mode 100644 index 0ff68490c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "strato/config/columns/onboarding:onboarding-strato-client", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSource.scala deleted file mode 100644 index e22fb465d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSource.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceParams.CandidateSourceEnabled -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceParams.LocaleToExcludeFromRecommendation -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.strato.generated.client.onboarding.UserPreferredLanguagesOnUserClientColumn -import com.twitter.strato.generated.client.onboarding.userrecs.LocaleFollowPpmiClientColumn -import com.twitter.timelines.configapi.HasParams - -/** - * Fetches candidates based on the Positive Pointwise Mutual Information (PPMI) statistic - * for a set of locales - * */ -@Singleton -class PPMILocaleFollowSource @Inject() ( - userPreferredLanguagesOnUserClientColumn: UserPreferredLanguagesOnUserClientColumn, - localeFollowPpmiClientColumn: LocaleFollowPpmiClientColumn, - statsReceiver: StatsReceiver) - extends CandidateSource[HasClientContext with HasParams, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = PPMILocaleFollowSource.Identifier - private val stats = statsReceiver.scope("PPMILocaleFollowSource") - - override def apply(target: HasClientContext with HasParams): Stitch[Seq[CandidateUser]] = { - (for { - countryCode <- target.getCountryCode - userId <- target.getOptionalUserId - } yield { - getPreferredLocales(userId, countryCode.toLowerCase()) - .flatMap { locale => - stats.addGauge("allLocale") { - locale.length - } - val filteredLocale = - locale.filter(!target.params(LocaleToExcludeFromRecommendation).contains(_)) - stats.addGauge("postFilterLocale") { - filteredLocale.length - } - if (target.params(CandidateSourceEnabled)) { - getPPMILocaleFollowCandidates(filteredLocale) - } else Stitch(Seq.empty) - } - .map(_.sortBy(_.score)(Ordering[Option[Double]].reverse) - .take(PPMILocaleFollowSource.DefaultMaxCandidatesToReturn)) - }).getOrElse(Stitch.Nil) - } - - private def getPPMILocaleFollowCandidates( - locales: Seq[String] - ): Stitch[Seq[CandidateUser]] = { - Stitch - .traverse(locales) { locale => - // Get PPMI candidates for each locale - localeFollowPpmiClientColumn.fetcher - .fetch(locale) - .map(_.v - .map(_.candidates).getOrElse(Nil).map { candidate => - CandidateUser(id = candidate.userId, score = Some(candidate.score)) - }.map(_.withCandidateSource(identifier))) - }.map(_.flatten) - } - - private def getPreferredLocales(userId: Long, countryCode: String): Stitch[Seq[String]] = { - userPreferredLanguagesOnUserClientColumn.fetcher - .fetch(userId) - .map(_.v.map(_.languages).getOrElse(Nil).map { lang => - s"$countryCode-$lang".toLowerCase - }) - } -} - -object PPMILocaleFollowSource { - val Identifier = CandidateSourceIdentifier(Algorithm.PPMILocaleFollow.toString) - val DefaultMaxCandidatesToReturn = 100 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceFSConfig.scala deleted file mode 100644 index 8a40ca92d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceFSConfig.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PPMILocaleFollowSourceFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - PPMILocaleFollowSourceParams.CandidateSourceEnabled, - ) - - override val stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Seq( - PPMILocaleFollowSourceParams.LocaleToExcludeFromRecommendation, - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - PPMILocaleFollowSourceParams.CandidateSourceWeight, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceParams.scala deleted file mode 100644 index 18aac545d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/PPMILocaleFollowSourceParams.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -class PPMILocaleFollowSourceParams {} -object PPMILocaleFollowSourceParams { - case object LocaleToExcludeFromRecommendation - extends FSParam[Seq[String]]( - "ppmilocale_follow_source_locales_to_exclude_from_recommendation", - default = Seq.empty) - - case object CandidateSourceEnabled - extends FSParam[Boolean]("ppmilocale_follow_source_enabled", true) - - case object CandidateSourceWeight - extends FSBoundedParam[Double]( - "ppmilocale_follow_source_candidate_source_weight", - default = 1, - min = 0.001, - max = 2000) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/README.md deleted file mode 100644 index 48cbee112..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# PPMI Locale Follow Candidate Source -Provides accounts based on PPMI ([Positive Pointwise Mutual Information](https://en.wikipedia.org/wiki/Pointwise_mutual_information#Positive_PMI)) using follow actions as a feature for a specific local (language + country) within a week. In simpler terms, it provides a list of the most followed accounts for a given country and language input, based on the PPMI algorithm. - -PPMI is a statistical measure of the association between two events. In this case, it measures the association between the follow actions and the accounts being followed. - -In summary, the service utilizes PPMI and follow actions to provide a list of the most followed accounts for a specific country and language input. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/BUILD deleted file mode 100644 index 27caedd22..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "src/thrift/com/twitter/socialgraph:thrift-scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/PromotedAccountsCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/PromotedAccountsCandidateSource.scala deleted file mode 100644 index ff2ad4cd9..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/PromotedAccountsCandidateSource.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts - -import com.twitter.adserver.thriftscala.AdServerException -import com.twitter.adserver.{thriftscala => adthrift} -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.adserver.AdRequest -import com.twitter.follow_recommendations.common.clients.adserver.AdserverClient -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.hermit.model.Algorithm -import com.twitter.inject.Logging -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -case class PromotedCandidateUser( - id: Long, - position: Int, - adImpression: adthrift.AdImpression, - followProof: FollowProof, - primaryCandidateSource: Option[CandidateSourceIdentifier]) - -@Singleton -class PromotedAccountsCandidateSource @Inject() ( - adserverClient: AdserverClient, - sgsClient: SocialGraphClient, - statsReceiver: StatsReceiver) - extends CandidateSource[AdRequest, PromotedCandidateUser] - with Logging { - - override val identifier: CandidateSourceIdentifier = - PromotedAccountsCandidateSource.Identifier - - val stats: StatsReceiver = statsReceiver.scope(identifier.name) - val failureStat: StatsReceiver = stats.scope("failures") - val adServerExceptionsCounter: Counter = failureStat.counter("AdServerException") - val timeoutCounter: Counter = failureStat.counter("TimeoutException") - - def apply(request: AdRequest): Stitch[Seq[PromotedCandidateUser]] = { - adserverClient - .getAdImpressions(request) - .rescue { - case e: TimeoutException => - timeoutCounter.incr() - logger.warn("Timeout on Adserver", e) - Stitch.Nil - case e: AdServerException => - adServerExceptionsCounter.incr() - logger.warn("Failed to fetch ads", e) - Stitch.Nil - } - .flatMap { adImpressions: Seq[adthrift.AdImpression] => - profileNumResults(adImpressions.size, "results_from_ad_server") - val idToImpMap = (for { - imp <- adImpressions - promotedAccountId <- imp.promotedAccountId - } yield promotedAccountId -> imp).toMap - request.clientContext.userId - .map { userId => - sgsClient - .getIntersections( - userId, - adImpressions.filter(shouldShowSocialContext).flatMap(_.promotedAccountId), - PromotedAccountsCandidateSource.NumIntersections - ).map { promotedAccountWithIntersections => - idToImpMap.map { - case (promotedAccountId, imp) => - PromotedCandidateUser( - promotedAccountId, - imp.insertionPosition - .map(_.toInt).getOrElse( - getInsertionPositionDefaultValue(request.isTest.getOrElse(false)) - ), - imp, - promotedAccountWithIntersections - .getOrElse(promotedAccountId, FollowProof(Nil, 0)), - Some(identifier) - ) - }.toSeq - }.onSuccess(result => profileNumResults(result.size, "final_results")) - }.getOrElse(Stitch.Nil) - } - } - - private def shouldShowSocialContext(imp: adthrift.AdImpression): Boolean = - imp.experimentValues.exists { expValues => - expValues.get("display.display_style").contains("show_social_context") - } - - private def getInsertionPositionDefaultValue(isTest: Boolean): Int = { - if (isTest) 0 else -1 - } - - private def profileNumResults(resultsSize: Int, statName: String): Unit = { - if (resultsSize <= 5) { - stats.scope(statName).counter(resultsSize.toString).incr() - } else { - stats.scope(statName).counter("more_than_5").incr() - } - } -} - -object PromotedAccountsCandidateSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.PromotedAccount.toString) - val NumIntersections = 3 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/README.md deleted file mode 100644 index 1091e1d88..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Promoted Accounts Candidate Source -Promoted accounts returned from Ads server. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/BUILD deleted file mode 100644 index 07c3c6665..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "strato/config/columns/onboarding/realGraph:realGraph-strato-client", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/config/columns/recommendations/twistly:twistly-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/README.md deleted file mode 100644 index 4ba5a8c11..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# RealGraph Candidate Source -Provides out-of-network RealGraph candidates for a given user. RealGraph is a user-user graph dataset that aims to measure the strength of the relationship between two users. - -RealGraph comprises two components: a real-time pipeline that tracks various counts and relationships between user-user edges (such as the number of favorites, replies, retweets, clicks, whether followed, muted, or blocked), and an offline pipeline of a larger set of such user-user edge counts and relationships. Currently, the top k in-network scores have been exported for use by various teams. - -The RealGraph dataset is used to predict user interactions at Twitter, and is based on the paper "[Realgraph: User interaction prediction at Twitter](http://www.ueo-workshop.com/wp-content/uploads/2014/04/sig-alternate.pdf)" by the UEO workshop at KDD'14. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonFSConfig.scala deleted file mode 100644 index df9c8ac68..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonFSConfig.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.real_graph - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RealGraphOonFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq( - RealGraphOonParams.IncludeRealGraphOonCandidates, - RealGraphOonParams.TryToReadRealGraphOonCandidates, - RealGraphOonParams.UseV2 - ) - override val doubleFSParams: Seq[FSBoundedParam[Double]] = - Seq( - RealGraphOonParams.ScoreThreshold - ) - override val intFSParams: Seq[FSBoundedParam[Int]] = - Seq( - RealGraphOonParams.RealGraphOonResultCountThreshold, - RealGraphOonParams.MaxResults, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonParams.scala deleted file mode 100644 index feaa1d7c7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonParams.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.real_graph - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object RealGraphOonParams { - case object IncludeRealGraphOonCandidates - extends FSParam[Boolean]( - "real_graph_oon_include_candidates", - false - ) - case object TryToReadRealGraphOonCandidates - extends FSParam[Boolean]( - "real_graph_oon_try_to_read_candidates", - false - ) - case object RealGraphOonResultCountThreshold - extends FSBoundedParam[Int]( - "real_graph_oon_result_count_threshold", - default = 1, - min = 0, - max = Integer.MAX_VALUE - ) - - case object UseV2 - extends FSParam[Boolean]( - "real_graph_oon_use_v2", - false - ) - - case object ScoreThreshold - extends FSBoundedParam[Double]( - "real_graph_oon_score_threshold", - default = 0.26, - min = 0, - max = 1.0 - ) - - case object MaxResults - extends FSBoundedParam[Int]( - "real_graph_oon_max_results", - default = 200, - min = 0, - max = 1000 - ) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonV2Source.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonV2Source.scala deleted file mode 100644 index 3c709770c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphOonV2Source.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.real_graph - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.realGraph.UserRealgraphOonV2ClientColumn -import com.twitter.timelines.configapi.HasParams -import com.twitter.wtf.candidate.thriftscala.CandidateSeq -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RealGraphOonV2Source @Inject() ( - realGraphClientColumn: UserRealgraphOonV2ClientColumn) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = - RealGraphOonV2Source.Identifier - - override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - request.getOptionalUserId - .map { userId => - realGraphClientColumn.fetcher - .fetch(userId) - .map { result => - result.v - .map { candidates => parseStratoResults(request, candidates) } - .getOrElse(Nil) - // returned candidates are sorted by score in descending order - .take(request.params(RealGraphOonParams.MaxResults)) - .map(_.withCandidateSource(identifier)) - } - }.getOrElse(Stitch(Seq.empty)) - } - - private def parseStratoResults( - request: HasParams with HasClientContext, - candidateSeqThrift: CandidateSeq - ): Seq[CandidateUser] = { - candidateSeqThrift.candidates.collect { - case candidate if candidate.score >= request.params(RealGraphOonParams.ScoreThreshold) => - CandidateUser( - candidate.userId, - Some(candidate.score) - ) - } - } - -} - -object RealGraphOonV2Source { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.RealGraphOonV2.toString - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphSource.scala deleted file mode 100644 index 7aa2aa8af..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph/RealGraphSource.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.real_graph - -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -/** - * This source gets the already followed edges from the real graph column as a candidate source. - */ -@Singleton -class RealGraphSource @Inject() ( - realGraph: RealTimeRealGraphClient) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] { - override val identifier: CandidateSourceIdentifier = RealGraphSource.Identifier - - override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - request.getOptionalUserId - .map { userId => - realGraph.getRealGraphWeights(userId).map { scoreMap => - scoreMap.map { - case (candidateId, realGraphScore) => - CandidateUser(id = candidateId, score = Some(realGraphScore)) - .withCandidateSource(identifier) - }.toSeq - } - }.getOrElse(Stitch.Nil) - } -} - -object RealGraphSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.RealGraphFollowed.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/BUILD deleted file mode 100644 index b2764c42e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "discovery-ds/src/main/thrift/com/twitter/dds/jobs/repeated_profile_visits:profile_visit-scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "src/thrift/com/twitter/experiments/general_metrics:general_metrics-scala", - "strato/config/columns/rux:rux-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/README.md deleted file mode 100644 index 616a3f7ed..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Recent Engagement Candidate Source -Provides recently engaged accounts for a given user: -* Explicit engagements: like, retweet, reply -* Implicit engagements: profile visit diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementDirectFollowSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementDirectFollowSource.scala deleted file mode 100644 index edfaac5e6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementDirectFollowSource.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement - -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecentEngagementDirectFollowSource @Inject() ( - realTimeRealGraphClient: RealTimeRealGraphClient) - extends CandidateSource[Long, CandidateUser] { - - val identifier: CandidateSourceIdentifier = - RecentEngagementDirectFollowSource.Identifier - - /** - * Generate a list of candidates for the target using RealtimeGraphClient - * and RecentEngagementStore. - */ - override def apply(targetUserId: Long): Stitch[Seq[CandidateUser]] = { - realTimeRealGraphClient - .getUsersRecentlyEngagedWith( - userId = targetUserId, - engagementScoreMap = RealTimeRealGraphClient.EngagementScoreMap, - includeDirectFollowCandidates = true, - includeNonDirectFollowCandidates = false - ) - .map(_.map(_.withCandidateSource(identifier)).sortBy(-_.score.getOrElse(0.0))) - } -} - -object RecentEngagementDirectFollowSource { - val Identifier = CandidateSourceIdentifier(Algorithm.RecentEngagementDirectFollow.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementNonDirectFollowSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementNonDirectFollowSource.scala deleted file mode 100644 index 46572da71..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RecentEngagementNonDirectFollowSource.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement - -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecentEngagementNonDirectFollowSource @Inject() ( - realTimeRealGraphClient: RealTimeRealGraphClient) - extends CandidateSource[Long, CandidateUser] { - - val identifier: CandidateSourceIdentifier = - RecentEngagementNonDirectFollowSource.Identifier - - /** - * Generate a list of candidates for the target using RealtimeGraphClient - * and RecentEngagementStore. - */ - override def apply(targetUserId: Long): Stitch[Seq[CandidateUser]] = { - realTimeRealGraphClient - .getUsersRecentlyEngagedWith( - userId = targetUserId, - engagementScoreMap = RealTimeRealGraphClient.EngagementScoreMap, - includeDirectFollowCandidates = false, - includeNonDirectFollowCandidates = true - ) - .map(_.map(_.withCandidateSource(identifier)).sortBy(-_.score.getOrElse(0.0))) - } -} - -object RecentEngagementNonDirectFollowSource { - val Identifier = CandidateSourceIdentifier(Algorithm.RecentEngagementNonDirectFollow.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsFSConfig.scala deleted file mode 100644 index 5aaccae2b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsFSConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RepeatedProfileVisitsFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq( - RepeatedProfileVisitsParams.IncludeCandidates, - RepeatedProfileVisitsParams.UseOnlineDataset, - ) - override val intFSParams: Seq[FSBoundedParam[Int]] = - Seq( - RepeatedProfileVisitsParams.RecommendationThreshold, - RepeatedProfileVisitsParams.BucketingThreshold, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsParams.scala deleted file mode 100644 index 5402d38f4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsParams.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object RepeatedProfileVisitsParams { - - // If RepeatedProfileVisitsSource is run and there are recommended candidates for the target user, whether or not - // to actually include such candidates in our output recommendations. This FS will be used to control bucketing of - // users into control vs treatment buckets. - case object IncludeCandidates - extends FSParam[Boolean](name = "repeated_profile_visits_include_candidates", default = false) - - // The threshold at or above which we will consider a profile to have been visited "frequently enough" to recommend - // the profile to the target user. - case object RecommendationThreshold - extends FSBoundedParam[Int]( - name = "repeated_profile_visits_recommendation_threshold", - default = 3, - min = 0, - max = Integer.MAX_VALUE) - - // The threshold at or above which we will consider a profile to have been visited "frequently enough" to recommend - // the profile to the target user. - case object BucketingThreshold - extends FSBoundedParam[Int]( - name = "repeated_profile_visits_bucketing_threshold", - default = 3, - min = 0, - max = Integer.MAX_VALUE) - - // Whether or not to use the online dataset (which has repeated profile visits information updated to within minutes) - // instead of the offline dataset (updated via offline jobs, which can have delays of hours to days). - case object UseOnlineDataset - extends FSParam[Boolean](name = "repeated_profile_visits_use_online_dataset", default = true) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsSource.scala deleted file mode 100644 index c4b4aa3e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsSource.scala +++ /dev/null @@ -1,157 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.recent_engagement - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.dds.jobs.repeated_profile_visits.thriftscala.ProfileVisitorInfo -import com.twitter.experiments.general_metrics.thriftscala.IdType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.Engagement -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params -import com.twitter.hermit.model.Algorithm -import com.twitter.inject.Logging -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.rux.RepeatedProfileVisitsAggregateClientColumn - -@Singleton -class RepeatedProfileVisitsSource @Inject() ( - repeatedProfileVisitsAggregateClientColumn: RepeatedProfileVisitsAggregateClientColumn, - realTimeRealGraphClient: RealTimeRealGraphClient, - statsReceiver: StatsReceiver) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] - with Logging { - - val identifier: CandidateSourceIdentifier = - RepeatedProfileVisitsSource.Identifier - - val sourceStatsReceiver = statsReceiver.scope("repeated_profile_visits_source") - val offlineFetchErrorCounter = sourceStatsReceiver.counter("offline_fetch_error") - val offlineFetchSuccessCounter = sourceStatsReceiver.counter("offline_fetch_success") - val onlineFetchErrorCounter = sourceStatsReceiver.counter("online_fetch_error") - val onlineFetchSuccessCounter = sourceStatsReceiver.counter("online_fetch_success") - val noRepeatedProfileVisitsAboveBucketingThresholdCounter = - sourceStatsReceiver.counter("no_repeated_profile_visits_above_bucketing_threshold") - val hasRepeatedProfileVisitsAboveBucketingThresholdCounter = - sourceStatsReceiver.counter("has_repeated_profile_visits_above_bucketing_threshold") - val noRepeatedProfileVisitsAboveRecommendationsThresholdCounter = - sourceStatsReceiver.counter("no_repeated_profile_visits_above_recommendations_threshold") - val hasRepeatedProfileVisitsAboveRecommendationsThresholdCounter = - sourceStatsReceiver.counter("has_repeated_profile_visits_above_recommendations_threshold") - val includeCandidatesCounter = sourceStatsReceiver.counter("include_candidates") - val noIncludeCandidatesCounter = sourceStatsReceiver.counter("no_include_candidates") - - // Returns visited user -> visit count, via off dataset. - def applyWithOfflineDataset(targetUserId: Long): Stitch[Map[Long, Int]] = { - repeatedProfileVisitsAggregateClientColumn.fetcher - .fetch(ProfileVisitorInfo(id = targetUserId, idType = IdType.User)).map(_.v) - .handle { - case e: Throwable => - logger.error("Strato fetch for RepeatedProfileVisitsAggregateClientColumn failed: " + e) - offlineFetchErrorCounter.incr() - None - }.onSuccess { result => - offlineFetchSuccessCounter.incr() - }.map { resultOption => - resultOption - .flatMap { result => - result.profileVisitSet.map { profileVisitSet => - profileVisitSet - .filter(profileVisit => profileVisit.totalTargetVisitsInLast14Days.getOrElse(0) > 0) - .filter(profileVisit => !profileVisit.doesSourceIdFollowTargetId.getOrElse(false)) - .flatMap { profileVisit => - (profileVisit.targetId, profileVisit.totalTargetVisitsInLast14Days) match { - case (Some(targetId), Some(totalVisitsInLast14Days)) => - Some(targetId -> totalVisitsInLast14Days) - case _ => None - } - }.toMap[Long, Int] - } - }.getOrElse(Map.empty) - } - } - - // Returns visited user -> visit count, via online dataset. - def applyWithOnlineData(targetUserId: Long): Stitch[Map[Long, Int]] = { - val visitedUserToEngagementsStitch: Stitch[Map[Long, Seq[Engagement]]] = - realTimeRealGraphClient.getRecentProfileViewEngagements(targetUserId) - visitedUserToEngagementsStitch - .onFailure { f => - onlineFetchErrorCounter.incr() - }.onSuccess { result => - onlineFetchSuccessCounter.incr() - }.map { visitedUserToEngagements => - visitedUserToEngagements - .mapValues(engagements => engagements.size) - } - } - - def getRepeatedVisitedAccounts(params: Params, targetUserId: Long): Stitch[Map[Long, Int]] = { - var results: Stitch[Map[Long, Int]] = Stitch.value(Map.empty) - if (params.getBoolean(RepeatedProfileVisitsParams.UseOnlineDataset)) { - results = applyWithOnlineData(targetUserId) - } else { - results = applyWithOfflineDataset(targetUserId) - } - // Only keep users that had non-zero engagement counts. - results.map(_.filter(input => input._2 > 0)) - } - - def getRecommendations(params: Params, userId: Long): Stitch[Seq[CandidateUser]] = { - val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold) - val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold) - - // Get the list of repeatedly visited profilts. Only keep accounts with >= bucketingThreshold visits. - val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] = - getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold)) - - repeatedVisitedAccountsStitch.map { candidates => - // Now check if we should includeCandidates (e.g. whether user is in control bucket or treatment buckets). - if (candidates.isEmpty) { - // User has not visited any accounts above bucketing threshold. We will not bucket user into experiment. Just - // don't return no candidates. - noRepeatedProfileVisitsAboveBucketingThresholdCounter.incr() - Seq.empty - } else { - hasRepeatedProfileVisitsAboveBucketingThresholdCounter.incr() - if (!params.getBoolean(RepeatedProfileVisitsParams.IncludeCandidates)) { - // User has reached bucketing criteria. We check whether to include candidates (e.g. checking which bucket - // the user is in for the experiment). In this case the user is in a bucket to not include any candidates. - noIncludeCandidatesCounter.incr() - Seq.empty - } else { - includeCandidatesCounter.incr() - // We should include candidates. Include any candidates above recommendation thresholds. - val outputCandidatesSeq = candidates - .filter(kv => kv._2 >= recommendationThreshold).map { kv => - val user = kv._1 - val visitCount = kv._2 - CandidateUser(user, Some(visitCount.toDouble)) - .withCandidateSource(RepeatedProfileVisitsSource.Identifier) - }.toSeq - if (outputCandidatesSeq.isEmpty) { - noRepeatedProfileVisitsAboveRecommendationsThresholdCounter.incr() - } else { - hasRepeatedProfileVisitsAboveRecommendationsThresholdCounter.incr() - } - outputCandidatesSeq - } - } - } - } - - override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - request.getOptionalUserId - .map { userId => - getRecommendations(request.params, userId) - }.getOrElse(Stitch.Nil) - } -} - -object RepeatedProfileVisitsSource { - val Identifier = CandidateSourceIdentifier(Algorithm.RepeatedProfileVisits.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/BUILD deleted file mode 100644 index 78a5729e2..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "src/thrift/com/twitter/onboarding/relevance/candidates:candidates-scala", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/README.md deleted file mode 100644 index fb6d032d8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# SALSA Candidate Source -Provides an account expansion based on the SALSA PYMK (People You May Know) algorithm for a given account. The algorithm focuses on the mutual follow and address book graph, making it highly effective at providing good mutual follow recommendations. - -The SALSA algorithm constructs a local graph and performs personalized random walks to identify the best recommendations for the user. The local graph represents the community of users that are most similar to or most relevant to the user, while the personalized random walk identifies the most popular interests among them. - -For each target user, the local graph is a bipartite graph with a left-hand side (LHS) and a right-hand side (RHS). The LHS is built from several sources, including the target user, forward and reverse address books, mutual follows, recent followings, and recent followers. We choose a specified number of top candidates from these sources for each target user with different weights assigned to each source to favor the corresponding source, and build the LHS using the target user and those top candidates. The RHS consists of two parts: the top candidates from the sources mentioned above for the target user and the mutual follows of the other entries in the LHS. - -The random walk starts from the target user in the LHS and adopts a restarting strategy to realize personalization. - -In summary, the SALSA Candidate Source provides an account expansion based on the SALSA PYMK algorithm, utilizing a bipartite graph with personalized random walks to identify the most relevant and interesting recommendations for the user. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/RecentEngagementDirectFollowSalsaExpansionSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/RecentEngagementDirectFollowSalsaExpansionSource.scala deleted file mode 100644 index 91fbba2ba..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/RecentEngagementDirectFollowSalsaExpansionSource.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.salsa - -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecentEngagementDirectFollowSalsaExpansionSource @Inject() ( - realTimeRealGraphClient: RealTimeRealGraphClient, - salsaExpander: SalsaExpander) - extends SalsaExpansionBasedCandidateSource[Long](salsaExpander) { - - override val identifier: CandidateSourceIdentifier = - RecentEngagementDirectFollowSalsaExpansionSource.Identifier - - override def firstDegreeNodes(target: Long): Stitch[Seq[Long]] = realTimeRealGraphClient - .getUsersRecentlyEngagedWith( - target, - RealTimeRealGraphClient.EngagementScoreMap, - includeDirectFollowCandidates = true, - includeNonDirectFollowCandidates = false - ).map { recentlyFollowed => - recentlyFollowed - .take(RecentEngagementDirectFollowSalsaExpansionSource.NumFirstDegreeNodesToRetrieve) - .map(_.id) - } - - override def maxResults(target: Long): Int = - RecentEngagementDirectFollowSalsaExpansionSource.OutputSize -} - -object RecentEngagementDirectFollowSalsaExpansionSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.RecentEngagementSarusOcCur.toString) - val NumFirstDegreeNodesToRetrieve = 10 - val OutputSize = 200 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpander.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpander.scala deleted file mode 100644 index a9390826e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpander.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.salsa - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.strato.generated.client.onboarding.userrecs.SalsaFirstDegreeOnUserClientColumn -import com.twitter.strato.generated.client.onboarding.userrecs.SalsaSecondDegreeOnUserClientColumn -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.stitch.Stitch -import com.twitter.wtf.candidate.thriftscala.Candidate -import javax.inject.Inject -import javax.inject.Singleton - -case class SalsaExpandedCandidate( - candidateId: Long, - numberOfConnections: Int, - totalScore: Double, - connectingUsers: Seq[Long]) { - def toCandidateUser: CandidateUser = - CandidateUser( - id = candidateId, - score = Some(totalScore), - reason = Some(Reason( - Some(AccountProof(followProof = Some(FollowProof(connectingUsers, connectingUsers.size)))))) - ) -} - -case class SimilarUserCandidate(candidateId: Long, score: Double, similarToCandidate: Long) - -/** - * Salsa expander uses pre-computed lists of candidates for each input user id and returns the highest scored candidates in the pre-computed lists as the expansion for the corresponding input id. - */ -@Singleton -class SalsaExpander @Inject() ( - statsReceiver: StatsReceiver, - firstDegreeClient: SalsaFirstDegreeOnUserClientColumn, - secondDegreeClient: SalsaSecondDegreeOnUserClientColumn, -) { - - val stats = statsReceiver.scope("salsa_expander") - - private def similarUsers( - input: Seq[Long], - neighbors: Seq[Option[Seq[Candidate]]] - ): Seq[SalsaExpandedCandidate] = { - input - .zip(neighbors).flatMap { - case (recId, Some(neighbors)) => - neighbors.map(neighbor => SimilarUserCandidate(neighbor.userId, neighbor.score, recId)) - case _ => Nil - }.groupBy(_.candidateId).map { - case (key, neighbors) => - val scores = neighbors.map(_.score) - val connectingUsers = neighbors - .sortBy(-_.score) - .take(SalsaExpander.MaxConnectingUsersToOutputPerExpandedCandidate) - .map(_.similarToCandidate) - - SalsaExpandedCandidate(key, scores.size, scores.sum, connectingUsers) - } - .filter( - _.numberOfConnections >= math - .min(SalsaExpander.MinConnectingUsersThreshold, input.size) - ) - .toSeq - } - - def apply( - firstDegreeInput: Seq[Long], - secondDegreeInput: Seq[Long], - maxNumOfCandidatesToReturn: Int - ): Stitch[Seq[CandidateUser]] = { - - val firstDegreeNeighborsStitch = - Stitch - .collect(firstDegreeInput.map(firstDegreeClient.fetcher - .fetch(_).map(_.v.map(_.candidates.take(SalsaExpander.MaxDirectNeighbors))))).onSuccess { - firstDegreeNeighbors => - stats.stat("first_degree_neighbors").add(firstDegreeNeighbors.flatten.size) - } - - val secondDegreeNeighborsStitch = - Stitch - .collect( - secondDegreeInput.map( - secondDegreeClient.fetcher - .fetch(_).map( - _.v.map(_.candidates.take(SalsaExpander.MaxIndirectNeighbors))))).onSuccess { - secondDegreeNeighbors => - stats.stat("second_degree_neighbors").add(secondDegreeNeighbors.flatten.size) - } - - val neighborStitches = - Stitch.join(firstDegreeNeighborsStitch, secondDegreeNeighborsStitch).map { - case (first, second) => first ++ second - } - - val similarUsersToInput = neighborStitches.map { neighbors => - similarUsers(firstDegreeInput ++ secondDegreeInput, neighbors) - } - - similarUsersToInput.map { - // Rank the candidate cot users by the combined weights from the connecting users. This is the default original implementation. It is unlikely to have weight ties and thus a second ranking function is not necessary. - _.sortBy(-_.totalScore) - .take(maxNumOfCandidatesToReturn) - .map(_.toCandidateUser) - } - } -} - -object SalsaExpander { - val MaxDirectNeighbors = 2000 - val MaxIndirectNeighbors = 2000 - val MinConnectingUsersThreshold = 2 - val MaxConnectingUsersToOutputPerExpandedCandidate = 3 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpansionBasedCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpansionBasedCandidateSource.scala deleted file mode 100644 index b299b966d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa/SalsaExpansionBasedCandidateSource.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.salsa - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.stitch.Stitch - -abstract class SalsaExpansionBasedCandidateSource[Target](salsaExpander: SalsaExpander) - extends CandidateSource[Target, CandidateUser] { - - // Define first/second degree as empty sequences in cases of subclasses - // that don't implement one or the other. - // Example: MagicRecs only uses first degree nodes, and can ignore implementing secondDegreeNodes - // - // This allows apply(target) to combine both in the base class - def firstDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq()) - - def secondDegreeNodes(target: Target): Stitch[Seq[Long]] = Stitch.value(Seq()) - - // max number output results - def maxResults(target: Target): Int - - override def apply(target: Target): Stitch[Seq[CandidateUser]] = { - val nodes = Stitch.join(firstDegreeNodes(target), secondDegreeNodes(target)) - - nodes.flatMap { - case (firstDegreeCandidates, secondDegreeCandidates) => { - salsaExpander(firstDegreeCandidates, secondDegreeCandidates, maxResults(target)) - .map(_.map(_.withCandidateSource(identifier)).sortBy(-_.score.getOrElse(0.0))) - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/BUILD deleted file mode 100644 index 15c4eb94d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/config/columns/recommendations/follow2vec:follow2vec-strato-client", - "strato/config/columns/recommendations/similarity:similarity-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/CacheBasedSimsStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/CacheBasedSimsStore.scala deleted file mode 100644 index 03894b533..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/CacheBasedSimsStore.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.hermit.candidate.thriftscala.Candidates -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration - -import java.lang.{Long => JLong} - -class CacheBasedSimsStore( - id: CandidateSourceIdentifier, - fetcher: Fetcher[Long, Unit, Candidates], - maxCacheSize: Int, - cacheTtl: Duration, - statsReceiver: StatsReceiver) - extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = id - private def getUsersFromSimsSource(userId: JLong): Stitch[Option[Candidates]] = { - fetcher - .fetch(userId) - .map(_.v) - } - - private val simsCache = StitchCache[JLong, Option[Candidates]]( - maxCacheSize = maxCacheSize, - ttl = cacheTtl, - statsReceiver = statsReceiver, - underlyingCall = getUsersFromSimsSource - ) - - override def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]] = { - Stitch - .traverse(request.similarToUserIds) { userId => - simsCache.readThrough(userId).map { candidatesOpt => - candidatesOpt - .map { candidates => - StratoBasedSimsCandidateSource.map(userId, candidates) - }.getOrElse(Nil) - } - }.map(_.flatten.distinct.map(_.withCandidateSource(identifier))) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsRefreshStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsRefreshStore.scala deleted file mode 100644 index 3d59e6e0d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsRefreshStore.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.generated.client.onboarding.userrecs.NewSimsRefreshOnUserClientColumn -import com.twitter.util.Duration - -import javax.inject.Inject - -@Singleton -class DBV2SimsRefreshStore @Inject() ( - newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn) - extends StratoBasedSimsCandidateSourceWithUnitView( - fetcher = newSimsRefreshOnUserClientColumn.fetcher, - identifier = DBV2SimsRefreshStore.Identifier) - -@Singleton -class CachedDBV2SimsRefreshStore @Inject() ( - newSimsRefreshOnUserClientColumn: NewSimsRefreshOnUserClientColumn, - statsReceiver: StatsReceiver) - extends CacheBasedSimsStore( - id = DBV2SimsRefreshStore.Identifier, - fetcher = newSimsRefreshOnUserClientColumn.fetcher, - maxCacheSize = DBV2SimsRefreshStore.MaxCacheSize, - cacheTtl = DBV2SimsRefreshStore.CacheTTL, - statsReceiver = statsReceiver.scope("CachedDBV2SimsRefreshStore", "cache") - ) - -object DBV2SimsRefreshStore { - val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) - val MaxCacheSize = 5000 - val CacheTTL: Duration = Duration.fromHours(24) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsStore.scala deleted file mode 100644 index ae291eddc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/DBV2SimsStore.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.hermit.candidate.thriftscala.Candidates -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.util.Duration - -import javax.inject.Inject - -@Singleton -class DBV2SimsStore @Inject() ( - @Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates]) - extends StratoBasedSimsCandidateSourceWithUnitView( - fetcher, - identifier = DBV2SimsStore.Identifier) - -@Singleton -class CachedDBV2SimsStore @Inject() ( - @Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates], - statsReceiver: StatsReceiver) - extends CacheBasedSimsStore( - id = DBV2SimsStore.Identifier, - fetcher = fetcher, - maxCacheSize = DBV2SimsStore.MaxCacheSize, - cacheTtl = DBV2SimsStore.CacheTTL, - statsReceiver = statsReceiver.scope("CachedDBV2SimsStore", "cache") - ) - -object DBV2SimsStore { - val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) - val MaxCacheSize = 1000 - val CacheTTL: Duration = Duration.fromHours(24) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/Follow2vecNearestNeighborsStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/Follow2vecNearestNeighborsStore.scala deleted file mode 100644 index 14131ffd3..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/Follow2vecNearestNeighborsStore.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.candidate_sources.sims.Follow2vecNearestNeighborsStore.NearestNeighborParamsType -import com.twitter.hermit.candidate.thriftscala.Candidate -import com.twitter.hermit.candidate.thriftscala.Candidates -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Fetch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.recommendations.follow2vec.LinearRegressionFollow2vecNearestNeighborsClientColumn -import com.twitter.util.Return -import com.twitter.util.Throw -import javax.inject.Inject - -@Singleton -class LinearRegressionFollow2vecNearestNeighborsStore @Inject() ( - linearRegressionFollow2vecNearestNeighborsClientColumn: LinearRegressionFollow2vecNearestNeighborsClientColumn) - extends StratoBasedSimsCandidateSource[NearestNeighborParamsType]( - Follow2vecNearestNeighborsStore.convertFetcher( - linearRegressionFollow2vecNearestNeighborsClientColumn.fetcher), - view = Follow2vecNearestNeighborsStore.defaultSearchParams, - identifier = Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression - ) - -object Follow2vecNearestNeighborsStore { - // (userid, feature store version for data) - type NearestNeighborKeyType = (Long, Long) - // (neighbors to be returned, ef value: accuracy / latency tradeoff, distance for filtering) - type NearestNeighborParamsType = (Option[Int], Option[Int], Option[Double]) - // (seq(found neighbor id, score), distance for filtering) - type NearestNeighborValueType = (Seq[(Long, Option[Double])], Option[Double]) - - val IdentifierF2vLinearRegression: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.LinearRegressionFollow2VecNearestNeighbors.toString) - - val defaultFeatureStoreVersion: Long = 20210708 - val defaultSearchParams: NearestNeighborParamsType = (None, None, None) - - def convertFetcher( - fetcher: Fetcher[NearestNeighborKeyType, NearestNeighborParamsType, NearestNeighborValueType] - ): Fetcher[Long, NearestNeighborParamsType, Candidates] = { - (key: Long, view: NearestNeighborParamsType) => - { - def toCandidates( - results: Option[NearestNeighborValueType] - ): Option[Candidates] = { - results.flatMap { r => - Some( - Candidates( - key, - r._1.map { neighbor => - Candidate(neighbor._1, neighbor._2.getOrElse(0)) - } - ) - ) - } - } - - val results: Stitch[Fetch.Result[NearestNeighborValueType]] = - fetcher.fetch(key = (key, defaultFeatureStoreVersion), view = view) - results.transform { - case Return(r) => Stitch.value(Fetch.Result(toCandidates(r.v))) - case Throw(e) => Stitch.exception(e) - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/README.md deleted file mode 100644 index 97bab500a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Sims Candidate Source -Offers various online sources for finding similar accounts based on a given user, whether it is the target user or an account candidate. - -## Sims -The objective is to identify a list of K users who are similar to a given user. In this scenario, we primarily focus on finding similar users as "producers" rather than "consumers." Sims has two steps: candidate generation and ranking. - -### Sims Candidate Generation - -With over 700 million users to consider, there are multiple ways to define similarities. Currently, we have three candidate sources for Sims: - -**CosineFollow** (based on user-user follow graph): The similarity between two users is defined as the cosine similarity between their followers. Despite sounding simple, computing all-pair similarity on the entire follow graph is computationally challenging. We are currently using the WHIMP algorithm to find the top 1000 similar users for each user ID. This candidate source has the largest coverage, as it can find similar user candidates for more than 700 million users. - -**CosineList** (based on user-list membership graph): The similarity between two users is defined as the cosine similarity between the lists they are included as members (e.g., [here](https://twitter.com/jack/lists/memberships) are the lists that @jack is on). The same algorithm as CosineFollow is used. - -**Follow2Vec** (essentially Word2Vec on user-user follow graph): We first train the Word2Vec model on follow sequence data to obtain users' embeddings and then find the most similar users based on the similarity of the embeddings. However, we need enough data for each user to learn a meaningful embedding for them, so we can only obtain embeddings for the top 10 million users (currently in production, testing 30 million users). Furthermore, Word2Vec model training is limited by memory and computation as it is trained on a single machine. - -##### Cosine Similarity -A crucial component in Sims is calculating cosine similarities between users based on a user-X (X can be a user, list, or other entities) bipartite graph. This problem is technically challenging and took several years of effort to solve. - -The current implementation uses the algorithm proposed in [When hashes met wedges: A distributed algorithm for finding high similarity vectors. WWW 2017](https://arxiv.org/pdf/1703.01054.pdf) - -### Sims Ranking -After the candidate generation step, we can obtain dozens to hundreds of similar user candidates for each user. However, since these candidates come from different algorithms, we need a way to rank them. To do this, we collect user feedback. - -We use the "Profile Sidebar Impressions & Follow" (a module with follow suggestions displayed when a user visits a profile page and scrolls down) to collect training data. To alleviate any system bias, we use 4% of traffic to show randomly shuffled candidates to users and collect positive (followed impression) and negative (impression only) data from this traffic. This data is used as an evaluation set. We use a portion of the remaining 96% of traffic for training data, filtering only for sets of impressions that had at least one follow, ensuring that the user taking action was paying attention to the impressions. - -The examples are in the format of (profile_user, candidate_user, label). We add features for profile_users and candidate_users based on some high-level aggregated statistics in a feature dataset provided by the Customer Journey team, as well as features that represent the similarity between the profile_user and candidate_user. - -We employ a multi-tower MLP model and optimize the logistic loss. The model is refreshed weekly using an ML workflow. - -We recompute the candidates and rank them daily. The ranked results are published to the Manhattan dataset. - diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsExperimentalStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsExperimentalStore.scala deleted file mode 100644 index d144640c7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsExperimentalStore.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.generated.client.recommendations.similarity.SimilarUsersBySimsExperimentalOnUserClientColumn -import com.twitter.util.Duration - -import javax.inject.Inject - -@Singleton -class SimsExperimentalStore @Inject() ( - simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn) - extends StratoBasedSimsCandidateSourceWithUnitView( - fetcher = simsExperimentalOnUserClientColumn.fetcher, - identifier = SimsExperimentalStore.Identifier - ) - -@Singleton -class CachedSimsExperimentalStore @Inject() ( - simsExperimentalOnUserClientColumn: SimilarUsersBySimsExperimentalOnUserClientColumn, - statsReceiver: StatsReceiver) - extends CacheBasedSimsStore( - id = SimsExperimentalStore.Identifier, - fetcher = simsExperimentalOnUserClientColumn.fetcher, - maxCacheSize = SimsExperimentalStore.MaxCacheSize, - cacheTtl = SimsExperimentalStore.CacheTTL, - statsReceiver = statsReceiver.scope("CachedSimsExperimentalStore", "cache") - ) - -object SimsExperimentalStore { - val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) - val MaxCacheSize = 1000 - val CacheTTL: Duration = Duration.fromHours(12) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceFSConfig.scala deleted file mode 100644 index 0bacb339f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceFSConfig.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SimsSourceFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( - SimsSourceParams.DisableHeavyRanker - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceParams.scala deleted file mode 100644 index e96775a6a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsSourceParams.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.twitter.timelines.configapi.FSParam - -object SimsSourceParams { - case object EnableDBV2SimsStore extends FSParam[Boolean]("sims_source_enable_dbv2_source", false) - - case object EnableDBV2SimsRefreshStore - extends FSParam[Boolean]("sims_source_enable_dbv2_refresh_source", false) - - case object EnableExperimentalSimsStore - extends FSParam[Boolean]("sims_source_enable_experimental_source", false) - - case object DisableHeavyRanker - extends FSParam[Boolean]("sims_source_disable_heavy_ranker", default = false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsStore.scala deleted file mode 100644 index 98a00a9a4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SimsStore.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.hermit.candidate.thriftscala.Candidates -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.util.Duration - -import javax.inject.Inject - -@Singleton -class SimsStore @Inject() ( - @Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates]) - extends StratoBasedSimsCandidateSourceWithUnitView(fetcher, identifier = SimsStore.Identifier) - -@Singleton -class CachedSimsStore @Inject() ( - @Named(GuiceNamedConstants.SIMS_FETCHER) fetcher: Fetcher[Long, Unit, Candidates], - statsReceiver: StatsReceiver) - extends CacheBasedSimsStore( - id = SimsStore.Identifier, - fetcher = fetcher, - maxCacheSize = SimsStore.MaxCacheSize, - cacheTtl = SimsStore.CacheTTL, - statsReceiver = statsReceiver.scope("CachedSimsStore", "cache") - ) - -object SimsStore { - val Identifier = CandidateSourceIdentifier(Algorithm.Sims.toString) - val MaxCacheSize = 50000 - val CacheTTL: Duration = Duration.fromHours(24) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSource.scala deleted file mode 100644 index 6d862c849..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSource.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.twitter.follow_recommendations.common.candidate_sources.base.StratoFetcherSource -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.follow_recommendations.common.models.SimilarToProof -import com.twitter.hermit.candidate.thriftscala.Candidates -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher - -abstract class StratoBasedSimsCandidateSource[U]( - fetcher: Fetcher[Long, U, Candidates], - view: U, - override val identifier: CandidateSourceIdentifier) - extends StratoFetcherSource[Long, U, Candidates](fetcher, view, identifier) { - - override def map(target: Long, candidates: Candidates): Seq[CandidateUser] = - StratoBasedSimsCandidateSource.map(target, candidates) -} - -object StratoBasedSimsCandidateSource { - def map(target: Long, candidates: Candidates): Seq[CandidateUser] = { - for { - candidate <- candidates.candidates - } yield CandidateUser( - id = candidate.userId, - score = Some(candidate.score), - reason = Some( - Reason( - Some( - AccountProof( - similarToProof = Some(SimilarToProof(Seq(target))) - ) - ) - ) - ) - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSourceWithUnitView.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSourceWithUnitView.scala deleted file mode 100644 index af1133893..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/StratoBasedSimsCandidateSourceWithUnitView.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.twitter.hermit.candidate.thriftscala.Candidates -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher - -abstract class StratoBasedSimsCandidateSourceWithUnitView( - fetcher: Fetcher[Long, Unit, Candidates], - override val identifier: CandidateSourceIdentifier) - extends StratoBasedSimsCandidateSource[Unit](fetcher, Unit, identifier) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SwitchingSimsSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SwitchingSimsSource.scala deleted file mode 100644 index 0d297a806..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/SwitchingSimsSource.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SwitchingSimsSource @Inject() ( - cachedDBV2SimsStore: CachedDBV2SimsStore, - cachedDBV2SimsRefreshStore: CachedDBV2SimsRefreshStore, - cachedSimsExperimentalStore: CachedSimsExperimentalStore, - cachedSimsStore: CachedSimsStore, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends CandidateSource[HasParams with HasSimilarToContext, CandidateUser] { - - override val identifier: CandidateSourceIdentifier = SwitchingSimsSource.Identifier - - private val stats = statsReceiver.scope("SwitchingSimsSource") - private val dbV2SimsStoreCounter = stats.counter("DBV2SimsStore") - private val dbV2SimsRefreshStoreCounter = stats.counter("DBV2SimsRefreshStore") - private val simsExperimentalStoreCounter = stats.counter("SimsExperimentalStore") - private val simsStoreCounter = stats.counter("SimsStore") - - override def apply(request: HasParams with HasSimilarToContext): Stitch[Seq[CandidateUser]] = { - val selectedSimsStore = - if (request.params(SimsSourceParams.EnableDBV2SimsStore)) { - dbV2SimsStoreCounter.incr() - cachedDBV2SimsStore - } else if (request.params(SimsSourceParams.EnableDBV2SimsRefreshStore)) { - dbV2SimsRefreshStoreCounter.incr() - cachedDBV2SimsRefreshStore - } else if (request.params(SimsSourceParams.EnableExperimentalSimsStore)) { - simsExperimentalStoreCounter.incr() - cachedSimsExperimentalStore - } else { - simsStoreCounter.incr() - cachedSimsStore - } - stats.counter("total").incr() - selectedSimsStore(request) - } -} - -object SwitchingSimsSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier(Algorithm.Sims.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/BUILD deleted file mode 100644 index f5ccb66e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/DBV2SimsExpansionParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/DBV2SimsExpansionParams.scala deleted file mode 100644 index c323ad1f3..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/DBV2SimsExpansionParams.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object DBV2SimsExpansionParams { - // Theses divisors are used to calibrate DBv2Sims extension candidates scores - case object RecentFollowingSimilarUsersDBV2CalibrateDivisor - extends FSBoundedParam[Double]( - "sims_expansion_recent_following_similar_users_dbv2_divisor", - default = 1.0d, - min = 0.1d, - max = 100d) - case object RecentEngagementSimilarUsersDBV2CalibrateDivisor - extends FSBoundedParam[Double]( - "sims_expansion_recent_engagement_similar_users_dbv2_divisor", - default = 1.0d, - min = 0.1d, - max = 100d) - case object DisableHeavyRanker - extends FSParam[Boolean]("sims_expansion_disable_heavy_ranker", default = false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/README.md deleted file mode 100644 index 6143d5868..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Sims Expansion Candidate Source -provides similar accounts based on the Sims algorithm for a given set of accounts. - -This is a 2nd-hop expansion, meaning that the input accounts could be a user's recently engaged, followed, or algorithm-generated (such as RealGraph) accounts. - -For more information on Sims and how it is utilized in the Follow Recommendations Service, please refer to the `follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims/README.md` file. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersFSConfig.scala deleted file mode 100644 index 5642f5852..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersFSConfig.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecentEngagementSimilarUsersFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = Seq( - RecentEngagementSimilarUsersParams.FirstDegreeSortEnabled - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersParams.scala deleted file mode 100644 index 4b1729702..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersParams.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam - -object RecentEngagementSimilarUsersParams { - - case object FirstDegreeSortEnabled - extends FSParam[Boolean]( - name = "sims_expansion_recent_engagement_first_degree_sort", - default = true) - case object Aggregator - extends FSEnumParam[SimsExpansionSourceAggregatorId.type]( - name = "sims_expansion_recent_engagement_aggregator_id", - default = SimsExpansionSourceAggregatorId.Sum, - enum = SimsExpansionSourceAggregatorId) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersSource.scala deleted file mode 100644 index 0d99c2dc2..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentEngagementSimilarUsersSource.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.follow_recommendations.common.models.SimilarToProof -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecentEngagementSimilarUsersSource @Inject() ( - realTimeRealGraphClient: RealTimeRealGraphClient, - switchingSimsSource: SwitchingSimsSource, - statsReceiver: StatsReceiver) - extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams]( - switchingSimsSource) { - override def maxSecondaryDegreeNodes(req: HasClientContext with HasParams): Int = Int.MaxValue - - override def maxResults(req: HasClientContext with HasParams): Int = - RecentEngagementSimilarUsersSource.MaxResults - - override val identifier: CandidateSourceIdentifier = RecentEngagementSimilarUsersSource.Identifier - private val stats = statsReceiver.scope(identifier.name) - private val calibratedScoreCounter = stats.counter("calibrated_scores_counter") - - override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double = { - sourceScore * similarToScore - } - - override def calibrateDivisor(req: HasClientContext with HasParams): Double = { - req.params(DBV2SimsExpansionParams.RecentEngagementSimilarUsersDBV2CalibrateDivisor) - } - - override def calibrateScore( - candidateScore: Double, - req: HasClientContext with HasParams - ): Double = { - calibratedScoreCounter.incr() - candidateScore / calibrateDivisor(req) - } - - /** - * fetch first degree nodes given request - */ - override def firstDegreeNodes( - target: HasClientContext with HasParams - ): Stitch[Seq[CandidateUser]] = { - target.getOptionalUserId - .map { userId => - realTimeRealGraphClient - .getUsersRecentlyEngagedWith( - userId, - RealTimeRealGraphClient.EngagementScoreMap, - includeDirectFollowCandidates = true, - includeNonDirectFollowCandidates = true - ).map(_.sortBy(-_.score.getOrElse(0.0d)) - .take(RecentEngagementSimilarUsersSource.MaxFirstDegreeNodes)) - }.getOrElse(Stitch.Nil) - } - - override def aggregateAndScore( - request: HasClientContext with HasParams, - firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]] - ): Stitch[Seq[CandidateUser]] = { - - val inputNodes = firstDegreeToSecondDegreeNodesMap.keys.map(_.id).toSet - val aggregator = request.params(RecentEngagementSimilarUsersParams.Aggregator) match { - case SimsExpansionSourceAggregatorId.Max => - SimsExpansionBasedCandidateSource.ScoreAggregator.Max - case SimsExpansionSourceAggregatorId.Sum => - SimsExpansionBasedCandidateSource.ScoreAggregator.Sum - case SimsExpansionSourceAggregatorId.MultiDecay => - SimsExpansionBasedCandidateSource.ScoreAggregator.MultiDecay - } - - val groupedCandidates = firstDegreeToSecondDegreeNodesMap.values.flatten - .filterNot(c => inputNodes.contains(c.candidateId)) - .groupBy(_.candidateId) - .map { - case (id, candidates) => - // Different aggregators for final score - val finalScore = aggregator(candidates.map(_.score).toSeq) - val proofs = candidates.map(_.similarTo).toSet - - CandidateUser( - id = id, - score = Some(finalScore), - reason = - Some(Reason(Some(AccountProof(similarToProof = Some(SimilarToProof(proofs.toSeq)))))) - ).withCandidateSource(identifier) - } - .toSeq - .sortBy(-_.score.getOrElse(0.0d)) - .take(maxResults(request)) - - Stitch.value(groupedCandidates) - } -} - -object RecentEngagementSimilarUsersSource { - val Identifier = CandidateSourceIdentifier(Algorithm.RecentEngagementSimilarUser.toString) - val MaxFirstDegreeNodes = 10 - val MaxResults = 200 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersParams.scala deleted file mode 100644 index 44b1378d4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersParams.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object RecentFollowingSimilarUsersParams { - case object MaxFirstDegreeNodes - extends FSBoundedParam[Int]( - name = "sims_expansion_recent_following_max_first_degree_nodes", - default = 10, - min = 0, - max = 200) - case object MaxSecondaryDegreeExpansionPerNode - extends FSBoundedParam[Int]( - name = "sims_expansion_recent_following_max_secondary_degree_nodes", - default = 40, - min = 0, - max = 200) - case object MaxResults - extends FSBoundedParam[Int]( - name = "sims_expansion_recent_following_max_results", - default = 200, - min = 0, - max = 200) - case object TimestampIntegrated - extends FSParam[Boolean]( - name = "sims_expansion_recent_following_integ_timestamp", - default = false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersSource.scala deleted file mode 100644 index b5a187fc5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentFollowingSimilarUsersSource.scala +++ /dev/null @@ -1,99 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import javax.inject.Inject - -object RecentFollowingSimilarUsersSource { - - val Identifier = CandidateSourceIdentifier(Algorithm.NewFollowingSimilarUser.toString) -} - -@Singleton -class RecentFollowingSimilarUsersSource @Inject() ( - socialGraph: SocialGraphClient, - switchingSimsSource: SwitchingSimsSource, - statsReceiver: StatsReceiver) - extends SimsExpansionBasedCandidateSource[ - HasParams with HasRecentFollowedUserIds with HasClientContext - ](switchingSimsSource) { - - val identifier = RecentFollowingSimilarUsersSource.Identifier - private val stats = statsReceiver.scope(identifier.name) - private val maxResultsStats = stats.scope("max_results") - private val calibratedScoreCounter = stats.counter("calibrated_scores_counter") - - override def firstDegreeNodes( - request: HasParams with HasRecentFollowedUserIds with HasClientContext - ): Stitch[Seq[CandidateUser]] = { - if (request.params(RecentFollowingSimilarUsersParams.TimestampIntegrated)) { - val recentFollowedUserIdsWithTimeStitch = - socialGraph.getRecentFollowedUserIdsWithTime(request.clientContext.userId.get) - - recentFollowedUserIdsWithTimeStitch.map { results => - val first_degree_nodes = results - .sortBy(-_.timeInMs).take( - request.params(RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes)) - val max_timestamp = first_degree_nodes.head.timeInMs - first_degree_nodes.map { - case userIdWithTime => - CandidateUser( - userIdWithTime.userId, - score = Some(userIdWithTime.timeInMs.toDouble / max_timestamp)) - } - } - } else { - Stitch.value( - request.recentFollowedUserIds - .getOrElse(Nil).take( - request.params(RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes)).map( - CandidateUser(_, score = Some(1.0))) - ) - } - } - - override def maxSecondaryDegreeNodes( - req: HasParams with HasRecentFollowedUserIds with HasClientContext - ): Int = { - req.params(RecentFollowingSimilarUsersParams.MaxSecondaryDegreeExpansionPerNode) - } - - override def maxResults( - req: HasParams with HasRecentFollowedUserIds with HasClientContext - ): Int = { - val firstDegreeNodes = req.params(RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes) - val maxResultsNum = req.params(RecentFollowingSimilarUsersParams.MaxResults) - maxResultsStats - .stat( - s"RecentFollowingSimilarUsersSource_firstDegreeNodes_${firstDegreeNodes}_maxResults_${maxResultsNum}") - .add(1) - maxResultsNum - } - - override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double = { - sourceScore * similarToScore - } - - override def calibrateDivisor( - req: HasParams with HasRecentFollowedUserIds with HasClientContext - ): Double = { - req.params(DBV2SimsExpansionParams.RecentFollowingSimilarUsersDBV2CalibrateDivisor) - } - - override def calibrateScore( - candidateScore: Double, - req: HasParams with HasRecentFollowedUserIds with HasClientContext - ): Double = { - calibratedScoreCounter.incr() - candidateScore / calibrateDivisor(req) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentStrongEngagementDirectFollowSimilarUsersSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentStrongEngagementDirectFollowSimilarUsersSource.scala deleted file mode 100644 index 0898aabfb..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/RecentStrongEngagementDirectFollowSimilarUsersSource.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -import javax.inject.Inject - -@Singleton -class RecentStrongEngagementDirectFollowSimilarUsersSource @Inject() ( - realTimeRealGraphClient: RealTimeRealGraphClient, - switchingSimsSource: SwitchingSimsSource) - extends SimsExpansionBasedCandidateSource[HasClientContext with HasParams]( - switchingSimsSource) { - - val identifier = RecentStrongEngagementDirectFollowSimilarUsersSource.Identifier - - override def firstDegreeNodes( - request: HasClientContext with HasParams - ): Stitch[Seq[CandidateUser]] = request.getOptionalUserId - .map { userId => - realTimeRealGraphClient - .getUsersRecentlyEngagedWith( - userId, - RealTimeRealGraphClient.StrongEngagementScoreMap, - includeDirectFollowCandidates = true, - includeNonDirectFollowCandidates = false - ).map(_.take(RecentStrongEngagementDirectFollowSimilarUsersSource.MaxFirstDegreeNodes)) - }.getOrElse(Stitch.Nil) - - override def maxSecondaryDegreeNodes(request: HasClientContext with HasParams): Int = Int.MaxValue - - override def maxResults(request: HasClientContext with HasParams): Int = - RecentStrongEngagementDirectFollowSimilarUsersSource.MaxResults - - override def scoreCandidate(sourceScore: Double, similarToScore: Double): Double = { - sourceScore * similarToScore - } - - override def calibrateDivisor(req: HasClientContext with HasParams): Double = 1.0d -} - -object RecentStrongEngagementDirectFollowSimilarUsersSource { - val Identifier = CandidateSourceIdentifier(Algorithm.RecentStrongEngagementSimilarUser.toString) - val MaxFirstDegreeNodes = 10 - val MaxResults = 200 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionBasedCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionBasedCandidateSource.scala deleted file mode 100644 index 018fe413b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionBasedCandidateSource.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.twitter.follow_recommendations.common.candidate_sources.base.TwoHopExpansionCandidateSource -import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.follow_recommendations.common.models.SimilarToProof -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import scala.math._ - -case class SimilarUser(candidateId: Long, similarTo: Long, score: Double) - -abstract class SimsExpansionBasedCandidateSource[-Target <: HasParams]( - switchingSimsSource: SwitchingSimsSource) - extends TwoHopExpansionCandidateSource[Target, CandidateUser, SimilarUser, CandidateUser] { - - // max number secondary degree nodes per first degree node - def maxSecondaryDegreeNodes(req: Target): Int - - // max number output results - def maxResults(req: Target): Int - - // scorer to score candidate based on first and second degree node scores - def scoreCandidate(source: Double, similarToScore: Double): Double - - def calibrateDivisor(req: Target): Double - - def calibrateScore(candidateScore: Double, req: Target): Double = { - candidateScore / calibrateDivisor(req) - } - - override def secondaryDegreeNodes(req: Target, node: CandidateUser): Stitch[Seq[SimilarUser]] = { - switchingSimsSource(new HasParams with HasSimilarToContext { - override val similarToUserIds = Seq(node.id) - override val params = (req.params) - }).map(_.take(maxSecondaryDegreeNodes(req)).map { candidate => - SimilarUser( - candidate.id, - node.id, - (node.score, candidate.score) match { - // only calibrated sims expanded candidates scores - case (Some(nodeScore), Some(candidateScore)) => - calibrateScore(scoreCandidate(nodeScore, candidateScore), req) - case (Some(nodeScore), _) => nodeScore - // NewFollowingSimilarUser will enter this case - case _ => calibrateScore(candidate.score.getOrElse(0.0), req) - } - ) - }) - } - - override def aggregateAndScore( - request: Target, - firstDegreeToSecondDegreeNodesMap: Map[CandidateUser, Seq[SimilarUser]] - ): Stitch[Seq[CandidateUser]] = { - - val inputNodes = firstDegreeToSecondDegreeNodesMap.keys.map(_.id).toSet - val aggregator = request.params(SimsExpansionSourceParams.Aggregator) match { - case SimsExpansionSourceAggregatorId.Max => - SimsExpansionBasedCandidateSource.ScoreAggregator.Max - case SimsExpansionSourceAggregatorId.Sum => - SimsExpansionBasedCandidateSource.ScoreAggregator.Sum - case SimsExpansionSourceAggregatorId.MultiDecay => - SimsExpansionBasedCandidateSource.ScoreAggregator.MultiDecay - } - - val groupedCandidates = firstDegreeToSecondDegreeNodesMap.values.flatten - .filterNot(c => inputNodes.contains(c.candidateId)) - .groupBy(_.candidateId) - .map { - case (id, candidates) => - // Different aggregators for final score - val finalScore = aggregator(candidates.map(_.score).toSeq) - val proofs = candidates.map(_.similarTo).toSet - - CandidateUser( - id = id, - score = Some(finalScore), - reason = - Some(Reason(Some(AccountProof(similarToProof = Some(SimilarToProof(proofs.toSeq)))))) - ).withCandidateSource(identifier) - } - .toSeq - .sortBy(-_.score.getOrElse(0.0d)) - .take(maxResults(request)) - - Stitch.value(groupedCandidates) - } -} - -object SimsExpansionBasedCandidateSource { - object ScoreAggregator { - val Max: Seq[Double] => Double = (candidateScores: Seq[Double]) => { - if (candidateScores.size > 0) candidateScores.max else 0.0 - } - val Sum: Seq[Double] => Double = (candidateScores: Seq[Double]) => { - candidateScores.sum - } - val MultiDecay: Seq[Double] => Double = (candidateScores: Seq[Double]) => { - val alpha = 0.1 - val beta = 0.1 - val gamma = 0.8 - val decay_scores: Seq[Double] = - candidateScores - .sorted(Ordering[Double].reverse) - .zipWithIndex - .map(x => x._1 * pow(gamma, x._2)) - alpha * candidateScores.max + decay_scores.sum + beta * candidateScores.size - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionFSConfig.scala deleted file mode 100644 index b145ee607..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionFSConfig.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SimsExpansionFSConfig @Inject() () extends FeatureSwitchConfig { - override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( - RecentFollowingSimilarUsersParams.MaxFirstDegreeNodes, - RecentFollowingSimilarUsersParams.MaxSecondaryDegreeExpansionPerNode, - RecentFollowingSimilarUsersParams.MaxResults - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - DBV2SimsExpansionParams.RecentFollowingSimilarUsersDBV2CalibrateDivisor, - DBV2SimsExpansionParams.RecentEngagementSimilarUsersDBV2CalibrateDivisor - ) - - override val booleanFSParams: Seq[FSParam[Boolean]] = Seq( - DBV2SimsExpansionParams.DisableHeavyRanker, - RecentFollowingSimilarUsersParams.TimestampIntegrated - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionSourceParams.scala deleted file mode 100644 index f03ccceea..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion/SimsExpansionSourceParams.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.sims_expansion -import com.twitter.timelines.configapi.FSEnumParam - -object SimsExpansionSourceParams { - case object Aggregator - extends FSEnumParam[SimsExpansionSourceAggregatorId.type]( - name = "sims_expansion_aggregator_id", - default = SimsExpansionSourceAggregatorId.Sum, - enum = SimsExpansionSourceAggregatorId) -} - -object SimsExpansionSourceAggregatorId extends Enumeration { - type AggregatorId = Value - val Sum: AggregatorId = Value("sum") - val Max: AggregatorId = Value("max") - val MultiDecay: AggregatorId = Value("multi_decay") -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/BUILD deleted file mode 100644 index e0392df45..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/README.md deleted file mode 100644 index bfdbacd25..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Social Graph Candidate Source -Provides candidate expansion based on the Twitter social graph. - -Currently, the expansion is mainly based on the "follow" social graph edge, which allows the service to identify recent following accounts for a given set of accounts. The input accounts could be a user's recent following, engaged, or other related accounts. - -In summary, the Social Graph Candidate Source utilizes the Twitter social graph to provide candidate expansion, primarily focusing on recent following accounts. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSource.scala deleted file mode 100644 index 577213cc1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSource.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.socialgraph - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.base.TwoHopExpansionCandidateSource -import com.twitter.follow_recommendations.common.clients.socialgraph.RecentEdgesQuery -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.hermit.model.Algorithm -import com.twitter.inject.Logging -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -/** - * This candidate source is a two hop expansion over the follow graph. The candidates returned from this source is the users that get followed by the target user's recent followings. It will call SocialGraph `n` + 1 times where `n` is the number of recent followings of the target user to be considered. - */ -@Singleton -class RecentFollowingRecentFollowingExpansionSource @Inject() ( - socialGraphClient: SocialGraphClient, - statsReceiver: StatsReceiver) - extends TwoHopExpansionCandidateSource[ - HasParams with HasRecentFollowedUserIds, - Long, - Long, - CandidateUser - ] - with Logging { - - override val identifier: CandidateSourceIdentifier = - RecentFollowingRecentFollowingExpansionSource.Identifier - - val stats = statsReceiver.scope(identifier.name) - - override def firstDegreeNodes( - target: HasParams with HasRecentFollowedUserIds - ): Stitch[Seq[Long]] = Stitch.value( - target.recentFollowedUserIds - .getOrElse(Nil).take( - RecentFollowingRecentFollowingExpansionSource.NumFirstDegreeNodesToRetrieve) - ) - - override def secondaryDegreeNodes( - target: HasParams with HasRecentFollowedUserIds, - node: Long - ): Stitch[Seq[Long]] = socialGraphClient - .getRecentEdgesCached( - RecentEdgesQuery( - node, - Seq(RelationshipType.Following), - Some(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)), - useCachedStratoColumn = - target.params(RecentFollowingRecentFollowingExpansionSourceParams.CallSgsCachedColumn) - ).map( - _.take(RecentFollowingRecentFollowingExpansionSource.NumSecondDegreeNodesToRetrieve)).rescue { - case exception: Exception => - logger.warn( - s"${this.getClass} fails to retrieve second degree nodes for first degree node $node", - exception) - stats.counter("second_degree_expansion_error").incr() - Stitch.Nil - } - - override def aggregateAndScore( - target: HasParams with HasRecentFollowedUserIds, - firstDegreeToSecondDegreeNodesMap: Map[Long, Seq[Long]] - ): Stitch[Seq[CandidateUser]] = { - val zipped = firstDegreeToSecondDegreeNodesMap.toSeq.flatMap { - case (firstDegreeId, secondDegreeIds) => - secondDegreeIds.map(secondDegreeId => firstDegreeId -> secondDegreeId) - } - val candidateAndConnections = zipped - .groupBy { case (_, secondDegreeId) => secondDegreeId } - .mapValues { v => v.map { case (firstDegreeId, _) => firstDegreeId } } - .toSeq - .sortBy { case (_, connections) => -connections.size } - .map { - case (candidateId, connections) => - CandidateUser( - id = candidateId, - score = Some(CandidateUser.DefaultCandidateScore), - reason = Some( - Reason( - Some(AccountProof(followProof = Some(FollowProof(connections, connections.size)))))) - ).withCandidateSource(identifier) - } - Stitch.value(candidateAndConnections) - } -} - -object RecentFollowingRecentFollowingExpansionSource { - val Identifier = CandidateSourceIdentifier(Algorithm.NewFollowingNewFollowingExpansion.toString) - - val NumFirstDegreeNodesToRetrieve = 5 - val NumSecondDegreeNodesToRetrieve = 20 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceFSConfig.scala deleted file mode 100644 index ac6403ebe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceFSConfig.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.socialgraph - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecentFollowingRecentFollowingExpansionSourceFSConfig @Inject() () - extends FeatureSwitchConfig { - - override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( - RecentFollowingRecentFollowingExpansionSourceParams.CallSgsCachedColumn, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceParams.scala deleted file mode 100644 index 4c51233ad..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph/RecentFollowingRecentFollowingExpansionSourceParams.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.socialgraph - -import com.twitter.timelines.configapi.FSParam - -object RecentFollowingRecentFollowingExpansionSourceParams { - object CallSgsCachedColumn - extends FSParam[Boolean]( - "recent_following_recent_following_source_call_sgs_cached_column", - true) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BUILD deleted file mode 100644 index e89bade43..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/scala/com/twitter/onboarding/relevance/features/strongtie", - "src/thrift/com/twitter/search/account_search/extended_network:extended_network_users-scala", - "strato/config/columns/hub:hub-strato-client", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/config/columns/search/account_search:account_search-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BaseOnlineSTPSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BaseOnlineSTPSource.scala deleted file mode 100644 index 9f4365642..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/BaseOnlineSTPSource.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.STPGraph -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPFeatureGenerator -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord - -abstract class BaseOnlineSTPSource( - stpGraphBuilder: STPGraphBuilder, - baseStatsReceiver: StatsReceiver) - extends CandidateSource[ - HasClientContext with HasParams with HasRecentFollowedUserIds, - CandidateUser - ] - with Logging { - - protected val statsReceiver: StatsReceiver = baseStatsReceiver.scope("online_stp") - - override val identifier: CandidateSourceIdentifier = BaseOnlineSTPSource.Identifier - - def getCandidates( - records: Seq[STPRecord], - request: HasClientContext with HasParams with HasRecentFollowedUserIds - ): Stitch[Seq[CandidateUser]] - - override def apply( - request: HasClientContext with HasParams with HasRecentFollowedUserIds - ): Stitch[Seq[CandidateUser]] = - request.getOptionalUserId - .map { userId => - stpGraphBuilder(request) - .flatMap { graph: STPGraph => - logger.debug(graph) - val records = STPFeatureGenerator.constructFeatures( - userId, - graph.firstDegreeEdgeInfoList, - graph.secondDegreeEdgeInfoList) - getCandidates(records, request) - } - }.getOrElse(Stitch.Nil) -} - -object BaseOnlineSTPSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.OnlineStrongTiePredictionRecNoCaching.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/Dbv2StpScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/Dbv2StpScorer.scala deleted file mode 100644 index 82308282a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/Dbv2StpScorer.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.cortex.deepbird.runtime.prediction_engine.TensorflowPredictionEngine -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.ml.api.Feature.Continuous -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.ml.prediction_service.PredictionRequest -import com.twitter.stitch.Stitch -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecordAdapter -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/** - * STP ML ranker trained using DeepBirdV2 - */ -@Singleton -class Dbv2StpScorer @Inject() ( - @Named(GuiceNamedConstants.STP_DBV2_SCORER) tfPredictionEngine: TensorflowPredictionEngine) { - def getScoredResponse(record: STPRecord): Stitch[Option[Double]] = { - val request: PredictionRequest = new PredictionRequest( - STPRecordAdapter.adaptToDataRecord(record)) - val responseStitch = Stitch.callFuture(tfPredictionEngine.getPrediction(request)) - responseStitch.map { response => - val richDr = SRichDataRecord(response.getPrediction) - richDr.getFeatureValueOpt(new Continuous("output")) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/EpStpScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/EpStpScorer.scala deleted file mode 100644 index d02259575..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/EpStpScorer.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.thrift.BinaryThriftCodec -import com.twitter.relevance.ep_model.scorer.EPScorer -import com.twitter.relevance.ep_model.scorer.ScorerUtil -import com.twitter.relevance.ep_model.thrift -import com.twitter.relevance.ep_model.thriftscala.EPScoringOptions -import com.twitter.relevance.ep_model.thriftscala.EPScoringRequest -import com.twitter.relevance.ep_model.thriftscala.EPScoringResponse -import com.twitter.relevance.ep_model.thriftscala.Record -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ -import scala.util.Success - -case class ScoredResponse(score: Double, featuresBreakdown: Option[String] = None) - -/** - * STP ML ranker trained using prehistoric ML framework - */ -@Singleton -class EpStpScorer @Inject() (epScorer: EPScorer) { - private def getScore(responses: List[EPScoringResponse]): Option[ScoredResponse] = - responses.headOption - .flatMap { response => - response.scores.flatMap { - _.headOption.map(score => ScoredResponse(ScorerUtil.normalize(score))) - } - } - - def getScoredResponse( - record: Record, - details: Boolean = false - ): Stitch[Option[ScoredResponse]] = { - val scoringOptions = EPScoringOptions( - addFeaturesBreakDown = details, - addTransformerIntermediateRecords = details - ) - val request = EPScoringRequest(auxFeatures = Some(Seq(record)), options = Some(scoringOptions)) - - Stitch.callFuture( - BinaryThriftCodec[thrift.EPScoringRequest] - .invert(BinaryScalaCodec(EPScoringRequest).apply(request)) - .map { thriftRequest: thrift.EPScoringRequest => - val responsesF = epScorer - .score(List(thriftRequest).asJava) - .map( - _.asScala.toList - .map(response => - BinaryScalaCodec(EPScoringResponse) - .invert(BinaryThriftCodec[thrift.EPScoringResponse].apply(response))) - .collect { case Success(response) => response } - ) - responsesF.map(getScore) - } - .getOrElse(Future(None))) - } -} - -object EpStpScorer { - val WithFeaturesBreakDown = false -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/MutualFollowStrongTiePredictionSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/MutualFollowStrongTiePredictionSource.scala deleted file mode 100644 index 10a269931..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/MutualFollowStrongTiePredictionSource.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.follow_recommendations.common.clients.socialgraph.RecentEdgesQuery -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.StrongTiePredictionFeaturesOnUserClientColumn -import javax.inject.Singleton -import javax.inject.Inject - -/** - * Returns mutual follows. It first gets mutual follows from recent 100 follows and followers, and then unions this - * with mutual follows from STP features dataset. - */ -@Singleton -class MutualFollowStrongTiePredictionSource @Inject() ( - sgsClient: SocialGraphClient, - strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn) - extends CandidateSource[HasClientContext with HasRecentFollowedUserIds, CandidateUser] { - val identifier: CandidateSourceIdentifier = - MutualFollowStrongTiePredictionSource.Identifier - - override def apply( - target: HasClientContext with HasRecentFollowedUserIds - ): Stitch[Seq[CandidateUser]] = { - target.getOptionalUserId match { - case Some(userId) => - val newFollowings = target.recentFollowedUserIds - .getOrElse(Nil) - .take(MutualFollowStrongTiePredictionSource.NumOfRecentFollowings) - val newFollowersStitch = - sgsClient - .getRecentEdges(RecentEdgesQuery(userId, Seq(RelationshipType.FollowedBy))).map( - _.take(MutualFollowStrongTiePredictionSource.NumOfRecentFollowers)) - val mutualFollowsStitch = - strongTiePredictionFeaturesOnUserClientColumn.fetcher - .fetch(userId).map(_.v.flatMap(_.topMutualFollows.map(_.map(_.userId))).getOrElse(Nil)) - - Stitch.join(newFollowersStitch, mutualFollowsStitch).map { - case (newFollowers, mutualFollows) => { - (newFollowings.intersect(newFollowers) ++ mutualFollows).distinct - .map(id => CandidateUser(id, Some(CandidateUser.DefaultCandidateScore))) - } - } - case _ => Stitch.Nil - } - } -} - -object MutualFollowStrongTiePredictionSource { - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.MutualFollowSTP.toString) - val NumOfRecentFollowings = 100 - val NumOfRecentFollowers = 100 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineMutualFollowExpansionSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineMutualFollowExpansionSource.scala deleted file mode 100644 index 37981f843..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineMutualFollowExpansionSource.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.google.inject.Singleton -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.generated.client.onboarding.userrecs.MutualFollowExpansionClientColumn -import javax.inject.Inject - -/** - * A source that finds the mutual follows of one's mutual follows that one isn't following already. - */ -@Singleton -class OfflineMutualFollowExpansionSource @Inject() ( - column: MutualFollowExpansionClientColumn) - extends OfflineStrongTiePredictionBaseSource(column.fetcher) { - override val identifier: CandidateSourceIdentifier = - OfflineMutualFollowExpansionSource.Identifier -} - -object OfflineMutualFollowExpansionSource { - val Identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(Algorithm.MutualFollowExpansion.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceFsConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceFsConfig.scala deleted file mode 100644 index 152fc97a6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceFsConfig.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class OfflineStpSourceFsConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( - OfflineStpSourceParams.UseDenserPmiMatrix - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceParams.scala deleted file mode 100644 index fb1672cdb..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceParams.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.timelines.configapi.FSParam - -object OfflineStpSourceParams { - // If enabled, we use the new, denser version of PMI matrix to generate OfflineSTP candidates. - case object UseDenserPmiMatrix - extends FSParam[Boolean]("offline_stp_source_use_denser_pmi_matrix", default = false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithDensePmiMatrix.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithDensePmiMatrix.scala deleted file mode 100644 index 6a37ff222..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithDensePmiMatrix.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.google.inject.Singleton -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.generated.client.hub.PpmiDenseMatrixCandidatesClientColumn -import javax.inject.Inject - -/** - * Main source for strong-tie-prediction candidates generated offline. - */ -@Singleton -class OfflineStpSourceWithDensePmiMatrix @Inject() ( - stpColumn: PpmiDenseMatrixCandidatesClientColumn) - extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher) { - override val identifier: CandidateSourceIdentifier = OfflineStpSourceWithDensePmiMatrix.Identifier -} - -object OfflineStpSourceWithDensePmiMatrix { - val Identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(Algorithm.StrongTiePredictionRec.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithLegacyPmiMatrix.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithLegacyPmiMatrix.scala deleted file mode 100644 index 7e17b2e8a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStpSourceWithLegacyPmiMatrix.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.google.inject.Singleton -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.generated.client.onboarding.userrecs.StrongTiePredictionClientColumn -import javax.inject.Inject - -/** - * Main source for strong-tie-prediction candidates generated offline. - */ -@Singleton -class OfflineStpSourceWithLegacyPmiMatrix @Inject() ( - stpColumn: StrongTiePredictionClientColumn) - extends OfflineStrongTiePredictionBaseSource(stpColumn.fetcher) { - override val identifier: CandidateSourceIdentifier = - OfflineStpSourceWithLegacyPmiMatrix.Identifier -} - -object OfflineStpSourceWithLegacyPmiMatrix { - val Identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(Algorithm.StrongTiePredictionRec.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionBaseSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionBaseSource.scala deleted file mode 100644 index a46d49662..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionBaseSource.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.timelines.configapi.HasParams - -/** Base class that all variants of our offline stp dataset can extend. Assumes the same STPResult - * value in the key and converts the result into the necessary internal model. - */ -abstract class OfflineStrongTiePredictionBaseSource( - fetcher: Fetcher[Long, Unit, STPResult]) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] { - - def fetch( - target: Long, - ): Stitch[Seq[CandidateUser]] = { - fetcher - .fetch(target) - .map { result => - result.v - .map { candidates => OfflineStrongTiePredictionBaseSource.map(target, candidates) } - .getOrElse(Nil) - .map(_.withCandidateSource(identifier)) - } - } - - override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - request.getOptionalUserId.map(fetch).getOrElse(Stitch.Nil) - } -} - -object OfflineStrongTiePredictionBaseSource { - def map(target: Long, candidates: STPResult): Seq[CandidateUser] = { - for { - candidate <- candidates.strongTieUsers.sortBy(-_.score) - } yield CandidateUser( - id = candidate.userId, - score = Some(candidate.score), - reason = Some( - Reason( - Some( - AccountProof( - followProof = candidate.socialProof.map(proof => FollowProof(proof, proof.size)) - ) - ) - ) - ) - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionSource.scala deleted file mode 100644 index 6a1cb3983..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OfflineStrongTiePredictionSource.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStpSourceParams.UseDenserPmiMatrix -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.util.logging.Logging -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject - -object OfflineStpScore extends Feature[UserCandidate, Option[Double]] - -/** - * Main source for strong-tie-prediction candidates generated offline. - */ -@Singleton -class OfflineStrongTiePredictionSource @Inject() ( - offlineStpSourceWithLegacyPmiMatrix: OfflineStpSourceWithLegacyPmiMatrix, - offlineStpSourceWithDensePmiMatrix: OfflineStpSourceWithDensePmiMatrix) - extends CandidateSource[HasParams with HasClientContext, CandidateUser] - with Logging { - override val identifier: CandidateSourceIdentifier = OfflineStrongTiePredictionSource.Identifier - - override def apply(request: HasParams with HasClientContext): Stitch[Seq[CandidateUser]] = { - if (request.params(UseDenserPmiMatrix)) { - logger.info("Using dense PMI matrix.") - offlineStpSourceWithDensePmiMatrix(request) - } else { - logger.info("Using legacy PMI matrix.") - offlineStpSourceWithLegacyPmiMatrix(request) - } - } -} - -object OfflineStrongTiePredictionSource { - val Identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(Algorithm.StrongTiePredictionRec.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceFSConfig.scala deleted file mode 100644 index ff61cc29c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceFSConfig.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class OnlineSTPSourceFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Seq( - OnlineSTPSourceParams.DisableHeavyRanker, - OnlineSTPSourceParams.UseDBv2Scorer, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceParams.scala deleted file mode 100644 index e6224d359..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceParams.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object OnlineSTPSourceParams { - // This replaces the old scorer module, located at EpStpScorer.scala, with the new scorer, located - // at Dbv2StpScorer.scala. - case object UseDBv2Scorer - extends FSParam[Boolean]("online_stp_source_dbv2_scorer_enabled", default = false) - - // For experiments that test the impact of an improved OnlineSTP source, this controls the usage - // of the PostNux heavy-ranker. Note that this FS should *NOT* trigger bucket impressions. - case object DisableHeavyRanker - extends FSParam[Boolean]("online_stp_source_disable_heavy_ranker", default = false) - - case object SetPredictionDetails extends Param[Boolean](default = false) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceScorer.scala deleted file mode 100644 index 16bc60776..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceScorer.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class OnlineSTPSourceScorer @Inject() ( - onlineSTPSourceWithEPScorer: OnlineSTPSourceWithEPScorer) - extends CandidateSource[ - HasClientContext with HasParams with HasRecentFollowedUserIds, - CandidateUser - ] { - - override def apply( - request: HasClientContext with HasParams with HasRecentFollowedUserIds - ): Stitch[Seq[CandidateUser]] = { - onlineSTPSourceWithEPScorer(request) - } - - override val identifier: CandidateSourceIdentifier = BaseOnlineSTPSource.Identifier -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithDeepbirdV2Scorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithDeepbirdV2Scorer.scala deleted file mode 100644 index b8f348fea..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithDeepbirdV2Scorer.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.onboarding.relevance.features.strongtie.{ - StrongTieFeatures => StrongTieFeaturesWrapper -} -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class OnlineSTPSourceWithDeepbirdV2Scorer @Inject() ( - dbv2StpScorer: Dbv2StpScorer, - stpGraphBuilder: STPGraphBuilder, - baseStatReceiver: StatsReceiver) - extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver) { - - private val dbv2ScorerUsedCounter = statsReceiver.counter("dbv2_scorer_used") - private val dbv2ScorerFailureCounter = statsReceiver.counter("dbv2_scorer_failure") - private val dbv2ScorerSuccessCounter = statsReceiver.counter("dbv2_scorer_success") - - override def getCandidates( - records: Seq[STPRecord], - request: HasClientContext with HasParams with HasRecentFollowedUserIds, - ): Stitch[Seq[CandidateUser]] = { - val possibleCandidates: Seq[Stitch[Option[CandidateUser]]] = records.map { trainingRecord => - dbv2ScorerUsedCounter.incr() - val score = dbv2StpScorer.getScoredResponse(trainingRecord) - score.map { - case None => - dbv2ScorerFailureCounter.incr() - None - case Some(scoreVal) => - dbv2ScorerSuccessCounter.incr() - Some( - CandidateUser( - id = trainingRecord.destinationId, - score = Some(OnlineSTPSourceWithDeepbirdV2Scorer.logitSubtraction(scoreVal)), - reason = Some( - Reason(Some( - AccountProof(followProof = - Some(FollowProof(trainingRecord.socialProof, trainingRecord.socialProof.size))) - ))) - ).withCandidateSourceAndFeatures( - identifier, - Seq(StrongTieFeaturesWrapper(trainingRecord.features))) - ) - } - } - Stitch.collect(possibleCandidates).map { _.flatten.sortBy(-_.score.getOrElse(0.0)) } - } -} - -object OnlineSTPSourceWithDeepbirdV2Scorer { - // The following two variables are the means for the distribution of scores coming from the legacy - // and DBv2 OnlineSTP models. We need this to calibrate the DBv2 scores and align the two means. - // BQ Link: https://console.cloud.google.com/bigquery?sq=213005704923:e06ac27e4db74385a77a4b538c531f82 - private val legacyMeanScore = 0.0478208871192468 - private val dbv2MeanScore = 0.238666097210261 - - // In below are the necessary functions to calibrate the scores such that the means are aligned. - private val EPS: Double = 1e-8 - private val e: Double = math.exp(1) - private def sigmoid(x: Double): Double = math.pow(e, x) / (math.pow(e, x) + 1) - // We add an EPS to the denominator to avoid division by 0. - private def logit(x: Double): Double = math.log(x / (1 - x + EPS)) - def logitSubtraction(x: Double): Double = sigmoid( - logit(x) - (logit(dbv2MeanScore) - logit(legacyMeanScore))) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithEPScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithEPScorer.scala deleted file mode 100644 index 1c6163852..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/OnlineSTPSourceWithEPScorer.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceParams.SetPredictionDetails -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.onboarding.relevance.features.strongtie.{ - StrongTieFeatures => StrongTieFeaturesWrapper -} -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.STPRecord -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class OnlineSTPSourceWithEPScorer @Inject() ( - epStpScorer: EpStpScorer, - stpGraphBuilder: STPGraphBuilder, - baseStatReceiver: StatsReceiver) - extends BaseOnlineSTPSource(stpGraphBuilder, baseStatReceiver) - with Logging { - private val epScorerUsedCounter = statsReceiver.counter("ep_scorer_used") - - override def getCandidates( - records: Seq[STPRecord], - request: HasClientContext with HasParams with HasRecentFollowedUserIds, - ): Stitch[Seq[CandidateUser]] = { - epScorerUsedCounter.incr() - - val possibleCandidates: Seq[Stitch[Option[CandidateUser]]] = records.map { trainingRecord => - val scoredResponse = - epStpScorer.getScoredResponse(trainingRecord.record, request.params(SetPredictionDetails)) - scoredResponse.map(_.map { response: ScoredResponse => - logger.debug(response) - CandidateUser( - id = trainingRecord.destinationId, - score = Some(response.score), - reason = Some( - Reason( - Some( - AccountProof(followProof = - Some(FollowProof(trainingRecord.socialProof, trainingRecord.socialProof.size))) - ))) - ).withCandidateSourceAndFeatures( - identifier, - Seq(StrongTieFeaturesWrapper(trainingRecord.features))) - }) - } - - Stitch.collect(possibleCandidates).map { _.flatten.sortBy(-_.score.getOrElse(0.0)) } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/README.md deleted file mode 100644 index f3d415d3a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Strong Tie Prediction (STP) Candidate Source -Provides accounts with a high probability of potential mutual follows between the target user and the candidates. - -## STP: Strong Tie Prediction -STP refers to the prediction of p(MutualFollow) for a given pair of users, which powers the concept of People You May Know (PYMK). - -For training, positives are existing mutual follows and negatives are mutual follows of your follows. Features help distinguish between friends and friends-of-friends. - -For inference, candidates are the topK mutuals of your follows. These are rescored, and we only send the topN to the product or next re-ranker. - - -### Online STP -Online STP generates a pool of candidates who are then ranked via a lightweight ranker. -It does this through a two-hop expansion of the mutual follow graph of users, where the first-degree neighbor is another user who has a link with the target user from following types: -* Mutual Follow -* Outbound phone contacts -* Outbound email contacts -* Inbound phone contacts -* Inbound email contacts - -The second-degree neighbor can only be a mutual follow link. - -Currently, online STP can only perform the two-hop expansions on new users (<= 30 days since signup) due to compute resource constraints. - -Features used for the lightweight ranker: -* realGraphWeight: real graph weight between user and first degree nodes -* isForwardEmail: whether the candidate is in the user's email book -* isReverseEmail: whether the user is in the candidate's email book -* isForwardPhonebook: whether the candidate is in the user's phone book -* isReversePhonebook: whether the user is in the candidate's phone book -* numMutualFollowPath: number of mutual follow path between the user and the candidate -* numLowTweepcredFollowPath: number of mutual low TweepCred path between the user and the candidate - * Tweepcred is a social network analysis tool that calculates the influence of Twitter users based on their interactions with other users. The tool uses the PageRank algorithm to rank users based on their influence. -* hasForwardEmailPath: is there a third user x in the user's email book that connect user -> x -> candidate? -* hasReverseEmailPath: is there a third user x in the user's reverse email book that connect user -> x -> candidate? -* hasForwardPhonebookPath: is there a third user x in the user's phonebook that connect user -> x -> candidate? -* hasReversePhonebookPath: is there a third user x in the user's reverse phonebook that connect user -> x -> candidate? - -### Offline STP -Offline STP is powered by Pointwise Mutual Information, which measures the association between two users based on Twitter's mutual follow graph. -An offline job generates candidates based on the overlap between their Mutual and Addressbook Follows and that of the target user. Candidates are then made available online. -Candidates in OfflineSTP are "accounts that have a high overlap of mutually-followed accounts with an account in your follow graph." -This can potentially mean that OfflineSTP has a bigger reach than OnlineSTP. -For example, in the provided diagram, B and C have a high overlap of mutual follows, so it would be considered a candidate for A that is three hops away. -![img.png](img.png) - -Overall, STP is a useful candidate source for generating potential follow recommendations based on strong ties between users, but it should be used in conjunction with other candidate sources and re-rankers to provide a well-rounded set of recommendations for the target user. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPFirstDegreeFetcher.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPFirstDegreeFetcher.scala deleted file mode 100644 index 477e19eb5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPFirstDegreeFetcher.scala +++ /dev/null @@ -1,155 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.clients.real_time_real_graph.RealTimeRealGraphClient -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.PotentialFirstDegreeEdge -import com.twitter.follow_recommendations.common.stores.LowTweepCredFollowStore -import com.twitter.hermit.model.Algorithm -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.inject.Logging -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration -import com.twitter.util.Timer -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdge -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdgeInfo -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdgeInfoMonoid -import javax.inject.Inject -import javax.inject.Singleton - -// Grabs FirstDegreeNodes from Candidate Sources -@Singleton -class STPFirstDegreeFetcher @Inject() ( - realTimeGraphClient: RealTimeRealGraphClient, - reversePhoneBookSource: ReversePhoneBookSource, - reverseEmailBookSource: ReverseEmailBookSource, - forwardEmailBookSource: ForwardEmailBookSource, - forwardPhoneBookSource: ForwardPhoneBookSource, - mutualFollowStrongTiePredictionSource: MutualFollowStrongTiePredictionSource, - lowTweepCredFollowStore: LowTweepCredFollowStore, - timer: Timer, - statsReceiver: StatsReceiver) - extends Logging { - - private val stats = statsReceiver.scope("STPFirstDegreeFetcher") - private val stitchRequests = stats.scope("stitchRequests") - private val allStitchRequests = stitchRequests.counter("all") - private val timeoutStitchRequests = stitchRequests.counter("timeout") - private val successStitchRequests = stitchRequests.counter("success") - - private implicit val firstDegreeEdgeInfoMonoid: FirstDegreeEdgeInfoMonoid = - new FirstDegreeEdgeInfoMonoid - - /** - * Used to map from algorithm to the correct fetcher and firstDegreeEdgeInfo. - * Afterward, uses fetcher to get candidates and construct the correct FirstDegreeEdgeInfo. - * */ - private def getPotentialFirstEdgesFromFetcher( - userId: Long, - target: HasClientContext with HasParams with HasRecentFollowedUserIds, - algorithm: Algorithm, - weight: Double - ): Stitch[Seq[PotentialFirstDegreeEdge]] = { - val (candidates, edgeInfo) = algorithm match { - case Algorithm.MutualFollowSTP => - ( - mutualFollowStrongTiePredictionSource(target), - Some(FirstDegreeEdgeInfo(mutualFollow = true))) - case Algorithm.ReverseEmailBookIbis => - (reverseEmailBookSource(target), Some(FirstDegreeEdgeInfo(reverseEmail = true))) - case Algorithm.ReversePhoneBook => - (reversePhoneBookSource(target), Some(FirstDegreeEdgeInfo(reversePhone = true))) - case Algorithm.ForwardEmailBook => - (forwardEmailBookSource(target), Some(FirstDegreeEdgeInfo(forwardEmail = true))) - case Algorithm.ForwardPhoneBook => - (forwardPhoneBookSource(target), Some(FirstDegreeEdgeInfo(forwardPhone = true))) - case Algorithm.LowTweepcredFollow => - ( - lowTweepCredFollowStore.getLowTweepCredUsers(target), - Some(FirstDegreeEdgeInfo(lowTweepcredFollow = true))) - case _ => - (Stitch.Nil, None) - } - candidates.map(_.flatMap { candidate => - edgeInfo.map(PotentialFirstDegreeEdge(userId, candidate.id, algorithm, weight, _)) - }) - } - - /** - * Using the DefaultMap (AlgorithmToScore) we iterate through algorithm/weights to get - * candidates with a set weight. Then, given repeating candidates (by candidate id). - * Given those candidates we group by the candidateId and sum all below weights and combine - * the edgeInfos of into one. Then we choose the candidates with most weight. Finally, - * we attach the realGraphWeight score to those candidates. - * */ - def getFirstDegreeEdges( - target: HasClientContext with HasParams with HasRecentFollowedUserIds - ): Stitch[Seq[FirstDegreeEdge]] = { - target.getOptionalUserId - .map { userId => - allStitchRequests.incr() - val firstEdgesQueryStitch = Stitch - .collect(STPFirstDegreeFetcher.DefaultGraphBuilderAlgorithmToScore.map { - case (algorithm, candidateWeight) => - getPotentialFirstEdgesFromFetcher(userId, target, algorithm, candidateWeight) - }.toSeq) - .map(_.flatten) - - val destinationIdsToEdges = firstEdgesQueryStitch - .map(_.groupBy(_.connectingId).map { - case (destinationId: Long, edges: Seq[PotentialFirstDegreeEdge]) => - val combinedDestScore = edges.map(_.score).sum - val combinedEdgeInfo: FirstDegreeEdgeInfo = - edges.map(_.edgeInfo).fold(firstDegreeEdgeInfoMonoid.zero) { - (aggregatedInfo, currentInfo) => - firstDegreeEdgeInfoMonoid.plus(aggregatedInfo, currentInfo) - } - (destinationId, combinedEdgeInfo, combinedDestScore) - }).map(_.toSeq) - - val topDestinationEdges = destinationIdsToEdges.map(_.sortBy { - case (_, _, combinedDestScore) => -combinedDestScore - }.take(STPFirstDegreeFetcher.MaxNumFirstDegreeEdges)) - - Stitch - .join(realTimeGraphClient.getRealGraphWeights(userId), topDestinationEdges).map { - case (realGraphWeights, topDestinationEdges) => - successStitchRequests.incr() - topDestinationEdges.map { - case (destinationId, combinedEdgeInfo, _) => - val updatedEdgeInfo = combinedEdgeInfo.copy( - realGraphWeight = realGraphWeights.getOrElse(destinationId, 0.0), - lowTweepcredFollow = - !combinedEdgeInfo.mutualFollow && combinedEdgeInfo.lowTweepcredFollow - ) - FirstDegreeEdge(userId, destinationId, updatedEdgeInfo) - } - }.within(STPFirstDegreeFetcher.LongTimeoutFetcher)(timer).rescue { - case ex => - timeoutStitchRequests.incr() - logger.error("Exception while loading direct edges in OnlineSTP: ", ex) - Stitch.Nil - } - }.getOrElse(Stitch.Nil) - } -} - -object STPFirstDegreeFetcher { - val MaxNumFirstDegreeEdges = 200 - val DefaultGraphBuilderAlgorithmToScore = Map( - Algorithm.MutualFollowSTP -> 10.0, - Algorithm.LowTweepcredFollow -> 6.0, - Algorithm.ForwardEmailBook -> 7.0, - Algorithm.ForwardPhoneBook -> 9.0, - Algorithm.ReverseEmailBookIbis -> 5.0, - Algorithm.ReversePhoneBook -> 8.0 - ) - val LongTimeoutFetcher: Duration = 300.millis -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPGraphBuilder.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPGraphBuilder.scala deleted file mode 100644 index 0d2fe7ffc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPGraphBuilder.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.STPGraph -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class STPGraphBuilder @Inject() ( - stpFirstDegreeFetcher: STPFirstDegreeFetcher, - stpSecondDegreeFetcher: STPSecondDegreeFetcher, - statsReceiver: StatsReceiver) { - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - private val firstDegreeStat: Stat = stats.stat("first_degree_edges") - private val secondDegreeStat: Stat = stats.stat("second_degree_edges") - def apply( - target: HasClientContext with HasParams with HasRecentFollowedUserIds - ): Stitch[STPGraph] = stpFirstDegreeFetcher - .getFirstDegreeEdges(target).flatMap { firstDegreeEdges => - firstDegreeStat.add(firstDegreeEdges.size) - stpSecondDegreeFetcher - .getSecondDegreeEdges(target, firstDegreeEdges).map { secondDegreeEdges => - secondDegreeStat.add(firstDegreeEdges.size) - STPGraph(firstDegreeEdges.toList, secondDegreeEdges.toList) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPSecondDegreeFetcher.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPSecondDegreeFetcher.scala deleted file mode 100644 index b7e996ab3..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/STPSecondDegreeFetcher.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.twitter.follow_recommendations.common.models.IntermediateSecondDegreeEdge -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.StrongTiePredictionFeaturesOnUserClientColumn -import com.twitter.timelines.configapi.HasParams -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdge -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.SecondDegreeEdge -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.SecondDegreeEdgeInfo -import javax.inject.Inject -import javax.inject.Singleton - -// Link to code functionality we're migrating -@Singleton -class STPSecondDegreeFetcher @Inject() ( - strongTiePredictionFeaturesOnUserClientColumn: StrongTiePredictionFeaturesOnUserClientColumn) { - - private def scoreSecondDegreeEdge(edge: SecondDegreeEdge): (Int, Int, Int) = { - def bool2int(b: Boolean): Int = if (b) 1 else 0 - ( - -edge.edgeInfo.numMutualFollowPath, - -edge.edgeInfo.numLowTweepcredFollowPath, - -(bool2int(edge.edgeInfo.forwardEmailPath) + bool2int(edge.edgeInfo.reverseEmailPath) + - bool2int(edge.edgeInfo.forwardPhonePath) + bool2int(edge.edgeInfo.reversePhonePath)) - ) - } - - // Use each first-degree edge(w/ candidateId) to expand and find mutual follows. - // Then, with the mutual follows, group-by candidateId and join edge information - // to create secondDegree edges. - def getSecondDegreeEdges( - target: HasClientContext with HasParams, - firstDegreeEdges: Seq[FirstDegreeEdge] - ): Stitch[Seq[SecondDegreeEdge]] = { - target.getOptionalUserId - .map { userId => - val firstDegreeConnectingIds = firstDegreeEdges.map(_.dstId) - val firstDegreeEdgeInfoMap = firstDegreeEdges.map(e => (e.dstId, e.edgeInfo)).toMap - - val intermediateSecondDegreeEdgesStitch = Stitch - .traverse(firstDegreeConnectingIds) { connectingId => - val stpFeaturesOptStitch = strongTiePredictionFeaturesOnUserClientColumn.fetcher - .fetch(connectingId) - .map(_.v) - stpFeaturesOptStitch.map { stpFeatureOpt => - val intermediateSecondDegreeEdges = for { - edgeInfo <- firstDegreeEdgeInfoMap.get(connectingId) - stpFeatures <- stpFeatureOpt - topSecondDegreeUserIds = - stpFeatures.topMutualFollows - .getOrElse(Nil) - .map(_.userId) - .take(STPSecondDegreeFetcher.MaxNumOfMutualFollows) - } yield topSecondDegreeUserIds.map( - IntermediateSecondDegreeEdge(connectingId, _, edgeInfo)) - intermediateSecondDegreeEdges.getOrElse(Nil) - } - }.map(_.flatten) - - intermediateSecondDegreeEdgesStitch.map { intermediateSecondDegreeEdges => - val secondaryDegreeEdges = intermediateSecondDegreeEdges.groupBy(_.candidateId).map { - case (candidateId, intermediateEdges) => - SecondDegreeEdge( - srcId = userId, - dstId = candidateId, - edgeInfo = SecondDegreeEdgeInfo( - numMutualFollowPath = intermediateEdges.count(_.edgeInfo.mutualFollow), - numLowTweepcredFollowPath = - intermediateEdges.count(_.edgeInfo.lowTweepcredFollow), - forwardEmailPath = intermediateEdges.exists(_.edgeInfo.forwardEmail), - reverseEmailPath = intermediateEdges.exists(_.edgeInfo.reverseEmail), - forwardPhonePath = intermediateEdges.exists(_.edgeInfo.forwardPhone), - reversePhonePath = intermediateEdges.exists(_.edgeInfo.reversePhone), - socialProof = intermediateEdges - .filter { e => e.edgeInfo.mutualFollow || e.edgeInfo.lowTweepcredFollow } - .sortBy(-_.edgeInfo.realGraphWeight) - .take(3) - .map { c => (c.connectingId, c.edgeInfo.realGraphWeight) } - ) - ) - } - secondaryDegreeEdges.toSeq - .sortBy(scoreSecondDegreeEdge) - .take(STPSecondDegreeFetcher.MaxNumSecondDegreeEdges) - } - }.getOrElse(Stitch.Nil) - } -} - -object STPSecondDegreeFetcher { - val MaxNumSecondDegreeEdges = 200 - val MaxNumOfMutualFollows = 50 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/SocialProofEnforcedOfflineStrongTiePredictionSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/SocialProofEnforcedOfflineStrongTiePredictionSource.scala deleted file mode 100644 index 140b8b156..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/SocialProofEnforcedOfflineStrongTiePredictionSource.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.stp - -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.base.SocialProofEnforcedCandidateSource -import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProof -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import javax.inject.Inject - -@Singleton -class SocialProofEnforcedOfflineStrongTiePredictionSource @Inject() ( - offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, - modifySocialProof: ModifySocialProof, - statsReceiver: StatsReceiver) - extends SocialProofEnforcedCandidateSource( - offlineStrongTiePredictionSource, - modifySocialProof, - SocialProofEnforcedOfflineStrongTiePredictionSource.MinNumSocialProofsRequired, - SocialProofEnforcedOfflineStrongTiePredictionSource.Identifier, - statsReceiver) - -object SocialProofEnforcedOfflineStrongTiePredictionSource { - val Identifier = CandidateSourceIdentifier( - Algorithm.StrongTiePredictionRecWithSocialProof.toString) - - val MinNumSocialProofsRequired = 1 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/img.png b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/img.png deleted file mode 100644 index 1a27bb70e..000000000 Binary files a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp/img.png and /dev/null differ diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/BUILD deleted file mode 100644 index 37d604a8f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/onboarding/relevance/organic_follows_accounts:organic_follows_accounts-scala", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/README.md deleted file mode 100644 index db8f13141..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Top Organic Follows Accounts -Provides the most organically followed (i.e. not followed through the Who-To-Follow module) accounts for a given country. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsFSConfig.scala deleted file mode 100644 index 8a179529b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsFSConfig.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TopOrganicFollowsAccountsFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - TopOrganicFollowsAccountsParams.CandidateSourceEnabled, - ) - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - TopOrganicFollowsAccountsParams.CandidateSourceWeight, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsParams.scala deleted file mode 100644 index 36778748e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsParams.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumSeqParam -import com.twitter.timelines.configapi.FSParam - -object TopOrganicFollowsAccountsParams { - // whether or not to fetch TopOrganicFollowsAccounts candidate sources - case object CandidateSourceEnabled - extends FSParam[Boolean]("top_organic_follows_accounts_candidate_source_enabled", false) - - /** - * Contains the logic key for account filtering and ranking. Currently we have 3 main logic keys - * - new_organic_follows: filtering top organically followed accounts followed by new users - * - non_new_organic_follows: filtering top organically followed accounts followed by non new users - * - organic_follows: filtering top organically followed accounts followed by all users - * Mapping of the Logic Id to Logic key is done via @enum AccountsFilteringAndRankingLogic - */ - case object AccountsFilteringAndRankingLogics - extends FSEnumSeqParam[AccountsFilteringAndRankingLogicId.type]( - name = "top_organic_follows_accounts_filtering_and_ranking_logic_ids", - default = Seq(AccountsFilteringAndRankingLogicId.OrganicFollows), - enum = AccountsFilteringAndRankingLogicId) - - case object CandidateSourceWeight - extends FSBoundedParam[Double]( - "top_organic_follows_accounts_candidate_source_weight", - default = 1200, - min = 0.001, - max = 2000) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsSource.scala deleted file mode 100644 index b7f60ece9..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts/TopOrganicFollowsAccountsSource.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts - -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.AccountsFilteringAndRankingLogics -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.CandidateSourceEnabled -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode -import com.twitter.hermit.model.Algorithm -import com.twitter.onboarding.relevance.organic_follows_accounts.thriftscala.OrganicFollowsAccounts -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.OrganicFollowsAccountsClientColumn -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -object AccountsFilteringAndRankingLogicId extends Enumeration { - type AccountsFilteringAndRankingLogicId = Value - - val NewOrganicFollows: AccountsFilteringAndRankingLogicId = Value("new_organic_follows") - val NonNewOrganicFollows: AccountsFilteringAndRankingLogicId = Value("non_new_organic_follows") - val OrganicFollows: AccountsFilteringAndRankingLogicId = Value("organic_follows") -} - -object TopOrganicFollowsAccountsSource { - val MaxCacheSize = 500 - val CacheTTL: Duration = Duration.fromHours(24) - - type Target = HasParams with HasClientContext with HasGeohashAndCountryCode - - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.OrganicFollowAccounts.toString) -} - -@Singleton -class TopOrganicFollowsAccountsSource @Inject() ( - organicFollowsAccountsClientColumn: OrganicFollowsAccountsClientColumn, - statsReceiver: StatsReceiver, -) extends CandidateSource[TopOrganicFollowsAccountsSource.Target, CandidateUser] - with Logging { - - /** @see [[CandidateSourceIdentifier]] */ - override val identifier: CandidateSourceIdentifier = - TopOrganicFollowsAccountsSource.Identifier - - private val stats = statsReceiver.scope(identifier.name) - private val requestsStats = stats.counter("requests") - private val noCountryCodeStats = stats.counter("no_country_code") - private val successStats = stats.counter("success") - private val errorStats = stats.counter("error") - - private val cache = StitchCache[String, Option[OrganicFollowsAccounts]]( - maxCacheSize = TopOrganicFollowsAccountsSource.MaxCacheSize, - ttl = TopOrganicFollowsAccountsSource.CacheTTL, - statsReceiver = statsReceiver.scope(identifier.name, "cache"), - underlyingCall = (k: String) => { - organicFollowsAccountsClientColumn.fetcher - .fetch(k) - .map { result => result.v } - } - ) - - /** returns a Seq of ''potential'' content */ - override def apply( - target: TopOrganicFollowsAccountsSource.Target - ): Stitch[Seq[CandidateUser]] = { - if (!target.params(CandidateSourceEnabled)) { - return Stitch.value(Seq[CandidateUser]()) - } - requestsStats.incr() - target.getCountryCode - .orElse(target.geohashAndCountryCode.flatMap(_.countryCode)).map { countryCode => - Stitch - .collect(target - .params(AccountsFilteringAndRankingLogics).map(logic => - cache.readThrough(countryCode.toUpperCase() + "-" + logic))) - .onSuccess(_ => { - successStats.incr() - }) - .onFailure(t => { - debug("candidate source failed identifier = %s".format(identifier), t) - errorStats.incr() - }) - .map(transformOrganicFollowAccountssToCandidateSource) - }.getOrElse { - noCountryCodeStats.incr() - Stitch.value(Seq[CandidateUser]()) - } - } - - private def transformOrganicFollowAccountssToCandidateSource( - organicFollowsAccounts: Seq[Option[OrganicFollowsAccounts]] - ): Seq[CandidateUser] = { - organicFollowsAccounts - .flatMap(opt => - opt - .map(accounts => - accounts.accounts.map(account => - CandidateUser( - id = account.accountId, - score = Some(account.followedCountScore), - ).withCandidateSource(identifier))) - .getOrElse(Seq[CandidateUser]())) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/BUILD deleted file mode 100644 index f69443cf6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/README.md deleted file mode 100644 index dcaec2298..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Triangular Loops Candidate Source -Provides account candidates based on the graph structures of the form u -> v -> w -> u, -where the arrow indicates a follow edge. In other words, it looks for triangular loops in the user-user graph. - -If the edge v -> u does not exist in the triangular loop, the Triangular Loops Candidate Source recommends u as a potential outbound mutual follow candidate for v. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsFSConfig.scala deleted file mode 100644 index 444fecd0e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsFSConfig.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.triangular_loops - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TriangularLoopsFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean] with FSName] = Nil -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsParams.scala deleted file mode 100644 index 9fb80235e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsParams.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.triangular_loops - -import com.twitter.timelines.configapi.FSParam - -object TriangularLoopsParams { - - object KeepOnlyCandidatesWhoFollowTargetUser - extends FSParam[Boolean]( - "triangular_loops_keep_only_candidates_who_follow_target_user", - false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsSource.scala deleted file mode 100644 index 73e187ba4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops/TriangularLoopsSource.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.triangular_loops - -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.HasRecentFollowedByUserIds -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.TriangularLoopsV2OnUserClientColumn -import com.twitter.timelines.configapi.HasParams -import com.twitter.wtf.triangular_loop.thriftscala.Candidates -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TriangularLoopsSource @Inject() ( - triangularLoopsV2Column: TriangularLoopsV2OnUserClientColumn) - extends CandidateSource[ - HasParams with HasClientContext with HasRecentFollowedByUserIds, - CandidateUser - ] { - - override val identifier: CandidateSourceIdentifier = TriangularLoopsSource.Identifier - - override def apply( - target: HasParams with HasClientContext with HasRecentFollowedByUserIds - ): Stitch[Seq[CandidateUser]] = { - val candidates = target.getOptionalUserId - .map { userId => - val fetcher = triangularLoopsV2Column.fetcher - fetcher - .fetch(userId) - .map { result => - result.v - .map(TriangularLoopsSource.mapCandidatesToCandidateUsers) - .getOrElse(Nil) - } - }.getOrElse(Stitch.Nil) - // Make sure recentFollowedByUserIds is populated within the RequestBuilder before enable it - if (target.params(TriangularLoopsParams.KeepOnlyCandidatesWhoFollowTargetUser)) - filterOutCandidatesNotFollowingTargetUser(candidates, target.recentFollowedByUserIds) - else - candidates - } - - def filterOutCandidatesNotFollowingTargetUser( - candidatesStitch: Stitch[Seq[CandidateUser]], - recentFollowings: Option[Seq[Long]] - ): Stitch[Seq[CandidateUser]] = { - candidatesStitch.map { candidates => - val recentFollowingIdsSet = recentFollowings.getOrElse(Nil).toSet - candidates.filter(candidate => recentFollowingIdsSet.contains(candidate.id)) - } - } -} - -object TriangularLoopsSource { - - val Identifier = CandidateSourceIdentifier(Algorithm.TriangularLoop.toString) - val NumResults = 100 - - def mapCandidatesToCandidateUsers(candidates: Candidates): Seq[CandidateUser] = { - candidates.candidates - .map { candidate => - CandidateUser( - id = candidate.incomingUserId, - score = Some(1.0 / math - .max(1, candidate.numFollowers.getOrElse(0) + candidate.numFollowings.getOrElse(0))), - reason = Some( - Reason( - Some( - AccountProof( - followProof = - if (candidate.socialProofUserIds.isEmpty) None - else - Some( - FollowProof( - candidate.socialProofUserIds, - candidate.numSocialProof.getOrElse(candidate.socialProofUserIds.size))) - ) - ) - ) - ) - ).withCandidateSource(Identifier) - }.sortBy(-_.score.getOrElse(0.0)).take(NumResults) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/BUILD deleted file mode 100644 index ad7d6805e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/README.md deleted file mode 100644 index 39c984926..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Two-hop Random Walk -The TwoHopRandomWalk algorithm re-ranks a user's second degree connections based on recent engagement strength. The algorithm works as follows: - -* Given a user `src`, find their top K first degree connections `fd(1)`, `fd(2)`, `fd(3)`,...,`fd(K)`. The ranking is based on real graph weights, which measure the recent engagement strength on the edges. -* For each of the first degree connections `fd(i)`, expand to their top L connections via real graph, `sd(i,1)`, `sd(i,2)`,...,`sd(i,L)`. Note that sd nodes can also be `src`'s first degree nodes. -* Aggregate all the nodes in step 2, filter out the first degree nodes, and calculate the weighted sum for the second degree. -* Re-rank the second degree nodes and select the top M results as the algorithm output. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/TwoHopRandomWalkSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/TwoHopRandomWalkSource.scala deleted file mode 100644 index 5ce2fee70..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk/TwoHopRandomWalkSource.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk - -import com.twitter.follow_recommendations.common.candidate_sources.base.StratoFetcherWithUnitViewSource -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.strato.client.Fetcher -import com.twitter.wtf.candidate.thriftscala.{CandidateSeq => TCandidateSeq} -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class TwoHopRandomWalkSource @Inject() ( - @Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) fetcher: Fetcher[ - Long, - Unit, - TCandidateSeq - ]) extends StratoFetcherWithUnitViewSource[Long, TCandidateSeq]( - fetcher, - TwoHopRandomWalkSource.Identifier) { - - override def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser] = - TwoHopRandomWalkSource.map(targetUserId, tCandidateSeq) - -} - -object TwoHopRandomWalkSource { - def map(targetUserId: Long, tCandidateSeq: TCandidateSeq): Seq[CandidateUser] = { - tCandidateSeq.candidates - .sortBy(-_.score) - .map { tCandidate => - CandidateUser(id = tCandidate.userId, score = Some(tCandidate.score)) - } - } - - val Identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier(Algorithm.TwoHopRandomWalk.toString) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/BUILD deleted file mode 100644 index 63b10be6c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/README.md b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/README.md deleted file mode 100644 index fa510fff7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# User-User Graph Candidate Source -Provides account candidates generated from the User-User Graph (UUG). -## User-User Graph (UUG) -The UUG algorithm reads User-Follow-User engagements that occurred in the past 24-48 hours, and provides accounts that the given user's recent followings have recently followed themselves. The UUG algorithm is implemented using the real-time graph processing library GraphJet. diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphCandidateSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphCandidateSource.scala deleted file mode 100644 index 4571f1471..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphCandidateSource.scala +++ /dev/null @@ -1,125 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.user_user_graph - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models._ -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.recos.recos_common.thriftscala.UserSocialProofType -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserDisplayLocation -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse -import com.twitter.recos.user_user_graph.thriftscala.RecommendedUser -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class UserUserGraphCandidateSource @Inject() ( - @Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER) - fetcher: Fetcher[RecommendUserRequest, Unit, RecommendUserResponse], - statsReceiver: StatsReceiver) - extends CandidateSource[ - UserUserGraphCandidateSource.Target, - CandidateUser - ] { - - override val identifier: CandidateSourceIdentifier = UserUserGraphCandidateSource.Identifier - val stats: StatsReceiver = statsReceiver.scope("UserUserGraph") - val requestCounter: Counter = stats.counter("requests") - - override def apply( - target: UserUserGraphCandidateSource.Target - ): Stitch[Seq[CandidateUser]] = { - if (target.params(UserUserGraphParams.UserUserGraphCandidateSourceEnabledInWeightMap)) { - requestCounter.incr() - buildRecommendUserRequest(target) - .map { request => - fetcher - .fetch(request) - .map(_.v) - .map { responseOpt => - responseOpt - .map { response => - response.recommendedUsers - .sortBy(-_.score) - .map(convertToCandidateUsers) - .map(_.withCandidateSource(identifier)) - }.getOrElse(Nil) - } - }.getOrElse(Stitch.Nil) - } else { - Stitch.Nil - } - } - - private[this] def buildRecommendUserRequest( - target: UserUserGraphCandidateSource.Target - ): Option[RecommendUserRequest] = { - (target.getOptionalUserId, target.recentFollowedUserIds) match { - case (Some(userId), Some(recentFollowedUserIds)) => - // use recentFollowedUserIds as seeds for initial experiment - val seedsWithWeights: Map[Long, Double] = recentFollowedUserIds.map { - recentFollowedUserId => - recentFollowedUserId -> UserUserGraphCandidateSource.DefaultSeedWeight - }.toMap - val request = RecommendUserRequest( - requesterId = userId, - displayLocation = UserUserGraphCandidateSource.DisplayLocation, - seedsWithWeights = seedsWithWeights, - excludedUserIds = Some(target.excludedUserIds), - maxNumResults = Some(target.params.getInt(UserUserGraphParams.MaxCandidatesToReturn)), - maxNumSocialProofs = Some(UserUserGraphCandidateSource.MaxNumSocialProofs), - minUserPerSocialProof = Some(UserUserGraphCandidateSource.MinUserPerSocialProof), - socialProofTypes = Some(Seq(UserUserGraphCandidateSource.SocialProofType)) - ) - Some(request) - case _ => None - } - } - - private[this] def convertToCandidateUsers( - recommendedUser: RecommendedUser - ): CandidateUser = { - val socialProofUserIds = - recommendedUser.socialProofs.getOrElse(UserUserGraphCandidateSource.SocialProofType, Nil) - val reasonOpt = if (socialProofUserIds.nonEmpty) { - Some( - Reason( - Some(AccountProof(followProof = - Some(FollowProof(socialProofUserIds, socialProofUserIds.size))))) - ) - } else { - None - } - CandidateUser( - id = recommendedUser.userId, - score = Some(recommendedUser.score), - reason = reasonOpt) - } -} - -object UserUserGraphCandidateSource { - type Target = HasParams - with HasClientContext - with HasRecentFollowedUserIds - with HasExcludedUserIds - - val Identifier: CandidateSourceIdentifier = CandidateSourceIdentifier( - Algorithm.UserUserGraph.toString) - //Use HomeTimeline for experiment - val DisplayLocation: RecommendUserDisplayLocation = RecommendUserDisplayLocation.HomeTimeLine - - //Default params used in MagicRecs - val DefaultSeedWeight: Double = 1.0 - val SocialProofType = UserSocialProofType.Follow - val MaxNumSocialProofs = 10 - val MinUserPerSocialProof: Map[UserSocialProofType, Int] = - Map[UserSocialProofType, Int]((SocialProofType, 2)) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphFSConfig.scala deleted file mode 100644 index 3d3015a36..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphFSConfig.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.user_user_graph - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class UserUserGraphFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - UserUserGraphParams.UserUserGraphCandidateSourceEnabledInWeightMap, - UserUserGraphParams.UserUserGraphCandidateSourceEnabledInTransform - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphParams.scala deleted file mode 100644 index d146c3459..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph/UserUserGraphParams.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.candidate_sources.user_user_graph - -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object UserUserGraphParams { - - // max number of candidates to return in total, 50 is the default param used in MagicRecs - object MaxCandidatesToReturn extends Param[Int](default = 50) - - // whether or not to include UserUserGraph candidate source in the weighted blending step - case object UserUserGraphCandidateSourceEnabledInWeightMap - extends FSParam[Boolean]("user_user_graph_candidate_source_enabled_in_weight_map", true) - - // whether or not to include UserUserGraph candidate source in the final transform step - case object UserUserGraphCandidateSourceEnabledInTransform - extends FSParam[Boolean]("user_user_graph_candidate_source_enabled_in_transform", true) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookClient.scala deleted file mode 100644 index 6fe1cee6f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookClient.scala +++ /dev/null @@ -1,221 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.addressbook - -import com.twitter.addressbook.datatypes.thriftscala.QueryType -import com.twitter.addressbook.thriftscala.AddressBookGetRequest -import com.twitter.addressbook.thriftscala.AddressBookGetResponse -import com.twitter.addressbook.thriftscala.Addressbook2 -import com.twitter.addressbook.thriftscala.ClientInfo -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.wtf.scalding.jobs.addressbook.thriftscala.STPResultFeature -import com.twitter.follow_recommendations.common.clients.addressbook.models.Contact -import com.twitter.follow_recommendations.common.clients.addressbook.models.EdgeType -import com.twitter.follow_recommendations.common.clients.addressbook.models.QueryOption -import com.twitter.follow_recommendations.common.clients.addressbook.models.RecordIdentifier -import com.twitter.wtf.scalding.jobs.address_book.ABUtil.hashContact -import com.twitter.wtf.scalding.jobs.address_book.ABUtil.normalizeEmail -import com.twitter.wtf.scalding.jobs.address_book.ABUtil.normalizePhoneNumber -import com.twitter.hermit.usercontacts.thriftscala.{UserContacts => tUserContacts} -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AddressbookClient @Inject() ( - addressbookService: Addressbook2.MethodPerEndpoint, - statsReceiver: StatsReceiver = NullStatsReceiver) { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - private[this] def getResponseFromService( - identifiers: Seq[RecordIdentifier], - batchSize: Int, - edgeType: EdgeType, - maxFetches: Int, - queryOption: Option[QueryOption] - ): Stitch[Seq[AddressBookGetResponse]] = { - Stitch - .collect( - identifiers.map { identifier => - Stitch.callFuture( - addressbookService.get(AddressBookGetRequest( - clientInfo = ClientInfo(None), - identifier = identifier.toThrift, - edgeType = edgeType.toThrift, - queryType = QueryType.UserId, - queryOption = queryOption.map(_.toThrift), - maxFetches = maxFetches, - resultBatchSize = batchSize - ))) - } - ) - } - - private[this] def getContactsResponseFromService( - identifiers: Seq[RecordIdentifier], - batchSize: Int, - edgeType: EdgeType, - maxFetches: Int, - queryOption: Option[QueryOption] - ): Stitch[Seq[AddressBookGetResponse]] = { - Stitch - .collect( - identifiers.map { identifier => - Stitch.callFuture( - addressbookService.get(AddressBookGetRequest( - clientInfo = ClientInfo(None), - identifier = identifier.toThrift, - edgeType = edgeType.toThrift, - queryType = QueryType.Contact, - queryOption = queryOption.map(_.toThrift), - maxFetches = maxFetches, - resultBatchSize = batchSize - ))) - } - ) - } - - /** Mode of addressbook resolving logic - * ManhattanThenABV2: fetching manhattan cached result and backfill with addressbook v2 - * ABV2Only: calling addressbook v2 directly without fetching manhattan cached result - * This can be controlled by passing a fetcher or not. Passing a fetcher will attempt to use it, - * if not then it won't. - */ - def getUsers( - userId: Long, - identifiers: Seq[RecordIdentifier], - batchSize: Int, - edgeType: EdgeType, - fetcherOption: Option[Fetcher[Long, Unit, tUserContacts]] = None, - maxFetches: Int = 1, - queryOption: Option[QueryOption] = None, - ): Stitch[Seq[Long]] = { - fetcherOption match { - case Some(fetcher) => - getUsersFromManhattan(userId, fetcher).flatMap { userContacts => - if (userContacts.isEmpty) { - stats.counter("mhEmptyThenFromAbService").incr() - getResponseFromService(identifiers, batchSize, edgeType, maxFetches, queryOption) - .map(_.flatMap(_.users).flatten.distinct) - } else { - stats.counter("fromManhattan").incr() - Stitch.value(userContacts) - } - } - case None => - stats.counter("fromAbService").incr() - getResponseFromService(identifiers, batchSize, edgeType, maxFetches, queryOption) - .map(_.flatMap(_.users).flatten.distinct) - } - } - - def getHashedContacts( - normalizeFn: String => String, - extractField: String, - )( - userId: Long, - identifiers: Seq[RecordIdentifier], - batchSize: Int, - edgeType: EdgeType, - fetcherOption: Option[Fetcher[String, Unit, STPResultFeature]] = None, - maxFetches: Int = 1, - queryOption: Option[QueryOption] = None, - ): Stitch[Seq[String]] = { - - fetcherOption match { - case Some(fetcher) => - getContactsFromManhattan(userId, fetcher).flatMap { userContacts => - if (userContacts.isEmpty) { - getContactsResponseFromService( - identifiers, - batchSize, - edgeType, - maxFetches, - queryOption) - .map { response => - for { - resp <- response - contacts <- resp.contacts - contactsThrift = contacts.map(Contact.fromThrift) - contactsSet = extractField match { - case "emails" => contactsThrift.flatMap(_.emails.toSeq.flatten) - case "phoneNumbers" => contactsThrift.flatMap(_.phoneNumbers.toSeq.flatten) - } - hashedAndNormalizedContacts = contactsSet.map(c => hashContact(normalizeFn(c))) - } yield hashedAndNormalizedContacts - }.map(_.flatten) - } else { - Stitch.Nil - } - } - case None => { - getContactsResponseFromService(identifiers, batchSize, edgeType, maxFetches, queryOption) - .map { response => - for { - resp <- response - contacts <- resp.contacts - contactsThrift = contacts.map(Contact.fromThrift) - contactsSet = extractField match { - case "emails" => contactsThrift.flatMap(_.emails.toSeq.flatten) - case "phoneNumbers" => contactsThrift.flatMap(_.phoneNumbers.toSeq.flatten) - } - hashedAndNormalizedContacts = contactsSet.map(c => hashContact(normalizeFn(c))) - } yield hashedAndNormalizedContacts - }.map(_.flatten) - } - } - } - - def getEmailContacts = getHashedContacts(normalizeEmail, "emails") _ - def getPhoneContacts = getHashedContacts(normalizePhoneNumber, "phoneNumbers") _ - - private def getUsersFromManhattan( - userId: Long, - fetcher: Fetcher[Long, Unit, tUserContacts], - ): Stitch[Seq[Long]] = fetcher - .fetch(userId) - .map(_.v.map(_.destinationIds).toSeq.flatten.distinct) - - private def getContactsFromManhattan( - userId: Long, - fetcher: Fetcher[String, Unit, STPResultFeature], - ): Stitch[Seq[String]] = fetcher - .fetch(userId.toString) - .map(_.v.map(_.strongTieUserFeature.map(_.destId)).toSeq.flatten.distinct) -} - -object AddressbookClient { - val AddressBook2BatchSize = 500 - - def createQueryOption(edgeType: EdgeType, isPhone: Boolean): Option[QueryOption] = - (edgeType, isPhone) match { - case (EdgeType.Reverse, _) => - None - case (EdgeType.Forward, true) => - Some( - QueryOption( - onlyDiscoverableInExpansion = false, - onlyConfirmedInExpansion = false, - onlyDiscoverableInResult = false, - onlyConfirmedInResult = false, - fetchGlobalApiNamespace = false, - isDebugRequest = false, - resolveEmails = false, - resolvePhoneNumbers = true - )) - case (EdgeType.Forward, false) => - Some( - QueryOption( - onlyDiscoverableInExpansion = false, - onlyConfirmedInExpansion = false, - onlyDiscoverableInResult = false, - onlyConfirmedInResult = false, - fetchGlobalApiNamespace = false, - isDebugRequest = false, - resolveEmails = true, - resolvePhoneNumbers = false - )) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookModule.scala deleted file mode 100644 index 97b841dd2..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/AddressbookModule.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.addressbook - -import com.twitter.addressbook.thriftscala.Addressbook2 -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule - -object AddressbookModule extends BaseClientModule[Addressbook2.MethodPerEndpoint] with MtlsClient { - override val label = "addressbook" - override val dest = "/s/addressbook/addressbook2" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/BUILD deleted file mode 100644 index b94cf3efe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "addressbook/thrift/src/thrift/com/twitter/addressbook:thrift-scala", - "addressbook/thrift/src/thrift/com/twitter/addressbook/datatypes:thrift-scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "src/scala/com/twitter/wtf/scalding/jobs/address_book:ab_util", - "src/thrift/com/twitter/hermit/usercontacts:hermit-usercontacts-scala", - "src/thrift/com/twitter/wtf/addressbook:addressbook-scala", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/BUILD deleted file mode 100644 index 885055e2c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "addressbook/thrift/src/thrift/com/twitter/addressbook:thrift-scala", - "addressbook/thrift/src/thrift/com/twitter/addressbook/datatypes:thrift-scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/Contact.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/Contact.scala deleted file mode 100644 index 3ff39fa32..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/Contact.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.addressbook.models - -import com.twitter.addressbook.{thriftscala => t} -import com.twitter.util.Time - -case class Contact( - id: Long, - emails: Option[Set[String]], - phoneNumbers: Option[Set[String]], - firstName: Option[String], - lastName: Option[String], - name: Option[String], - appId: Option[Long], - appIds: Option[Set[Long]], - importedTimestamp: Option[Time]) - -object Contact { - def fromThrift(thriftContact: t.Contact): Contact = Contact( - thriftContact.id, - thriftContact.emails.map(_.toSet), - thriftContact.phoneNumbers.map(_.toSet), - thriftContact.firstName, - thriftContact.lastName, - thriftContact.name, - thriftContact.appId, - thriftContact.appIds.map(_.toSet), - thriftContact.importedTimestamp.map(Time.fromMilliseconds) - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/EdgeType.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/EdgeType.scala deleted file mode 100644 index 6cfcd65cd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/EdgeType.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.addressbook.models - -import com.twitter.addressbook.datatypes.{thriftscala => t} - -sealed trait EdgeType { - def toThrift: t.EdgeType -} - -object EdgeType { - case object Forward extends EdgeType { - override val toThrift: t.EdgeType = t.EdgeType.Forward - } - case object Reverse extends EdgeType { - override val toThrift: t.EdgeType = t.EdgeType.Reverse - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/QueryOption.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/QueryOption.scala deleted file mode 100644 index a17c163cb..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/QueryOption.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.addressbook.models - -import com.twitter.addressbook.{thriftscala => t} - -case class QueryOption( - onlyDiscoverableInExpansion: Boolean, - onlyConfirmedInExpansion: Boolean, - onlyDiscoverableInResult: Boolean, - onlyConfirmedInResult: Boolean, - fetchGlobalApiNamespace: Boolean, - isDebugRequest: Boolean, - resolveEmails: Boolean, - resolvePhoneNumbers: Boolean) { - def toThrift: t.QueryOption = t.QueryOption( - onlyDiscoverableInExpansion, - onlyConfirmedInExpansion, - onlyDiscoverableInResult, - onlyConfirmedInResult, - fetchGlobalApiNamespace, - isDebugRequest, - resolveEmails, - resolvePhoneNumbers - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/RecordIdentifier.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/RecordIdentifier.scala deleted file mode 100644 index 0154c71dc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook/models/RecordIdentifier.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.addressbook.models - -import com.twitter.addressbook.datatypes.{thriftscala => t} - -case class RecordIdentifier( - userId: Option[Long], - email: Option[String], - phoneNumber: Option[String]) { - def toThrift: t.RecordIdentifier = t.RecordIdentifier(userId, email, phoneNumber) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdRequest.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdRequest.scala deleted file mode 100644 index 07cfcdaf5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdRequest.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.adserver - -import com.twitter.adserver.{thriftscala => t} -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext - -case class AdRequest( - clientContext: ClientContext, - displayLocation: DisplayLocation, - isTest: Option[Boolean], - profileUserId: Option[Long]) { - def toThrift: t.AdRequestParams = { - - val request = t.AdRequest( - displayLocation = displayLocation.toAdDisplayLocation.getOrElse( - throw new MissingAdDisplayLocation(displayLocation)), - isTest = isTest, - countImpressionsOnCallback = Some(true), - numOrganicItems = Some(AdRequest.DefaultNumOrganicItems.toShort), - profileUserId = profileUserId - ) - - val clientInfo = t.ClientInfo( - clientId = clientContext.appId.map(_.toInt), - userIp = clientContext.ipAddress, - userId64 = clientContext.userId, - guestId = clientContext.guestId, - userAgent = clientContext.userAgent, - referrer = None, - deviceId = clientContext.deviceId, - languageCode = clientContext.languageCode, - countryCode = clientContext.countryCode - ) - - t.AdRequestParams(request, clientInfo) - } -} - -object AdRequest { - val DefaultNumOrganicItems = 10 -} - -class MissingAdDisplayLocation(displayLocation: DisplayLocation) - extends Exception( - s"Display Location ${displayLocation.toString} has no mapped AdsDisplayLocation set.") diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverClient.scala deleted file mode 100644 index 927c0784c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverClient.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.adserver - -import com.twitter.adserver.thriftscala.NewAdServer -import com.twitter.adserver.{thriftscala => t} -import com.twitter.stitch.Stitch -import javax.inject.{Inject, Singleton} - -@Singleton -class AdserverClient @Inject() (adserverService: NewAdServer.MethodPerEndpoint) { - def getAdImpressions(adRequest: AdRequest): Stitch[Seq[t.AdImpression]] = { - Stitch - .callFuture( - adserverService.makeAdRequest(adRequest.toThrift) - ).map(_.impressions) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverModule.scala deleted file mode 100644 index 9a425c058..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/AdserverModule.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.adserver - -import com.twitter.adserver.thriftscala.NewAdServer -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.ThriftMux -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule - -object AdserverModule extends BaseClientModule[NewAdServer.MethodPerEndpoint] with MtlsClient { - override val label = "adserver" - override val dest = "/s/ads/adserver" - - override def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = - client.withRequestTimeout(500.millis) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/BUILD deleted file mode 100644 index c90656afe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "stitch/stitch-core", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/BUILD deleted file mode 100644 index a32739651..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "cache/client", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "stitch/stitch-core", - "util/util-core:scala", - "util/util-thrift", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheClient.scala deleted file mode 100644 index d00aef475..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheClient.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.cache - -import com.twitter.bijection.Bijection -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.io.Buf -import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time -import java.security.MessageDigest - -object MemcacheClient { - def apply[V]( - client: Client, - dest: String, - valueBijection: Bijection[Buf, V], - ttl: Duration, - statsReceiver: StatsReceiver - ): MemcacheClient[V] = { - new MemcacheClient(client, dest, valueBijection, ttl, statsReceiver) - } -} - -class MemcacheClient[V]( - client: Client, - dest: String, - valueBijection: Bijection[Buf, V], - ttl: Duration, - statsReceiver: StatsReceiver) { - val cache = client.newRichClient(dest).adapt[V](valueBijection) - val cacheTtl = Time.fromSeconds(ttl.inSeconds) - - /** - * If cache contains key, return value from cache. Otherwise, run the underlying call - * to fetch the value, store it in cache, and then return the value. - */ - def readThrough( - key: String, - underlyingCall: () => Stitch[V] - ): Stitch[V] = { - val cachedResult: Stitch[Option[V]] = Stitch - .callFuture(getIfPresent(key)) - .within(70.millisecond)(DefaultTimer) - .rescue { - case e: Exception => - statsReceiver.scope("rescued").counter(e.getClass.getSimpleName).incr() - Stitch(None) - } - val resultStitch = cachedResult.map { resultOption => - resultOption match { - case Some(cacheValue) => Stitch.value(cacheValue) - case None => - val underlyingCallStitch = profileStitch( - underlyingCall(), - statsReceiver.scope("underlyingCall") - ) - underlyingCallStitch.map { result => - put(key, result) - result - } - } - }.flatten - // profile the overall Stitch, and return the result - profileStitch(resultStitch, statsReceiver.scope("readThrough")) - } - - def getIfPresent(key: String): Future[Option[V]] = { - cache - .get(hashString(key)) - .onSuccess { - case Some(value) => statsReceiver.counter("cache_hits").incr() - case None => statsReceiver.counter("cache_misses").incr() - } - .onFailure { - case e: Exception => - statsReceiver.counter("cache_misses").incr() - statsReceiver.scope("rescued").counter(e.getClass.getSimpleName).incr() - } - .rescue { - case _ => Future.None - } - } - - def put(key: String, value: V): Future[Unit] = { - cache.set(hashString(key), 0, cacheTtl, value) - } - - /** - * Hash the input key string to a fixed length format using SHA-256 hash function. - */ - def hashString(input: String): String = { - val bytes = MessageDigest.getInstance("SHA-256").digest(input.getBytes("UTF-8")) - bytes.map("%02x".format(_)).mkString - } - - /** - * Helper function for timing a stitch, returning the original stitch. - * - * Defining the profiling function here to keep the dependencies of this class - * generic and easy to export (i.e. copy-and-paste) into other services or packages. - */ - def profileStitch[T](stitch: Stitch[T], stat: StatsReceiver): Stitch[T] = { - Stitch - .time(stitch) - .map { - case (response, stitchRunDuration) => - stat.counter("requests").incr() - stat.stat("latency_ms").add(stitchRunDuration.inMilliseconds) - response - .onSuccess { _ => stat.counter("success").incr() } - .onFailure { e => - stat.counter("failures").incr() - stat.scope("failures").counter(e.getClass.getSimpleName).incr() - } - } - .lowerFromTry - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheModule.scala deleted file mode 100644 index e85d12f20..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/MemcacheModule.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.cache - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.Memcached -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.service.Retries -import com.twitter.finagle.service.RetryPolicy -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import javax.inject.Singleton - -object MemcacheModule extends TwitterModule { - @Provides - @Singleton - def provideMemcacheClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver, - ): Client = { - Memcached.client - .withMutualTls(serviceIdentifier) - .withStatsReceiver(statsReceiver.scope("twemcache")) - .withTransport.connectTimeout(1.seconds) - .withRequestTimeout(1.seconds) - .withSession.acquisitionTimeout(10.seconds) - .configured(Retries.Policy(RetryPolicy.tries(1))) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/ThriftBijection.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/ThriftBijection.scala deleted file mode 100644 index 62e36bc33..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache/ThriftBijection.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.cache - -import com.twitter.bijection.Bijection -import com.twitter.io.Buf -import com.twitter.scrooge.CompactThriftSerializer -import com.twitter.scrooge.ThriftEnum -import com.twitter.scrooge.ThriftStruct -import java.nio.ByteBuffer - -abstract class ThriftBijection[T <: ThriftStruct] extends Bijection[Buf, T] { - val serializer: CompactThriftSerializer[T] - - override def apply(b: Buf): T = { - val byteArray = Buf.ByteArray.Owned.extract(b) - serializer.fromBytes(byteArray) - } - - override def invert(a: T): Buf = { - val byteArray = serializer.toBytes(a) - Buf.ByteArray.Owned(byteArray) - } -} - -abstract class ThriftOptionBijection[T <: ThriftStruct] extends Bijection[Buf, Option[T]] { - val serializer: CompactThriftSerializer[T] - - override def apply(b: Buf): Option[T] = { - if (b.isEmpty) { - None - } else { - val byteArray = Buf.ByteArray.Owned.extract(b) - Some(serializer.fromBytes(byteArray)) - } - } - - override def invert(a: Option[T]): Buf = { - a match { - case Some(t) => - val byteArray = serializer.toBytes(t) - Buf.ByteArray.Owned(byteArray) - case None => Buf.Empty - } - } -} - -class ThriftEnumBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, T] { - override def apply(b: Buf): T = { - val byteArray = Buf.ByteArray.Owned.extract(b) - val byteBuffer = ByteBuffer.wrap(byteArray) - constructor(byteBuffer.getInt()) - } - - override def invert(a: T): Buf = { - val byteBuffer: ByteBuffer = ByteBuffer.allocate(4) - byteBuffer.putInt(a.getValue) - Buf.ByteArray.Owned(byteBuffer.array()) - } -} - -class ThriftEnumOptionBijection[T <: ThriftEnum](constructor: Int => T) extends Bijection[Buf, Option[T]] { - override def apply(b: Buf): Option[T] = { - if (b.isEmpty) { - None - } else { - val byteArray = Buf.ByteArray.Owned.extract(b) - val byteBuffer = ByteBuffer.wrap(byteArray) - Some(constructor(byteBuffer.getInt())) - } - } - - override def invert(a: Option[T]): Buf = { - a match { - case Some(obj) => { - val byteBuffer: ByteBuffer = ByteBuffer.allocate(4) - byteBuffer.putInt(obj.getValue) - Buf.ByteArray.Owned(byteBuffer.array()) - } - case None => Buf.Empty - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BUILD deleted file mode 100644 index 330981d80..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BaseClientModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BaseClientModule.scala deleted file mode 100644 index 43949ea19..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common/BaseClientModule.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.common - -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.thrift.Protocols -import com.twitter.follow_recommendations.common.constants.ServiceConstants._ -import com.twitter.inject.thrift.modules.ThriftClientModule -import scala.reflect.ClassTag - -/** - * basic client configurations that we apply for all of our clients go in here - */ -abstract class BaseClientModule[T: ClassTag] extends ThriftClientModule[T] { - def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = { - client - .withProtocolFactory( - Protocols.binaryFactory( - stringLengthLimit = StringLengthLimit, - containerLengthLimit = ContainerLengthLimit)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD deleted file mode 100644 index daf4c63fa..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "cortex-deepbird/prediction/src/main/scala/com/twitter/cortex/deepbird/prediction", - "cortex-deepbird/thrift/src/main/thrift:thrift-java", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "src/scala/com/twitter/ml/api/util", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.scala deleted file mode 100644 index 8f707910b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2/DeepBirdV2PredictionServiceClientModule.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.deepbirdv2 - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.bijection.scrooge.TBinaryProtocol -import com.twitter.conversions.DurationOps._ -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.builder.ClientBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.thrift.RichClientParam -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.inject.TwitterModule - -/** - * Module that provides multiple deepbirdv2 prediction service clients - * We use the java api since data records are native java objects and we want to reduce overhead - * while serializing/deserializing data. - */ -object DeepBirdV2PredictionServiceClientModule extends TwitterModule { - - val RequestTimeout = 300.millis - - private def getDeepbirdPredictionServiceClient( - clientId: ClientId, - label: String, - dest: String, - statsReceiver: StatsReceiver, - serviceIdentifier: ServiceIdentifier - ): DeepbirdPredictionService.ServiceToClient = { - val clientStatsReceiver = statsReceiver.scope("clnt") - val mTlsClient = ThriftMux.client.withClientId(clientId).withMutualTls(serviceIdentifier) - new DeepbirdPredictionService.ServiceToClient( - ClientBuilder() - .name(label) - .stack(mTlsClient) - .dest(dest) - .requestTimeout(RequestTimeout) - .reportHostStats(NullStatsReceiver) - .build(), - RichClientParam( - new TBinaryProtocol.Factory(), - clientStats = clientStatsReceiver - ) - ) - } - - @Provides - @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) - def providesWtfProdDeepbirdV2PredictionService( - clientId: ClientId, - statsReceiver: StatsReceiver, - serviceIdentifier: ServiceIdentifier - ): DeepbirdPredictionService.ServiceToClient = { - getDeepbirdPredictionServiceClient( - clientId = clientId, - label = "WtfProdDeepbirdV2PredictionService", - dest = "/s/cassowary/deepbirdv2-hermit-wtf", - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD deleted file mode 100644 index 33d8ba841..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "src/thrift/com/twitter/onboarding/relevance/store:store-scala", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.scala deleted file mode 100644 index 1c8c9f8fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store/DismissStore.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.dismiss_store - -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Scanner -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/** - * this store gets the list of dismissed candidates since a certain time - * primarily used for filtering out accounts that a user has explicitly dismissed - * - * we fail open on timeouts, but loudly on other errors - */ -@Singleton -class DismissStore @Inject() ( - @Named(GuiceNamedConstants.DISMISS_STORE_SCANNER) - scanner: Scanner[(Long, Slice[ - (Long, Long) - ]), Unit, (Long, (Long, Long)), WhoToFollowDismissEventDetails], - stats: StatsReceiver) - extends Logging { - - private val MaxCandidatesToReturn = 100 - - // gets a list of dismissed candidates. if numCandidatesToFetchOption is none, we will fetch the default number of candidates - def get( - userId: Long, - negStartTimeMs: Long, - maxCandidatesToFetchOption: Option[Int] - ): Stitch[Seq[Long]] = { - - val maxCandidatesToFetch = maxCandidatesToFetchOption.getOrElse(MaxCandidatesToReturn) - - scanner - .scan( - ( - userId, - Slice( - from = None, - to = Some((negStartTimeMs, Long.MaxValue)), - limit = Some(maxCandidatesToFetch) - ) - ) - ) - .map { - case s: Seq[((Long, (Long, Long)), WhoToFollowDismissEventDetails)] if s.nonEmpty => - s.map { - case ((_: Long, (_: Long, candidateId: Long)), _: WhoToFollowDismissEventDetails) => - candidateId - } - case _ => Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD deleted file mode 100644 index 78a06d536..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "emailstorage/server/src/main/thrift/com/twitter/emailstorage/api:email-storage-service-scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "stitch/stitch-core", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.scala deleted file mode 100644 index 4cf66c658..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceClient.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.email_storage_service - -import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing -import com.twitter.emailstorage.api.thriftscala.EmailStorageService -import com.twitter.emailstorage.api.thriftscala.GetUsersEmailsRequest -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class EmailStorageServiceClient @Inject() ( - val emailStorageService: EmailStorageService.MethodPerEndpoint) { - - def getVerifiedEmail( - userId: Long, - purposeOfProcessing: PurposeOfProcessing - ): Stitch[Option[String]] = { - val req = GetUsersEmailsRequest( - userIds = Seq(userId), - clientIdentifier = Some("follow-recommendations-service"), - purposesOfProcessing = Some(Seq(purposeOfProcessing)) - ) - - Stitch.callFuture(emailStorageService.getUsersEmails(req)) map { - _.usersEmails.map(_.confirmedEmail.map(_.email)).head - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.scala deleted file mode 100644 index 0a1be9d1b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service/EmailStorageServiceModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.email_storage_service - -import com.twitter.emailstorage.api.thriftscala.EmailStorageService -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule - -object EmailStorageServiceModule - extends BaseClientModule[EmailStorageService.MethodPerEndpoint] - with MtlsClient { - override val label = "email-storage-service" - override val dest = "/s/email-server/email-server" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD deleted file mode 100644 index d67fa01d8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "src/thrift/com/twitter/geoduck:geoduck-scala", - "src/thrift/com/twitter/geoduck:geoduckpartnerplaces-thrift-scala", - "stitch/stitch-core", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.scala deleted file mode 100644 index 69207537c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceClient.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.geoduck.common.thriftscala.LocationSource -import com.twitter.geoduck.common.thriftscala.PlaceQuery -import com.twitter.geoduck.common.thriftscala.TransactionLocation -import com.twitter.geoduck.common.thriftscala.UserLocationRequest -import com.twitter.geoduck.thriftscala.LocationService -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class LocationServiceClient @Inject() (locationService: LocationService.MethodPerEndpoint) { - def getGeohashAndCountryCode(userId: Long): Stitch[GeohashAndCountryCode] = { - Stitch - .callFuture { - locationService - .userLocation( - UserLocationRequest( - Seq(userId), - Some(PlaceQuery(allPlaceTypes = Some(true))), - simpleReverseGeocode = true)) - .map(_.found.get(userId)).map { transactionLocationOpt => - val geohashOpt = transactionLocationOpt.flatMap(getGeohashFromTransactionLocation) - val countryCodeOpt = - transactionLocationOpt.flatMap(_.simpleRgcResult.flatMap(_.countryCodeAlpha2)) - GeohashAndCountryCode(geohashOpt, countryCodeOpt) - } - } - } - - private[this] def getGeohashFromTransactionLocation( - transactionLocation: TransactionLocation - ): Option[String] = { - transactionLocation.geohash.flatMap { geohash => - val geohashPrefixLength = transactionLocation.locationSource match { - // if location source is logical, keep the first 4 chars in geohash - case Some(LocationSource.Logical) => Some(4) - // if location source is physical, keep the prefix according to accuracy - // accuracy is the accuracy of GPS readings in the unit of meter - case Some(LocationSource.Physical) => - transactionLocation.coordinate.flatMap { coordinate => - coordinate.accuracy match { - case Some(accuracy) if (accuracy < 50) => Some(7) - case Some(accuracy) if (accuracy < 200) => Some(6) - case Some(accuracy) if (accuracy < 1000) => Some(5) - case Some(accuracy) if (accuracy < 50000) => Some(4) - case Some(accuracy) if (accuracy < 100000) => Some(3) - case _ => None - } - } - case Some(LocationSource.Model) => Some(4) - case _ => None - } - geohashPrefixLength match { - case Some(l: Int) => geohash.stringGeohash.map(_.take(l)) - case _ => None - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.scala deleted file mode 100644 index a5fc79a80..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/LocationServiceModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.geoduck.thriftscala.LocationService - -object LocationServiceModule - extends BaseClientModule[LocationService.MethodPerEndpoint] - with MtlsClient { - override val label = "geoduck_locationservice" - override val dest = "/s/geo/geoduck_locationservice" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.scala deleted file mode 100644 index 576359128..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/ReverseGeocodeClient.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.geoduck.common.thriftscala.Location -import com.twitter.geoduck.common.thriftscala.PlaceQuery -import com.twitter.geoduck.common.thriftscala.ReverseGeocodeIPRequest -import com.twitter.geoduck.service.thriftscala.GeoContext -import com.twitter.geoduck.thriftscala.ReverseGeocoder -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ReverseGeocodeClient @Inject() (rgcService: ReverseGeocoder.MethodPerEndpoint) { - def getGeohashAndCountryCode(ipAddress: String): Stitch[GeohashAndCountryCode] = { - Stitch - .callFuture { - rgcService - .reverseGeocodeIp( - ReverseGeocodeIPRequest( - Seq(ipAddress), - PlaceQuery(None), - simpleReverseGeocode = true - ) // note: simpleReverseGeocode means that country code will be included in response - ).map { response => - response.found.get(ipAddress) match { - case Some(location) => getGeohashAndCountryCodeFromLocation(location) - case _ => GeohashAndCountryCode(None, None) - } - } - } - } - - private def getGeohashAndCountryCodeFromLocation(location: Location): GeohashAndCountryCode = { - val countryCode: Option[String] = location.simpleRgcResult.flatMap { _.countryCodeAlpha2 } - - val geohashString: Option[String] = location.geohash.flatMap { hash => - hash.stringGeohash.flatMap { hashString => - Some(ReverseGeocodeClient.truncate(hashString)) - } - } - - GeohashAndCountryCode(geohashString, countryCode) - } - -} - -object ReverseGeocodeClient { - - val DefaultGeoduckIPRequestContext: GeoContext = - GeoContext(allPlaceTypes = true, includeGeohash = true, includeCountryCode = true) - - // All these geohashes are guessed by IP (Logical Location Source). - // So take the four letters to make sure it is consistent with LocationServiceClient - val GeohashLengthAfterTruncation = 4 - def truncate(geohash: String): String = geohash.take(GeohashLengthAfterTruncation) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.scala deleted file mode 100644 index 706ae5143..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck/UserLocationFetcher.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.geoduck - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.stitch.Stitch - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class UserLocationFetcher @Inject() ( - locationServiceClient: LocationServiceClient, - reverseGeocodeClient: ReverseGeocodeClient, - statsReceiver: StatsReceiver) { - - private val stats: StatsReceiver = statsReceiver.scope("user_location_fetcher") - private val totalRequestsCounter = stats.counter("requests") - private val emptyResponsesCounter = stats.counter("empty") - private val locationServiceExceptionCounter = stats.counter("location_service_exception") - private val reverseGeocodeExceptionCounter = stats.counter("reverse_geocode_exception") - - def getGeohashAndCountryCode( - userId: Option[Long], - ipAddress: Option[String] - ): Stitch[Option[GeohashAndCountryCode]] = { - totalRequestsCounter.incr() - val lscLocationStitch = Stitch - .collect { - userId.map(locationServiceClient.getGeohashAndCountryCode) - }.rescue { - case _: Exception => - locationServiceExceptionCounter.incr() - Stitch.None - } - - val ipLocationStitch = Stitch - .collect { - ipAddress.map(reverseGeocodeClient.getGeohashAndCountryCode) - }.rescue { - case _: Exception => - reverseGeocodeExceptionCounter.incr() - Stitch.None - } - - Stitch.join(lscLocationStitch, ipLocationStitch).map { - case (lscLocation, ipLocation) => { - val geohash = lscLocation.flatMap(_.geohash).orElse(ipLocation.flatMap(_.geohash)) - val countryCode = - lscLocation.flatMap(_.countryCode).orElse(ipLocation.flatMap(_.countryCode)) - (geohash, countryCode) match { - case (None, None) => - emptyResponsesCounter.incr() - None - case _ => Some(GeohashAndCountryCode(geohash, countryCode)) - } - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD deleted file mode 100644 index 77cb553c0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "stitch/stitch-gizmoduck", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.scala deleted file mode 100644 index 25b5f0d75..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckClient.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.gizmoduck - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.PerspectiveEdge -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GizmoduckClient @Inject() (gizmoduckStitchClient: Gizmoduck, statsReceiver: StatsReceiver) { - val stats = statsReceiver.scope("gizmoduck_client") - val getByIdStats = stats.scope("get_by_id") - val getUserById = stats.scope("get_user_by_id") - - def isProtected(userId: Long): Stitch[Boolean] = { - // get latency metrics with StatsUtil.profileStitch when calling .getById - val response = StatsUtil.profileStitch( - gizmoduckStitchClient.getById(userId, Set(QueryFields.Safety)), - getByIdStats - ) - response.map { result => - result.user.flatMap(_.safety).map(_.isProtected).getOrElse(true) - } - } - - def getUserName(userId: Long, forUserId: Long): Stitch[Option[String]] = { - val queryFields = GizmoduckClient.GetUserByIdUserNameQueryFields - val lookupContext = LookupContext( - forUserId = Some(forUserId), - perspectiveEdges = Some(GizmoduckClient.DefaultPerspectiveEdges) - ) - // get latency metrics with StatsUtil.profileStitch when calling .getUserById - val response = StatsUtil.profileStitch( - gizmoduckStitchClient.getUserById(userId, queryFields, lookupContext), - getUserById - ) - response.map(_.profile.map(_.name)) - } -} - -object GizmoduckClient { - // Similar to GizmoduckUserRepository.DefaultPerspectiveEdges - val DefaultPerspectiveEdges: Set[PerspectiveEdge] = - Set( - PerspectiveEdge.Blocking, - PerspectiveEdge.BlockedBy, - PerspectiveEdge.DeviceFollowing, - PerspectiveEdge.FollowRequestSent, - PerspectiveEdge.Following, - PerspectiveEdge.FollowedBy, - PerspectiveEdge.LifelineFollowing, - PerspectiveEdge.LifelineFollowedBy, - PerspectiveEdge.Muting, - PerspectiveEdge.NoRetweetsFrom - ) - - // From GizmoduckUserRepository.DefaultQueryFields - val GetUserByIdQueryFields: Set[QueryFields] = Set( - QueryFields.Account, - QueryFields.Counts, - QueryFields.ExtendedProfile, - QueryFields.Perspective, - QueryFields.Profile, - QueryFields.ProfileDesign, - QueryFields.ProfileLocation, - QueryFields.Safety, - QueryFields.Roles, - QueryFields.Takedowns, - QueryFields.UrlEntities, - QueryFields.DirectMessageView, - QueryFields.MediaView - ) - - val GetUserByIdUserNameQueryFields: Set[QueryFields] = Set( - QueryFields.Profile - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.scala deleted file mode 100644 index 9a5efea06..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck/GizmoduckModule.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.gizmoduck - -import com.google.inject.Provides -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.gizmoduck.thriftscala.UserService -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Singleton - -object GizmoduckModule extends BaseClientModule[UserService.MethodPerEndpoint] with MtlsClient { - override val label = "gizmoduck" - override val dest = "/s/gizmoduck/gizmoduck" - - @Provides - @Singleton - def provideExtraGizmoduckQueryFields: Set[QueryFields] = Set.empty - - @Provides - @Singleton - def providesStitchClient(futureIface: UserService.MethodPerEndpoint): Gizmoduck = { - Gizmoduck(futureIface) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD deleted file mode 100644 index ec1139cfe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "graph-feature-service/src/main/thrift/com/twitter/graph_feature_service:graph_feature_service_thrift-scala", - "stitch/stitch-core", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.scala deleted file mode 100644 index f333d895f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureServiceClient.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.graph_feature_service - -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.graph_feature_service.thriftscala.PresetFeatureTypes.WtfTwoHop -import com.twitter.graph_feature_service.thriftscala.EdgeType -import com.twitter.graph_feature_service.thriftscala.GfsIntersectionResponse -import com.twitter.graph_feature_service.thriftscala.GfsPresetIntersectionRequest -import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService} -import com.twitter.stitch.Stitch -import javax.inject.{Inject, Singleton} - -@Singleton -class GraphFeatureServiceClient @Inject() ( - graphFeatureService: GraphFeatureService.MethodPerEndpoint) { - - import GraphFeatureServiceClient._ - def getIntersections( - userId: Long, - candidateIds: Seq[Long], - numIntersectionIds: Int - ): Stitch[Map[Long, FollowProof]] = { - Stitch - .callFuture( - graphFeatureService.getPresetIntersection( - GfsPresetIntersectionRequest(userId, candidateIds, WtfTwoHop, Some(numIntersectionIds)) - ) - ).map { - case GfsIntersectionResponse(gfsIntersectionResults) => - (for { - candidateId <- candidateIds - gfsIntersectionResultForCandidate = - gfsIntersectionResults.filter(_.candidateUserId == candidateId) - followProof <- for { - result <- gfsIntersectionResultForCandidate - intersection <- result.intersectionValues - if leftEdgeTypes.contains(intersection.featureType.leftEdgeType) - if rightEdgeTypes.contains(intersection.featureType.rightEdgeType) - intersectionIds <- intersection.intersectionIds.toSeq - } yield FollowProof(intersectionIds, intersection.count.getOrElse(0)) - } yield { - candidateId -> followProof - }).toMap - } - } -} - -object GraphFeatureServiceClient { - val leftEdgeTypes: Set[EdgeType] = Set(EdgeType.Following) - val rightEdgeTypes: Set[EdgeType] = Set(EdgeType.FollowedBy) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.scala deleted file mode 100644 index 8d15358cd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service/GraphFeatureStoreModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.graph_feature_service - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.graph_feature_service.thriftscala.{Server => GraphFeatureService} - -object GraphFeatureStoreModule - extends BaseClientModule[GraphFeatureService.MethodPerEndpoint] - with MtlsClient { - override val label = "graph_feature_service" - override val dest = "/s/cassowary/graph_feature_service-server" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD deleted file mode 100644 index 7432ce040..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "stitch/stitch-socialgraph", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.scala deleted file mode 100644 index 35b5d7885..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/ImpressionStoreModule.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.impression_store - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.thriftscala.DisplayLocation -import com.twitter.inject.TwitterModule -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Client -import com.twitter.strato.thrift.ScroogeConvImplicits._ - -object ImpressionStoreModule extends TwitterModule { - - val columnPath: String = "onboarding/userrecs/wtfImpressionCountsStore" - - type PKey = (Long, DisplayLocation) - type LKey = Long - type Value = (Long, Int) - - @Provides - @Singleton - def providesImpressionStore(stratoClient: Client): WtfImpressionStore = { - new WtfImpressionStore( - stratoClient.scanner[ - (PKey, Slice[LKey]), - Unit, - (PKey, LKey), - Value - ](columnPath) - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.scala deleted file mode 100644 index 68692dc28..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store/WtfImpressionStore.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.impression_store - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.WtfImpression -import com.twitter.follow_recommendations.thriftscala.{DisplayLocation => TDisplayLocation} -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Scanner -import com.twitter.util.Time -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class WtfImpressionStore @Inject() ( - scanner: Scanner[ - ((Long, TDisplayLocation), Slice[Long]), - Unit, - ((Long, TDisplayLocation), Long), - (Long, Int) - ]) extends Logging { - def get(userId: Long, dl: DisplayLocation): Stitch[Seq[WtfImpression]] = { - val thriftDl = dl.toThrift - scanner.scan(((userId, thriftDl), Slice.all[Long])).map { impressionsPerDl => - val wtfImpressions = - for { - (((_, _), candidateId), (latestTs, counts)) <- impressionsPerDl - } yield WtfImpression( - candidateId = candidateId, - displayLocation = dl, - latestTime = Time.fromMilliseconds(latestTs), - counts = counts - ) - wtfImpressions - } rescue { - // fail open so that the request can still go through - case ex: Throwable => - logger.warn(s"$dl WtfImpressionsStore warn: " + ex.getMessage) - Stitch.Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD deleted file mode 100644 index 0835d840b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - name = "interests_service", - sources = ["InterestServiceClient.scala"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", - "interests-service/thrift/src/main/thrift:thrift-scala", - "strato/src/main/scala/com/twitter/strato/catalog", - "strato/src/main/scala/com/twitter/strato/client", - "strato/src/main/scala/com/twitter/strato/data", - "strato/src/main/scala/com/twitter/strato/thrift", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.scala deleted file mode 100644 index 3904cf515..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service/InterestServiceClient.scala +++ /dev/null @@ -1,115 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.interests_service - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.InterestedInInterestsFetchKey -import com.twitter.inject.Logging -import com.twitter.interests.thriftscala.InterestId -import com.twitter.interests.thriftscala.InterestRelationship -import com.twitter.interests.thriftscala.InterestedInInterestModel -import com.twitter.interests.thriftscala.UserInterest -import com.twitter.interests.thriftscala.UserInterestData -import com.twitter.interests.thriftscala.UserInterestsResponse -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Client -import com.twitter.strato.thrift.ScroogeConvImplicits._ - -@Singleton -class InterestServiceClient @Inject() ( - stratoClient: Client, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends Logging { - - val interestsServiceStratoColumnPath = "interests/interestedInInterests" - val stats = statsReceiver.scope("interest_service_client") - val errorCounter = stats.counter("error") - - private val interestsFetcher = - stratoClient.fetcher[InterestedInInterestsFetchKey, UserInterestsResponse]( - interestsServiceStratoColumnPath, - checkTypes = true - ) - - def fetchUttInterestIds( - userId: Long - ): Stitch[Seq[Long]] = { - fetchInterestRelationships(userId) - .map(_.toSeq.flatten.flatMap(extractUttInterest)) - } - - def extractUttInterest( - interestRelationShip: InterestRelationship - ): Option[Long] = { - interestRelationShip match { - case InterestRelationship.V1(relationshipV1) => - relationshipV1.interestId match { - case InterestId.SemanticCore(semanticCoreInterest) => Some(semanticCoreInterest.id) - case _ => None - } - case _ => None - } - } - - def fetchCustomInterests( - userId: Long - ): Stitch[Seq[String]] = { - fetchInterestRelationships(userId) - .map(_.toSeq.flatten.flatMap(extractCustomInterest)) - } - - def extractCustomInterest( - interestRelationShip: InterestRelationship - ): Option[String] = { - interestRelationShip match { - case InterestRelationship.V1(relationshipV1) => - relationshipV1.interestId match { - case InterestId.FreeForm(freeFormInterest) => Some(freeFormInterest.interest) - case _ => None - } - case _ => None - } - } - - def fetchInterestRelationships( - userId: Long - ): Stitch[Option[Seq[InterestRelationship]]] = { - interestsFetcher - .fetch( - InterestedInInterestsFetchKey( - userId = userId, - labels = None, - None - )) - .map(_.v) - .map { - case Some(response) => - response.interests.interests.map { interests => - interests.collect { - case UserInterest(_, Some(interestData)) => - getInterestRelationship(interestData) - }.flatten - } - case _ => None - } - .rescue { - case e: Throwable => // we are swallowing all errors - logger.warn(s"interests could not be retrieved for user $userId due to ${e.getCause}") - errorCounter.incr - Stitch.None - } - } - - private def getInterestRelationship( - interestData: UserInterestData - ): Seq[InterestRelationship] = { - interestData match { - case UserInterestData.InterestedIn(interestModels) => - interestModels.collect { - case InterestedInInterestModel.ExplicitModel(model) => model - } - case _ => Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD deleted file mode 100644 index 37a7f5906..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "phonestorage/server/src/main/thrift/com/twitter/phonestorage/api:phone-storage-service-scala", - "stitch/stitch-core", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.scala deleted file mode 100644 index 46e76b162..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceClient.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.phone_storage_service - -import com.twitter.cds.contact_consent_state.thriftscala.PurposeOfProcessing -import com.twitter.phonestorage.api.thriftscala.GetUserPhonesByUsersRequest -import com.twitter.phonestorage.api.thriftscala.PhoneStorageService -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PhoneStorageServiceClient @Inject() ( - val phoneStorageService: PhoneStorageService.MethodPerEndpoint) { - - /** - * PSS can potentially return multiple phone records. - * The current implementation of getUserPhonesByUsers returns only a single phone for a single user_id but - * we can trivially support handling multiple in case that changes in the future. - */ - def getPhoneNumbers( - userId: Long, - purposeOfProcessing: PurposeOfProcessing, - forceCarrierLookup: Option[Boolean] = None - ): Stitch[Seq[String]] = { - val req = GetUserPhonesByUsersRequest( - userIds = Seq(userId), - forceCarrierLookup = forceCarrierLookup, - purposesOfProcessing = Some(Seq(purposeOfProcessing)) - ) - - Stitch.callFuture(phoneStorageService.getUserPhonesByUsers(req)) map { - _.userPhones.map(_.phoneNumber) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.scala deleted file mode 100644 index 90767a509..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service/PhoneStorageServiceModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.phone_storage_service - -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.phonestorage.api.thriftscala.PhoneStorageService - -object PhoneStorageServiceModule - extends BaseClientModule[PhoneStorageService.MethodPerEndpoint] - with MtlsClient { - override val label = "phone-storage-service" - override val dest = "/s/ibis-ds-api/ibis-ds-api:thrift2" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD deleted file mode 100644 index f711bc3e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.scala deleted file mode 100644 index 98c0555e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/Engagement.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.real_time_real_graph - -sealed trait EngagementType - -// We do not include SoftFollow since it's deprecated -object EngagementType { - object Click extends EngagementType - object Like extends EngagementType - object Mention extends EngagementType - object Retweet extends EngagementType - object ProfileView extends EngagementType -} - -case class Engagement(engagementType: EngagementType, timestamp: Long) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.scala deleted file mode 100644 index c41fcc98e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/EngagementScorer.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.real_time_real_graph - -import com.twitter.conversions.DurationOps._ -import com.twitter.util.Time - -object EngagementScorer { - private[real_time_real_graph] val MemoryDecayHalfLife = 24.hour - private val ScoringFunctionBase = 0.5 - - def apply( - engagements: Map[Long, Seq[Engagement]], - engagementScoreMap: Map[EngagementType, Double], - minScore: Double = 0.0 - ): Seq[(Long, Double, Seq[EngagementType])] = { - val now = Time.now - engagements - .mapValues { engags => - val totalScore = engags.map { engagement => score(engagement, now, engagementScoreMap) }.sum - val engagementProof = getEngagementProof(engags, engagementScoreMap) - (totalScore, engagementProof) - } - .collect { case (uid, (score, proof)) if score > minScore => (uid, score, proof) } - .toSeq - .sortBy(-_._2) - } - - /** - * The engagement score is the base score decayed via timestamp, loosely model the human memory forgetting - * curve, see https://en.wikipedia.org/wiki/Forgetting_curve - */ - private[real_time_real_graph] def score( - engagement: Engagement, - now: Time, - engagementScoreMap: Map[EngagementType, Double] - ): Double = { - val timeLapse = math.max(now.inMillis - engagement.timestamp, 0) - val engagementScore = engagementScoreMap.getOrElse(engagement.engagementType, 0.0) - engagementScore * math.pow( - ScoringFunctionBase, - timeLapse.toDouble / MemoryDecayHalfLife.inMillis) - } - - private def getEngagementProof( - engagements: Seq[Engagement], - engagementScoreMap: Map[EngagementType, Double] - ): Seq[EngagementType] = { - - val filteredEngagement = engagements - .collectFirst { - case engagement - if engagement.engagementType != EngagementType.Click - && engagementScoreMap.get(engagement.engagementType).exists(_ > 0.0) => - engagement.engagementType - } - - Seq(filteredEngagement.getOrElse(EngagementType.Click)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.scala deleted file mode 100644 index 16fedc8a0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/real_time_real_graph/RealTimeRealGraphClient.scala +++ /dev/null @@ -1,128 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.real_time_real_graph - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.ml.featureStore.TimelinesUserVertexOnUserClientColumn -import com.twitter.strato.generated.client.onboarding.userrecs.RealGraphScoresMhOnUserClientColumn -import com.twitter.util.Duration -import com.twitter.util.Time -import com.twitter.wtf.real_time_interaction_graph.thriftscala._ - -@Singleton -class RealTimeRealGraphClient @Inject() ( - timelinesUserVertexOnUserClientColumn: TimelinesUserVertexOnUserClientColumn, - realGraphScoresMhOnUserClientColumn: RealGraphScoresMhOnUserClientColumn) { - - def mapUserVertexToEngagementAndFilter(userVertex: UserVertex): Map[Long, Seq[Engagement]] = { - val minTimestamp = (Time.now - RealTimeRealGraphClient.MaxEngagementAge).inMillis - userVertex.outgoingInteractionMap.mapValues { interactions => - interactions - .flatMap { interaction => RealTimeRealGraphClient.toEngagement(interaction) }.filter( - _.timestamp >= minTimestamp) - }.toMap - } - - def getRecentProfileViewEngagements(userId: Long): Stitch[Map[Long, Seq[Engagement]]] = { - timelinesUserVertexOnUserClientColumn.fetcher - .fetch(userId).map(_.v).map { input => - input - .map { userVertex => - val targetToEngagements = mapUserVertexToEngagementAndFilter(userVertex) - targetToEngagements.mapValues { engagements => - engagements.filter(engagement => - engagement.engagementType == EngagementType.ProfileView) - } - }.getOrElse(Map.empty) - } - } - - def getUsersRecentlyEngagedWith( - userId: Long, - engagementScoreMap: Map[EngagementType, Double], - includeDirectFollowCandidates: Boolean, - includeNonDirectFollowCandidates: Boolean - ): Stitch[Seq[CandidateUser]] = { - val isNewUser = - SnowflakeId.timeFromIdOpt(userId).exists { signupTime => - (Time.now - signupTime) < RealTimeRealGraphClient.MaxNewUserAge - } - val updatedEngagementScoreMap = - if (isNewUser) - engagementScoreMap + (EngagementType.ProfileView -> RealTimeRealGraphClient.ProfileViewScore) - else engagementScoreMap - - Stitch - .join( - timelinesUserVertexOnUserClientColumn.fetcher.fetch(userId).map(_.v), - realGraphScoresMhOnUserClientColumn.fetcher.fetch(userId).map(_.v)).map { - case (Some(userVertex), Some(neighbors)) => - val engagements = mapUserVertexToEngagementAndFilter(userVertex) - - val candidatesAndScores: Seq[(Long, Double, Seq[EngagementType])] = - EngagementScorer.apply(engagements, engagementScoreMap = updatedEngagementScoreMap) - - val directNeighbors = neighbors.candidates.map(_._1).toSet - val (directFollows, nonDirectFollows) = candidatesAndScores - .partition { - case (id, _, _) => directNeighbors.contains(id) - } - - val candidates = - (if (includeNonDirectFollowCandidates) nonDirectFollows else Seq.empty) ++ - (if (includeDirectFollowCandidates) - directFollows.take(RealTimeRealGraphClient.MaxNumDirectFollow) - else Seq.empty) - - candidates.map { - case (id, score, proof) => - CandidateUser(id, Some(score)) - } - - case _ => Nil - } - } - - def getRealGraphWeights(userId: Long): Stitch[Map[Long, Double]] = - realGraphScoresMhOnUserClientColumn.fetcher - .fetch(userId) - .map( - _.v - .map(_.candidates.map(candidate => (candidate.userId, candidate.score)).toMap) - .getOrElse(Map.empty[Long, Double])) -} - -object RealTimeRealGraphClient { - private def toEngagement(interaction: Interaction): Option[Engagement] = { - // We do not include SoftFollow since it's deprecated - interaction match { - case Interaction.Retweet(Retweet(timestamp)) => - Some(Engagement(EngagementType.Retweet, timestamp)) - case Interaction.Favorite(Favorite(timestamp)) => - Some(Engagement(EngagementType.Like, timestamp)) - case Interaction.Click(Click(timestamp)) => Some(Engagement(EngagementType.Click, timestamp)) - case Interaction.Mention(Mention(timestamp)) => - Some(Engagement(EngagementType.Mention, timestamp)) - case Interaction.ProfileView(ProfileView(timestamp)) => - Some(Engagement(EngagementType.ProfileView, timestamp)) - case _ => None - } - } - - val MaxNumDirectFollow = 50 - val MaxEngagementAge: Duration = 14.days - val MaxNewUserAge: Duration = 30.days - val ProfileViewScore = 0.4 - val EngagementScoreMap = Map( - EngagementType.Like -> 1.0, - EngagementType.Retweet -> 1.0, - EngagementType.Mention -> 1.0 - ) - val StrongEngagementScoreMap = Map( - EngagementType.Like -> 1.0, - EngagementType.Retweet -> 1.0, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD deleted file mode 100644 index 58b5001fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/BUILD +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/github/nscala_time", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-thrift-client/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "socialgraph/server/src/main/scala/com/twitter/socialgraph/util", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-socialgraph", - "strato/config/columns/onboarding/socialGraphService:socialGraphService-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-core:scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.scala deleted file mode 100644 index 3ad90b5ed..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphClient.scala +++ /dev/null @@ -1,421 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.socialgraph - -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.common.models.UserIdWithTimestamp -import com.twitter.inject.Logging -import com.twitter.socialgraph.thriftscala.EdgesRequest -import com.twitter.socialgraph.thriftscala.IdsRequest -import com.twitter.socialgraph.thriftscala.IdsResult -import com.twitter.socialgraph.thriftscala.LookupContext -import com.twitter.socialgraph.thriftscala.OverCapacity -import com.twitter.socialgraph.thriftscala.PageRequest -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.socialgraph.thriftscala.SrcRelationship -import com.twitter.socialgraph.util.ByteBufferUtil -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.onboarding.socialGraphService.IdsClientColumn -import com.twitter.util.Duration -import com.twitter.util.Time -import java.nio.ByteBuffer -import javax.inject.Inject -import javax.inject.Singleton - -case class RecentEdgesQuery( - userId: Long, - relations: Seq[RelationshipType], - // prefer to default value to better utilize the caching function of stitch - count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), - performUnion: Boolean = true, - recentEdgesWindowOpt: Option[Duration] = None, - targets: Option[Seq[Long]] = None) - -case class EdgeRequestQuery( - userId: Long, - relation: RelationshipType, - count: Option[Int] = Some(SocialGraphClient.MaxQuerySize), - performUnion: Boolean = true, - recentEdgesWindowOpt: Option[Duration] = None, - targets: Option[Seq[Long]] = None) - -@Singleton -class SocialGraphClient @Inject() ( - socialGraph: SocialGraph, - idsClientColumn: IdsClientColumn, - statsReceiver: StatsReceiver = NullStatsReceiver) - extends Logging { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val cacheStats = stats.scope("cache") - private val getIntersectionsStats = stats.scope("getIntersections") - private val getIntersectionsFromCachedColumnStats = - stats.scope("getIntersectionsFromCachedColumn") - private val getRecentEdgesStats = stats.scope("getRecentEdges") - private val getRecentEdgesCachedStats = stats.scope("getRecentEdgesCached") - private val getRecentEdgesFromCachedColumnStats = stats.scope("getRecentEdgesFromCachedColumn") - private val getRecentEdgesCachedInternalStats = stats.scope("getRecentEdgesCachedInternal") - private val getRecentEdgesWithTimeStats = stats.scope("getRecentEdgesWithTime") - - val sgsIdsFetcher: Fetcher[IdsRequest, Unit, IdsResult] = idsClientColumn.fetcher - - private val recentEdgesCache = StitchCache[RecentEdgesQuery, Seq[Long]]( - maxCacheSize = SocialGraphClient.MaxCacheSize, - ttl = SocialGraphClient.CacheTTL, - statsReceiver = cacheStats, - underlyingCall = getRecentEdges - ) - - def getRecentEdgesCached( - rq: RecentEdgesQuery, - useCachedStratoColumn: Boolean = true - ): Stitch[Seq[Long]] = { - getRecentEdgesCachedStats.counter("requests").incr() - if (useCachedStratoColumn) { - getRecentEdgesFromCachedColumn(rq) - } else { - StatsUtil.profileStitch( - getRecentEdgesCachedInternal(rq), - getRecentEdgesCachedInternalStats - ) - } - } - - def getRecentEdgesCachedInternal(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { - recentEdgesCache.readThrough(rq) - } - - def getRecentEdgesFromCachedColumn(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { - val pageRequest = rq.recentEdgesWindowOpt match { - case Some(recentEdgesWindow) => - PageRequest( - count = rq.count, - cursor = Some(getEdgeCursor(recentEdgesWindow)), - selectAll = Some(true) - ) - case _ => PageRequest(count = rq.count) - } - val idsRequest = IdsRequest( - rq.relations.map { relationshipType => - SrcRelationship( - source = rq.userId, - relationshipType = relationshipType, - targets = rq.targets - ) - }, - pageRequest = Some(pageRequest), - context = Some(LookupContext(performUnion = Some(rq.performUnion))) - ) - - val socialGraphStitch = sgsIdsFetcher - .fetch(idsRequest, Unit) - .map(_.v) - .map { result => - result - .map { idResult => - val userIds: Seq[Long] = idResult.ids - getRecentEdgesFromCachedColumnStats.stat("num_edges").add(userIds.size) - userIds - }.getOrElse(Seq.empty) - } - .rescue { - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch.Nil - } - - StatsUtil.profileStitch( - socialGraphStitch, - getRecentEdgesFromCachedColumnStats - ) - } - - def getRecentEdges(rq: RecentEdgesQuery): Stitch[Seq[Long]] = { - val pageRequest = rq.recentEdgesWindowOpt match { - case Some(recentEdgesWindow) => - PageRequest( - count = rq.count, - cursor = Some(getEdgeCursor(recentEdgesWindow)), - selectAll = Some(true) - ) - case _ => PageRequest(count = rq.count) - } - val socialGraphStitch = socialGraph - .ids( - IdsRequest( - rq.relations.map { relationshipType => - SrcRelationship( - source = rq.userId, - relationshipType = relationshipType, - targets = rq.targets - ) - }, - pageRequest = Some(pageRequest), - context = Some(LookupContext(performUnion = Some(rq.performUnion))) - ) - ) - .map { idsResult => - val userIds: Seq[Long] = idsResult.ids - getRecentEdgesStats.stat("num_edges").add(userIds.size) - userIds - } - .rescue { - case e: OverCapacity => - stats.counter(e.getClass.getSimpleName).incr() - logger.warn("SGS Over Capacity", e) - Stitch.Nil - } - StatsUtil.profileStitch( - socialGraphStitch, - getRecentEdgesStats - ) - } - - // This method return recent edges of (userId, timeInMs) - def getRecentEdgesWithTime(rq: EdgeRequestQuery): Stitch[Seq[UserIdWithTimestamp]] = { - val pageRequest = rq.recentEdgesWindowOpt match { - case Some(recentEdgesWindow) => - PageRequest( - count = rq.count, - cursor = Some(getEdgeCursor(recentEdgesWindow)), - selectAll = Some(true) - ) - case _ => PageRequest(count = rq.count) - } - - val socialGraphStitch = socialGraph - .edges( - EdgesRequest( - SrcRelationship( - source = rq.userId, - relationshipType = rq.relation, - targets = rq.targets - ), - pageRequest = Some(pageRequest), - context = Some(LookupContext(performUnion = Some(rq.performUnion))) - ) - ) - .map { edgesResult => - val userIds = edgesResult.edges.map { socialEdge => - UserIdWithTimestamp(socialEdge.target, socialEdge.updatedAt) - } - getRecentEdgesWithTimeStats.stat("num_edges").add(userIds.size) - userIds - } - .rescue { - case e: OverCapacity => - stats.counter(e.getClass.getSimpleName).incr() - logger.warn("SGS Over Capacity", e) - Stitch.Nil - } - StatsUtil.profileStitch( - socialGraphStitch, - getRecentEdgesWithTimeStats - ) - } - - // This method returns the cursor for a time duration, such that all the edges returned by SGS will be created - // in the range (now-window, now) - def getEdgeCursor(window: Duration): ByteBuffer = { - val cursorInLong = (-(Time.now - window).inMilliseconds) << 20 - ByteBufferUtil.fromLong(cursorInLong) - } - - // notice that this is more expensive but more realtime than the GFS one - def getIntersections( - userId: Long, - candidateIds: Seq[Long], - numIntersectionIds: Int - ): Stitch[Map[Long, FollowProof]] = { - val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch - .collect(candidateIds.map { candidateId => - socialGraph - .ids( - IdsRequest( - Seq( - SrcRelationship(userId, RelationshipType.Following), - SrcRelationship(candidateId, RelationshipType.FollowedBy) - ), - pageRequest = Some(PageRequest(count = Some(numIntersectionIds))) - ) - ).map { idsResult => - getIntersectionsStats.stat("num_edges").add(idsResult.ids.size) - (candidateId -> FollowProof(idsResult.ids, idsResult.ids.size)) - } - }).map(_.toMap) - .rescue { - case e: OverCapacity => - stats.counter(e.getClass.getSimpleName).incr() - logger.warn("social graph over capacity in hydrating social proof", e) - Stitch.value(Map.empty) - } - StatsUtil.profileStitch( - socialGraphStitch, - getIntersectionsStats - ) - } - - def getIntersectionsFromCachedColumn( - userId: Long, - candidateIds: Seq[Long], - numIntersectionIds: Int - ): Stitch[Map[Long, FollowProof]] = { - val socialGraphStitch: Stitch[Map[Long, FollowProof]] = Stitch - .collect(candidateIds.map { candidateId => - val idsRequest = IdsRequest( - Seq( - SrcRelationship(userId, RelationshipType.Following), - SrcRelationship(candidateId, RelationshipType.FollowedBy) - ), - pageRequest = Some(PageRequest(count = Some(numIntersectionIds))) - ) - - sgsIdsFetcher - .fetch(idsRequest, Unit) - .map(_.v) - .map { resultOpt => - resultOpt.map { idsResult => - getIntersectionsFromCachedColumnStats.stat("num_edges").add(idsResult.ids.size) - candidateId -> FollowProof(idsResult.ids, idsResult.ids.size) - } - } - }).map(_.flatten.toMap) - .rescue { - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch.value(Map.empty) - } - StatsUtil.profileStitch( - socialGraphStitch, - getIntersectionsFromCachedColumnStats - ) - } - - def getInvalidRelationshipUserIds( - userId: Long, - maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship - ): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - SocialGraphClient.InvalidRelationshipTypes, - Some(maxNumRelationship) - ) - ) - } - - def getInvalidRelationshipUserIdsFromCachedColumn( - userId: Long, - maxNumRelationship: Int = SocialGraphClient.MaxNumInvalidRelationship - ): Stitch[Seq[Long]] = { - getRecentEdgesFromCachedColumn( - RecentEdgesQuery( - userId, - SocialGraphClient.InvalidRelationshipTypes, - Some(maxNumRelationship) - ) - ) - } - - def getRecentFollowedUserIds(userId: Long): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - Seq(RelationshipType.Following) - ) - ) - } - - def getRecentFollowedUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = { - getRecentEdgesFromCachedColumn( - RecentEdgesQuery( - userId, - Seq(RelationshipType.Following) - ) - ) - } - - def getRecentFollowedUserIdsWithTime(userId: Long): Stitch[Seq[UserIdWithTimestamp]] = { - getRecentEdgesWithTime( - EdgeRequestQuery( - userId, - RelationshipType.Following - ) - ) - } - - def getRecentFollowedByUserIds(userId: Long): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - Seq(RelationshipType.FollowedBy) - ) - ) - } - - def getRecentFollowedByUserIdsFromCachedColumn(userId: Long): Stitch[Seq[Long]] = { - getRecentEdgesFromCachedColumn( - RecentEdgesQuery( - userId, - Seq(RelationshipType.FollowedBy) - ) - ) - } - - def getRecentFollowedUserIdsWithTimeWindow( - userId: Long, - timeWindow: Duration - ): Stitch[Seq[Long]] = { - getRecentEdges( - RecentEdgesQuery( - userId, - Seq(RelationshipType.Following), - recentEdgesWindowOpt = Some(timeWindow) - ) - ) - } -} - -object SocialGraphClient { - - val MaxQuerySize: Int = 500 - val MaxCacheSize: Int = 5000000 - // Ref: src/thrift/com/twitter/socialgraph/social_graph_service.thrift - val MaxNumInvalidRelationship: Int = 5000 - val CacheTTL: Duration = Duration.fromHours(24) - - val InvalidRelationshipTypes: Seq[RelationshipType] = Seq( - RelationshipType.HideRecommendations, - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.Muting, - RelationshipType.MutedBy, - RelationshipType.ReportedAsSpam, - RelationshipType.ReportedAsSpamBy, - RelationshipType.ReportedAsAbuse, - RelationshipType.ReportedAsAbuseBy, - RelationshipType.FollowRequestOutgoing, - RelationshipType.Following, - RelationshipType.UsedToFollow, - ) - - /** - * - * Whether to call SGS to validate each candidate based on the number of invalid relationship users - * prefetched during request building step. This aims to not omit any invalid candidates that are - * not filtered out in previous steps. - * If the number is 0, this might be a fail-opened SGS call. - * If the number is larger or equal to 5000, this could hit SGS page size limit. - * Both cases account for a small percentage of the total traffic (<5%). - * - * @param numInvalidRelationshipUsers number of invalid relationship users fetched from getInvalidRelationshipUserIds - * @return whether to enable post-ranker SGS predicate - */ - def enablePostRankerSgsPredicate(numInvalidRelationshipUsers: Int): Boolean = { - numInvalidRelationshipUsers == 0 || numInvalidRelationshipUsers >= MaxNumInvalidRelationship - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.scala deleted file mode 100644 index 5a97448d1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph/SocialGraphModule.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.socialgraph - -import com.google.inject.Provides -import com.twitter.finagle.ThriftMux -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.common.clients.common.BaseClientModule -import com.twitter.socialgraph.thriftscala.SocialGraphService -import com.twitter.stitch.socialgraph.SocialGraph -import javax.inject.Singleton - -object SocialGraphModule - extends BaseClientModule[SocialGraphService.MethodPerEndpoint] - with MtlsClient { - override val label = "social-graph-service" - override val dest = "/s/socialgraph/socialgraph" - - override def configureThriftMuxClient(client: ThriftMux.Client): ThriftMux.Client = - client.withSessionQualifier.noFailFast - - @Provides - @Singleton - def providesStitchClient(futureIface: SocialGraphService.MethodPerEndpoint): SocialGraph = { - SocialGraph(futureIface) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD deleted file mode 100644 index 3661887c8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "src/scala/com/twitter/onboarding/relevance/candidate_generation/utt/models", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/frigate/data_pipeline:frigate-user-history-thrift-scala", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/hermit/pop_geo:hermit-pop-geo-scala", - "src/thrift/com/twitter/onboarding/relevance/relatable_accounts:relatable_accounts-scala", - "src/thrift/com/twitter/onboarding/relevance/store:store-scala", - "src/thrift/com/twitter/recos/user_user_graph:user_user_graph-scala", - "src/thrift/com/twitter/search/account_search/extended_network:extended_network_users-scala", - "src/thrift/com/twitter/service/metastore/gen:thrift-scala", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "src/thrift/com/twitter/wtf/ml:wtf-ml-output-thrift-scala", - "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-scala", - "src/thrift/com/twitter/wtf/triangular_loop:triangular_loop-scala", - "strato/src/main/scala/com/twitter/strato/client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.scala deleted file mode 100644 index 4046ac754..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato/StratoClientModule.scala +++ /dev/null @@ -1,249 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.strato - -import com.google.inject.name.Named -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState -import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserKey -import com.twitter.search.account_search.extended_network.thriftscala.ExtendedNetworkUserVal -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.thrift.Protocols -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.constants.ServiceConstants._ -import com.twitter.frigate.data_pipeline.candidate_generation.thriftscala.LatestEvents -import com.twitter.hermit.candidate.thriftscala.{Candidates => HermitCandidates} -import com.twitter.hermit.pop_geo.thriftscala.PopUsersInPlace -import com.twitter.onboarding.relevance.relatable_accounts.thriftscala.RelatableAccounts -import com.twitter.inject.TwitterModule -import com.twitter.onboarding.relevance.candidates.thriftscala.InterestBasedUserRecommendations -import com.twitter.onboarding.relevance.candidates.thriftscala.UTTInterest -import com.twitter.onboarding.relevance.store.thriftscala.WhoToFollowDismissEventDetails -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserRequest -import com.twitter.recos.user_user_graph.thriftscala.RecommendUserResponse -import com.twitter.service.metastore.gen.thriftscala.UserRecommendabilityFeatures -import com.twitter.strato.catalog.Scan.Slice -import com.twitter.strato.client.Strato.{Client => StratoClient} -import com.twitter.strato.client.Client -import com.twitter.strato.client.Fetcher -import com.twitter.strato.client.Scanner -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.wtf.candidate.thriftscala.CandidateSeq -import com.twitter.wtf.ml.thriftscala.CandidateFeatures -import com.twitter.wtf.real_time_interaction_graph.thriftscala.Interaction -import com.twitter.wtf.triangular_loop.thriftscala.{Candidates => TriangularLoopCandidates} -import com.twitter.strato.opcontext.Attribution._ - -object StratoClientModule extends TwitterModule { - - // column paths - val CosineFollowPath = "recommendations/similarity/similarUsersByFollowGraph.User" - val CosineListPath = "recommendations/similarity/similarUsersByListGraph.User" - val CuratedCandidatesPath = "onboarding/curatedAccounts" - val CuratedFilteredAccountsPath = "onboarding/filteredAccountsFromRecommendations" - val PopUsersInPlacePath = "onboarding/userrecs/popUsersInPlace" - val ProfileSidebarBlacklistPath = "recommendations/hermit/profile-sidebar-blacklist" - val RealTimeInteractionsPath = "hmli/realTimeInteractions" - val SimsPath = "recommendations/similarity/similarUsersBySims.User" - val DBV2SimsPath = "onboarding/userrecs/newSims.User" - val TriangularLoopsPath = "onboarding/userrecs/triangularLoops.User" - val TwoHopRandomWalkPath = "onboarding/userrecs/twoHopRandomWalk.User" - val UserRecommendabilityPath = "onboarding/userRecommendabilityWithLongKeys.User" - val UTTAccountRecommendationsPath = "onboarding/userrecs/utt_account_recommendations" - val UttSeedAccountsRecommendationPath = "onboarding/userrecs/utt_seed_accounts" - val UserStatePath = "onboarding/userState.User" - val WTFPostNuxFeaturesPath = "ml/featureStore/onboarding/wtfPostNuxFeatures.User" - val ElectionCandidatesPath = "onboarding/electionAccounts" - val UserUserGraphPath = "recommendations/userUserGraph" - val WtfDissmissEventsPath = "onboarding/wtfDismissEvents" - val RelatableAccountsPath = "onboarding/userrecs/relatableAccounts" - val ExtendedNetworkCandidatesPath = "search/account_search/extendedNetworkCandidatesMH" - val LabeledNotificationPath = "frigate/magicrecs/labeledPushRecsAggregated.User" - - @Provides - @Singleton - def stratoClient(serviceIdentifier: ServiceIdentifier): Client = { - val timeoutBudget = 500.milliseconds - StratoClient( - ThriftMux.client - .withRequestTimeout(timeoutBudget) - .withProtocolFactory(Protocols.binaryFactory( - stringLengthLimit = StringLengthLimit, - containerLengthLimit = ContainerLengthLimit))) - .withMutualTls(serviceIdentifier) - .build() - } - - // add strato putters, fetchers, scanners below: - @Provides - @Singleton - @Named(GuiceNamedConstants.COSINE_FOLLOW_FETCHER) - def cosineFollowFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](CosineFollowPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.COSINE_LIST_FETCHER) - def cosineListFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](CosineListPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) - def curatedBlacklistedAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = - stratoClient.fetcher[String, Unit, Seq[Long]](CuratedFilteredAccountsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.CURATED_CANDIDATES_FETCHER) - def curatedCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = - stratoClient.fetcher[String, Unit, Seq[Long]](CuratedCandidatesPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.POP_USERS_IN_PLACE_FETCHER) - def popUsersInPlaceFetcher(stratoClient: Client): Fetcher[String, Unit, PopUsersInPlace] = - stratoClient.fetcher[String, Unit, PopUsersInPlace](PopUsersInPlacePath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.RELATABLE_ACCOUNTS_FETCHER) - def relatableAccountsFetcher(stratoClient: Client): Fetcher[String, Unit, RelatableAccounts] = - stratoClient.fetcher[String, Unit, RelatableAccounts](RelatableAccountsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.PROFILE_SIDEBAR_BLACKLIST_SCANNER) - def profileSidebarBlacklistScanner( - stratoClient: Client - ): Scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit] = - stratoClient.scanner[(Long, Slice[Long]), Unit, (Long, Long), Unit](ProfileSidebarBlacklistPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.REAL_TIME_INTERACTIONS_FETCHER) - def realTimeInteractionsFetcher( - stratoClient: Client - ): Fetcher[(Long, Long), Unit, Seq[Interaction]] = - stratoClient.fetcher[(Long, Long), Unit, Seq[Interaction]](RealTimeInteractionsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.SIMS_FETCHER) - def simsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](SimsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.DBV2_SIMS_FETCHER) - def dbv2SimsFetcher(stratoClient: Client): Fetcher[Long, Unit, HermitCandidates] = - stratoClient.fetcher[Long, Unit, HermitCandidates](DBV2SimsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.TRIANGULAR_LOOPS_FETCHER) - def triangularLoopsFetcher(stratoClient: Client): Fetcher[Long, Unit, TriangularLoopCandidates] = - stratoClient.fetcher[Long, Unit, TriangularLoopCandidates](TriangularLoopsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.TWO_HOP_RANDOM_WALK_FETCHER) - def twoHopRandomWalkFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateSeq] = - stratoClient.fetcher[Long, Unit, CandidateSeq](TwoHopRandomWalkPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) - def userRecommendabilityFetcher( - stratoClient: Client - ): Fetcher[Long, Unit, UserRecommendabilityFeatures] = - stratoClient.fetcher[Long, Unit, UserRecommendabilityFeatures](UserRecommendabilityPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.USER_STATE_FETCHER) - def userStateFetcher(stratoClient: Client): Fetcher[Long, Unit, CondensedUserState] = - stratoClient.fetcher[Long, Unit, CondensedUserState](UserStatePath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.UTT_ACCOUNT_RECOMMENDATIONS_FETCHER) - def uttAccountRecommendationsFetcher( - stratoClient: Client - ): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = - stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( - UTTAccountRecommendationsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.UTT_SEED_ACCOUNTS_FETCHER) - def uttSeedAccountRecommendationsFetcher( - stratoClient: Client - ): Fetcher[UTTInterest, Unit, InterestBasedUserRecommendations] = - stratoClient.fetcher[UTTInterest, Unit, InterestBasedUserRecommendations]( - UttSeedAccountsRecommendationPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.ELECTION_CANDIDATES_FETCHER) - def electionCandidatesFetcher(stratoClient: Client): Fetcher[String, Unit, Seq[Long]] = - stratoClient.fetcher[String, Unit, Seq[Long]](ElectionCandidatesPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.USER_USER_GRAPH_FETCHER) - def userUserGraphFetcher( - stratoClient: Client - ): Fetcher[RecommendUserRequest, Unit, RecommendUserResponse] = - stratoClient.fetcher[RecommendUserRequest, Unit, RecommendUserResponse](UserUserGraphPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.POST_NUX_WTF_FEATURES_FETCHER) - def wtfPostNuxFeaturesFetcher(stratoClient: Client): Fetcher[Long, Unit, CandidateFeatures] = { - val attribution = ManhattanAppId("starbuck", "wtf_starbuck") - stratoClient - .fetcher[Long, Unit, CandidateFeatures](WTFPostNuxFeaturesPath) - .withAttribution(attribution) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.EXTENDED_NETWORK) - def extendedNetworkFetcher( - stratoClient: Client - ): Fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal] = { - stratoClient - .fetcher[ExtendedNetworkUserKey, Unit, ExtendedNetworkUserVal](ExtendedNetworkCandidatesPath) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.DISMISS_STORE_SCANNER) - def dismissStoreScanner( - stratoClient: Client - ): Scanner[ - (Long, Slice[(Long, Long)]), - Unit, - (Long, (Long, Long)), - WhoToFollowDismissEventDetails - ] = - stratoClient.scanner[ - (Long, Slice[(Long, Long)]), // PKEY: userId, LKEY: (-ts, candidateId) - Unit, - (Long, (Long, Long)), - WhoToFollowDismissEventDetails - ](WtfDissmissEventsPath) - - @Provides - @Singleton - @Named(GuiceNamedConstants.LABELED_NOTIFICATION_FETCHER) - def labeledNotificationFetcher( - stratoClient: Client - ): Fetcher[Long, Unit, LatestEvents] = { - stratoClient - .fetcher[Long, Unit, LatestEvents](LabeledNotificationPath) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD deleted file mode 100644 index 16a82a302..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "stitch/stitch-core", - "strato/src/main/scala/com/twitter/strato/client", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.scala deleted file mode 100644 index fe8101261..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state/UserStateClient.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.follow_recommendations.common.clients.user_state - -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps._ -import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient -import com.twitter.follow_recommendations.common.clients.cache.ThriftEnumOptionBijection -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton -import java.lang.{Long => JLong} - -@Singleton -class UserStateClient @Inject() ( - @Named(GuiceNamedConstants.USER_STATE_FETCHER) userStateFetcher: Fetcher[ - Long, - Unit, - CondensedUserState - ], - client: Client, - statsReceiver: StatsReceiver, - decider: Decider = Decider.False) { - - private val stats: StatsReceiver = statsReceiver.scope("user_state_client") - - // client to memcache cluster - val bijection = new ThriftEnumOptionBijection[UserState](UserState.apply) - val memcacheClient = MemcacheClient[Option[UserState]]( - client = client, - dest = "/s/cache/follow_recos_service:twemcaches", - valueBijection = bijection, - ttl = UserStateClient.CacheTTL, - statsReceiver = stats.scope("twemcache") - ) - - def getUserState(userId: Long): Stitch[Option[UserState]] = { - val deciderKey: String = DeciderKey.EnableDistributedCaching.toString - val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - val userStateStitch: Stitch[Option[UserState]] = - enableDistributedCaching match { - // read from memcache - case true => memcacheClient.readThrough( - // add a key prefix to address cache key collisions - key = "UserStateClient" + userId.toString, - underlyingCall = () => fetchUserState(userId) - ) - case false => fetchUserState(userId) - } - val userStateStitchWithTimeout: Stitch[Option[UserState]] = - userStateStitch - // set a 150ms timeout limit for user state fetches - .within(150.milliseconds)(DefaultTimer) - .rescue { - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - Stitch(None) - } - // profile the latency of stitch call and return the result - StatsUtil.profileStitch( - userStateStitchWithTimeout, - stats.scope("getUserState") - ) - } - - def fetchUserState(userId: JLong): Stitch[Option[UserState]] = { - userStateFetcher.fetch(userId).map(_.v.flatMap(_.userState)) - } -} - -object UserStateClient { - val CacheTTL: Duration = Duration.fromHours(6) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD deleted file mode 100644 index dc9335d5b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.scala deleted file mode 100644 index 7c7892c02..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/CandidateAlgorithmTypeConstants.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.follow_recommendations.common.constants - -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.hermit.model.Algorithm._ -import com.twitter.follow_recommendations.common.models.AlgorithmType - -object CandidateAlgorithmTypeConstants { - - /** - * Each algorithm is based on one, or more, of the 4 types of information we have on users, - * described in [[AlgorithmType]]. Assignment of algorithms to these categories are based on - */ - private val AlgorithmIdToType: Map[String, Set[AlgorithmType.Value]] = Map( - // Activity Algorithms: - AlgorithmToFeedbackTokenMap(NewFollowingSimilarUser).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(Sims).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(NewFollowingSimilarUserSalsa).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentEngagementNonDirectFollow).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentEngagementSimilarUser).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentEngagementSarusOcCur).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RecentSearchBasedRec).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TwistlyTweetAuthors).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(Follow2VecNearestNeighbors).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(EmailTweetClick).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RepeatedProfileVisits).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(GoodTweetClickEngagements).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TweetShareEngagements).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TweetSharerToShareRecipientEngagements).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TweetAuthorToShareRecipientEngagements).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(LinearRegressionFollow2VecNearestNeighbors).toString -> Set( - AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(NUXLOHistory).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(TrafficAttributionAccounts).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(RealGraphOonV2).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(MagicRecsRecentEngagements).toString -> Set(AlgorithmType.Activity), - AlgorithmToFeedbackTokenMap(NotificationEngagement).toString -> Set(AlgorithmType.Activity), - // Social Algorithms: - AlgorithmToFeedbackTokenMap(TwoHopRandomWalk).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(RealTimeMutualFollow).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ForwardPhoneBook).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ForwardEmailBook).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(NewFollowingNewFollowingExpansion).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(NewFollowingSarusCoOccurSocialProof).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ReverseEmailBookIbis).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(ReversePhoneBook).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(StrongTiePredictionRec).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(StrongTiePredictionRecWithSocialProof).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRec).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRecNoCaching).toString -> Set( - AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(TriangularLoop).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(StrongTiePredictionPmi).toString -> Set(AlgorithmType.Social), - AlgorithmToFeedbackTokenMap(OnlineStrongTiePredictionRAB).toString -> Set(AlgorithmType.Social), - // Geo Algorithms: - AlgorithmToFeedbackTokenMap(PopCountryBackFill).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PopCountry).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PopGeohash).toString -> Set(AlgorithmType.Geo), -// AlgorithmToFeedbackTokenMap(PopGeohashRealGraph).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(EngagedFollowerRatio).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(CrowdSearchAccounts).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(OrganicFollowAccounts).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PopGeohashQualityFollow).toString -> Set(AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(PPMILocaleFollow).toString -> Set(AlgorithmType.Geo), - // Interest Algorithms: - AlgorithmToFeedbackTokenMap(TttInterest).toString -> Set(AlgorithmType.Interest), - AlgorithmToFeedbackTokenMap(UttInterestRelatedUsers).toString -> Set(AlgorithmType.Interest), - AlgorithmToFeedbackTokenMap(UttSeedAccounts).toString -> Set(AlgorithmType.Interest), - AlgorithmToFeedbackTokenMap(UttProducerExpansion).toString -> Set(AlgorithmType.Interest), - // Hybrid (more than one type) Algorithms: - AlgorithmToFeedbackTokenMap(UttProducerOfflineMbcgV1).toString -> Set( - AlgorithmType.Interest, - AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(CuratedAccounts).toString -> Set( - AlgorithmType.Interest, - AlgorithmType.Geo), - AlgorithmToFeedbackTokenMap(UserUserGraph).toString -> Set( - AlgorithmType.Social, - AlgorithmType.Activity), - ) - def getAlgorithmTypes(algoId: String): Set[String] = { - AlgorithmIdToType.get(algoId).map(_.map(_.toString)).getOrElse(Set.empty) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.scala deleted file mode 100644 index d3d61fa43..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/GuiceNamedConstants.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.follow_recommendations.common.constants - -object GuiceNamedConstants { - final val PRODUCER_SIDE_FEATURE_SWITCHES = "PRODUCER_SIDE_FEATURE_SWITCHES" - final val CLIENT_EVENT_LOGGER = "CLIENT_EVENT_LOGGER" - final val COSINE_FOLLOW_FETCHER = "cosine_follow_fetcher" - final val COSINE_LIST_FETCHER = "cosine_list_fetcher" - final val CURATED_CANDIDATES_FETCHER = "curated_candidates_fetcher" - final val CURATED_COMPETITOR_ACCOUNTS_FETCHER = "curated_competitor_accounts_fetcher" - final val POP_USERS_IN_PLACE_FETCHER = "pop_users_in_place_fetcher" - final val PROFILE_SIDEBAR_BLACKLIST_SCANNER = "profile_sidebar_blacklist_scanner" - final val REQUEST_LOGGER = "REQUEST_LOGGER" - final val FLOW_LOGGER = "FLOW_LOGGER" - final val REAL_TIME_INTERACTIONS_FETCHER = "real_time_interactions_fetcher" - final val SIMS_FETCHER = "sims_fetcher" - final val DBV2_SIMS_FETCHER = "dbv2_sims_fetcher" - - final val TRIANGULAR_LOOPS_FETCHER = "triangular_loops_fetcher" - final val TWO_HOP_RANDOM_WALK_FETCHER = "two_hop_random_walk_fetcher" - final val USER_RECOMMENDABILITY_FETCHER = "user_recommendability_fetcher" - final val USER_STATE_FETCHER = "user_state_fetcher" - final val UTT_ACCOUNT_RECOMMENDATIONS_FETCHER = "utt_account_recomendations_fetcher" - final val UTT_SEED_ACCOUNTS_FETCHER = "utt_seed_accounts_fetcher" - - final val ELECTION_CANDIDATES_FETCHER = "election_candidates_fetcher" - final val POST_NUX_WTF_FEATURES_FETCHER = "post_nux_wtf_features_fetcher" - - final val USER_USER_GRAPH_FETCHER = "user_user_graph_fetcher" - final val DISMISS_STORE_SCANNER = "dismiss_store_scanner" - final val LABELED_NOTIFICATION_FETCHER = "labeled_notification_scanner" - - final val STP_EP_SCORER = "stp_ep_scorer" - final val STP_DBV2_SCORER = "stp_dbv2_scorer" - final val STP_RAB_DBV2_SCORER = "stp_rab_dbv2_scorer" - - final val EXTENDED_NETWORK = "extended_network_candidates" - - // scoring client constants - final val WTF_PROD_DEEPBIRDV2_CLIENT = "wtf_prod_deepbirdv2_client" - - // ann clients - final val RELATABLE_ACCOUNTS_FETCHER = "relatable_accounts_fetcher" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.scala deleted file mode 100644 index 6aade704c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants/ServiceConstants.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.constants - -import com.twitter.conversions.StorageUnitOps._ - -object ServiceConstants { - - /** thrift client response size limits - * these were estimated using monitoring dashboard - * 3MB network usage per second / 25 rps ~ 120KB/req << 1MB - * we give some buffer here in case some requests require more data than others - */ - val StringLengthLimit: Long = - 10.megabyte.inBytes - val ContainerLengthLimit: Long = 1.megabyte.inBytes -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD deleted file mode 100644 index b4afee590..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/BUILD +++ /dev/null @@ -1,82 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":candidate-algorithm-adapter", - ":client-context-adapter", - ":post-nux-algorithm-adapter", - ":pre-fetched-feature-adapter", - ], -) - -target( - name = "common", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/onboarding/relevance/util/metadata", - "util/util-slf4j-api/src/main/scala", - ], -) - -scala_library( - name = "candidate-algorithm-adapter", - sources = [ - "CandidateAlgorithmAdapter.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - ], -) - -scala_library( - name = "client-context-adapter", - sources = [ - "ClientContextAdapter.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - "snowflake/src/main/scala/com/twitter/snowflake/id", - ], -) - -scala_library( - name = "post-nux-algorithm-adapter", - sources = [ - "PostNuxAlgorithmAdapter.scala", - ], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - "src/scala/com/twitter/ml/featurestore/catalog/features/customer_journey:post-nux-algorithm-aggregate", - ], -) - -scala_library( - name = "pre-fetched-feature-adapter", - sources = [ - "PreFetchedFeatureAdapter.scala", - ], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":common", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.scala deleted file mode 100644 index 7a487a95b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/CandidateAlgorithmAdapter.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.hermit.model.Algorithm -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.hermit.model.Algorithm.UttProducerOfflineMbcgV1 -import com.twitter.hermit.model.Algorithm.UttProducerOnlineMbcgV1 -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature.SparseBinary -import com.twitter.ml.api.Feature.SparseContinuous -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.api.util.FDsl._ - -object CandidateAlgorithmAdapter - extends IRecordOneToOneAdapter[Option[UserCandidateSourceDetails]] { - - val CANDIDATE_ALGORITHMS: SparseBinary = new SparseBinary("candidate.source.algorithm_ids") - val CANDIDATE_SOURCE_SCORES: SparseContinuous = - new SparseContinuous("candidate.source.scores") - val CANDIDATE_SOURCE_RANKS: SparseContinuous = - new SparseContinuous("candidate.source.ranks") - - override val getFeatureContext: FeatureContext = new FeatureContext( - CANDIDATE_ALGORITHMS, - CANDIDATE_SOURCE_SCORES, - CANDIDATE_SOURCE_RANKS - ) - - /** list of candidate source remaps to avoid creating different features for experimental sources. - * the LHS should contain the experimental source, and the RHS should contain the prod source. - */ - def remapCandidateSource(a: Algorithm): Algorithm = a match { - case UttProducerOnlineMbcgV1 => UttProducerOfflineMbcgV1 - case _ => a - } - - // add the list of algorithm feedback tokens (integers) as a sparse binary feature - override def adaptToDataRecord( - userCandidateSourceDetailsOpt: Option[UserCandidateSourceDetails] - ): DataRecord = { - val dr = new DataRecord() - userCandidateSourceDetailsOpt.foreach { userCandidateSourceDetails => - val scoreMap = for { - (source, scoreOpt) <- userCandidateSourceDetails.candidateSourceScores - score <- scoreOpt - algo <- Algorithm.withNameOpt(source.name) - algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - } yield algoId.toString -> score - val rankMap = for { - (source, rank) <- userCandidateSourceDetails.candidateSourceRanks - algo <- Algorithm.withNameOpt(source.name) - algoId <- AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - } yield algoId.toString -> rank.toDouble - - val algoIds = scoreMap.keys.toSet ++ rankMap.keys.toSet - - // hydrate if not empty - if (rankMap.nonEmpty) { - dr.setFeatureValue(CANDIDATE_SOURCE_RANKS, rankMap) - } - if (scoreMap.nonEmpty) { - dr.setFeatureValue(CANDIDATE_SOURCE_SCORES, scoreMap) - } - if (algoIds.nonEmpty) { - dr.setFeatureValue(CANDIDATE_ALGORITHMS, algoIds) - } - } - dr - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.scala deleted file mode 100644 index 9aa4bdb0d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/ClientContextAdapter.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.ml.api.Feature.Binary -import com.twitter.ml.api.Feature.Continuous -import com.twitter.ml.api.Feature.Discrete -import com.twitter.ml.api.Feature.Text -import com.twitter.ml.api.util.FDsl._ -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.onboarding.relevance.util.metadata.LanguageUtil -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.snowflake.id.SnowflakeId - -object ClientContextAdapter extends IRecordOneToOneAdapter[(ClientContext, DisplayLocation)] { - - // we name features with `user.account` for relatively static user-related features - val USER_COUNTRY: Text = new Text("user.account.country") - val USER_LANGUAGE: Text = new Text("user.account.language") - // we name features with `user.context` for more dynamic user-related features - val USER_LANGUAGE_PREFIX: Text = new Text("user.context.language_prefix") - val USER_CLIENT: Discrete = new Discrete("user.context.client") - val USER_AGE: Continuous = new Continuous("user.context.age") - val USER_IS_RECENT: Binary = new Binary("user.is.recent") - // we name features with `meta` for meta info about the WTF recommendation request - val META_DISPLAY_LOCATION: Text = new Text("meta.display_location") - val META_POSITION: Discrete = new Discrete("meta.position") - // This indicates whether a data point is from a random serving policy - val META_IS_RANDOM: Binary = new Binary("prediction.engine.is_random") - - val RECENT_WIN_IN_DAYS: Int = 30 - val GOAL_META_POSITION: Long = 1L - val GOAL_META_IS_RANDOM: Boolean = true - - override val getFeatureContext: FeatureContext = new FeatureContext( - USER_COUNTRY, - USER_LANGUAGE, - USER_AGE, - USER_LANGUAGE_PREFIX, - USER_CLIENT, - USER_IS_RECENT, - META_DISPLAY_LOCATION, - META_POSITION, - META_IS_RANDOM - ) - - /** - * we only want to set the relevant fields iff they exist to eliminate redundant information - * we do some simple normalization on the language code - * we set META_POSITION to 1 always - * we set META_IS_RANDOM to true always to simulate a random serving distribution - * @param record ClientContext and DisplayLocation from the request - */ - override def adaptToDataRecord(target: (ClientContext, DisplayLocation)): DataRecord = { - val dr = new DataRecord() - val cc = target._1 - val dl = target._2 - cc.countryCode.foreach(countryCode => dr.setFeatureValue(USER_COUNTRY, countryCode)) - cc.languageCode.foreach(rawLanguageCode => { - val userLanguage = LanguageUtil.simplifyLanguage(rawLanguageCode) - val userLanguagePrefix = userLanguage.take(2) - dr.setFeatureValue(USER_LANGUAGE, userLanguage) - dr.setFeatureValue(USER_LANGUAGE_PREFIX, userLanguagePrefix) - }) - cc.appId.foreach(appId => dr.setFeatureValue(USER_CLIENT, appId)) - cc.userId.foreach(id => - SnowflakeId.timeFromIdOpt(id).map { signupTime => - val userAge = signupTime.untilNow.inMillis.toDouble - dr.setFeatureValue(USER_AGE, userAge) - dr.setFeatureValue(USER_IS_RECENT, signupTime.untilNow.inDays <= RECENT_WIN_IN_DAYS) - signupTime.untilNow.inDays - }) - dr.setFeatureValue(META_DISPLAY_LOCATION, dl.toFsName) - dr.setFeatureValue(META_POSITION, GOAL_META_POSITION) - dr.setFeatureValue(META_IS_RANDOM, GOAL_META_IS_RANDOM) - dr - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.scala deleted file mode 100644 index e8fe745a0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PostNuxAlgorithmAdapter.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.Feature.Continuous -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.api.util.FDsl._ -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmFeatures -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmIdAggregateFeatureGroup -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmTypeAggregateFeatureGroup -import scala.collection.JavaConverters._ - -object PostNuxAlgorithmIdAdapter extends PostNuxAlgorithmAdapter { - override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures = - PostNuxAlgorithmIdAggregateFeatureGroup - - // To keep the length of feature names reasonable, we remove the prefix added by FeatureStore. - override val FeatureStorePrefix: String = - "wtf_algorithm_id.customer_journey.post_nux_algorithm_id_aggregate_feature_group." -} - -object PostNuxAlgorithmTypeAdapter extends PostNuxAlgorithmAdapter { - override val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures = - PostNuxAlgorithmTypeAggregateFeatureGroup - - // To keep the length of feature names reasonable, we remove the prefix added by FeatureStore. - override val FeatureStorePrefix: String = - "wtf_algorithm_type.customer_journey.post_nux_algorithm_type_aggregate_feature_group." -} - -trait PostNuxAlgorithmAdapter extends IRecordOneToOneAdapter[DataRecord] { - - val PostNuxAlgorithmFeatureGroup: PostNuxAlgorithmFeatures - - // The string that is attached to the feature name when it is fetched from feature store. - val FeatureStorePrefix: String - - /** - * - * This stores transformed aggregate features for PostNux algorithm aggregate features. The - * transformation here is log-ratio, where ratio is the raw value divided by # of impressions. - */ - case class TransformedAlgorithmFeatures( - ratioLog: Continuous) { - def getFeatures: Seq[Continuous] = Seq(ratioLog) - } - - private def applyFeatureStorePrefix(feature: Continuous) = new Continuous( - s"$FeatureStorePrefix${feature.getFeatureName}") - - // The list of input features WITH the prefix assigned to them by FeatureStore. - lazy val allInputFeatures: Seq[Seq[Continuous]] = Seq( - PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures.map(applyFeatureStorePrefix), - PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures.map(applyFeatureStorePrefix) - ) - - // This is a list of the features WITHOUT the prefix assigned to them by FeatureStore. - lazy val outputBaseFeatureNames: Seq[Seq[Continuous]] = Seq( - PostNuxAlgorithmFeatureGroup.Aggregate7DayFeatures, - PostNuxAlgorithmFeatureGroup.Aggregate30DayFeatures - ) - - // We use backend impression to calculate ratio values. - lazy val ratioDenominators: Seq[Continuous] = Seq( - applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions7Days), - applyFeatureStorePrefix(PostNuxAlgorithmFeatureGroup.BackendImpressions30Days) - ) - - /** - * A mapping from an original feature's ID to the corresponding set of transformed features. - * This is used to compute the transformed features for each of the original ones. - */ - private lazy val TransformedFeaturesMap: Map[Continuous, TransformedAlgorithmFeatures] = - outputBaseFeatureNames.flatten.map { feature => - ( - // The input feature would have the FeatureStore prefix attached to it. - new Continuous(s"$FeatureStorePrefix${feature.getFeatureName}"), - // We don't keep the FeatureStore prefix to keep the length of feature names reasonable. - TransformedAlgorithmFeatures( - new Continuous(s"${feature.getFeatureName}-ratio-log") - )) - }.toMap - - /** - * Given a denominator, number of impressions, this function returns another function that adds - * transformed features (log1p and ratio) of an input feature to a DataRecord. - */ - private def addTransformedFeaturesToDataRecordFunc( - originalDr: DataRecord, - numImpressions: Double, - ): (DataRecord, Continuous) => DataRecord = { (record: DataRecord, feature: Continuous) => - { - Option(originalDr.getFeatureValue(feature)) foreach { featureValue => - TransformedFeaturesMap.get(feature).foreach { transformedFeatures => - record.setFeatureValue( - transformedFeatures.ratioLog, - // We don't use log1p here since the values are ratios and adding 1 to the _ratio_ would - // lead to logarithm of values between 1 and 2, essentially making all values the same. - math.log((featureValue + 1) / numImpressions) - ) - } - } - record - } - } - - /** - * @param record: The input record whose PostNuxAlgorithm aggregates are to be transformed. - * @return the input [[DataRecord]] with transformed aggregates added. - */ - override def adaptToDataRecord(record: DataRecord): DataRecord = { - if (record.continuousFeatures == null) { - // There are no base features available, and hence no transformations. - record - } else { - - /** - * The `foldLeft` below goes through pairs of (1) Feature groups, such as those calculated over - * 7 days or 30 days, and (2) the number of impressions for each of these groups, which is the - * denominator when ratio is calculated. - */ - ratioDenominators - .zip(allInputFeatures).foldLeft( /* initial empty DataRecord */ record)( - ( - /* DataRecord with transformed features up to here */ transformedRecord, - /* A tuple with the denominator (#impressions) and features to be transformed */ numImpressionsAndFeatures - ) => { - val (numImpressionsFeature, features) = numImpressionsAndFeatures - Option(record.getFeatureValue(numImpressionsFeature)) match { - case Some(numImpressions) if numImpressions > 0.0 => - /** - * With the number of impressions fixed, we generate a function that adds log-ratio - * for each feature in the current [[DataRecord]]. The `foldLeft` goes through all - * such features and applies that function while updating the kept DataRecord. - */ - features.foldLeft(transformedRecord)( - addTransformedFeaturesToDataRecordFunc(record, numImpressions)) - case _ => - transformedRecord - } - }) - } - } - - def getFeatures: Seq[Feature[_]] = TransformedFeaturesMap.values.flatMap(_.getFeatures).toSeq - - override def getFeatureContext: FeatureContext = - new FeatureContext() - .addFeatures(this.getFeatures.asJava) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.scala deleted file mode 100644 index f24ed1efe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters/PreFetchedFeatureAdapter.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.adapters - -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.ml.api.Feature.Continuous -import com.twitter.ml.api.util.FDsl._ -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.util.Time - -/** - * This adapter mimics UserRecentWTFImpressionsAndFollowsAdapter (for user) and - * RecentWTFImpressionsFeatureAdapter (for candidate) for extracting recent impression - * and follow features. This adapter extracts user, candidate, and pair-wise features. - */ -object PreFetchedFeatureAdapter - extends IRecordOneToOneAdapter[ - (HasPreFetchedFeature, CandidateUser) - ] { - - // impression features - val USER_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous( - "user.prefetch.num_recent_impressions" - ) - val USER_LAST_IMPRESSION_DURATION: Continuous = new Continuous( - "user.prefetch.last_impression_duration" - ) - val CANDIDATE_NUM_RECENT_IMPRESSIONS: Continuous = new Continuous( - "user-candidate.prefetch.num_recent_impressions" - ) - val CANDIDATE_LAST_IMPRESSION_DURATION: Continuous = new Continuous( - "user-candidate.prefetch.last_impression_duration" - ) - // follow features - val USER_NUM_RECENT_FOLLOWERS: Continuous = new Continuous( - "user.prefetch.num_recent_followers" - ) - val USER_NUM_RECENT_FOLLOWED_BY: Continuous = new Continuous( - "user.prefetch.num_recent_followed_by" - ) - val USER_NUM_RECENT_MUTUAL_FOLLOWS: Continuous = new Continuous( - "user.prefetch.num_recent_mutual_follows" - ) - // impression + follow features - val USER_NUM_RECENT_FOLLOWED_IMPRESSIONS: Continuous = new Continuous( - "user.prefetch.num_recent_followed_impression" - ) - val USER_LAST_FOLLOWED_IMPRESSION_DURATION: Continuous = new Continuous( - "user.prefetch.last_followed_impression_duration" - ) - - override def adaptToDataRecord( - record: (HasPreFetchedFeature, CandidateUser) - ): DataRecord = { - val (target, candidate) = record - val dr = new DataRecord() - val t = Time.now - // set impression features for user, optionally for candidate - dr.setFeatureValue(USER_NUM_RECENT_IMPRESSIONS, target.numWtfImpressions.toDouble) - dr.setFeatureValue( - USER_LAST_IMPRESSION_DURATION, - (t - target.latestImpressionTime).inMillis.toDouble) - target.getCandidateImpressionCounts(candidate.id).foreach { counts => - dr.setFeatureValue(CANDIDATE_NUM_RECENT_IMPRESSIONS, counts.toDouble) - } - target.getCandidateLatestTime(candidate.id).foreach { latestTime: Time => - dr.setFeatureValue(CANDIDATE_LAST_IMPRESSION_DURATION, (t - latestTime).inMillis.toDouble) - } - // set recent follow features for user - dr.setFeatureValue(USER_NUM_RECENT_FOLLOWERS, target.numRecentFollowedUserIds.toDouble) - dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_BY, target.numRecentFollowedByUserIds.toDouble) - dr.setFeatureValue(USER_NUM_RECENT_MUTUAL_FOLLOWS, target.numRecentMutualFollows.toDouble) - dr.setFeatureValue(USER_NUM_RECENT_FOLLOWED_IMPRESSIONS, target.numFollowedImpressions.toDouble) - dr.setFeatureValue( - USER_LAST_FOLLOWED_IMPRESSION_DURATION, - target.lastFollowedImpressionDurationMs.getOrElse(Long.MaxValue).toDouble) - dr - } - override def getFeatureContext: FeatureContext = new FeatureContext( - USER_NUM_RECENT_IMPRESSIONS, - USER_LAST_IMPRESSION_DURATION, - CANDIDATE_NUM_RECENT_IMPRESSIONS, - CANDIDATE_LAST_IMPRESSION_DURATION, - USER_NUM_RECENT_FOLLOWERS, - USER_NUM_RECENT_FOLLOWED_BY, - USER_NUM_RECENT_MUTUAL_FOLLOWS, - USER_NUM_RECENT_FOLLOWED_IMPRESSIONS, - USER_LAST_FOLLOWED_IMPRESSION_DURATION, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD deleted file mode 100644 index 93ddb1191..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "src/java/com/twitter/ml/api:api-base", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.scala deleted file mode 100644 index 9d43f0c13..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSource.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.common - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -trait FeatureSource { - def id: FeatureSourceId - def featureContext: FeatureContext - def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.scala deleted file mode 100644 index 66b883120..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/FeatureSourceId.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.common - -sealed trait FeatureSourceId - -object FeatureSourceId { - object CandidateAlgorithmSourceId extends FeatureSourceId - object ClientContextSourceId extends FeatureSourceId - object FeatureStoreSourceId extends FeatureSourceId - object FeatureStoreTimelinesAuthorSourceId extends FeatureSourceId - object FeatureStoreGizmoduckSourceId extends FeatureSourceId - object FeatureStoreUserMetricCountsSourceId extends FeatureSourceId - object FeatureStoreNotificationSourceId extends FeatureSourceId - - object FeatureStorePrecomputedNotificationSourceId extends FeatureSourceId - object FeatureStorePostNuxAlgorithmSourceId extends FeatureSourceId - @deprecated object StratoFeatureHydrationSourceId extends FeatureSourceId - object PreFetchedFeatureSourceId extends FeatureSourceId - object UserScoringFeatureSourceId extends FeatureSourceId -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.scala deleted file mode 100644 index 8f4bae887..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common/HasPreFetchedFeature.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.common - -import com.twitter.follow_recommendations.common.models.HasMutualFollowedUserIds -import com.twitter.follow_recommendations.common.models.HasWtfImpressions -import com.twitter.follow_recommendations.common.models.WtfImpression -import com.twitter.util.Time - -trait HasPreFetchedFeature extends HasMutualFollowedUserIds with HasWtfImpressions { - - lazy val followedImpressions: Seq[WtfImpression] = { - for { - wtfImprList <- wtfImpressions.toSeq - wtfImpr <- wtfImprList - if recentFollowedUserIds.exists(_.contains(wtfImpr.candidateId)) - } yield wtfImpr - } - - lazy val numFollowedImpressions: Int = followedImpressions.size - - lazy val lastFollowedImpressionDurationMs: Option[Long] = { - if (followedImpressions.nonEmpty) { - Some((Time.now - followedImpressions.map(_.latestTime).max).inMillis) - } else None - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD deleted file mode 100644 index c0538240f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/BUILD +++ /dev/null @@ -1,59 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/adapters", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/core:socialgraph", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/core:usersource", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:mc-user-counting", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:user-wtf-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-impression", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-post-nux", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/onboarding:wtf-user-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/datasets/timelines:timelines-author-features", - "src/scala/com/twitter/ml/featurestore/catalog/entities/core", - "src/scala/com/twitter/ml/featurestore/catalog/entities/onboarding", - "src/scala/com/twitter/ml/featurestore/catalog/features/core:socialgraph", - "src/scala/com/twitter/ml/featurestore/catalog/features/core:user", - "src/scala/com/twitter/ml/featurestore/catalog/features/interests_discovery:user-topic-relationships", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmaries", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:non-mr-notif-summmary-aggregates", - "src/scala/com/twitter/ml/featurestore/catalog/features/magicrecs:nonmr-ntab-summaries", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:mc-user-counting", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:post-nux-offline-edge", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:ratio", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:simcluster-user-interested-in-candidate-known-for", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:user-wtf-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:wtf-impression", - "src/scala/com/twitter/ml/featurestore/catalog/features/onboarding:wtf-user-algorithm-aggregate", - "src/scala/com/twitter/ml/featurestore/catalog/features/rux:user-resurrection", - "src/scala/com/twitter/ml/featurestore/catalog/features/timelines:aggregate", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/ml/featurestore/lib/dynamic", - "src/scala/com/twitter/ml/featurestore/lib/embedding", - "src/scala/com/twitter/ml/featurestore/lib/feature", - "src/scala/com/twitter/ml/featurestore/lib/online", - "src/scala/com/twitter/ml/featurestore/lib/params", - "src/scala/com/twitter/onboarding/relevance/adapters/features/featurestore", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "strato/config/columns/ml/featureStore/onboarding:onboarding-strato-client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.scala deleted file mode 100644 index 0838fb98d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/CandidateAlgorithmSource.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Inject -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This source only takes features from the candidate's source, - * which is all the information we have about the candidate pre-feature-hydration - */ - -@Provides -@Singleton -class CandidateAlgorithmSource @Inject() (stats: StatsReceiver) extends FeatureSource { - - override val id: FeatureSourceId = FeatureSourceId.CandidateAlgorithmSourceId - - override val featureContext: FeatureContext = CandidateAlgorithmAdapter.getFeatureContext - - override def hydrateFeatures( - t: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, // we don't use the target here - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - val featureHydrationStats = stats.scope("candidate_alg_source") - val hasSourceDetailsStat = featureHydrationStats.counter("has_source_details") - val noSourceDetailsStat = featureHydrationStats.counter("no_source_details") - val noSourceRankStat = featureHydrationStats.counter("no_source_rank") - val hasSourceRankStat = featureHydrationStats.counter("has_source_rank") - val noSourceScoreStat = featureHydrationStats.counter("no_source_score") - val hasSourceScoreStat = featureHydrationStats.counter("has_source_score") - - val candidatesToAlgoMap = for { - candidate <- candidates - } yield { - if (candidate.userCandidateSourceDetails.nonEmpty) { - hasSourceDetailsStat.incr() - candidate.userCandidateSourceDetails.foreach { details => - if (details.candidateSourceRanks.isEmpty) { - noSourceRankStat.incr() - } else { - hasSourceRankStat.incr() - } - if (details.candidateSourceScores.isEmpty) { - noSourceScoreStat.incr() - } else { - hasSourceScoreStat.incr() - } - } - } else { - noSourceDetailsStat.incr() - } - candidate -> CandidateAlgorithmAdapter.adaptToDataRecord(candidate.userCandidateSourceDetails) - } - Stitch.value(candidatesToAlgoMap.toMap) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.scala deleted file mode 100644 index 718502d44..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/ClientContextSource.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.feature_hydration.adapters.ClientContextAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This source only takes features from the request (e.g. client context, WTF display location) - * No external calls are made. - */ -@Provides -@Singleton -class ClientContextSource() extends FeatureSource { - - override val id: FeatureSourceId = FeatureSourceId.ClientContextSourceId - - override val featureContext: FeatureContext = ClientContextAdapter.getFeatureContext - - override def hydrateFeatures( - t: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - Stitch.value( - candidates - .map(_ -> ((t.clientContext, t.displayLocation))).toMap.mapValues( - ClientContextAdapter.adaptToDataRecord)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.scala deleted file mode 100644 index f78ec17cc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFSConfig.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FeatureHydrationSourcesFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures, - FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures, - FeatureStoreSourceParams.EnableCandidateClientFeatures, - FeatureStoreSourceParams.EnableCandidatePrecomputedNotificationFeatures, - FeatureStoreSourceParams.EnableCandidateUserAuthorRealTimeAggregateFeatures, - FeatureStoreSourceParams.EnableCandidateUserFeatures, - FeatureStoreSourceParams.EnableCandidateUserResurrectionFeatures, - FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures, - FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors, - FeatureStoreSourceParams.EnableSeparateClientForGizmoduck, - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting, - FeatureStoreSourceParams.EnableSeparateClientForNotifications, - FeatureStoreSourceParams.EnableSimilarToUserFeatures, - FeatureStoreSourceParams.EnableTargetUserFeatures, - FeatureStoreSourceParams.EnableTargetUserResurrectionFeatures, - FeatureStoreSourceParams.EnableTargetUserWtfImpressionFeatures, - FeatureStoreSourceParams.EnableTopicAggregateFeatures, - FeatureStoreSourceParams.EnableUserCandidateEdgeFeatures, - FeatureStoreSourceParams.EnableUserCandidateWtfImpressionCandidateFeatures, - FeatureStoreSourceParams.EnableUserClientFeatures, - FeatureStoreSourceParams.EnableUserTopicFeatures, - FeatureStoreSourceParams.EnableUserWtfAlgEdgeFeatures, - ) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - FeatureStoreSourceParams.GlobalFetchTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.scala deleted file mode 100644 index fb2232927..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureHydrationSourcesFeatureSwitchKeys.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -object FeatureHydrationSourcesFeatureSwitchKeys { - val EnableAlgorithmAggregateFeatures = "feature_store_source_enable_algorithm_aggregate_features" - val EnableAuthorTopicAggregateFeatures = - "feature_store_source_enable_author_topic_aggregate_features" - val EnableCandidateClientFeatures = "feature_store_source_enable_candidate_client_features" - val EnableCandidateNotificationFeatures = - "feature_store_source_enable_candidate_notification_features" - val EnableCandidatePrecomputedNotificationFeatures = - "feature_store_source_enable_candidate_precomputed_notification_features" - val EnableCandidateUserFeatures = "feature_store_source_enable_candidate_user_features" - val EnableCandidateUserAuthorRealTimeAggregateFeatures = - "feature_store_source_enable_candidate_user_author_rta_features" - val EnableCandidateUserResurrectionFeatures = - "feature_store_source_enable_candidate_user_resurrection_features" - val EnableCandidateUserTimelinesAuthorAggregateFeatures = - "feature_store_source_enable_candidate_user_timelines_author_aggregate_features" - val EnableSimilarToUserFeatures = "feature_store_source_enable_similar_to_user_features" - val EnableTargetUserFeatures = "feature_store_source_enable_target_user_features" - val EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature = - "feature_store_source_enable_target_user_user_author_user_state_rta_features" - val EnableTargetUserResurrectionFeatures = - "feature_store_source_enable_target_user_resurrection_features" - val EnableTargetUserWtfImpressionFeatures = - "feature_store_source_enable_target_user_wtf_impression_features" - val EnableTopicAggregateFeatures = "feature_store_source_enable_topic_aggregate_features" - val EnableUserCandidateEdgeFeatures = "feature_store_source_enable_user_candidate_edge_features" - val EnableUserCandidateWtfImpressionCandidateFeatures = - "feature_store_source_enable_user_candidate_wtf_impression_features" - val EnableUserClientFeatures = "feature_store_source_enable_user_client_features" - val EnableUserNotificationFeatures = "feature_store_source_enable_user_notification_features" - val EnableUserTopicFeatures = "feature_store_source_enable_user_topic_features" - val EnableUserWtfAlgEdgeFeatures = "feature_store_source_enable_user_wtf_alg_edge_features" - val FeatureHydrationTimeout = "feature_store_source_hydration_timeout_in_millis" - val UseSeparateClientForTimelinesAuthor = - "feature_store_source_separate_client_for_timelines_author_data" - val UseSeparateClientMetricCenterUserCounting = - "feature_store_source_separate_client_for_mc_user_counting_data" - val UseSeparateClientForNotifications = "feature_store_source_separate_client_for_notifications" - val UseSeparateClientForGizmoduck = "feature_store_source_separate_client_for_gizmoduck" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.scala deleted file mode 100644 index 6c849f7a7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreFeatures.scala +++ /dev/null @@ -1,342 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{Topic => TopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{UserCandidate => UserCandidateEntity} -import com.twitter.ml.featurestore.catalog.entities.onboarding.UserWtfAlgorithmEntity -import com.twitter.ml.featurestore.catalog.entities.onboarding.{ - WtfAlgorithm => WtfAlgorithmIdEntity -} -import com.twitter.ml.featurestore.catalog.entities.onboarding.{ - WtfAlgorithmType => WtfAlgorithmTypeEntity -} -import com.twitter.ml.featurestore.catalog.features.core.UserClients.FullPrimaryClientVersion -import com.twitter.ml.featurestore.catalog.features.core.UserClients.NumClients -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryClient -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryClientVersion -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryDeviceManufacturer -import com.twitter.ml.featurestore.catalog.features.core.UserClients.PrimaryMobileSdkVersion -import com.twitter.ml.featurestore.catalog.features.core.UserClients.SecondaryClient -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Favorites -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Followers -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Following -import com.twitter.ml.featurestore.catalog.features.core.UserCounts.Tweets -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmIdAggregateFeatureGroup -import com.twitter.ml.featurestore.catalog.features.customer_journey.PostNuxAlgorithmTypeAggregateFeatureGroup -import com.twitter.ml.featurestore.catalog.features.customer_journey.{Utils => FeatureGroupUtils} -import com.twitter.ml.featurestore.catalog.features.interests_discovery.UserTopicRelationships.FollowedTopics -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFavorites -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFavoritesReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollowBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollows -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumFollowsReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumLoginDays -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumLoginTweetImpressions -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumMuteBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumMuted -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumOriginalTweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQualityFollowReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQuoteRetweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumQuoteRetweetsReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumReplies -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRepliesReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRetweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumRetweetsReceived -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumSpamBlocked -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumSpamBlockedBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumTweetImpressions -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumTweets -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUnfollowBacks -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUnfollows -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumUserActiveMinutes -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasMutualFollowed -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasMutualUnfollowed -import com.twitter.ml.featurestore.catalog.features.onboarding.MetricCenterUserCounts.NumWasUnfollowed -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.Country -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.FollowersOverFollowingRatio -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.Language -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.MutualFollowsOverFollowersRatio -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.MutualFollowsOverFollowingRatio -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumFollowers -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumFollowings -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.NumMutualFollows -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.TweepCred -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOffline.UserState -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameCountry -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameLanguage -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.HaveSameUserState -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumFollowersGap -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumFollowingsGap -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.NumMutualFollowsGap -import com.twitter.ml.featurestore.catalog.features.onboarding.PostNuxOfflineEdge.TweepCredGap -import com.twitter.ml.featurestore.catalog.features.onboarding.Ratio.FollowersFollowings -import com.twitter.ml.featurestore.catalog.features.onboarding.Ratio.MutualFollowsFollowing -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.HasIntersection -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionCandidateKnownForScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionClusterIds -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFavCandidateKnownForScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFavScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFollowCandidateKnownForScore -import com.twitter.ml.featurestore.catalog.features.onboarding.SimclusterUserInterestedInCandidateKnownFor.IntersectionUserFollowScore -import com.twitter.ml.featurestore.catalog.features.onboarding.UserWtfAlgorithmAggregate -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateCounts -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateImpressionCounts -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfCandidateImpressionLatestTimestamp -import com.twitter.ml.featurestore.catalog.features.onboarding.WhoToFollowImpression.HomeTimelineWtfLatestTimestamp -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.Follows -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetFavRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetReplies -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetReplyRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetRetweetRate -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsTweetRetweets -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsWithTweetFavs -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.FollowsWithTweetImpressions -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasAnyEngagements -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasForwardEngagements -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.HasReverseEngagements -import com.twitter.ml.featurestore.catalog.features.onboarding.WtfUserAlgorithmAggregate.Impressions -import com.twitter.ml.featurestore.catalog.features.rux.UserResurrection.DaysSinceRecentResurrection -import com.twitter.ml.featurestore.catalog.features.timelines.AuthorTopicAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.EngagementsReceivedByAuthorRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.NegativeEngagementsReceivedByAuthorRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.OriginalAuthorAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.TopicEngagementRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.TopicEngagementUserStateRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.TopicNegativeEngagementUserStateRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.UserEngagementAuthorUserStateRealTimeAggregates -import com.twitter.ml.featurestore.catalog.features.timelines.UserNegativeEngagementAuthorUserStateRealTimeAggregates -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.Feature - -object FeatureStoreFeatures { - import FeatureStoreRawFeatures._ - ///////////////////////////// Target user features //////////////////////// - val targetUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = - (userKeyedFeatures ++ userAlgorithmAggregateFeatures).map(_.bind(UserEntity)) - - val targetUserResurrectionFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userResurrectionFeatures.map(_.bind(UserEntity)) - val targetUserWtfImpressionFeatures: Set[BoundFeature[_ <: EntityId, _]] = - wtfImpressionUserFeatures.map(_.bind(UserEntity)) - val targetUserUserAuthorUserStateRealTimeAggregatesFeature: Set[BoundFeature[_ <: EntityId, _]] = - userAuthorUserStateRealTimeAggregatesFeature.map(_.bind(UserEntity)) - - val targetUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userStatusFeatures.map(_.bind(UserEntity).logarithm1p) - val targetUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = - mcFeatures.map(_.bind(UserEntity).logarithm1p) - - val targetUserClientFeatures: Set[BoundFeature[_ <: EntityId, _]] = - clientFeatures.map(_.bind(UserEntity)) - - ///////////////////////////// Candidate user features //////////////////////// - val candidateUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userKeyedFeatures.map(_.bind(CandidateUserEntity)) - val candidateUserAuthorRealTimeAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - authorAggregateFeatures.map(_.bind(CandidateUserEntity)) - val candidateUserResurrectionFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userResurrectionFeatures.map(_.bind(CandidateUserEntity)) - - val candidateUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userStatusFeatures.map(_.bind(CandidateUserEntity).logarithm1p) - val candidateUserTimelinesAuthorAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(timelinesAuthorAggregateFeatures.bind(CandidateUserEntity)) - val candidateUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = - mcFeatures.map(_.bind(CandidateUserEntity).logarithm1p) - - val candidateUserClientFeatures: Set[BoundFeature[_ <: EntityId, _]] = - clientFeatures.map(_.bind(CandidateUserEntity)) - - val similarToUserFeatures: Set[BoundFeature[_ <: EntityId, _]] = - (userKeyedFeatures ++ authorAggregateFeatures).map(_.bind(AuthorEntity)) - - val similarToUserStatusFeatures: Set[BoundFeature[_ <: EntityId, _]] = - userStatusFeatures.map(_.bind(AuthorEntity).logarithm1p) - val similarToUserTimelinesAuthorAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(timelinesAuthorAggregateFeatures.bind(AuthorEntity)) - val similarToUserMetricCountFeatures: Set[BoundFeature[_ <: EntityId, _]] = - mcFeatures.map(_.bind(AuthorEntity).logarithm1p) - - val userCandidateEdgeFeatures: Set[BoundFeature[_ <: EntityId, _]] = - (simclusterUVIntersectionFeatures ++ userCandidatePostNuxEdgeFeatures).map( - _.bind(UserCandidateEntity)) - val userCandidateWtfImpressionCandidateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - wtfImpressionCandidateFeatures.map(_.bind(UserCandidateEntity)) - - /** - * Aggregate features based on candidate source algorithms. - */ - val postNuxAlgorithmIdAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(PostNuxAlgorithmIdAggregateFeatureGroup.FeaturesAsDataRecord) - .map(_.bind(WtfAlgorithmIdEntity)) - - /** - * Aggregate features based on candidate source algorithm types. There are 4 at the moment: - * Geo, Social, Activity and Interest. - */ - val postNuxAlgorithmTypeAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = - Set(PostNuxAlgorithmTypeAggregateFeatureGroup.FeaturesAsDataRecord) - .map(_.bind(WtfAlgorithmTypeEntity)) - - // user wtf-Algorithm features - val userWtfAlgorithmEdgeFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureGroupUtils.getTimelinesAggregationFrameworkCombinedFeatures( - UserWtfAlgorithmAggregate, - UserWtfAlgorithmEntity, - FeatureGroupUtils.getMaxSumAvgAggregate(UserWtfAlgorithmAggregate) - ) - - /** - * We have to add the max/sum/avg-aggregated features to the set of all features so that we can - * register them using FRS's [[FrsFeatureJsonExporter]]. - * - * Any additional such aggregated features that are included in [[FeatureStoreSource]] client - * should be registered here as well. - */ - val maxSumAvgAggregatedFeatureContext: FeatureContext = new FeatureContext() - .addFeatures( - UserWtfAlgorithmAggregate.getSecondaryAggregatedFeatureContext - ) - - // topic features - val topicAggregateFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set( - TopicEngagementRealTimeAggregates.FeaturesAsDataRecord, - TopicNegativeEngagementUserStateRealTimeAggregates.FeaturesAsDataRecord, - TopicEngagementUserStateRealTimeAggregates.FeaturesAsDataRecord - ).map(_.bind(TopicEntity)) - val userTopicFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set(FollowedTopics.bind(UserEntity)) - val authorTopicFeatures: Set[BoundFeature[_ <: EntityId, _]] = Set( - AuthorTopicAggregates.FeaturesAsDataRecord.bind(AuthorTopicEntity)) - val topicFeatures = topicAggregateFeatures ++ userTopicFeatures ++ authorTopicFeatures - -} - -object FeatureStoreRawFeatures { - val mcFeatures = Set( - NumTweets, - NumRetweets, - NumOriginalTweets, - NumRetweetsReceived, - NumFavoritesReceived, - NumRepliesReceived, - NumQuoteRetweetsReceived, - NumFollowsReceived, - NumFollowBacks, - NumFollows, - NumUnfollows, - NumUnfollowBacks, - NumQualityFollowReceived, - NumQuoteRetweets, - NumFavorites, - NumReplies, - NumLoginTweetImpressions, - NumTweetImpressions, - NumLoginDays, - NumUserActiveMinutes, - NumMuted, - NumSpamBlocked, - NumMuteBacks, - NumSpamBlockedBacks, - NumWasMutualFollowed, - NumWasMutualUnfollowed, - NumWasUnfollowed - ) - // based off usersource, and each feature represents the cumulative 'sent' counts - val userStatusFeatures = Set( - Favorites, - Followers, - Following, - Tweets - ) - // ratio features created from combining other features - val userRatioFeatures = Set(MutualFollowsFollowing, FollowersFollowings) - // features related to user login history - val userResurrectionFeatures: Set[Feature[UserId, Int]] = Set( - DaysSinceRecentResurrection - ) - - // real-time aggregate features borrowed from timelines - val authorAggregateFeatures = Set( - EngagementsReceivedByAuthorRealTimeAggregates.FeaturesAsDataRecord, - NegativeEngagementsReceivedByAuthorRealTimeAggregates.FeaturesAsDataRecord, - ) - - val timelinesAuthorAggregateFeatures = OriginalAuthorAggregates.FeaturesAsDataRecord - - val userAuthorUserStateRealTimeAggregatesFeature: Set[Feature[UserId, DataRecord]] = Set( - UserEngagementAuthorUserStateRealTimeAggregates.FeaturesAsDataRecord, - UserNegativeEngagementAuthorUserStateRealTimeAggregates.FeaturesAsDataRecord - ) - // post nux per-user offline features - val userOfflineFeatures = Set( - NumFollowings, - NumFollowers, - NumMutualFollows, - TweepCred, - UserState, - Language, - Country, - MutualFollowsOverFollowingRatio, - MutualFollowsOverFollowersRatio, - FollowersOverFollowingRatio, - ) - // matched post nux offline features between user and candidate - val userCandidatePostNuxEdgeFeatures = Set( - HaveSameUserState, - HaveSameLanguage, - HaveSameCountry, - NumFollowingsGap, - NumFollowersGap, - NumMutualFollowsGap, - TweepCredGap, - ) - // user algorithm aggregate features - val userAlgorithmAggregateFeatures = Set( - Impressions, - Follows, - FollowRate, - FollowsWithTweetImpressions, - FollowsWithTweetFavs, - FollowsTweetFavRate, - FollowsTweetReplies, - FollowsTweetReplyRate, - FollowsTweetRetweets, - FollowsTweetRetweetRate, - HasForwardEngagements, - HasReverseEngagements, - HasAnyEngagements, - ) - val userKeyedFeatures = userRatioFeatures ++ userOfflineFeatures - val wtfImpressionUserFeatures = - Set(HomeTimelineWtfCandidateCounts, HomeTimelineWtfLatestTimestamp) - val wtfImpressionCandidateFeatures = - Set(HomeTimelineWtfCandidateImpressionCounts, HomeTimelineWtfCandidateImpressionLatestTimestamp) - val simclusterUVIntersectionFeatures = Set( - IntersectionClusterIds, - HasIntersection, - IntersectionUserFollowScore, - IntersectionUserFavScore, - IntersectionCandidateKnownForScore, - IntersectionUserFollowCandidateKnownForScore, - IntersectionUserFavCandidateKnownForScore - ) - - // Client features - val clientFeatures = Set( - NumClients, - PrimaryClient, - PrimaryClientVersion, - FullPrimaryClientVersion, - PrimaryDeviceManufacturer, - PrimaryMobileSdkVersion, - SecondaryClient - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.scala deleted file mode 100644 index bb6b58857..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreGizmoduckSource.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -class FeatureStoreGizmoduckSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreGizmoduckSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_gizmoduck") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStoreGizmoduckSourceId - override def featureContext: FeatureContext = getFeatureContext - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - UsersourceEntityDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userEntityId = UserEntity.withId(UserId(targetUserId)) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val authorTopicEntity = if (topicProof.isDefined) { - backupSourceStats.counter("candidates_with_topic_proof").incr() - Set( - AuthorTopicEntity.withId( - EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) - } else Nil - - val entities = - Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStoreGizmoduckSource { - private val DatasetCacheScope = "feature_store_local_cache_gizmoduck" - private val DefaultCacheMaxKeys = 20000 - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureStoreFeatures.candidateUserStatusFeatures ++ - FeatureStoreFeatures.similarToUserStatusFeatures ++ - FeatureStoreFeatures.targetUserStatusFeatures - - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.targetUserStatusFeatures), - gate = HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.candidateUserStatusFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.similarToUserStatusFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck) & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - )) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.scala deleted file mode 100644 index 468a15ff6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreParameters.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.conversions.DurationOps._ -import com.twitter.ml.featurestore.catalog.datasets.core.UserMobileSdkDataset -import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmIdAggregateDataset -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmTypeAggregateDataset -import com.twitter.ml.featurestore.catalog.datasets.magicrecs.NotificationSummariesEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.UserWtfAlgorithmAggregateFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.WhoToFollowPostNuxFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.rux.UserRecentReactivationTimeDataset -import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset -import com.twitter.ml.featurestore.lib.dataset.DatasetParams -import com.twitter.ml.featurestore.lib.dataset.online.BatchingPolicy -import com.twitter.ml.featurestore.lib.params.FeatureStoreParams -import com.twitter.strato.opcontext.Attribution.ManhattanAppId -import com.twitter.strato.opcontext.ServeWithin - -object FeatureStoreParameters { - - private val FeatureServiceBatchSize = 100 - - val featureStoreParams = FeatureStoreParams( - global = DatasetParams( - serveWithin = Some(ServeWithin(duration = 240.millis, roundTripAllowance = None)), - attributions = Seq( - ManhattanAppId("omega", "wtf_impression_store"), - ManhattanAppId("athena", "wtf_athena"), - ManhattanAppId("starbuck", "wtf_starbuck"), - ManhattanAppId("apollo", "wtf_apollo") - ), - batchingPolicy = Some(BatchingPolicy.Isolated(FeatureServiceBatchSize)) - ), - perDataset = Map( - MetricCenterUserCountingFeaturesDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding"), - batchingPolicy = Some(BatchingPolicy.Isolated(200)) - ), - UsersourceEntityDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding") - ), - WhoToFollowPostNuxFeaturesDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding"), - batchingPolicy = Some(BatchingPolicy.Isolated(200)) - ), - AuthorFeaturesEntityDataset.id -> - DatasetParams( - stratoSuffix = Some("onboarding"), - batchingPolicy = Some(BatchingPolicy.Isolated(10)) - ), - UserRecentReactivationTimeDataset.id -> DatasetParams( - stratoSuffix = - None // removed due to low hit rate. we should use a negative cache in the future - ), - UserWtfAlgorithmAggregateFeaturesDataset.id -> DatasetParams( - stratoSuffix = None - ), - NotificationSummariesEntityDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding"), - serveWithin = Some(ServeWithin(duration = 45.millis, roundTripAllowance = None)), - batchingPolicy = Some(BatchingPolicy.Isolated(10)) - ), - UserMobileSdkDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding") - ), - PostNuxAlgorithmIdAggregateDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding") - ), - PostNuxAlgorithmTypeAggregateDataset.id -> DatasetParams( - stratoSuffix = Some("onboarding") - ), - ), - enableFeatureGenerationStats = true - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.scala deleted file mode 100644 index 6821e6a5f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStorePostNuxAlgorithmSource.scala +++ /dev/null @@ -1,232 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.CandidateAlgorithmTypeConstants -import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter.remapCandidateSource -import com.twitter.follow_recommendations.common.feature_hydration.adapters.PostNuxAlgorithmIdAdapter -import com.twitter.follow_recommendations.common.feature_hydration.adapters.PostNuxAlgorithmTypeAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmIdAggregateDataset -import com.twitter.ml.featurestore.catalog.datasets.customer_journey.PostNuxAlgorithmTypeAggregateDataset -import com.twitter.ml.featurestore.catalog.entities.onboarding.{WtfAlgorithm => OnboardingWtfAlgoId} -import com.twitter.ml.featurestore.catalog.entities.onboarding.{ - WtfAlgorithmType => OnboardingWtfAlgoType -} -import com.twitter.ml.featurestore.catalog.features.customer_journey.CombineAllFeaturesPolicy -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.WtfAlgorithmId -import com.twitter.ml.featurestore.lib.WtfAlgorithmType -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.entity.EntityWithId -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters._ - -class FeatureStorePostNuxAlgorithmSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStorePostNuxAlgorithmSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_post_nux_algorithm") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStorePostNuxAlgorithmSourceId - override def featureContext: FeatureContext = getFeatureContext - - private val dataRecordMerger = new DataRecordMerger - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - PostNuxAlgorithmIdAggregateDataset, - PostNuxAlgorithmTypeAggregateDataset, - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapterToDataRecord: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - // These two calculate the rate for each feature by dividing it by the number of impressions, then - // apply a log transformation. - private val transformAdapters = Seq(PostNuxAlgorithmIdAdapter, PostNuxAlgorithmTypeAdapter) - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { _: Long => - val candidateAlgoIdEntities = candidates.map { candidate => - candidate.id -> candidate.getAllAlgorithms - .flatMap { algo => - AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - }.map(algoId => OnboardingWtfAlgoId.withId(WtfAlgorithmId(algoId))) - }.toMap - - val candidateAlgoTypeEntities = candidateAlgoIdEntities.map { - case (candidateId, algoIdEntities) => - candidateId -> algoIdEntities - .map(_.id.algoId) - .flatMap(algoId => CandidateAlgorithmTypeConstants.getAlgorithmTypes(algoId.toString)) - .distinct - .map(algoType => OnboardingWtfAlgoType.withId(WtfAlgorithmType(algoType))) - } - - val entities = { - candidateAlgoIdEntities.values.flatten ++ candidateAlgoTypeEntities.values.flatten - }.toSeq.distinct - val requests = entities.map(entity => FeatureStoreRequest(Seq(entity))) - - val predictionRecordsFut = dynamicFeatureStoreClient(requests, target) - val candidateFeatureMap = predictionRecordsFut.map { - predictionRecords: Seq[PredictionRecord] => - val entityFeatureMap: Map[EntityWithId[_], DataRecord] = entities - .zip(predictionRecords).map { - case (entity, predictionRecord) => - entity -> adaptAdditionalFeaturesToDataRecord( - adapterToDataRecord.adaptToDataRecord(predictionRecord), - adapterStats, - transformAdapters) - }.toMap - - // In case we have more than one algorithm ID, or type, for a candidate, we merge the - // resulting DataRecords using the two merging policies below. - val algoIdMergeFn = - CombineAllFeaturesPolicy(PostNuxAlgorithmIdAdapter.getFeatures).getMergeFn - val algoTypeMergeFn = - CombineAllFeaturesPolicy(PostNuxAlgorithmTypeAdapter.getFeatures).getMergeFn - - val candidateAlgoIdFeaturesMap = candidateAlgoIdEntities.mapValues { entities => - val features = entities.flatMap(e => Option(entityFeatureMap.getOrElse(e, null))) - algoIdMergeFn(features) - } - - val candidateAlgoTypeFeaturesMap = candidateAlgoTypeEntities.mapValues { entities => - val features = entities.flatMap(e => Option(entityFeatureMap.getOrElse(e, null))) - algoTypeMergeFn(features) - } - - candidates.map { candidate => - val idDrOpt = candidateAlgoIdFeaturesMap.getOrElse(candidate.id, None) - val typeDrOpt = candidateAlgoTypeFeaturesMap.getOrElse(candidate.id, None) - - val featureDr = (idDrOpt, typeDrOpt) match { - case (None, Some(typeDataRecord)) => typeDataRecord - case (Some(idDataRecord), None) => idDataRecord - case (None, None) => new DataRecord() - case (Some(idDataRecord), Some(typeDataRecord)) => - dataRecordMerger.merge(idDataRecord, typeDataRecord) - idDataRecord - } - candidate -> featureDr - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStorePostNuxAlgorithmSource { - private val DatasetCacheScope = "feature_store_local_cache_post_nux_algorithm" - private val DefaultCacheMaxKeys = 1000 // Both of these datasets have <50 keys total. - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureStoreFeatures.postNuxAlgorithmIdAggregateFeatures ++ - FeatureStoreFeatures.postNuxAlgorithmTypeAggregateFeatures - - val algoIdFinalFeatures = CombineAllFeaturesPolicy( - PostNuxAlgorithmIdAdapter.getFeatures).outputFeaturesPostMerge.toSeq - val algoTypeFinalFeatures = CombineAllFeaturesPolicy( - PostNuxAlgorithmTypeAdapter.getFeatures).outputFeaturesPostMerge.toSeq - - val getFeatureContext: FeatureContext = - new FeatureContext().addFeatures((algoIdFinalFeatures ++ algoTypeFinalFeatures).asJava) - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = - BoundFeatureSet(FeatureStoreFeatures.postNuxAlgorithmIdAggregateFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = - BoundFeatureSet(FeatureStoreFeatures.postNuxAlgorithmTypeAggregateFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableAlgorithmAggregateFeatures) - ), - )) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.scala deleted file mode 100644 index 991bdee86..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSource.scala +++ /dev/null @@ -1,368 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.adapters.CandidateAlgorithmAdapter.remapCandidateSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.core.UsersourceEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.magicrecs.NotificationSummariesEntityDataset -import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset -import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{Topic => TopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{UserCandidate => UserCandidateEntity} -import com.twitter.ml.featurestore.catalog.entities.onboarding.UserWtfAlgorithmEntity -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dynamic._ -import com.twitter.ml.featurestore.lib.feature._ -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.WtfAlgorithmId -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateAuthorTopicAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicEngagementRealTimeAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicEngagementUserStateRealTimeAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.CandidateTopicNegativeEngagementUserStateRealTimeAggregatesAdapter -import com.twitter.onboarding.relevance.adapters.features.featurestore.FeatureStoreAdapter -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -import java.util.concurrent.TimeUnit - -@Singleton -class FeatureStoreSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreSource._ - - override val id: FeatureSourceId = FeatureSourceId.FeatureStoreSourceId - override val featureContext: FeatureContext = FeatureStoreSource.getFeatureContext - val hydrateFeaturesStats = stats.scope("hydrate_features") - val adapterStats = stats.scope("adapters") - val featureSet: BoundFeatureSet = BoundFeatureSet(FeatureStoreSource.allFeatures) - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = FeatureStoreSource.dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - MetricCenterUserCountingFeaturesDataset, - UsersourceEntityDataset, - AuthorFeaturesEntityDataset, - NotificationSummariesEntityDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - stats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(stats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userId = UserId(targetUserId) - val userEntityId = UserEntity.withId(userId) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val userCandidateEdgeEntityId = - UserCandidateEntity.withId(EdgeEntityId(userId, UserId(candidate.id))) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val topicEntities = if (topicProof.isDefined) { - hydrateFeaturesStats.counter("candidates_with_topic_proof").incr() - val topicId = topicProof.get.topicId - val topicEntityId = TopicEntity.withId(TopicId(topicId)) - val authorTopicEntityId = - AuthorTopicEntity.withId(EdgeEntityId(UserId(candidate.id), TopicId(topicId))) - Seq(topicEntityId, authorTopicEntityId) - } else Nil - - val candidateAlgorithmsWithScores = candidate.getAllAlgorithms - val userWtfAlgEdgeEntities = - candidateAlgorithmsWithScores.flatMap(algo => { - val algoId = AlgorithmToFeedbackTokenMap.get(remapCandidateSource(algo)) - algoId.map(id => - UserWtfAlgorithmEntity.withId(EdgeEntityId(userId, WtfAlgorithmId(id)))) - }) - - val entities = Seq( - userEntityId, - candidateEntityId, - userCandidateEdgeEntityId) ++ similarToUserId ++ topicEntities ++ userWtfAlgEdgeEntities - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -// list of features that we will be fetching, even if we are only scribing but not scoring with them -object FeatureStoreSource { - - private val DatasetCacheScope = "feature_store_local_cache" - private val DefaultCacheMaxKeys = 70000 - - import FeatureStoreFeatures._ - - ///////////////////// ALL hydrated features ///////////////////// - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - //target user - targetUserFeatures ++ - targetUserUserAuthorUserStateRealTimeAggregatesFeature ++ - targetUserResurrectionFeatures ++ - targetUserWtfImpressionFeatures ++ - targetUserStatusFeatures ++ - targetUserMetricCountFeatures ++ - //candidate user - candidateUserFeatures ++ - candidateUserResurrectionFeatures ++ - candidateUserAuthorRealTimeAggregateFeatures ++ - candidateUserStatusFeatures ++ - candidateUserMetricCountFeatures ++ - candidateUserTimelinesAuthorAggregateFeatures ++ - candidateUserClientFeatures ++ - //similar to user - similarToUserFeatures ++ - similarToUserStatusFeatures ++ - similarToUserMetricCountFeatures ++ - similarToUserTimelinesAuthorAggregateFeatures ++ - //other - userCandidateEdgeFeatures ++ - userCandidateWtfImpressionCandidateFeatures ++ - topicFeatures ++ - userWtfAlgorithmEdgeFeatures ++ - targetUserClientFeatures - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(topicAggregateFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTopicAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(authorTopicFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userTopicFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserTopicFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserUserAuthorUserStateRealTimeAggregatesFeature), - gate = HasParams.paramGate( - FeatureStoreSourceParams.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserResurrectionFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserResurrectionFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserWtfImpressionFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserWtfImpressionFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserStatusFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserMetricCountFeatures), - gate = HasParams - .paramGate( - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserAuthorRealTimeAggregateFeatures), - gate = HasParams.paramGate( - FeatureStoreSourceParams.EnableCandidateUserAuthorRealTimeAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserResurrectionFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserResurrectionFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserStatusFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & - HasParams.paramGate( - FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserMetricCountFeatures), - gate = - HasParams - .paramGate( - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userCandidateEdgeFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserCandidateEdgeFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userCandidateWtfImpressionCandidateFeatures), - gate = HasParams.paramGate( - FeatureStoreSourceParams.EnableUserCandidateWtfImpressionCandidateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(userWtfAlgorithmEdgeFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserWtfAlgEdgeFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserStatusFeatures), - gate = - HasParams.paramGate(FeatureStoreSourceParams.EnableSeparateClientForGizmoduck).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserMetricCountFeatures), - gate = - HasParams - .paramGate( - FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting).unary_! & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserClientFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateClientFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(targetUserClientFeatures), - gate = HasParams.paramGate(FeatureStoreSourceParams.EnableUserClientFeatures) - ), - ) - ) - // for calibrating features, e.g. add log transformed topic features - val featureAdapters: Seq[FeatureStoreAdapter] = Seq( - CandidateTopicEngagementRealTimeAggregatesAdapter, - CandidateTopicNegativeEngagementUserStateRealTimeAggregatesAdapter, - CandidateTopicEngagementUserStateRealTimeAggregatesAdapter, - CandidateAuthorTopicAggregatesAdapter - ) - val additionalFeatureContext: FeatureContext = FeatureContext.merge( - featureAdapters - .foldRight(new FeatureContext())((adapter, context) => - context - .addFeatures(adapter.getFeatureContext)) - ) - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - .addFeatures(additionalFeatureContext) - // The below are aggregated features that are aggregated for a second time over multiple keys. - .addFeatures(maxSumAvgAggregatedFeatureContext) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.scala deleted file mode 100644 index 15488ce90..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreSourceParams.scala +++ /dev/null @@ -1,148 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object FeatureStoreSourceParams { - case object EnableTopicAggregateFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTopicAggregateFeatures, - default = true - ) - case object EnableAlgorithmAggregateFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAlgorithmAggregateFeatures, - default = false - ) - case object EnableAuthorTopicAggregateFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableAuthorTopicAggregateFeatures, - default = true - ) - case object EnableUserTopicFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserTopicFeatures, - default = false - ) - case object EnableTargetUserFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserFeatures, - default = true - ) - case object EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserUserAuthorUserStateRealTimeAggregatesFeature, - default = true - ) - case object EnableTargetUserResurrectionFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserResurrectionFeatures, - default = true - ) - case object EnableTargetUserWtfImpressionFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableTargetUserWtfImpressionFeatures, - default = true - ) - case object EnableCandidateUserFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserFeatures, - default = true - ) - case object EnableCandidateUserAuthorRealTimeAggregateFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserAuthorRealTimeAggregateFeatures, - default = true - ) - case object EnableCandidateUserResurrectionFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserResurrectionFeatures, - default = true - ) - case object EnableCandidateUserTimelinesAuthorAggregateFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateUserTimelinesAuthorAggregateFeatures, - default = true - ) - case object EnableUserCandidateEdgeFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateEdgeFeatures, - default = true - ) - case object EnableUserCandidateWtfImpressionCandidateFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableUserCandidateWtfImpressionCandidateFeatures, - default = true - ) - case object EnableUserWtfAlgEdgeFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserWtfAlgEdgeFeatures, - default = false - ) - case object EnableSimilarToUserFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableSimilarToUserFeatures, - default = true - ) - - case object EnableCandidatePrecomputedNotificationFeatures - extends FSParam[Boolean]( - name = - FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidatePrecomputedNotificationFeatures, - default = false - ) - - case object EnableCandidateClientFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableCandidateClientFeatures, - default = false - ) - - case object EnableUserClientFeatures - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.EnableUserClientFeatures, - default = false - ) - - case object EnableSeparateClientForTimelinesAuthors - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForTimelinesAuthor, - default = false - ) - - case object EnableSeparateClientForMetricCenterUserCounting - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientMetricCenterUserCounting, - default = false - ) - - case object EnableSeparateClientForNotifications - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForNotifications, - default = false - ) - - case object EnableSeparateClientForGizmoduck - extends FSParam[Boolean]( - name = FeatureHydrationSourcesFeatureSwitchKeys.UseSeparateClientForGizmoduck, - default = false - ) - - case object GlobalFetchTimeout - extends FSBoundedParam[Duration]( - name = FeatureHydrationSourcesFeatureSwitchKeys.FeatureHydrationTimeout, - default = 240.millisecond, - min = 100.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.scala deleted file mode 100644 index 179ae7081..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreTimelinesAuthorSource.scala +++ /dev/null @@ -1,191 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.timelines.AuthorFeaturesEntityDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -class FeatureStoreTimelinesAuthorSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreTimelinesAuthorSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_timelines_author") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStoreTimelinesAuthorSourceId - override def featureContext: FeatureContext = getFeatureContext - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - AuthorFeaturesEntityDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userEntityId = UserEntity.withId(UserId(targetUserId)) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val authorTopicEntity = if (topicProof.isDefined) { - backupSourceStats.counter("candidates_with_topic_proof").incr() - Set( - AuthorTopicEntity.withId( - EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) - } else Nil - - val entities = - Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStoreTimelinesAuthorSource { - private val DatasetCacheScope = "feature_store_local_cache_timelines_author" - private val DefaultCacheMaxKeys = 20000 - - import FeatureStoreFeatures._ - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - similarToUserTimelinesAuthorAggregateFeatures ++ - candidateUserTimelinesAuthorAggregateFeatures ++ - authorTopicFeatures - - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(authorTopicFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & - HasParams.paramGate(FeatureStoreSourceParams.EnableAuthorTopicAggregateFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(similarToUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(candidateUserTimelinesAuthorAggregateFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForTimelinesAuthors) & - HasParams.paramGate( - FeatureStoreSourceParams.EnableCandidateUserTimelinesAuthorAggregateFeatures) - ), - )) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.scala deleted file mode 100644 index 110985c92..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/FeatureStoreUserMetricCountsSource.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.github.benmanes.caffeine.cache.Caffeine -import com.google.inject.Inject -import com.twitter.finagle.TimeoutException -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.adaptAdditionalFeaturesToDataRecord -import com.twitter.follow_recommendations.common.feature_hydration.sources.Utils.randomizedTTL -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.ml.featurestore.catalog.datasets.onboarding.MetricCenterUserCountingFeaturesDataset -import com.twitter.ml.featurestore.catalog.entities.core.{Author => AuthorEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{AuthorTopic => AuthorTopicEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{CandidateUser => CandidateUserEntity} -import com.twitter.ml.featurestore.catalog.entities.core.{User => UserEntity} -import com.twitter.ml.featurestore.lib.EdgeEntityId -import com.twitter.ml.featurestore.lib.EntityId -import com.twitter.ml.featurestore.lib.TopicId -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.ml.featurestore.lib.data.PredictionRecord -import com.twitter.ml.featurestore.lib.data.PredictionRecordAdapter -import com.twitter.ml.featurestore.lib.dataset.DatasetId -import com.twitter.ml.featurestore.lib.dataset.online.Hydrator.HydrationResponse -import com.twitter.ml.featurestore.lib.dataset.online.OnlineAccessDataset -import com.twitter.ml.featurestore.lib.dynamic.ClientConfig -import com.twitter.ml.featurestore.lib.dynamic.DynamicFeatureStoreClient -import com.twitter.ml.featurestore.lib.dynamic.DynamicHydrationConfig -import com.twitter.ml.featurestore.lib.dynamic.FeatureStoreParamsConfig -import com.twitter.ml.featurestore.lib.dynamic.GatedFeatures -import com.twitter.ml.featurestore.lib.feature.BoundFeature -import com.twitter.ml.featurestore.lib.feature.BoundFeatureSet -import com.twitter.ml.featurestore.lib.online.DatasetValuesCache -import com.twitter.ml.featurestore.lib.online.FeatureStoreRequest -import com.twitter.ml.featurestore.lib.online.OnlineFeatureGenerationStats -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import java.util.concurrent.TimeUnit -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -class FeatureStoreUserMetricCountsSource @Inject() ( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver) - extends FeatureSource { - import FeatureStoreUserMetricCountsSource._ - - val backupSourceStats = stats.scope("feature_store_hydration_mc_counting") - val adapterStats = backupSourceStats.scope("adapters") - override def id: FeatureSourceId = FeatureSourceId.FeatureStoreUserMetricCountsSourceId - override def featureContext: FeatureContext = getFeatureContext - - val clientConfig: ClientConfig[HasParams] = ClientConfig( - dynamicHydrationConfig = dynamicHydrationConfig, - featureStoreParamsConfig = - FeatureStoreParamsConfig(FeatureStoreParameters.featureStoreParams, Map.empty), - /** - * The smaller one between `timeoutProvider` and `FeatureStoreSourceParams.GlobalFetchTimeout` - * used below takes effect. - */ - timeoutProvider = Function.const(800.millis), - serviceIdentifier = serviceIdentifier - ) - - private val datasetsToCache = Set( - MetricCenterUserCountingFeaturesDataset - ).asInstanceOf[Set[OnlineAccessDataset[_ <: EntityId, _]]] - - private val datasetValuesCache: DatasetValuesCache = - DatasetValuesCache( - Caffeine - .newBuilder() - .expireAfterWrite(randomizedTTL(12.hours.inSeconds), TimeUnit.SECONDS) - .maximumSize(DefaultCacheMaxKeys) - .build[(_ <: EntityId, DatasetId), Stitch[HydrationResponse[_]]] - .asMap, - datasetsToCache, - DatasetCacheScope - ) - - private val dynamicFeatureStoreClient = DynamicFeatureStoreClient( - clientConfig, - backupSourceStats, - Set(datasetValuesCache) - ) - - private val adapter: IRecordOneToOneAdapter[PredictionRecord] = - PredictionRecordAdapter.oneToOne( - BoundFeatureSet(allFeatures), - OnlineFeatureGenerationStats(backupSourceStats) - ) - - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - target.getOptionalUserId - .map { targetUserId => - val featureRequests = candidates.map { candidate => - val userEntityId = UserEntity.withId(UserId(targetUserId)) - val candidateEntityId = CandidateUserEntity.withId(UserId(candidate.id)) - val similarToUserId = target.similarToUserIds.map(id => AuthorEntity.withId(UserId(id))) - val topicProof = candidate.reason.flatMap(_.accountProof.flatMap(_.topicProof)) - val authorTopicEntity = if (topicProof.isDefined) { - backupSourceStats.counter("candidates_with_topic_proof").incr() - Set( - AuthorTopicEntity.withId( - EdgeEntityId(UserId(candidate.id), TopicId(topicProof.get.topicId)))) - } else Nil - - val entities = - Seq(userEntityId, candidateEntityId) ++ similarToUserId ++ authorTopicEntity - FeatureStoreRequest(entities) - } - - val predictionRecordsFut = dynamicFeatureStoreClient(featureRequests, target) - val candidateFeatureMap = predictionRecordsFut.map { predictionRecords => - // we can zip predictionRecords with candidates as the order is preserved in the client - candidates - .zip(predictionRecords).map { - case (candidate, predictionRecord) => - candidate -> adaptAdditionalFeaturesToDataRecord( - adapter.adaptToDataRecord(predictionRecord), - adapterStats, - FeatureStoreSource.featureAdapters) - }.toMap - } - Stitch - .callFuture(candidateFeatureMap) - .within(target.params(FeatureStoreSourceParams.GlobalFetchTimeout))( - com.twitter.finagle.util.DefaultTimer) - .rescue { - case _: TimeoutException => - Stitch.value(Map.empty[CandidateUser, DataRecord]) - } - }.getOrElse(Stitch.value(Map.empty[CandidateUser, DataRecord])) - } -} - -object FeatureStoreUserMetricCountsSource { - private val DatasetCacheScope = "feature_store_local_cache_mc_user_counting" - private val DefaultCacheMaxKeys = 20000 - - val allFeatures: Set[BoundFeature[_ <: EntityId, _]] = - FeatureStoreFeatures.candidateUserMetricCountFeatures ++ - FeatureStoreFeatures.similarToUserMetricCountFeatures ++ - FeatureStoreFeatures.targetUserMetricCountFeatures - - val getFeatureContext: FeatureContext = - BoundFeatureSet(allFeatures).toFeatureContext - - val dynamicHydrationConfig: DynamicHydrationConfig[HasParams] = - DynamicHydrationConfig( - Set( - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.targetUserMetricCountFeatures), - gate = HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & - HasParams.paramGate(FeatureStoreSourceParams.EnableTargetUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.candidateUserMetricCountFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & - HasParams.paramGate(FeatureStoreSourceParams.EnableCandidateUserFeatures) - ), - GatedFeatures( - boundFeatureSet = BoundFeatureSet(FeatureStoreFeatures.similarToUserMetricCountFeatures), - gate = - HasParams - .paramGate(FeatureStoreSourceParams.EnableSeparateClientForMetricCenterUserCounting) & - HasParams.paramGate(FeatureStoreSourceParams.EnableSimilarToUserFeatures) - ), - )) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.scala deleted file mode 100644 index 59e3ea186..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/HydrationSourcesModule.scala +++ /dev/null @@ -1,152 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection -import com.twitter.storage.client.manhattan.kv.Guarantee -import com.twitter.storage.client.manhattan.kv.ManhattanKVClient -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder -import com.twitter.storage.client.manhattan.kv.impl.Component -import com.twitter.storage.client.manhattan.kv.impl.Component0 -import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.strato.generated.client.ml.featureStore.McUserCountingOnUserClientColumn -import com.twitter.strato.generated.client.ml.featureStore.onboarding.TimelinesAuthorFeaturesOnUserClientColumn -import com.twitter.timelines.author_features.v1.thriftscala.AuthorFeatures -import com.twitter.conversions.DurationOps._ -import com.twitter.onboarding.relevance.features.thriftscala.MCUserCountingFeatures -import java.lang.{Long => JLong} -import scala.util.Random - -object HydrationSourcesModule extends TwitterModule { - - val readFromManhattan = flag( - "feature_hydration_enable_reading_from_manhattan", - false, - "Whether to read the data from Manhattan or Strato") - - val manhattanAppId = - flag("frs_readonly.appId", "ml_features_athena", "RO App Id used by the RO FRS service") - val manhattanDestName = flag( - "frs_readonly.destName", - "/s/manhattan/athena.native-thrift", - "manhattan Dest Name used by the RO FRS service") - - @Provides - @Singleton - def providesAthenaManhattanClient( - serviceIdentifier: ServiceIdentifier - ): ManhattanKVEndpoint = { - val client = ManhattanKVClient( - manhattanAppId(), - manhattanDestName(), - ManhattanKVClientMtlsParams(serviceIdentifier) - ) - ManhattanKVEndpointBuilder(client) - .defaultGuarantee(Guarantee.Weak) - .build() - } - - val manhattanAuthorDataset = "timelines_author_features" - private val defaultCacheMaxKeys = 60000 - private val cacheTTL = 12.hours - private val earlyExpiration = 0.2 - - val authorKeyDesc = KeyDescriptor(Component(LongInjection), Component0) - val authorDatasetKey = authorKeyDesc.withDataset(manhattanAuthorDataset) - val authorValDesc = ValueDescriptor(BinaryCompactScalaInjection(AuthorFeatures)) - - @Provides - @Singleton - def timelinesAuthorStitchCache( - manhattanReadOnlyEndpoint: ManhattanKVEndpoint, - timelinesAuthorFeaturesColumn: TimelinesAuthorFeaturesOnUserClientColumn, - stats: StatsReceiver - ): StitchCache[JLong, Option[AuthorFeatures]] = { - - val stitchCacheStats = - stats - .scope("direct_ds_source_feature_hydration_module").scope("timelines_author") - - val stStat = stitchCacheStats.counter("readFromStrato-each") - val mhtStat = stitchCacheStats.counter("readFromManhattan-each") - - val timelinesAuthorUnderlyingCall = if (readFromManhattan()) { - stitchCacheStats.counter("readFromManhattan").incr() - val authorCacheUnderlyingManhattanCall: JLong => Stitch[Option[AuthorFeatures]] = id => { - mhtStat.incr() - val key = authorDatasetKey.withPkey(id) - manhattanReadOnlyEndpoint - .get(key = key, valueDesc = authorValDesc).map(_.map(value => - clearUnsedFieldsForAuthorFeature(value.contents))) - } - authorCacheUnderlyingManhattanCall - } else { - stitchCacheStats.counter("readFromStrato").incr() - val authorCacheUnderlyingStratoCall: JLong => Stitch[Option[AuthorFeatures]] = id => { - stStat.incr() - val timelinesAuthorFeaturesFetcher = timelinesAuthorFeaturesColumn.fetcher - timelinesAuthorFeaturesFetcher - .fetch(id).map(result => result.v.map(clearUnsedFieldsForAuthorFeature)) - } - authorCacheUnderlyingStratoCall - } - - StitchCache[JLong, Option[AuthorFeatures]]( - underlyingCall = timelinesAuthorUnderlyingCall, - maxCacheSize = defaultCacheMaxKeys, - ttl = randomizedTTL(cacheTTL.inSeconds).seconds, - statsReceiver = stitchCacheStats - ) - - } - - // Not adding manhattan since it didn't seem useful for Author Data, we can add in another phab - // if deemed helpful - @Provides - @Singleton - def metricCenterUserCountingStitchCache( - mcUserCountingFeaturesColumn: McUserCountingOnUserClientColumn, - stats: StatsReceiver - ): StitchCache[JLong, Option[MCUserCountingFeatures]] = { - - val stitchCacheStats = - stats - .scope("direct_ds_source_feature_hydration_module").scope("mc_user_counting") - - val stStat = stitchCacheStats.counter("readFromStrato-each") - stitchCacheStats.counter("readFromStrato").incr() - - val mcUserCountingCacheUnderlyingCall: JLong => Stitch[Option[MCUserCountingFeatures]] = id => { - stStat.incr() - val mcUserCountingFeaturesFetcher = mcUserCountingFeaturesColumn.fetcher - mcUserCountingFeaturesFetcher.fetch(id).map(_.v) - } - - StitchCache[JLong, Option[MCUserCountingFeatures]]( - underlyingCall = mcUserCountingCacheUnderlyingCall, - maxCacheSize = defaultCacheMaxKeys, - ttl = randomizedTTL(cacheTTL.inSeconds).seconds, - statsReceiver = stitchCacheStats - ) - - } - - // clear out fields we don't need to save cache space - private def clearUnsedFieldsForAuthorFeature(entry: AuthorFeatures): AuthorFeatures = { - entry.unsetUserTopics.unsetUserHealth.unsetAuthorCountryCodeAggregates.unsetOriginalAuthorCountryCodeAggregates - } - - // To avoid a cache stampede. See https://en.wikipedia.org/wiki/Cache_stampede - private def randomizedTTL(ttl: Long): Long = { - (ttl - ttl * earlyExpiration * Random.nextDouble()).toLong - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.scala deleted file mode 100644 index 51975c487..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/PreFetchedFeatureSource.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Inject -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.feature_hydration.adapters.PreFetchedFeatureAdapter -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -@Provides -@Singleton -class PreFetchedFeatureSource @Inject() () extends FeatureSource { - override def id: FeatureSourceId = FeatureSourceId.PreFetchedFeatureSourceId - override def featureContext: FeatureContext = PreFetchedFeatureAdapter.getFeatureContext - override def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - Stitch.value(candidates.map { candidate => - candidate -> PreFetchedFeatureAdapter.adaptToDataRecord((target, candidate)) - }.toMap) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.scala deleted file mode 100644 index 155d9e442..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/UserScoringFeatureSource.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.google.inject.Inject -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSource -import com.twitter.follow_recommendations.common.feature_hydration.common.FeatureSourceId -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This source wraps around the separate sources that we hydrate features from - * @param featureStoreSource gets features that require a RPC call to feature store - * @param stratoFeatureHydrationSource gets features that require a RPC call to strato columns - * @param clientContextSource gets features that are already present in the request context - * @param candidateAlgorithmSource gets features that are already present from candidate generation - * @param preFetchedFeatureSource gets features that were prehydrated (shared in request lifecycle) - */ -@Provides -@Singleton -class UserScoringFeatureSource @Inject() ( - featureStoreSource: FeatureStoreSource, - featureStoreGizmoduckSource: FeatureStoreGizmoduckSource, - featureStorePostNuxAlgorithmSource: FeatureStorePostNuxAlgorithmSource, - featureStoreTimelinesAuthorSource: FeatureStoreTimelinesAuthorSource, - featureStoreUserMetricCountsSource: FeatureStoreUserMetricCountsSource, - clientContextSource: ClientContextSource, - candidateAlgorithmSource: CandidateAlgorithmSource, - preFetchedFeatureSource: PreFetchedFeatureSource) - extends FeatureSource { - - override val id: FeatureSourceId = FeatureSourceId.UserScoringFeatureSourceId - - override val featureContext: FeatureContext = FeatureContext.merge( - featureStoreSource.featureContext, - featureStoreGizmoduckSource.featureContext, - featureStorePostNuxAlgorithmSource.featureContext, - featureStoreTimelinesAuthorSource.featureContext, - featureStoreUserMetricCountsSource.featureContext, - clientContextSource.featureContext, - candidateAlgorithmSource.featureContext, - preFetchedFeatureSource.featureContext, - ) - - val sources = - Seq( - featureStoreSource, - featureStorePostNuxAlgorithmSource, - featureStoreTimelinesAuthorSource, - featureStoreUserMetricCountsSource, - featureStoreGizmoduckSource, - clientContextSource, - candidateAlgorithmSource, - preFetchedFeatureSource - ) - - val dataRecordMerger = new DataRecordMerger - - def hydrateFeatures( - target: HasClientContext - with HasPreFetchedFeature - with HasParams - with HasSimilarToContext - with HasDisplayLocation, - candidates: Seq[CandidateUser] - ): Stitch[Map[CandidateUser, DataRecord]] = { - Stitch.collect(sources.map(_.hydrateFeatures(target, candidates))).map { featureMaps => - (for { - candidate <- candidates - } yield { - val combinedDataRecord = new DataRecord - featureMaps - .flatMap(_.get(candidate).toSeq).foreach(dataRecordMerger.merge(combinedDataRecord, _)) - candidate -> combinedDataRecord - }).toMap - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.scala deleted file mode 100644 index 99bd71310..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources/Utils.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.feature_hydration.sources - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.IRecordOneToOneAdapter -import scala.util.Random - -/** - * Helper functions for FeatureStoreSource operations in FRS are available here. - */ -object Utils { - - private val EarlyExpiration = 0.2 - - private[common] def adaptAdditionalFeaturesToDataRecord( - record: DataRecord, - adapterStats: StatsReceiver, - featureAdapters: Seq[IRecordOneToOneAdapter[DataRecord]] - ): DataRecord = { - featureAdapters.foldRight(record) { (adapter, record) => - adapterStats.counter(adapter.getClass.getSimpleName).incr() - adapter.adaptToDataRecord(record) - } - } - - // To avoid a cache stampede. See https://en.wikipedia.org/wiki/Cache_stampede - private[common] def randomizedTTL(ttl: Long): Long = { - (ttl - ttl * EarlyExpiration * Random.nextDouble()).toLong - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD deleted file mode 100644 index b5ece498a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.scala deleted file mode 100644 index 5325c6b56..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/LocationFeature.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.features - -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object LocationFeature - extends FeatureWithDefaultOnFailure[PipelineQuery, Option[GeohashAndCountryCode]] { - override val defaultValue: Option[GeohashAndCountryCode] = None -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.scala deleted file mode 100644 index 23571c8df..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/TrackingTokenFeature.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.features - -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object TrackingTokenFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Option[Int]] { - override val defaultValue: Option[Int] = None -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.scala deleted file mode 100644 index 73072b295..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features/UserStateFeature.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.features - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -case object UserStateFeature extends Feature[PipelineQuery, Option[UserState]] {} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.scala deleted file mode 100644 index 303417d23..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AddressBookMetadata.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -/** - * contains information if a candidate is from a candidate source generated using the following signals. - */ -case class AddressBookMetadata( - inForwardPhoneBook: Boolean, - inReversePhoneBook: Boolean, - inForwardEmailBook: Boolean, - inReverseEmailBook: Boolean) - -object AddressBookMetadata { - - val ForwardPhoneBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ForwardPhoneBook.toString) - - val ReversePhoneBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ReversePhoneBook.toString) - - val ForwardEmailBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ForwardEmailBook.toString) - - val ReverseEmailBookCandidateSource = CandidateSourceIdentifier( - Algorithm.ReverseEmailBookIbis.toString) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.scala deleted file mode 100644 index b60afb8b3..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/AlgorithmType.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -/** - * Each candidate source algorithm could be based on one, or more, of the 4 general type of - * information we have on a user: - * 1. Social: the user's connections in Twitter's social graph. - * 2. Geo: the user's geographical information. - * 3. Interest: information on the user's chosen interests. - * 4. Activity: information on the user's past activity. - * - * Note that an algorithm can fall under more than one of these categories. - */ -object AlgorithmType extends Enumeration { - type AlgorithmType = Value - - val Social: Value = Value("social") - val Geo: Value = Value("geo") - val Activity: Value = Value("activity") - val Interest: Value = Value("interest") -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD deleted file mode 100644 index c4916b6d0..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/model", - "hermit/hermit-ml/src/main/scala/com/twitter/hermit/ml/models", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "scrooge/scrooge-serializer/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/wtf/scalding/jobs/strong_tie_prediction", - "src/thrift/com/twitter/ads/adserver:adserver_rpc-scala", - "src/thrift/com/twitter/timelines/author_features/user_health:thrift-scala", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - ], - exports = [ - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.scala deleted file mode 100644 index 178f34b30..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/CandidateUser.scala +++ /dev/null @@ -1,192 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.hermit.constants.AlgorithmFeedbackTokens -import com.twitter.ml.api.thriftscala.{DataRecord => TDataRecord} -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -trait FollowableEntity extends UniversalNoun[Long] - -trait Recommendation - extends FollowableEntity - with HasReason - with HasAdMetadata - with HasTrackingToken { - val score: Option[Double] - - def toThrift: t.Recommendation - - def toOfflineThrift: offline.OfflineRecommendation -} - -case class CandidateUser( - override val id: Long, - override val score: Option[Double] = None, - override val reason: Option[Reason] = None, - override val userCandidateSourceDetails: Option[UserCandidateSourceDetails] = None, - override val adMetadata: Option[AdMetadata] = None, - override val trackingToken: Option[TrackingToken] = None, - override val dataRecord: Option[RichDataRecord] = None, - override val scores: Option[Scores] = None, - override val infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] = None, - override val params: Params = Params.Invalid, - override val engagements: Seq[EngagementType] = Nil, - override val recommendationFlowIdentifier: Option[String] = None) - extends Recommendation - with HasUserCandidateSourceDetails - with HasDataRecord - with HasScores - with HasParams - with HasEngagements - with HasRecommendationFlowIdentifier - with HasInfoPerRankingStage { - - val rankerIdsStr: Option[Seq[String]] = { - val strs = scores.map(_.scores.flatMap(_.rankerId.map(_.toString))) - if (strs.exists(_.nonEmpty)) strs else None - } - - val thriftDataRecord: Option[TDataRecord] = for { - richDataRecord <- dataRecord - dr <- richDataRecord.dataRecord - } yield { - ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(dr) - } - - val toOfflineUserThrift: offline.OfflineUserRecommendation = { - val scoringDetails = - if (userCandidateSourceDetails.isEmpty && score.isEmpty && thriftDataRecord.isEmpty) { - None - } else { - Some( - offline.ScoringDetails( - candidateSourceDetails = userCandidateSourceDetails.map(_.toOfflineThrift), - score = score, - dataRecord = thriftDataRecord, - rankerIds = rankerIdsStr, - infoPerRankingStage = infoPerRankingStage.map(_.mapValues(_.toOfflineThrift)) - ) - ) - } - offline - .OfflineUserRecommendation( - id, - reason.map(_.toOfflineThrift), - adMetadata.map(_.adImpression), - trackingToken.map(_.toOfflineThrift), - scoringDetails = scoringDetails - ) - } - - override val toOfflineThrift: offline.OfflineRecommendation = - offline.OfflineRecommendation.User(toOfflineUserThrift) - - val toUserThrift: t.UserRecommendation = { - val scoringDetails = - if (userCandidateSourceDetails.isEmpty && score.isEmpty && thriftDataRecord.isEmpty && scores.isEmpty) { - None - } else { - Some( - t.ScoringDetails( - candidateSourceDetails = userCandidateSourceDetails.map(_.toThrift), - score = score, - dataRecord = thriftDataRecord, - rankerIds = rankerIdsStr, - debugDataRecord = dataRecord.flatMap(_.debugDataRecord), - infoPerRankingStage = infoPerRankingStage.map(_.mapValues(_.toThrift)) - ) - ) - } - t.UserRecommendation( - userId = id, - reason = reason.map(_.toThrift), - adImpression = adMetadata.map(_.adImpression), - trackingInfo = trackingToken.map(TrackingToken.serialize), - scoringDetails = scoringDetails, - recommendationFlowIdentifier = recommendationFlowIdentifier - ) - } - - override val toThrift: t.Recommendation = - t.Recommendation.User(toUserThrift) - - def setFollowProof(followProofOpt: Option[FollowProof]): CandidateUser = { - this.copy( - reason = reason - .map { reason => - reason.copy( - accountProof = reason.accountProof - .map { accountProof => - accountProof.copy(followProof = followProofOpt) - }.orElse(Some(AccountProof(followProof = followProofOpt))) - ) - }.orElse(Some(Reason(Some(AccountProof(followProof = followProofOpt))))) - ) - } - - def addScore(score: Score): CandidateUser = { - val newScores = scores match { - case Some(existingScores) => existingScores.copy(scores = existingScores.scores :+ score) - case None => Scores(Seq(score)) - } - this.copy(scores = Some(newScores)) - } -} - -object CandidateUser { - val DefaultCandidateScore = 1.0 - - // for converting candidate in ScoringUserRequest - def fromUserRecommendation(candidate: t.UserRecommendation): CandidateUser = { - // we only use the primary candidate source for now - val userCandidateSourceDetails = for { - scoringDetails <- candidate.scoringDetails - candidateSourceDetails <- scoringDetails.candidateSourceDetails - } yield UserCandidateSourceDetails( - primaryCandidateSource = candidateSourceDetails.primarySource - .flatMap(AlgorithmFeedbackTokens.TokenToAlgorithmMap.get).map { algo => - CandidateSourceIdentifier(algo.toString) - }, - candidateSourceScores = fromThriftScoreMap(candidateSourceDetails.candidateSourceScores), - candidateSourceRanks = fromThriftRankMap(candidateSourceDetails.candidateSourceRanks), - addressBookMetadata = None - ) - CandidateUser( - id = candidate.userId, - score = candidate.scoringDetails.flatMap(_.score), - reason = candidate.reason.map(Reason.fromThrift), - userCandidateSourceDetails = userCandidateSourceDetails, - trackingToken = candidate.trackingInfo.map(TrackingToken.deserialize), - recommendationFlowIdentifier = candidate.recommendationFlowIdentifier, - infoPerRankingStage = candidate.scoringDetails.flatMap( - _.infoPerRankingStage.map(_.mapValues(RankingInfo.fromThrift))) - ) - } - - def fromThriftScoreMap( - thriftMapOpt: Option[scala.collection.Map[String, Double]] - ): Map[CandidateSourceIdentifier, Option[Double]] = { - (for { - thriftMap <- thriftMapOpt.toSeq - (algoName, score) <- thriftMap.toSeq - } yield { - CandidateSourceIdentifier(algoName) -> Some(score) - }).toMap - } - - def fromThriftRankMap( - thriftMapOpt: Option[scala.collection.Map[String, Int]] - ): Map[CandidateSourceIdentifier, Int] = { - (for { - thriftMap <- thriftMapOpt.toSeq - (algoName, rank) <- thriftMap.toSeq - } yield { - CandidateSourceIdentifier(algoName) -> rank - }).toMap - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.scala deleted file mode 100644 index ac601371c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/ClientContextConverter.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => frs} -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext - -object ClientContextConverter { - def toFRSOfflineClientContextThrift( - productMixerClientContext: ClientContext - ): offline.OfflineClientContext = - offline.OfflineClientContext( - productMixerClientContext.userId, - productMixerClientContext.guestId, - productMixerClientContext.appId, - productMixerClientContext.countryCode, - productMixerClientContext.languageCode, - productMixerClientContext.guestIdAds, - productMixerClientContext.guestIdMarketing - ) - - def fromThrift(clientContext: frs.ClientContext): ClientContext = ClientContext( - userId = clientContext.userId, - guestId = clientContext.guestId, - appId = clientContext.appId, - ipAddress = clientContext.ipAddress, - userAgent = clientContext.userAgent, - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - isTwoffice = clientContext.isTwoffice, - userRoles = clientContext.userRoles.map(_.toSet), - deviceId = clientContext.deviceId, - guestIdAds = clientContext.guestIdAds, - guestIdMarketing = clientContext.guestIdMarketing, - mobileDeviceId = None, - mobileDeviceAdId = None, - limitAdTracking = None - ) - - def toThrift(clientContext: ClientContext): frs.ClientContext = frs.ClientContext( - userId = clientContext.userId, - guestId = clientContext.guestIdAds, - appId = clientContext.appId, - ipAddress = clientContext.ipAddress, - userAgent = clientContext.userAgent, - countryCode = clientContext.countryCode, - languageCode = clientContext.languageCode, - isTwoffice = clientContext.isTwoffice, - userRoles = clientContext.userRoles, - deviceId = clientContext.deviceId, - guestIdAds = clientContext.guestIdAds, - guestIdMarketing = clientContext.guestIdMarketing - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala deleted file mode 100644 index b49baf034..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala +++ /dev/null @@ -1,420 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.adserver.thriftscala.{DisplayLocation => AdDisplayLocation} -import com.twitter.follow_recommendations.logging.thriftscala.{ - OfflineDisplayLocation => TOfflineDisplayLocation -} -import com.twitter.follow_recommendations.thriftscala.{DisplayLocation => TDisplayLocation} - -sealed trait DisplayLocation { - def toThrift: TDisplayLocation - - def toOfflineThrift: TOfflineDisplayLocation - - def toFsName: String - - // corresponding display location in adserver if available - // make sure to be consistent with the definition here - def toAdDisplayLocation: Option[AdDisplayLocation] = None -} - -/** - * Make sure you add the new DL to the following files and redeploy our attribution jobs - * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift - * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift - * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala - */ - -object DisplayLocation { - - case object ProfileSidebar extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileSidebar - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ProfileSidebar - override val toFsName: String = "ProfileSidebar" - - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - AdDisplayLocation.ProfileAccountsSidebar - ) - } - - case object HomeTimeline extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimeline - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.HomeTimeline - override val toFsName: String = "HomeTimeline" - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - // it is based on the logic that HTL DL should correspond to Sidebar: - AdDisplayLocation.WtfSidebar - ) - } - - case object ReactiveFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ReactiveFollow - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ReactiveFollow - override val toFsName: String = "ReactiveFollow" - } - - case object ExploreTab extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ExploreTab - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.ExploreTab - override val toFsName: String = "ExploreTab" - } - - case object MagicRecs extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.MagicRecs - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.MagicRecs - override val toFsName: String = "MagicRecs" - } - - case object AbUploadInjection extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.AbUploadInjection - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.AbUploadInjection - override val toFsName: String = "AbUploadInjection" - } - - case object RuxLandingPage extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.RuxLandingPage - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.RuxLandingPage - override val toFsName: String = "RuxLandingPage" - } - - case object ProfileBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileBonusFollow - override val toFsName: String = "ProfileBonusFollow" - } - - case object ElectionExploreWtf extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ElectionExploreWtf - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ElectionExploreWtf - override val toFsName: String = "ElectionExploreWtf" - } - - case object ClusterFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ClusterFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ClusterFollow - override val toFsName: String = "ClusterFollow" - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - AdDisplayLocation.ClusterFollow - ) - } - - case object HtlBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HtlBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HtlBonusFollow - override val toFsName: String = "HtlBonusFollow" - } - - case object TopicLandingPageHeader extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TopicLandingPageHeader - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TopicLandingPageHeader - override val toFsName: String = "TopicLandingPageHeader" - } - - case object NewUserSarusBackfill extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NewUserSarusBackfill - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NewUserSarusBackfill - override val toFsName: String = "NewUserSarusBackfill" - } - - case object NuxPymk extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxPymk - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxPymk - override val toFsName: String = "NuxPymk" - } - - case object NuxInterests extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxInterests - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxInterests - override val toFsName: String = "NuxInterests" - } - - case object NuxTopicBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxTopicBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxTopicBonusFollow - override val toFsName: String = "NuxTopicBonusFollow" - } - - case object Sidebar extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.Sidebar - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.Sidebar - override val toFsName: String = "Sidebar" - - override val toAdDisplayLocation: Option[AdDisplayLocation] = Some( - AdDisplayLocation.WtfSidebar - ) - } - - case object CampaignForm extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.CampaignForm - override val toOfflineThrift: TOfflineDisplayLocation = TOfflineDisplayLocation.CampaignForm - override val toFsName: String = "CampaignForm" - } - - case object ProfileTopFollowers extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileTopFollowers - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileTopFollowers - override val toFsName: String = "ProfileTopFollowers" - } - - case object ProfileTopFollowing extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileTopFollowing - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileTopFollowing - override val toFsName: String = "ProfileTopFollowing" - } - - case object RuxPymk extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.RuxPymk - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.RuxPymk - override val toFsName: String = "RuxPymk" - } - - case object IndiaCovid19CuratedAccountsWtf extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.IndiaCovid19CuratedAccountsWtf - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.IndiaCovid19CuratedAccountsWtf - override val toFsName: String = "IndiaCovid19CuratedAccountsWtf" - } - - case object PeoplePlusPlus extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.PeoplePlusPlus - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.PeoplePlusPlus - override val toFsName: String = "PeoplePlusPlus" - } - - case object TweetNotificationRecs extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TweetNotificationRecs - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TweetNotificationRecs - override val toFsName: String = "TweetNotificationRecs" - } - - case object ProfileDeviceFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ProfileDeviceFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ProfileDeviceFollow - override val toFsName: String = "ProfileDeviceFollow" - } - - case object RecosBackfill extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.RecosBackfill - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.RecosBackfill - override val toFsName: String = "RecosBackfill" - } - - case object HtlSpaceHosts extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HtlSpaceHosts - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HtlSpaceHosts - override val toFsName: String = "HtlSpaceHosts" - } - - case object PostNuxFollowTask extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.PostNuxFollowTask - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.PostNuxFollowTask - override val toFsName: String = "PostNuxFollowTask" - } - - case object TopicLandingPage extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TopicLandingPage - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TopicLandingPage - override val toFsName: String = "TopicLandingPage" - } - - case object UserTypeaheadPrefetch extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.UserTypeaheadPrefetch - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.UserTypeaheadPrefetch - override val toFsName: String = "UserTypeaheadPrefetch" - } - - case object HomeTimelineRelatableAccounts extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineRelatableAccounts - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HomeTimelineRelatableAccounts - override val toFsName: String = "HomeTimelineRelatableAccounts" - } - - case object NuxGeoCategory extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxGeoCategory - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxGeoCategory - override val toFsName: String = "NuxGeoCategory" - } - - case object NuxInterestsCategory extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxInterestsCategory - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxInterestsCategory - override val toFsName: String = "NuxInterestsCategory" - } - - case object TopArticles extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.TopArticles - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.TopArticles - override val toFsName: String = "TopArticles" - } - - case object NuxPymkCategory extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxPymkCategory - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxPymkCategory - override val toFsName: String = "NuxPymkCategory" - } - - case object HomeTimelineTweetRecs extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineTweetRecs - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HomeTimelineTweetRecs - override val toFsName: String = "HomeTimelineTweetRecs" - } - - case object HtlBulkFriendFollows extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HtlBulkFriendFollows - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HtlBulkFriendFollows - override val toFsName: String = "HtlBulkFriendFollows" - } - - case object NuxAutoFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.NuxAutoFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.NuxAutoFollow - override val toFsName: String = "NuxAutoFollow" - } - - case object SearchBonusFollow extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.SearchBonusFollow - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.SearchBonusFollow - override val toFsName: String = "SearchBonusFollow" - } - - case object ContentRecommender extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.ContentRecommender - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.ContentRecommender - override val toFsName: String = "ContentRecommender" - } - - case object HomeTimelineReverseChron extends DisplayLocation { - override val toThrift: TDisplayLocation = TDisplayLocation.HomeTimelineReverseChron - override val toOfflineThrift: TOfflineDisplayLocation = - TOfflineDisplayLocation.HomeTimelineReverseChron - override val toFsName: String = "HomeTimelineReverseChron" - } - - def fromThrift(displayLocation: TDisplayLocation): DisplayLocation = displayLocation match { - case TDisplayLocation.ProfileSidebar => ProfileSidebar - case TDisplayLocation.HomeTimeline => HomeTimeline - case TDisplayLocation.MagicRecs => MagicRecs - case TDisplayLocation.AbUploadInjection => AbUploadInjection - case TDisplayLocation.RuxLandingPage => RuxLandingPage - case TDisplayLocation.ProfileBonusFollow => ProfileBonusFollow - case TDisplayLocation.ElectionExploreWtf => ElectionExploreWtf - case TDisplayLocation.ClusterFollow => ClusterFollow - case TDisplayLocation.HtlBonusFollow => HtlBonusFollow - case TDisplayLocation.ReactiveFollow => ReactiveFollow - case TDisplayLocation.TopicLandingPageHeader => TopicLandingPageHeader - case TDisplayLocation.NewUserSarusBackfill => NewUserSarusBackfill - case TDisplayLocation.NuxPymk => NuxPymk - case TDisplayLocation.NuxInterests => NuxInterests - case TDisplayLocation.NuxTopicBonusFollow => NuxTopicBonusFollow - case TDisplayLocation.ExploreTab => ExploreTab - case TDisplayLocation.Sidebar => Sidebar - case TDisplayLocation.CampaignForm => CampaignForm - case TDisplayLocation.ProfileTopFollowers => ProfileTopFollowers - case TDisplayLocation.ProfileTopFollowing => ProfileTopFollowing - case TDisplayLocation.RuxPymk => RuxPymk - case TDisplayLocation.IndiaCovid19CuratedAccountsWtf => IndiaCovid19CuratedAccountsWtf - case TDisplayLocation.PeoplePlusPlus => PeoplePlusPlus - case TDisplayLocation.TweetNotificationRecs => TweetNotificationRecs - case TDisplayLocation.ProfileDeviceFollow => ProfileDeviceFollow - case TDisplayLocation.RecosBackfill => RecosBackfill - case TDisplayLocation.HtlSpaceHosts => HtlSpaceHosts - case TDisplayLocation.PostNuxFollowTask => PostNuxFollowTask - case TDisplayLocation.TopicLandingPage => TopicLandingPage - case TDisplayLocation.UserTypeaheadPrefetch => UserTypeaheadPrefetch - case TDisplayLocation.HomeTimelineRelatableAccounts => HomeTimelineRelatableAccounts - case TDisplayLocation.NuxGeoCategory => NuxGeoCategory - case TDisplayLocation.NuxInterestsCategory => NuxInterestsCategory - case TDisplayLocation.TopArticles => TopArticles - case TDisplayLocation.NuxPymkCategory => NuxPymkCategory - case TDisplayLocation.HomeTimelineTweetRecs => HomeTimelineTweetRecs - case TDisplayLocation.HtlBulkFriendFollows => HtlBulkFriendFollows - case TDisplayLocation.NuxAutoFollow => NuxAutoFollow - case TDisplayLocation.SearchBonusFollow => SearchBonusFollow - case TDisplayLocation.ContentRecommender => ContentRecommender - case TDisplayLocation.HomeTimelineReverseChron => HomeTimelineReverseChron - case TDisplayLocation.EnumUnknownDisplayLocation(i) => - throw new UnknownDisplayLocationException( - s"Unknown display location thrift enum with value: ${i}") - } - - def fromOfflineThrift(displayLocation: TOfflineDisplayLocation): DisplayLocation = - displayLocation match { - case TOfflineDisplayLocation.ProfileSidebar => ProfileSidebar - case TOfflineDisplayLocation.HomeTimeline => HomeTimeline - case TOfflineDisplayLocation.MagicRecs => MagicRecs - case TOfflineDisplayLocation.AbUploadInjection => AbUploadInjection - case TOfflineDisplayLocation.RuxLandingPage => RuxLandingPage - case TOfflineDisplayLocation.ProfileBonusFollow => ProfileBonusFollow - case TOfflineDisplayLocation.ElectionExploreWtf => ElectionExploreWtf - case TOfflineDisplayLocation.ClusterFollow => ClusterFollow - case TOfflineDisplayLocation.HtlBonusFollow => HtlBonusFollow - case TOfflineDisplayLocation.TopicLandingPageHeader => TopicLandingPageHeader - case TOfflineDisplayLocation.NewUserSarusBackfill => NewUserSarusBackfill - case TOfflineDisplayLocation.NuxPymk => NuxPymk - case TOfflineDisplayLocation.NuxInterests => NuxInterests - case TOfflineDisplayLocation.NuxTopicBonusFollow => NuxTopicBonusFollow - case TOfflineDisplayLocation.ExploreTab => ExploreTab - case TOfflineDisplayLocation.ReactiveFollow => ReactiveFollow - case TOfflineDisplayLocation.Sidebar => Sidebar - case TOfflineDisplayLocation.CampaignForm => CampaignForm - case TOfflineDisplayLocation.ProfileTopFollowers => ProfileTopFollowers - case TOfflineDisplayLocation.ProfileTopFollowing => ProfileTopFollowing - case TOfflineDisplayLocation.RuxPymk => RuxPymk - case TOfflineDisplayLocation.IndiaCovid19CuratedAccountsWtf => IndiaCovid19CuratedAccountsWtf - case TOfflineDisplayLocation.PeoplePlusPlus => PeoplePlusPlus - case TOfflineDisplayLocation.TweetNotificationRecs => TweetNotificationRecs - case TOfflineDisplayLocation.ProfileDeviceFollow => ProfileDeviceFollow - case TOfflineDisplayLocation.RecosBackfill => RecosBackfill - case TOfflineDisplayLocation.HtlSpaceHosts => HtlSpaceHosts - case TOfflineDisplayLocation.PostNuxFollowTask => PostNuxFollowTask - case TOfflineDisplayLocation.TopicLandingPage => TopicLandingPage - case TOfflineDisplayLocation.UserTypeaheadPrefetch => UserTypeaheadPrefetch - case TOfflineDisplayLocation.HomeTimelineRelatableAccounts => HomeTimelineRelatableAccounts - case TOfflineDisplayLocation.NuxGeoCategory => NuxGeoCategory - case TOfflineDisplayLocation.NuxInterestsCategory => NuxInterestsCategory - case TOfflineDisplayLocation.TopArticles => TopArticles - case TOfflineDisplayLocation.NuxPymkCategory => NuxPymkCategory - case TOfflineDisplayLocation.HomeTimelineTweetRecs => HomeTimelineTweetRecs - case TOfflineDisplayLocation.HtlBulkFriendFollows => HtlBulkFriendFollows - case TOfflineDisplayLocation.NuxAutoFollow => NuxAutoFollow - case TOfflineDisplayLocation.SearchBonusFollow => SearchBonusFollow - case TOfflineDisplayLocation.ContentRecommender => ContentRecommender - case TOfflineDisplayLocation.HomeTimelineReverseChron => HomeTimelineReverseChron - case TOfflineDisplayLocation.EnumUnknownOfflineDisplayLocation(i) => - throw new UnknownDisplayLocationException( - s"Unknown offline display location thrift enum with value: ${i}") - } -} - -class UnknownDisplayLocationException(message: String) extends Exception(message) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.scala deleted file mode 100644 index b12a4404c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/EngagementType.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.thriftscala.{EngagementType => TEngagementType} -import com.twitter.follow_recommendations.logging.thriftscala.{ - EngagementType => OfflineEngagementType -} -sealed trait EngagementType { - def toThrift: TEngagementType - def toOfflineThrift: OfflineEngagementType -} - -object EngagementType { - object Click extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Click - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Click - } - object Like extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Like - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Like - } - object Mention extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Mention - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Mention - } - object Retweet extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.Retweet - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.Retweet - } - object ProfileView extends EngagementType { - override val toThrift: TEngagementType = TEngagementType.ProfileView - - override val toOfflineThrift: OfflineEngagementType = OfflineEngagementType.ProfileView - } - - def fromThrift(engagementType: TEngagementType): EngagementType = engagementType match { - case TEngagementType.Click => Click - case TEngagementType.Like => Like - case TEngagementType.Mention => Mention - case TEngagementType.Retweet => Retweet - case TEngagementType.ProfileView => ProfileView - case TEngagementType.EnumUnknownEngagementType(i) => - throw new UnknownEngagementTypeException( - s"Unknown engagement type thrift enum with value: ${i}") - } - - def fromOfflineThrift(engagementType: OfflineEngagementType): EngagementType = - engagementType match { - case OfflineEngagementType.Click => Click - case OfflineEngagementType.Like => Like - case OfflineEngagementType.Mention => Mention - case OfflineEngagementType.Retweet => Retweet - case OfflineEngagementType.ProfileView => ProfileView - case OfflineEngagementType.EnumUnknownEngagementType(i) => - throw new UnknownEngagementTypeException( - s"Unknown engagement type offline thrift enum with value: ${i}") - } -} -class UnknownEngagementTypeException(message: String) extends Exception(message) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.scala deleted file mode 100644 index 86b496776..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FilterReason.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -sealed trait FilterReason { - def reason: String -} - -object FilterReason { - - case object NoReason extends FilterReason { - override val reason: String = "no_reason" - } - - case class ParamReason(paramName: String) extends FilterReason { - override val reason: String = s"param_$paramName" - } - - case object ExcludedId extends FilterReason { - override val reason: String = "excluded_id_from_request" - } - - case object ProfileSidebarBlacklist extends FilterReason { - override val reason: String = "profile_sidebar_blacklisted_id" - } - - case object CuratedAccountsCompetitorList extends FilterReason { - override val reason: String = "curated_blacklisted_id" - } - - case class InvalidRelationshipTypes(relationshipTypes: String) extends FilterReason { - override val reason: String = s"invalid_relationship_types $relationshipTypes" - } - - case object ProfileId extends FilterReason { - override val reason: String = "candidate_has_same_id_as_profile" - } - - case object DismissedId extends FilterReason { - override val reason: String = s"dismissed_candidate" - } - - case object OptedOutId extends FilterReason { - override val reason: String = s"candidate_opted_out_from_criteria_in_request" - } - - // gizmoduck predicates - case object NoUser extends FilterReason { - override val reason: String = "no_user_result_from_gizmoduck" - } - - case object AddressBookUndiscoverable extends FilterReason { - override val reason: String = "not_discoverable_via_address_book" - } - - case object PhoneBookUndiscoverable extends FilterReason { - override val reason: String = "not_discoverable_via_phone_book" - } - - case object Deactivated extends FilterReason { - override val reason: String = "deactivated" - } - - case object Suspended extends FilterReason { - override val reason: String = "suspended" - } - - case object Restricted extends FilterReason { - override val reason: String = "restricted" - } - - case object NsfwUser extends FilterReason { - override val reason: String = "nsfwUser" - } - - case object NsfwAdmin extends FilterReason { - override val reason: String = "nsfwAdmin" - } - - case object HssSignal extends FilterReason { - override val reason: String = "hssSignal" - } - - case object IsProtected extends FilterReason { - override val reason: String = "isProtected" - } - - case class CountryTakedown(countryCode: String) extends FilterReason { - override val reason: String = s"takedown_in_$countryCode" - } - - case object Blink extends FilterReason { - override val reason: String = "blink" - } - - case object AlreadyFollowed extends FilterReason { - override val reason: String = "already_followed" - } - - case object InvalidRelationship extends FilterReason { - override val reason: String = "invalid_relationship" - } - - case object NotFollowingTargetUser extends FilterReason { - override val reason: String = "not_following_target_user" - } - - case object CandidateSideHoldback extends FilterReason { - override val reason: String = "candidate_side_holdback" - } - - case object Inactive extends FilterReason { - override val reason: String = "inactive" - } - - case object MissingRecommendabilityData extends FilterReason { - override val reason: String = "missing_recommendability_data" - } - - case object HighTweetVelocity extends FilterReason { - override val reason: String = "high_tweet_velocity" - } - - case object AlreadyRecommended extends FilterReason { - override val reason: String = "already_recommended" - } - - case object MinStateNotMet extends FilterReason { - override val reason: String = "min_state_user_not_met" - } - - case object FailOpen extends FilterReason { - override val reason: String = "fail_open" - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.scala deleted file mode 100644 index 15e36321e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowContext.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class FlowContext(steps: Seq[RecommendationStep]) { - - def toThrift: t.FlowContext = t.FlowContext(steps = steps.map(_.toThrift)) - - def toOfflineThrift: offline.OfflineFlowContext = - offline.OfflineFlowContext(steps = steps.map(_.toOfflineThrift)) -} - -object FlowContext { - - def fromThrift(flowContext: t.FlowContext): FlowContext = { - FlowContext(steps = flowContext.steps.map(RecommendationStep.fromThrift)) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.scala deleted file mode 100644 index 118ff258d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/FlowRecommendation.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class FlowRecommendation(userId: Long) { - - def toThrift: t.FlowRecommendation = - t.FlowRecommendation(userId = userId) - - def toOfflineThrift: offline.OfflineFlowRecommendation = - offline.OfflineFlowRecommendation(userId = userId) - -} - -object FlowRecommendation { - def fromThrift(flowRecommendation: t.FlowRecommendation): FlowRecommendation = { - FlowRecommendation( - userId = flowRecommendation.userId - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.scala deleted file mode 100644 index 782d3fc9e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/GeohashAndCountryCode.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -case class GeohashAndCountryCode(geohash: Option[String], countryCode: Option[String]) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.scala deleted file mode 100644 index 57979e376..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasAdMetadata.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.adserver.{thriftscala => t} - -case class AdMetadata( - insertPosition: Int, - // use original ad impression info to avoid losing data in domain model translations - adImpression: t.AdImpression) - -trait HasAdMetadata { - - def adMetadata: Option[AdMetadata] - - def adImpression: Option[t.AdImpression] = { - adMetadata.map(_.adImpression) - } - - def insertPosition: Option[Int] = { - adMetadata.map(_.insertPosition) - } - - def isPromotedAccount: Boolean = adMetadata.isDefined -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.scala deleted file mode 100644 index d4cbdcee8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasByfSeedUserIds.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasByfSeedUserIds { - def byfSeedUserIds: Option[Seq[Long]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.scala deleted file mode 100644 index 4e7047b4e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDataRecord.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.thriftscala.DebugDataRecord -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.util.Try -import com.twitter.util.logging.Logging -import scala.collection.convert.ImplicitConversions._ - -// contains the standard dataRecord struct, and the debug version if required -case class RichDataRecord( - dataRecord: Option[DataRecord] = None, - debugDataRecord: Option[DebugDataRecord] = None, -) - -trait HasDataRecord extends Logging { - def dataRecord: Option[RichDataRecord] - - def toDebugDataRecord(dr: DataRecord, featureContext: FeatureContext): DebugDataRecord = { - - val binaryFeatures: Option[Set[String]] = if (dr.isSetBinaryFeatures) { - Some(dr.getBinaryFeatures.flatMap { id => - Try(featureContext.getFeature(id).getFeatureName).toOption - }.toSet) - } else None - - val continuousFeatures: Option[Map[String, Double]] = if (dr.isSetContinuousFeatures) { - Some(dr.getContinuousFeatures.flatMap { - case (id, value) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> value.toDouble - } - }.toMap) - } else None - - val discreteFeatures: Option[Map[String, Long]] = if (dr.isSetDiscreteFeatures) { - Some(dr.getDiscreteFeatures.flatMap { - case (id, value) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> value.toLong - } - }.toMap) - } else None - - val stringFeatures: Option[Map[String, String]] = if (dr.isSetStringFeatures) { - Some(dr.getStringFeatures.flatMap { - case (id, value) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> value - } - }.toMap) - } else None - - val sparseBinaryFeatures: Option[Map[String, Set[String]]] = if (dr.isSetSparseBinaryFeatures) { - Some(dr.getSparseBinaryFeatures.flatMap { - case (id, values) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> values.toSet - } - }.toMap) - } else None - - val sparseContinuousFeatures: Option[Map[String, Map[String, Double]]] = - if (dr.isSetSparseContinuousFeatures) { - Some(dr.getSparseContinuousFeatures.flatMap { - case (id, values) => - Try(featureContext.getFeature(id).getFeatureName).toOption.map { id => - id -> values.map { - case (str, value) => - str -> value.toDouble - }.toMap - } - }.toMap) - } else None - - DebugDataRecord( - binaryFeatures = binaryFeatures, - continuousFeatures = continuousFeatures, - discreteFeatures = discreteFeatures, - stringFeatures = stringFeatures, - sparseBinaryFeatures = sparseBinaryFeatures, - sparseContinuousFeatures = sparseContinuousFeatures, - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.scala deleted file mode 100644 index 0956ca34e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDebugOptions.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.thriftscala.DebugParams - -case class DebugOptions( - randomizationSeed: Option[Long] = None, - fetchDebugInfo: Boolean = false, - doNotLog: Boolean = false) - -object DebugOptions { - def fromDebugParamsThrift(debugParams: DebugParams): DebugOptions = { - DebugOptions( - debugParams.randomizationSeed, - debugParams.includeDebugInfoInResults.getOrElse(false), - debugParams.doNotLog.getOrElse(false) - ) - } -} - -trait HasDebugOptions { - def debugOptions: Option[DebugOptions] - - def getRandomizationSeed: Option[Long] = debugOptions.flatMap(_.randomizationSeed) - - def fetchDebugInfo: Option[Boolean] = debugOptions.map(_.fetchDebugInfo) -} - -trait HasFrsDebugOptions { - def frsDebugOptions: Option[DebugOptions] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.scala deleted file mode 100644 index 3f2154992..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDismissedUserIds.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasDismissedUserIds { - // user ids that are recently followed by the target user - def dismissedUserIds: Option[Seq[Long]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.scala deleted file mode 100644 index e74ae83e1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasDisplayLocation.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasDisplayLocation { - def displayLocation: DisplayLocation -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.scala deleted file mode 100644 index de59e4479..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasEngagements.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasEngagements { - - def engagements: Seq[EngagementType] - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.scala deleted file mode 100644 index 3addcef83..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasExcludedUserIds.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasExcludedUserIds { - // user ids that are going to be excluded from recommendations - def excludedUserIds: Seq[Long] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.scala deleted file mode 100644 index a4364bbf4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasGeohashAndCountryCode.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasGeohashAndCountryCode { - def geohashAndCountryCode: Option[GeohashAndCountryCode] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.scala deleted file mode 100644 index c4d277412..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInfoPerRankingStage.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasInfoPerRankingStage { - def infoPerRankingStage: Option[scala.collection.Map[String, RankingInfo]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.scala deleted file mode 100644 index 69f97b673..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInterestIds.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasCustomInterests { - def customInterests: Option[Seq[String]] -} - -trait HasUttInterests { - def uttInterestIds: Option[Seq[Long]] -} - -trait HasInterestIds extends HasCustomInterests with HasUttInterests {} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.scala deleted file mode 100644 index 3cf3f66db..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasInvalidRelationshipUserIds.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasInvalidRelationshipUserIds { - // user ids that have invalid relationship with the target user - def invalidRelationshipUserIds: Option[Set[Long]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.scala deleted file mode 100644 index 8cf1532ce..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasIsSoftUser.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasIsSoftUser { - def isSoftUser: Boolean -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.scala deleted file mode 100644 index c5e1e16cc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasMutualFollowedUserIds.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -// intersection of recent followers and followed by -trait HasMutualFollowedUserIds extends HasRecentFollowedUserIds with HasRecentFollowedByUserIds { - - lazy val recentMutualFollows: Seq[Long] = - recentFollowedUserIds.getOrElse(Nil).intersect(recentFollowedByUserIds.getOrElse(Nil)) - - lazy val numRecentMutualFollows: Int = recentMutualFollows.size -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.scala deleted file mode 100644 index 2480faaad..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasPreviousRecommendationsContext.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasPreviousRecommendationsContext { - - def previouslyRecommendedUserIDs: Set[Long] - - def previouslyFollowedUserIds: Set[Long] - - def skippedFollows: Set[Long] = { - previouslyRecommendedUserIDs.diff(previouslyFollowedUserIds) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.scala deleted file mode 100644 index 10cd7c02f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasProfileId.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasProfileId { - def profileId: Long -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.scala deleted file mode 100644 index 96527b2ba..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasQualityFactor.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasQualityFactor { - def qualityFactor: Option[Double] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.scala deleted file mode 100644 index bc15e8bd8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedByUserIds.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentFollowedByUserIds { - // user ids that have recently followed the target user; target user has been "followed by" them. - def recentFollowedByUserIds: Option[Seq[Long]] - - lazy val numRecentFollowedByUserIds: Int = recentFollowedByUserIds.map(_.size).getOrElse(0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.scala deleted file mode 100644 index 67ada7c66..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIds.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentFollowedUserIds { - // user ids that are recently followed by the target user - def recentFollowedUserIds: Option[Seq[Long]] - - // user ids that are recently followed by the target user in set data-structure - lazy val recentFollowedUserIdsSet: Option[Set[Long]] = recentFollowedUserIds match { - case Some(users) => Some(users.toSet) - case None => Some(Set.empty) - } - - lazy val numRecentFollowedUserIds: Int = recentFollowedUserIds.map(_.size).getOrElse(0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.scala deleted file mode 100644 index 7e3cba4a7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentFollowedUserIdsWithTime.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentFollowedUserIdsWithTime { - // user ids that are recently followed by the target user - def recentFollowedUserIdsWithTime: Option[Seq[UserIdWithTimestamp]] - - lazy val numRecentFollowedUserIdsWithTime: Int = - recentFollowedUserIdsWithTime.map(_.size).getOrElse(0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.scala deleted file mode 100644 index 44420ec27..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecentlyEngagedUserIds.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecentlyEngagedUserIds { - val recentlyEngagedUserIds: Option[Seq[RecentlyEngagedUserId]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.scala deleted file mode 100644 index 5706c7a29..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasRecommendationFlowIdentifier.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasRecommendationFlowIdentifier { - def recommendationFlowIdentifier: Option[String] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.scala deleted file mode 100644 index e8a6698ee..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasScores.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasScores { - def scores: Option[Scores] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.scala deleted file mode 100644 index bbe2ac258..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasSimilarToContext.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasSimilarToContext { - - // user ids that are used to generate similar to recommendations - def similarToUserIds: Seq[Long] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.scala deleted file mode 100644 index 4bd6e63e7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasTopicId.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -trait HasTopicId { - def topicId: Long -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.scala deleted file mode 100644 index e0e363449..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserCandidateSourceDetails.scala +++ /dev/null @@ -1,162 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.hermit.ml.models.Feature -import com.twitter.hermit.model.Algorithm -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -/** - * Used to keep track of a candidate's source not so much as a feature but for filtering candidate - * from specific sources (eg. GizmoduckPredicate) - */ -trait HasUserCandidateSourceDetails { candidateUser: CandidateUser => - def userCandidateSourceDetails: Option[UserCandidateSourceDetails] - - def getAlgorithm: Algorithm = { - val algorithm = for { - details <- userCandidateSourceDetails - identifier <- details.primaryCandidateSource - algorithm <- Algorithm.withNameOpt(identifier.name) - } yield algorithm - - algorithm.getOrElse(throw new Exception("Algorithm missing on candidate user!")) - } - - def getAllAlgorithms: Seq[Algorithm] = { - getCandidateSources.keys - .flatMap(identifier => Algorithm.withNameOpt(identifier.name)).toSeq - } - - def getAddressBookMetadata: Option[AddressBookMetadata] = { - userCandidateSourceDetails.flatMap(_.addressBookMetadata) - } - - def getCandidateSources: Map[CandidateSourceIdentifier, Option[Double]] = { - userCandidateSourceDetails.map(_.candidateSourceScores).getOrElse(Map.empty) - } - - def getCandidateRanks: Map[CandidateSourceIdentifier, Int] = { - userCandidateSourceDetails.map(_.candidateSourceRanks).getOrElse(Map.empty) - } - - def getCandidateFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = { - userCandidateSourceDetails.map(_.candidateSourceFeatures).getOrElse(Map.empty) - } - - def getPrimaryCandidateSource: Option[CandidateSourceIdentifier] = { - userCandidateSourceDetails.flatMap(_.primaryCandidateSource) - } - - def withCandidateSource(source: CandidateSourceIdentifier): CandidateUser = { - withCandidateSourceAndScore(source, candidateUser.score) - } - - def withCandidateSourceAndScore( - source: CandidateSourceIdentifier, - score: Option[Double] - ): CandidateUser = { - withCandidateSourceScoreAndFeatures(source, score, Nil) - } - - def withCandidateSourceAndFeatures( - source: CandidateSourceIdentifier, - features: Seq[Feature] - ): CandidateUser = { - withCandidateSourceScoreAndFeatures(source, candidateUser.score, features) - } - - def withCandidateSourceScoreAndFeatures( - source: CandidateSourceIdentifier, - score: Option[Double], - features: Seq[Feature] - ): CandidateUser = { - val candidateSourceDetails = - candidateUser.userCandidateSourceDetails - .map { details => - details.copy( - primaryCandidateSource = Some(source), - candidateSourceScores = details.candidateSourceScores + (source -> score), - candidateSourceFeatures = details.candidateSourceFeatures + (source -> features) - ) - }.getOrElse( - UserCandidateSourceDetails( - Some(source), - Map(source -> score), - Map.empty, - None, - Map(source -> features))) - candidateUser.copy( - userCandidateSourceDetails = Some(candidateSourceDetails) - ) - } - - def addCandidateSourceScoresMap( - scoreMap: Map[CandidateSourceIdentifier, Option[Double]] - ): CandidateUser = { - val candidateSourceDetails = candidateUser.userCandidateSourceDetails - .map { details => - details.copy(candidateSourceScores = details.candidateSourceScores ++ scoreMap) - }.getOrElse(UserCandidateSourceDetails(scoreMap.keys.headOption, scoreMap, Map.empty, None)) - candidateUser.copy( - userCandidateSourceDetails = Some(candidateSourceDetails) - ) - } - - def addCandidateSourceRanksMap( - rankMap: Map[CandidateSourceIdentifier, Int] - ): CandidateUser = { - val candidateSourceDetails = candidateUser.userCandidateSourceDetails - .map { details => - details.copy(candidateSourceRanks = details.candidateSourceRanks ++ rankMap) - }.getOrElse(UserCandidateSourceDetails(rankMap.keys.headOption, Map.empty, rankMap, None)) - candidateUser.copy( - userCandidateSourceDetails = Some(candidateSourceDetails) - ) - } - - def addInfoPerRankingStage( - rankingStage: String, - scores: Option[Scores], - rank: Int - ): CandidateUser = { - val scoresOpt: Option[Scores] = scores.orElse(candidateUser.scores) - val originalInfoPerRankingStage = - candidateUser.infoPerRankingStage.getOrElse(Map[String, RankingInfo]()) - candidateUser.copy( - infoPerRankingStage = - Some(originalInfoPerRankingStage + (rankingStage -> RankingInfo(scoresOpt, Some(rank)))) - ) - } - - def addAddressBookMetadataIfAvailable( - candidateSources: Seq[CandidateSourceIdentifier] - ): CandidateUser = { - - val addressBookMetadata = AddressBookMetadata( - inForwardPhoneBook = - candidateSources.contains(AddressBookMetadata.ForwardPhoneBookCandidateSource), - inReversePhoneBook = - candidateSources.contains(AddressBookMetadata.ReversePhoneBookCandidateSource), - inForwardEmailBook = - candidateSources.contains(AddressBookMetadata.ForwardEmailBookCandidateSource), - inReverseEmailBook = - candidateSources.contains(AddressBookMetadata.ReverseEmailBookCandidateSource) - ) - - val newCandidateSourceDetails = candidateUser.userCandidateSourceDetails - .map { details => - details.copy(addressBookMetadata = Some(addressBookMetadata)) - }.getOrElse( - UserCandidateSourceDetails( - None, - Map.empty, - Map.empty, - Some(addressBookMetadata), - Map.empty)) - - candidateUser.copy( - userCandidateSourceDetails = Some(newCandidateSourceDetails) - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.scala deleted file mode 100644 index bf0df46f7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasUserState.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.core_workflows.user_model.thriftscala.UserState - -trait HasUserState { - def userState: Option[UserState] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.scala deleted file mode 100644 index d840ed6fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/HasWtfImpressions.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.util.Time - -trait HasWtfImpressions { - - def wtfImpressions: Option[Seq[WtfImpression]] - - lazy val numWtfImpressions: Int = wtfImpressions.map(_.size).getOrElse(0) - - lazy val candidateImpressions: Map[Long, WtfImpression] = wtfImpressions - .map { imprMap => - imprMap.map { i => - i.candidateId -> i - }.toMap - }.getOrElse(Map.empty) - - lazy val latestImpressionTime: Time = { - if (wtfImpressions.exists(_.nonEmpty)) { - wtfImpressions.get.map(_.latestTime).max - } else Time.Top - } - - def getCandidateImpressionCounts(id: Long): Option[Int] = - candidateImpressions.get(id).map(_.counts) - - def getCandidateLatestTime(id: Long): Option[Time] = { - candidateImpressions.get(id).map(_.latestTime) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.scala deleted file mode 100644 index df4c228e1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/OptimusRequest.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams - -/** -Convenience trait to group together all traits needed for optimus ranking - */ -trait OptimusRequest - extends HasParams - with HasClientContext - with HasDisplayLocation - with HasInterestIds - with HasDebugOptions - with HasPreviousRecommendationsContext {} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.scala deleted file mode 100644 index f37cff56e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Product.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.{Product => ProductMixerProduct} - -object Product { - case object MagicRecs extends ProductMixerProduct { - override val identifier: ProductIdentifier = ProductIdentifier("MagicRecs") - override val stringCenterProject: Option[String] = Some("people-discovery") - } - - case object PlaceholderProductMixerProduct extends ProductMixerProduct { - override val identifier: ProductIdentifier = ProductIdentifier("PlaceholderProductMixerProduct") - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.scala deleted file mode 100644 index 02eb46b5a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RankingInfo.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} - -case class RankingInfo( - scores: Option[Scores], - rank: Option[Int]) { - - def toThrift: t.RankingInfo = { - t.RankingInfo(scores.map(_.toThrift), rank) - } - - def toOfflineThrift: offline.RankingInfo = { - offline.RankingInfo(scores.map(_.toOfflineThrift), rank) - } -} - -object RankingInfo { - - def fromThrift(rankingInfo: t.RankingInfo): RankingInfo = { - RankingInfo( - scores = rankingInfo.scores.map(Scores.fromThrift), - rank = rankingInfo.rank - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.scala deleted file mode 100644 index b7c9c6c75..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Reason.scala +++ /dev/null @@ -1,206 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} - -case class FollowProof(followedBy: Seq[Long], numIds: Int) { - def toThrift: t.FollowProof = { - t.FollowProof(followedBy, numIds) - } - - def toOfflineThrift: offline.FollowProof = offline.FollowProof(followedBy, numIds) -} - -object FollowProof { - - def fromThrift(proof: t.FollowProof): FollowProof = { - FollowProof(proof.userIds, proof.numIds) - } -} - -case class SimilarToProof(similarTo: Seq[Long]) { - def toThrift: t.SimilarToProof = { - t.SimilarToProof(similarTo) - } - - def toOfflineThrift: offline.SimilarToProof = offline.SimilarToProof(similarTo) -} - -object SimilarToProof { - def fromThrift(proof: t.SimilarToProof): SimilarToProof = { - SimilarToProof(proof.userIds) - } -} - -case class PopularInGeoProof(location: String) { - def toThrift: t.PopularInGeoProof = { - t.PopularInGeoProof(location) - } - - def toOfflineThrift: offline.PopularInGeoProof = offline.PopularInGeoProof(location) -} - -object PopularInGeoProof { - - def fromThrift(proof: t.PopularInGeoProof): PopularInGeoProof = { - PopularInGeoProof(proof.location) - } -} - -case class TttInterestProof(interestId: Long, interestDisplayName: String) { - def toThrift: t.TttInterestProof = { - t.TttInterestProof(interestId, interestDisplayName) - } - - def toOfflineThrift: offline.TttInterestProof = - offline.TttInterestProof(interestId, interestDisplayName) -} - -object TttInterestProof { - - def fromThrift(proof: t.TttInterestProof): TttInterestProof = { - TttInterestProof(proof.interestId, proof.interestDisplayName) - } -} - -case class TopicProof(topicId: Long) { - def toThrift: t.TopicProof = { - t.TopicProof(topicId) - } - - def toOfflineThrift: offline.TopicProof = - offline.TopicProof(topicId) -} - -object TopicProof { - def fromThrift(proof: t.TopicProof): TopicProof = { - TopicProof(proof.topicId) - } -} - -case class CustomInterest(query: String) { - def toThrift: t.CustomInterestProof = { - t.CustomInterestProof(query) - } - - def toOfflineThrift: offline.CustomInterestProof = - offline.CustomInterestProof(query) -} - -object CustomInterest { - def fromThrift(proof: t.CustomInterestProof): CustomInterest = { - CustomInterest(proof.query) - } -} - -case class TweetsAuthorProof(tweetIds: Seq[Long]) { - def toThrift: t.TweetsAuthorProof = { - t.TweetsAuthorProof(tweetIds) - } - - def toOfflineThrift: offline.TweetsAuthorProof = - offline.TweetsAuthorProof(tweetIds) -} - -object TweetsAuthorProof { - def fromThrift(proof: t.TweetsAuthorProof): TweetsAuthorProof = { - TweetsAuthorProof(proof.tweetIds) - } -} - -case class DeviceFollowProof(isDeviceFollow: Boolean) { - def toThrift: t.DeviceFollowProof = { - t.DeviceFollowProof(isDeviceFollow) - } - def toOfflineThrift: offline.DeviceFollowProof = - offline.DeviceFollowProof(isDeviceFollow) -} - -object DeviceFollowProof { - def fromThrift(proof: t.DeviceFollowProof): DeviceFollowProof = { - DeviceFollowProof(proof.isDeviceFollow) - } - -} - -case class AccountProof( - followProof: Option[FollowProof] = None, - similarToProof: Option[SimilarToProof] = None, - popularInGeoProof: Option[PopularInGeoProof] = None, - tttInterestProof: Option[TttInterestProof] = None, - topicProof: Option[TopicProof] = None, - customInterestProof: Option[CustomInterest] = None, - tweetsAuthorProof: Option[TweetsAuthorProof] = None, - deviceFollowProof: Option[DeviceFollowProof] = None) { - def toThrift: t.AccountProof = { - t.AccountProof( - followProof.map(_.toThrift), - similarToProof.map(_.toThrift), - popularInGeoProof.map(_.toThrift), - tttInterestProof.map(_.toThrift), - topicProof.map(_.toThrift), - customInterestProof.map(_.toThrift), - tweetsAuthorProof.map(_.toThrift), - deviceFollowProof.map(_.toThrift) - ) - } - - def toOfflineThrift: offline.AccountProof = { - offline.AccountProof( - followProof.map(_.toOfflineThrift), - similarToProof.map(_.toOfflineThrift), - popularInGeoProof.map(_.toOfflineThrift), - tttInterestProof.map(_.toOfflineThrift), - topicProof.map(_.toOfflineThrift), - customInterestProof.map(_.toOfflineThrift), - tweetsAuthorProof.map(_.toOfflineThrift), - deviceFollowProof.map(_.toOfflineThrift) - ) - } -} - -object AccountProof { - def fromThrift(proof: t.AccountProof): AccountProof = { - AccountProof( - proof.followProof.map(FollowProof.fromThrift), - proof.similarToProof.map(SimilarToProof.fromThrift), - proof.popularInGeoProof.map(PopularInGeoProof.fromThrift), - proof.tttInterestProof.map(TttInterestProof.fromThrift), - proof.topicProof.map(TopicProof.fromThrift), - proof.customInterestProof.map(CustomInterest.fromThrift), - proof.tweetsAuthorProof.map(TweetsAuthorProof.fromThrift), - proof.deviceFollowProof.map(DeviceFollowProof.fromThrift) - ) - } -} - -case class Reason(accountProof: Option[AccountProof]) { - def toThrift: t.Reason = { - t.Reason(accountProof.map(_.toThrift)) - } - - def toOfflineThrift: offline.Reason = { - offline.Reason(accountProof.map(_.toOfflineThrift)) - } -} - -object Reason { - - def fromThrift(reason: t.Reason): Reason = { - Reason(reason.accountProof.map(AccountProof.fromThrift)) - } -} - -trait HasReason { - - def reason: Option[Reason] - // helper methods below - - def followedBy: Option[Seq[Long]] = { - for { - reason <- reason - accountProof <- reason.accountProof - followProof <- accountProof.followProof - } yield { followProof.followedBy } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.scala deleted file mode 100644 index b20d5d6fb..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecentlyEngagedUserId.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class RecentlyEngagedUserId(id: Long, engagementType: EngagementType) { - def toThrift: t.RecentlyEngagedUserId = - t.RecentlyEngagedUserId(id = id, engagementType = engagementType.toThrift) - - def toOfflineThrift: offline.RecentlyEngagedUserId = - offline.RecentlyEngagedUserId(id = id, engagementType = engagementType.toOfflineThrift) -} - -object RecentlyEngagedUserId { - def fromThrift(recentlyEngagedUserId: t.RecentlyEngagedUserId): RecentlyEngagedUserId = { - RecentlyEngagedUserId( - id = recentlyEngagedUserId.id, - engagementType = EngagementType.fromThrift(recentlyEngagedUserId.engagementType) - ) - } - - def fromOfflineThrift( - recentlyEngagedUserId: offline.RecentlyEngagedUserId - ): RecentlyEngagedUserId = { - RecentlyEngagedUserId( - id = recentlyEngagedUserId.id, - engagementType = EngagementType.fromOfflineThrift(recentlyEngagedUserId.engagementType) - ) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.scala deleted file mode 100644 index 4fd4bc70e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/RecommendationStep.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} - -case class RecommendationStep( - recommendations: Seq[FlowRecommendation], - followedUserIds: Set[Long]) { - - def toThrift: t.RecommendationStep = t.RecommendationStep( - recommendations = recommendations.map(_.toThrift), - followedUserIds = followedUserIds - ) - - def toOfflineThrift: offline.OfflineRecommendationStep = - offline.OfflineRecommendationStep( - recommendations = recommendations.map(_.toOfflineThrift), - followedUserIds = followedUserIds) - -} - -object RecommendationStep { - - def fromThrift(recommendationStep: t.RecommendationStep): RecommendationStep = { - RecommendationStep( - recommendations = recommendationStep.recommendations.map(FlowRecommendation.fromThrift), - followedUserIds = recommendationStep.followedUserIds.toSet) - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.scala deleted file mode 100644 index 2d577e21d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/STPGraph.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.hermit.model.Algorithm.Algorithm -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdge -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.FirstDegreeEdgeInfo -import com.twitter.wtf.scalding.jobs.strong_tie_prediction.SecondDegreeEdge - -case class PotentialFirstDegreeEdge( - userId: Long, - connectingId: Long, - algorithm: Algorithm, - score: Double, - edgeInfo: FirstDegreeEdgeInfo) - -case class IntermediateSecondDegreeEdge( - connectingId: Long, - candidateId: Long, - edgeInfo: FirstDegreeEdgeInfo) - -case class STPGraph( - firstDegreeEdgeInfoList: List[FirstDegreeEdge], - secondDegreeEdgeInfoList: List[SecondDegreeEdge]) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.scala deleted file mode 100644 index 10c21704a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SafetyLevel.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.spam.rtf.thriftscala.{SafetyLevel => ThriftSafetyLevel} - -sealed trait SafetyLevel { - def toThrift: ThriftSafetyLevel -} - -object SafetyLevel { - case object Recommendations extends SafetyLevel { - override val toThrift = ThriftSafetyLevel.Recommendations - } - - case object TopicsLandingPageTopicRecommendations extends SafetyLevel { - override val toThrift = ThriftSafetyLevel.TopicsLandingPageTopicRecommendations - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.scala deleted file mode 100644 index 1007c5827..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Score.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -/** - * Type of Score. This is used to differentiate scores. - * - * Define it as a trait so it is possible to add more information for different score types. - */ -sealed trait ScoreType { - def getName: String -} - -/** - * Existing Score Types - */ -object ScoreType { - - /** - * the score is calculated based on heuristics and most likely not normalized - */ - case object HeuristicBasedScore extends ScoreType { - override def getName: String = "HeuristicBasedScore" - } - - /** - * probability of follow after the candidate is recommended to the user - */ - case object PFollowGivenReco extends ScoreType { - override def getName: String = "PFollowGivenReco" - } - - /** - * probability of engage after the user follows the candidate - */ - case object PEngagementGivenFollow extends ScoreType { - override def getName: String = "PEngagementGivenFollow" - } - - /** - * probability of engage per tweet impression - */ - case object PEngagementPerImpression extends ScoreType { - override def getName: String = "PEngagementPerImpression" - } - - /** - * probability of engage per tweet impression - */ - case object PEngagementGivenReco extends ScoreType { - override def getName: String = "PEngagementGivenReco" - } - - def fromScoreTypeString(scoreTypeName: String): ScoreType = scoreTypeName match { - case "HeuristicBasedScore" => HeuristicBasedScore - case "PFollowGivenReco" => PFollowGivenReco - case "PEngagementGivenFollow" => PEngagementGivenFollow - case "PEngagementPerImpression" => PEngagementPerImpression - case "PEngagementGivenReco" => PEngagementGivenReco - } -} - -/** - * Represent the output from a certain ranker or scorer. All the fields are optional - * - * @param value value of the score - * @param rankerId ranker id - * @param scoreType score type - */ -final case class Score( - value: Double, - rankerId: Option[RankerId] = None, - scoreType: Option[ScoreType] = None) { - - def toThrift: t.Score = t.Score( - value = value, - rankerId = rankerId.map(_.toString), - scoreType = scoreType.map(_.getName) - ) - - def toOfflineThrift: offline.Score = - offline.Score( - value = value, - rankerId = rankerId.map(_.toString), - scoreType = scoreType.map(_.getName) - ) -} - -object Score { - - val RandomScore = Score(0.0d, Some(RankerId.RandomRanker)) - - def optimusScore(score: Double, scoreType: ScoreType): Score = { - Score(value = score, scoreType = Some(scoreType)) - } - - def predictionScore(score: Double, rankerId: RankerId): Score = { - Score(value = score, rankerId = Some(rankerId)) - } - - def fromThrift(thriftScore: t.Score): Score = - Score( - value = thriftScore.value, - rankerId = thriftScore.rankerId.flatMap(RankerId.getRankerByName), - scoreType = thriftScore.scoreType.map(ScoreType.fromScoreTypeString) - ) -} - -/** - * a list of scores - */ -final case class Scores( - scores: Seq[Score], - selectedRankerId: Option[RankerId] = None, - isInProducerScoringExperiment: Boolean = false) { - - def toThrift: t.Scores = - t.Scores( - scores = scores.map(_.toThrift), - selectedRankerId = selectedRankerId.map(_.toString), - isInProducerScoringExperiment = isInProducerScoringExperiment - ) - - def toOfflineThrift: offline.Scores = - offline.Scores( - scores = scores.map(_.toOfflineThrift), - selectedRankerId = selectedRankerId.map(_.toString), - isInProducerScoringExperiment = isInProducerScoringExperiment - ) -} - -object Scores { - val Empty: Scores = Scores(Nil) - - def fromThrift(thriftScores: t.Scores): Scores = - Scores( - scores = thriftScores.scores.map(Score.fromThrift), - selectedRankerId = thriftScores.selectedRankerId.flatMap(RankerId.getRankerByName), - isInProducerScoringExperiment = thriftScores.isInProducerScoringExperiment - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.scala deleted file mode 100644 index 25ff00b48..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/Session.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.finagle.tracing.Trace - -object Session { - - /** - * The sessionId in FRS is the finagle trace id which is static within the lifetime of a single - * request. - * - * It is used when generating per-candidate tokens (in TrackingTokenTransform) and is also passed - * in to downstream Optimus ranker requests. - * - */ - def getSessionId: Long = Trace.id.traceId.toLong -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.scala deleted file mode 100644 index 3af5c6e1d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/SignalData.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.usersignalservice.thriftscala.Signal - -trait SignalData { - val userId: Long - val signalType: SignalType -} - -case class RecentFollowsSignal( - override val userId: Long, - override val signalType: SignalType, - followedUserId: Long, - timestamp: Long) - extends SignalData - -object RecentFollowsSignal { - - def fromUssSignal(targetUserId: Long, signal: Signal): RecentFollowsSignal = { - val InternalId.UserId(followedUserId) = signal.targetInternalId.getOrElse( - throw new IllegalArgumentException("RecentFollow Signal does not have internalId")) - - RecentFollowsSignal( - userId = targetUserId, - followedUserId = followedUserId, - timestamp = signal.timestamp, - signalType = signal.signalType - ) - } - - def getRecentFollowedUserIds( - signalDataMap: Option[Map[SignalType, Seq[SignalData]]] - ): Option[Seq[Long]] = { - signalDataMap.map(_.getOrElse(SignalType.AccountFollow, default = Seq.empty).flatMap { - case RecentFollowsSignal(userId, signalType, followedUserId, timestamp) => - Some(followedUserId) - case _ => None - }) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.scala deleted file mode 100644 index 177e08f65..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TrackingToken.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.finagle.tracing.Trace -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.scrooge.BinaryThriftStructSerializer -import com.twitter.suggests.controller_data.thriftscala.ControllerData -import com.twitter.util.Base64StringEncoder - -/** - * used for attribution per target-candidate pair - * @param sessionId trace-id of the finagle request - * @param controllerData 64-bit encoded binary attributes of our recommendation - * @param algorithmId id for identifying a candidate source. maintained for backwards compatibility - */ -case class TrackingToken( - sessionId: Long, - displayLocation: Option[DisplayLocation], - controllerData: Option[ControllerData], - algorithmId: Option[Int]) { - - def toThrift: t.TrackingToken = { - Trace.id.traceId.toLong - t.TrackingToken( - sessionId = sessionId, - displayLocation = displayLocation.map(_.toThrift), - controllerData = controllerData, - algoId = algorithmId - ) - } - - def toOfflineThrift: offline.TrackingToken = { - offline.TrackingToken( - sessionId = sessionId, - displayLocation = displayLocation.map(_.toOfflineThrift), - controllerData = controllerData, - algoId = algorithmId - ) - } -} - -object TrackingToken { - val binaryThriftSerializer = BinaryThriftStructSerializer[t.TrackingToken](t.TrackingToken) - def serialize(trackingToken: TrackingToken): String = { - Base64StringEncoder.encode(binaryThriftSerializer.toBytes(trackingToken.toThrift)) - } - def deserialize(trackingTokenStr: String): TrackingToken = { - fromThrift(binaryThriftSerializer.fromBytes(Base64StringEncoder.decode(trackingTokenStr))) - } - def fromThrift(token: t.TrackingToken): TrackingToken = { - TrackingToken( - sessionId = token.sessionId, - displayLocation = token.displayLocation.map(DisplayLocation.fromThrift), - controllerData = token.controllerData, - algorithmId = token.algoId - ) - } -} - -trait HasTrackingToken { - def trackingToken: Option[TrackingToken] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.scala deleted file mode 100644 index 1957e28cc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/TweetCandidate.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -case class TweetCandidate( - tweetId: Long, - authorId: Long, - score: Option[Double]) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.scala deleted file mode 100644 index 73b766232..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserCandidateSourceDetails.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.hermit.constants.AlgorithmFeedbackTokens._ -import com.twitter.hermit.ml.models.Feature -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier - -/** - * primaryCandidateSource param is showing the candidate source that responsible for generating this - * candidate, as the candidate might have gone through multiple candidate sources to get generated - * (for example if it has generated by a composite source). WeightedCandidateSourceRanker uses this - * field to do the sampling over candidate sources. All the sources used for generating this - * candidate (including the primary source) and their corresponding score exist in the - * candidateSourceScores field. - */ -case class UserCandidateSourceDetails( - primaryCandidateSource: Option[CandidateSourceIdentifier], - candidateSourceScores: Map[CandidateSourceIdentifier, Option[Double]] = Map.empty, - candidateSourceRanks: Map[CandidateSourceIdentifier, Int] = Map.empty, - addressBookMetadata: Option[AddressBookMetadata] = None, - candidateSourceFeatures: Map[CandidateSourceIdentifier, Seq[Feature]] = Map.empty, -) { - - def toThrift: t.CandidateSourceDetails = { - t.CandidateSourceDetails( - candidateSourceScores = Some(candidateSourceScores.map { - case (identifier, score) => - (identifier.name, score.getOrElse(0.0d)) - }), - primarySource = for { - identifier <- primaryCandidateSource - algo <- Algorithm.withNameOpt(identifier.name) - feedbackToken <- AlgorithmToFeedbackTokenMap.get(algo) - } yield feedbackToken - ) - } - - def toOfflineThrift: offline.CandidateSourceDetails = { - offline.CandidateSourceDetails( - candidateSourceScores = Some(candidateSourceScores.map { - case (identifier, score) => - (identifier.name, score.getOrElse(0.0d)) - }), - primarySource = for { - identifier <- primaryCandidateSource - algo <- Algorithm.withNameOpt(identifier.name) - feedbackToken <- AlgorithmToFeedbackTokenMap.get(algo) - } yield feedbackToken - ) - } -} - -object UserCandidateSourceDetails { - val algorithmNameMap: Map[String, Algorithm.Value] = Algorithm.values.map { - algorithmValue: Algorithm.Value => - (algorithmValue.toString, algorithmValue) - }.toMap - - /** - * This method is used to parse the candidate source of the candidates, which is only passed from - * the scoreUserCandidates endpoint. We create custom candidate source identifiers which - * CandidateAlgorithmSource will read from to hydrate the algorithm id feature. - * candidateSourceScores will not be populated from the endpoint, but we add the conversion for - * completeness. Note that the conversion uses the raw string of the Algorithm rather than the - * assigned strings that we give to our own candidate sources in the FRS. - */ - def fromThrift(details: t.CandidateSourceDetails): UserCandidateSourceDetails = { - val primaryCandidateSource: Option[CandidateSourceIdentifier] = for { - primarySourceToken <- details.primarySource - algo <- TokenToAlgorithmMap.get(primarySourceToken) - } yield CandidateSourceIdentifier(algo.toString) - - val candidateSourceScores = for { - scoreMap <- details.candidateSourceScores.toSeq - (name, score) <- scoreMap - algo <- algorithmNameMap.get(name) - } yield { - CandidateSourceIdentifier(algo.toString) -> Some(score) - } - val candidateSourceRanks = for { - rankMap <- details.candidateSourceRanks.toSeq - (name, rank) <- rankMap - algo <- algorithmNameMap.get(name) - } yield { - CandidateSourceIdentifier(algo.toString) -> rank - } - UserCandidateSourceDetails( - primaryCandidateSource = primaryCandidateSource, - candidateSourceScores = candidateSourceScores.toMap, - candidateSourceRanks = candidateSourceRanks.toMap, - addressBookMetadata = None, - candidateSourceFeatures = Map.empty - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.scala deleted file mode 100644 index 74f33eb40..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/UserIdAndTimestamp.scala +++ /dev/null @@ -1,3 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -case class UserIdWithTimestamp(userId: Long, timeInMs: Long) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.scala deleted file mode 100644 index 39e0561a2..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/WtfImpression.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.models - -import com.twitter.util.Time - -/** - * Domain model for representing impressions on wtf recommendations in the past 16 days - */ -case class WtfImpression( - candidateId: Long, - displayLocation: DisplayLocation, - latestTime: Time, - counts: Int) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD deleted file mode 100644 index ffcbe65a7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/features", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.scala deleted file mode 100644 index 0713f728f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateParamPredicate.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Param - -class CandidateParamPredicate[A <: HasParams]( - param: Param[Boolean], - reason: FilterReason) - extends Predicate[A] { - override def apply(candidate: A): Stitch[PredicateResult] = { - if (candidate.params(param)) { - Stitch.value(PredicateResult.Valid) - } else { - Stitch.value(PredicateResult.Invalid(Set(reason))) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.scala deleted file mode 100644 index cf08f5623..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CandidateSourceParamPredicate.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Param - -/** - * This predicate allows us to filter candidates given its source. - * To avoid bucket dilution, we only want to evaluate the param (which would implicitly trigger - * bucketing for FSParams) only if the candidate source fn yields true. - * The param provided should be true when we want to keep the candidate and false otherwise. - */ -class CandidateSourceParamPredicate( - val param: Param[Boolean], - val reason: FilterReason, - candidateSources: Set[CandidateSourceIdentifier]) - extends Predicate[CandidateUser] { - override def apply(candidate: CandidateUser): Stitch[PredicateResult] = { - // we want to avoid evaluating the param if the candidate source fn yields false - if (candidate.getCandidateSources.keys.exists(candidateSources.contains) && !candidate.params( - param)) { - Stitch.value(PredicateResult.Invalid(Set(reason))) - } else { - Stitch.value(PredicateResult.Valid) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.scala deleted file mode 100644 index 16d76ce44..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/CuratedCompetitorListPredicate.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.FilterReason.CuratedAccountsCompetitorList -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.escherbird.util.stitchcache.StitchCache - -@Singleton -case class CuratedCompetitorListPredicate @Inject() ( - statsReceiver: StatsReceiver, - @Named(GuiceNamedConstants.CURATED_COMPETITOR_ACCOUNTS_FETCHER) competitorAccountFetcher: Fetcher[ - String, - Unit, - Seq[Long] - ]) extends Predicate[CandidateUser] { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - private val cacheStats = stats.scope("cache") - - private val cache = StitchCache[String, Set[Long]]( - maxCacheSize = CuratedCompetitorListPredicate.CacheNumberOfEntries, - ttl = CuratedCompetitorListPredicate.CacheTTL, - statsReceiver = cacheStats, - underlyingCall = (competitorListPrefix: String) => query(competitorListPrefix) - ) - - private def query(prefix: String): Stitch[Set[Long]] = - competitorAccountFetcher.fetch(prefix).map(_.v.getOrElse(Nil).toSet) - - /** - * Caveat here is that though the similarToUserIds allows for a Seq[Long], in practice we would - * only return 1 userId. Multiple userId's would result in filtering candidates associated with - * a different similarToUserId. For example: - * - similarToUser1 -> candidate1, candidate2 - * - similarToUser2 -> candidate3 - * and in the competitorList store we have: - * - similarToUser1 -> candidate3 - * we'll be filtering candidate3 on account of similarToUser1, even though it was generated - * with similarToUser2. This might still be desirable at a product level (since we don't want - * to show these accounts anyway), but might not achieve what you intend to code-wise. - */ - override def apply(candidate: CandidateUser): Stitch[PredicateResult] = { - cache.readThrough(CuratedCompetitorListPredicate.DefaultKey).map { competitorListAccounts => - if (competitorListAccounts.contains(candidate.id)) { - PredicateResult.Invalid(Set(CuratedAccountsCompetitorList)) - } else { - PredicateResult.Valid - } - } - } -} - -object CuratedCompetitorListPredicate { - val DefaultKey: String = "default_list" - val CacheTTL = 5.minutes - val CacheNumberOfEntries = 5 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.scala deleted file mode 100644 index 01a5f96fd..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/ExcludedUserIdPredicate.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.FilterReason.ExcludedId -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasExcludedUserIds -import com.twitter.stitch.Stitch - -object ExcludedUserIdPredicate extends Predicate[(HasExcludedUserIds, CandidateUser)] { - - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val ExcludedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(ExcludedId))) - - override def apply(pair: (HasExcludedUserIds, CandidateUser)): Stitch[PredicateResult] = { - val (excludedUserIds, candidate) = pair - if (excludedUserIds.excludedUserIds.contains(candidate.id)) { - ExcludedStitch - } else { - ValidStitch - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.scala deleted file mode 100644 index c77538a99..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicate.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.google.inject.name.Named -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.predicates.InactivePredicateParams._ -import com.twitter.service.metastore.gen.thriftscala.UserRecommendabilityFeatures -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.follow_recommendations.common.models.HasUserState -import com.twitter.follow_recommendations.common.predicates.InactivePredicateParams.DefaultInactivityThreshold -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext - -import java.lang.{Long => JLong} - -@Singleton -case class InactivePredicate @Inject() ( - statsReceiver: StatsReceiver, - @Named(GuiceNamedConstants.USER_RECOMMENDABILITY_FETCHER) userRecommendabilityFetcher: Fetcher[ - Long, - Unit, - UserRecommendabilityFeatures - ]) extends Predicate[(HasParams with HasClientContext with HasUserState, CandidateUser)] { - - private val stats: StatsReceiver = statsReceiver.scope("InactivePredicate") - private val cacheStats = stats.scope("cache") - - private def queryUserRecommendable(userId: Long): Stitch[Option[UserRecommendabilityFeatures]] = - userRecommendabilityFetcher.fetch(userId).map(_.v) - - private val userRecommendableCache = - StitchCache[JLong, Option[UserRecommendabilityFeatures]]( - maxCacheSize = 100000, - ttl = 12.hours, - statsReceiver = cacheStats.scope("UserRecommendable"), - underlyingCall = (userId: JLong) => queryUserRecommendable(userId) - ) - - override def apply( - targetAndCandidate: (HasParams with HasClientContext with HasUserState, CandidateUser) - ): Stitch[PredicateResult] = { - val (target, candidate) = targetAndCandidate - - userRecommendableCache - .readThrough(candidate.id).map { - case recFeaturesFetchResult => - recFeaturesFetchResult match { - case None => - PredicateResult.Invalid(Set(FilterReason.MissingRecommendabilityData)) - case Some(recFeatures) => - if (disableInactivityPredicate(target, target.userState, recFeatures.userState)) { - PredicateResult.Valid - } else { - val defaultInactivityThreshold = target.params(DefaultInactivityThreshold).days - val hasBeenActiveRecently = recFeatures.lastStatusUpdateMs - .map(Time.now - Time.fromMilliseconds(_)).getOrElse( - Duration.Top) < defaultInactivityThreshold - stats - .scope(defaultInactivityThreshold.toString).counter( - if (hasBeenActiveRecently) - "active" - else - "inactive" - ).incr() - if (hasBeenActiveRecently && (!target - .params(UseEggFilter) || recFeatures.isNotEgg.contains(1))) { - PredicateResult.Valid - } else { - PredicateResult.Invalid(Set(FilterReason.Inactive)) - } - } - } - }.rescue { - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FilterReason.FailOpen))) - } - } - - private[this] def disableInactivityPredicate( - target: HasParams, - consumerState: Option[UserState], - candidateState: Option[UserState] - ): Boolean = { - target.params(MightBeDisabled) && - consumerState.exists(InactivePredicate.ValidConsumerStates.contains) && - ( - ( - candidateState.exists(InactivePredicate.ValidCandidateStates.contains) && - !target.params(OnlyDisableForNewUserStateCandidates) - ) || - ( - candidateState.contains(UserState.New) && - target.params(OnlyDisableForNewUserStateCandidates) - ) - ) - } -} - -object InactivePredicate { - val ValidConsumerStates: Set[UserState] = Set( - UserState.HeavyNonTweeter, - UserState.MediumNonTweeter, - UserState.HeavyTweeter, - UserState.MediumTweeter - ) - val ValidCandidateStates: Set[UserState] = - Set(UserState.New, UserState.VeryLight, UserState.Light, UserState.NearZero) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.scala deleted file mode 100644 index 0bb52caa8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/InactivePredicateParams.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object InactivePredicateParams { - case object DefaultInactivityThreshold - extends FSBoundedParam[Int]( - name = "inactive_predicate_default_inactivity_threshold", - default = 60, - min = 1, - max = 500 - ) - case object UseEggFilter extends Param[Boolean](true) - case object MightBeDisabled extends FSParam[Boolean]("inactive_predicate_might_be_disabled", true) - case object OnlyDisableForNewUserStateCandidates - extends FSParam[Boolean]( - "inactive_predicate_only_disable_for_new_user_state_candidates", - false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.scala deleted file mode 100644 index 7879860ac..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/PreviouslyRecommendedUserIdsPredicate.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.HasPreviousRecommendationsContext -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class PreviouslyRecommendedUserIdsPredicate - extends Predicate[(HasPreviousRecommendationsContext, CandidateUser)] { - override def apply( - pair: (HasPreviousRecommendationsContext, CandidateUser) - ): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - - val previouslyRecommendedUserIDs = targetUser.previouslyRecommendedUserIDs - - if (!previouslyRecommendedUserIDs.contains(candidate.id)) { - PreviouslyRecommendedUserIdsPredicate.ValidStitch - } else { - PreviouslyRecommendedUserIdsPredicate.AlreadyRecommendedStitch - } - } -} - -object PreviouslyRecommendedUserIdsPredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val AlreadyRecommendedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(FilterReason.AlreadyRecommended))) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD deleted file mode 100644 index 9d1ca9b40..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/dismiss_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.scala deleted file mode 100644 index 550017b95..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicate.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.dismiss - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.FilterReason.DismissedId -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDismissedUserIds -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class DismissedCandidatePredicate extends Predicate[(HasDismissedUserIds, CandidateUser)] { - - override def apply(pair: (HasDismissedUserIds, CandidateUser)): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - targetUser.dismissedUserIds - .map { dismissedUserIds => - if (!dismissedUserIds.contains(candidate.id)) { - DismissedCandidatePredicate.ValidStitch - } else { - DismissedCandidatePredicate.DismissedStitch - } - }.getOrElse(DismissedCandidatePredicate.ValidStitch) - } -} - -object DismissedCandidatePredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val DismissedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(DismissedId))) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.scala deleted file mode 100644 index 7f1d51a07..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss/DismissedCandidatePredicateParams.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.dismiss - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object DismissedCandidatePredicateParams { - case object LookBackDuration extends Param[Duration](180.days) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD deleted file mode 100644 index a154121e6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "stitch/stitch-gizmoduck", - "util/util-slf4j-api/src/main/scala", - "util/util-thrift", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.scala deleted file mode 100644 index 2ca3e2fc5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicate.scala +++ /dev/null @@ -1,284 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.escherbird.util.stitchcache.StitchCache -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient -import com.twitter.follow_recommendations.common.clients.cache.ThriftBijection -import com.twitter.follow_recommendations.common.models.FilterReason._ -import com.twitter.follow_recommendations.common.models.AddressBookMetadata -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate._ -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateParams._ -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.gizmoduck.thriftscala.LabelValue.BlinkBad -import com.twitter.gizmoduck.thriftscala.LabelValue.BlinkWorst -import com.twitter.gizmoduck.thriftscala.LabelValue -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.QueryFields -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserResult -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.scrooge.CompactThriftSerializer -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Duration -import com.twitter.util.logging.Logging -import java.lang.{Long => JLong} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * In this filter, we want to check 4 categories of conditions: - * - if candidate is discoverable given that it's from an address-book/phone-book based source - * - if candidate is unsuitable based on it's safety sub-fields in gizmoduck - * - if candidate is withheld because of country-specific take-down policies - * - if candidate is marked as bad/worst based on blink labels - * We fail close on the query as this is a product-critical filter - */ -@Singleton -case class GizmoduckPredicate @Inject() ( - gizmoduck: Gizmoduck, - client: Client, - statsReceiver: StatsReceiver, - decider: Decider = Decider.False) - extends Predicate[(HasClientContext with HasParams, CandidateUser)] - with Logging { - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - - // track # of Gizmoduck predicate queries that yielded valid & invalid predicate results - private val validPredicateResultCounter = stats.counter("predicate_valid") - private val invalidPredicateResultCounter = stats.counter("predicate_invalid") - - // track # of cases where no Gizmoduck user was found - private val noGizmoduckUserCounter = stats.counter("no_gizmoduck_user_found") - - private val gizmoduckCache = StitchCache[JLong, UserResult]( - maxCacheSize = MaxCacheSize, - ttl = CacheTTL, - statsReceiver = stats.scope("cache"), - underlyingCall = getByUserId - ) - - // Distributed Twemcache to store UserResult objects keyed on user IDs - val bijection = new ThriftBijection[UserResult] { - override val serializer = CompactThriftSerializer(UserResult) - } - val memcacheClient = MemcacheClient[UserResult]( - client = client, - dest = "/s/cache/frs:twemcaches", - valueBijection = bijection, - ttl = CacheTTL, - statsReceiver = stats.scope("twemcache") - ) - - // main method used to apply GizmoduckPredicate to a candidate user - override def apply( - pair: (HasClientContext with HasParams, CandidateUser) - ): Stitch[PredicateResult] = { - val (request, candidate) = pair - // measure the latency of the getGizmoduckPredicateResult, since this predicate - // check is product-critical and relies on querying a core service (Gizmoduck) - StatsUtil.profileStitch( - getGizmoduckPredicateResult(request, candidate), - stats.scope("getGizmoduckPredicateResult") - ) - } - - private def getGizmoduckPredicateResult( - request: HasClientContext with HasParams, - candidate: CandidateUser - ): Stitch[PredicateResult] = { - val timeout: Duration = request.params(GizmoduckGetTimeout) - - val deciderKey: String = DeciderKey.EnableGizmoduckCaching.toString - val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - - // try getting an existing UserResult from cache if possible - val userResultStitch: Stitch[UserResult] = - enableDistributedCaching match { - // read from memcache - case true => memcacheClient.readThrough( - // add a key prefix to address cache key collisions - key = "GizmoduckPredicate" + candidate.id.toString, - underlyingCall = () => getByUserId(candidate.id) - ) - // read from local cache - case false => gizmoduckCache.readThrough(candidate.id) - } - - val predicateResultStitch = userResultStitch.map { - userResult => { - val predicateResult = getPredicateResult(request, candidate, userResult) - if (enableDistributedCaching) { - predicateResult match { - case PredicateResult.Valid => - stats.scope("twemcache").counter("predicate_valid").incr() - case PredicateResult.Invalid(reasons) => - stats.scope("twemcache").counter("predicate_invalid").incr() - } - // log metrics to check if local cache value matches distributed cache value - logPredicateResultEquality( - request, - candidate, - predicateResult - ) - } else { - predicateResult match { - case PredicateResult.Valid => - stats.scope("cache").counter("predicate_valid").incr() - case PredicateResult.Invalid(reasons) => - stats.scope("cache").counter("predicate_invalid").incr() - } - } - predicateResult - } - } - predicateResultStitch - .within(timeout)(DefaultTimer) - .rescue { // fail-open when timeout or exception - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - invalidPredicateResultCounter.incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - } - } - - private def logPredicateResultEquality( - request: HasClientContext with HasParams, - candidate: CandidateUser, - predicateResult: PredicateResult - ): Unit = { - val localCachedUserResult = Option(gizmoduckCache.cache.getIfPresent(candidate.id)) - if (localCachedUserResult.isDefined) { - val localPredicateResult = getPredicateResult(request, candidate, localCachedUserResult.get) - localPredicateResult.equals(predicateResult) match { - case true => stats.scope("has_equal_predicate_value").counter("true").incr() - case false => stats.scope("has_equal_predicate_value").counter("false").incr() - } - } else { - stats.scope("has_equal_predicate_value").counter("undefined").incr() - } - } - - // method to get PredicateResult from UserResult - def getPredicateResult( - request: HasClientContext with HasParams, - candidate: CandidateUser, - userResult: UserResult, - ): PredicateResult = { - userResult.user match { - case Some(user) => - val abPbReasons = getAbPbReason(user, candidate.getAddressBookMetadata) - val safetyReasons = getSafetyReasons(user) - val countryTakedownReasons = getCountryTakedownReasons(user, request.getCountryCode) - val blinkReasons = getBlinkReasons(user) - val allReasons = - abPbReasons ++ safetyReasons ++ countryTakedownReasons ++ blinkReasons - if (allReasons.nonEmpty) { - invalidPredicateResultCounter.incr() - PredicateResult.Invalid(allReasons) - } else { - validPredicateResultCounter.incr() - PredicateResult.Valid - } - case None => - noGizmoduckUserCounter.incr() - invalidPredicateResultCounter.incr() - PredicateResult.Invalid(Set(NoUser)) - } - } - - private def getByUserId(userId: JLong): Stitch[UserResult] = { - StatsUtil.profileStitch( - gizmoduck.getById(userId = userId, queryFields = queryFields, context = lookupContext), - stats.scope("getByUserId") - ) - } -} - -object GizmoduckPredicate { - - private[gizmoduck] val lookupContext: LookupContext = - LookupContext(`includeDeactivated` = true, `safetyLevel` = Some(SafetyLevel.Recommendations)) - - private[gizmoduck] val queryFields: Set[QueryFields] = - Set( - QueryFields.Discoverability, // needed for Address Book / Phone Book discoverability checks in getAbPbReason - QueryFields.Safety, // needed for user state safety checks in getSafetyReasons, getCountryTakedownReasons - QueryFields.Labels, // needed for user label checks in getBlinkReasons - QueryFields.Takedowns // needed for checking takedown labels for a user in getCountryTakedownReasons - ) - - private[gizmoduck] val BlinkLabels: Set[LabelValue] = Set(BlinkBad, BlinkWorst) - - private[gizmoduck] def getAbPbReason( - user: User, - abMetadataOpt: Option[AddressBookMetadata] - ): Set[FilterReason] = { - (for { - discoverability <- user.discoverability - abMetadata <- abMetadataOpt - } yield { - val AddressBookMetadata(fwdPb, rvPb, fwdAb, rvAb) = abMetadata - val abReason: Set[FilterReason] = - if ((!discoverability.discoverableByEmail) && (fwdAb || rvAb)) - Set(AddressBookUndiscoverable) - else Set.empty - val pbReason: Set[FilterReason] = - if ((!discoverability.discoverableByMobilePhone) && (fwdPb || rvPb)) - Set(PhoneBookUndiscoverable) - else Set.empty - abReason ++ pbReason - }).getOrElse(Set.empty) - } - - private[gizmoduck] def getSafetyReasons(user: User): Set[FilterReason] = { - user.safety - .map { s => - val deactivatedReason: Set[FilterReason] = - if (s.deactivated) Set(Deactivated) else Set.empty - val suspendedReason: Set[FilterReason] = if (s.suspended) Set(Suspended) else Set.empty - val restrictedReason: Set[FilterReason] = if (s.restricted) Set(Restricted) else Set.empty - val nsfwUserReason: Set[FilterReason] = if (s.nsfwUser) Set(NsfwUser) else Set.empty - val nsfwAdminReason: Set[FilterReason] = if (s.nsfwAdmin) Set(NsfwAdmin) else Set.empty - val isProtectedReason: Set[FilterReason] = if (s.isProtected) Set(IsProtected) else Set.empty - deactivatedReason ++ suspendedReason ++ restrictedReason ++ nsfwUserReason ++ nsfwAdminReason ++ isProtectedReason - }.getOrElse(Set.empty) - } - - private[gizmoduck] def getCountryTakedownReasons( - user: User, - countryCodeOpt: Option[String] - ): Set[FilterReason] = { - (for { - safety <- user.safety.toSeq - if safety.hasTakedown - takedowns <- user.takedowns.toSeq - takedownCountry <- takedowns.countryCodes - requestingCountry <- countryCodeOpt - if takedownCountry.toLowerCase == requestingCountry.toLowerCase - } yield Set(CountryTakedown(takedownCountry.toLowerCase))).flatten.toSet - } - - private[gizmoduck] def getBlinkReasons(user: User): Set[FilterReason] = { - user.labels - .map(_.labels.map(_.labelValue)) - .getOrElse(Nil) - .exists(BlinkLabels.contains) - for { - labels <- user.labels.toSeq - label <- labels.labels - if (BlinkLabels.contains(label.labelValue)) - } yield Set(Blink) - }.flatten.toSet -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.scala deleted file mode 100644 index 36fb2f20f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateCache.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import java.util.concurrent.TimeUnit - -import com.google.common.base.Ticker -import com.google.common.cache.CacheBuilder -import com.google.common.cache.Cache -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Time -import com.twitter.util.Duration - -/** - * In-memory cache used for caching GizmoduckPredicate query calls in - * com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate. - * - * References the cache implementation in com.twitter.escherbird.util.stitchcache, - * but without the underlying Stitch call. - */ -object GizmoduckPredicateCache { - - private[GizmoduckPredicateCache] class TimeTicker extends Ticker { - override def read(): Long = Time.now.inNanoseconds - } - - def apply[K, V]( - maxCacheSize: Int, - ttl: Duration, - statsReceiver: StatsReceiver - ): Cache[K, V] = { - - val cache: Cache[K, V] = - CacheBuilder - .newBuilder() - .maximumSize(maxCacheSize) - .asInstanceOf[CacheBuilder[K, V]] - .expireAfterWrite(ttl.inSeconds, TimeUnit.SECONDS) - .recordStats() - .ticker(new TimeTicker()) - .build() - - // metrics for tracking cache usage - statsReceiver.provideGauge("cache_size") { cache.size.toFloat } - statsReceiver.provideGauge("cache_hits") { cache.stats.hitCount.toFloat } - statsReceiver.provideGauge("cache_misses") { cache.stats.missCount.toFloat } - statsReceiver.provideGauge("cache_hit_rate") { cache.stats.hitRate.toFloat } - statsReceiver.provideGauge("cache_evictions") { cache.stats.evictionCount.toFloat } - - cache - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.scala deleted file mode 100644 index 447eac835..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateFSConfig.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateParams._ -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GizmoduckPredicateFSConfig @Inject() () extends FeatureSwitchConfig { - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - GizmoduckGetTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.scala deleted file mode 100644 index 811897e27..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck/GizmoduckPredicateParams.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.gizmoduck - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object GizmoduckPredicateParams { - case object GizmoduckGetTimeout - extends FSBoundedParam[Duration]( - name = "gizmoduck_predicate_timeout_in_millis", - default = 200.millisecond, - min = 1.millisecond, - max = 500.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } - val MaxCacheSize: Int = 250000 - val CacheTTL: Duration = Duration.fromHours(6) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD deleted file mode 100644 index d0e9e1015..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "escherbird/src/scala/com/twitter/escherbird/util/stitchcache", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "strato/config/columns/hss/user_signals/api:api-strato-client", - "util/util-slf4j-api/src/main/scala", - "util/util-thrift", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.scala deleted file mode 100644 index 7464be7df..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicate.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.hss - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.FilterReason.FailOpen -import com.twitter.hss.api.thriftscala.SignalValue -import com.twitter.hss.api.thriftscala.UserHealthSignal.AgathaCseDouble -import com.twitter.hss.api.thriftscala.UserHealthSignal.NsfwAgathaUserScoreDouble -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.hss.user_signals.api.HealthSignalsOnUserClientColumn -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Filter out candidates based on Health Signal Store (HSS) health signals - */ -@Singleton -case class HssPredicate @Inject() ( - healthSignalsOnUserClientColumn: HealthSignalsOnUserClientColumn, - statsReceiver: StatsReceiver) - extends Predicate[(HasClientContext with HasParams, CandidateUser)] - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - - override def apply( - pair: (HasClientContext with HasParams, CandidateUser) - ): Stitch[PredicateResult] = { - val (request, candidate) = pair - StatsUtil.profileStitch( - getHssPredicateResult(request, candidate), - stats.scope("getHssPredicateResult") - ) - } - - private def getHssPredicateResult( - request: HasClientContext with HasParams, - candidate: CandidateUser - ): Stitch[PredicateResult] = { - - val hssCseScoreThreshold: Double = request.params(HssPredicateParams.HssCseScoreThreshold) - val hssNsfwScoreThreshold: Double = request.params(HssPredicateParams.HssNsfwScoreThreshold) - val timeout: Duration = request.params(HssPredicateParams.HssApiTimeout) - - healthSignalsOnUserClientColumn.fetcher - .fetch(candidate.id, Seq(AgathaCseDouble, NsfwAgathaUserScoreDouble)) - .map { fetchResult => - fetchResult.v match { - case Some(response) => - val agathaCseScoreDouble: Double = userHealthSignalValueToDoubleOpt( - response.signalValues.get(AgathaCseDouble)).getOrElse(0d) - val agathaNsfwScoreDouble: Double = userHealthSignalValueToDoubleOpt( - response.signalValues.get(NsfwAgathaUserScoreDouble)).getOrElse(0d) - - stats.stat("agathaCseScoreDistribution").add(agathaCseScoreDouble.toFloat) - stats.stat("agathaNsfwScoreDistribution").add(agathaNsfwScoreDouble.toFloat) - - /** - * Only filter out the candidate when it has both high Agatha CSE score and NSFW score, as the Agatha CSE - * model is an old one that may not be precise or have high recall. - */ - if (agathaCseScoreDouble >= hssCseScoreThreshold && agathaNsfwScoreDouble >= hssNsfwScoreThreshold) { - PredicateResult.Invalid(Set(FilterReason.HssSignal)) - } else { - PredicateResult.Valid - } - case None => - PredicateResult.Valid - } - } - .within(timeout)(DefaultTimer) - .rescue { - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - } - } - - private def userHealthSignalValueToDoubleOpt(signalValue: Option[SignalValue]): Option[Double] = { - signalValue match { - case Some(SignalValue.DoubleValue(value)) => Some(value) - case _ => None - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.scala deleted file mode 100644 index 8cc1620a9..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateFSConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.hss - -import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateParams._ -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HssPredicateFSConfig @Inject() () extends FeatureSwitchConfig { - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - HssCseScoreThreshold, - HssNsfwScoreThreshold, - ) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - HssApiTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.scala deleted file mode 100644 index ac6e14bbe..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health/HssPredicateParams.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.hss - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -object HssPredicateParams { - object HssCseScoreThreshold - extends FSBoundedParam[Double]( - "hss_predicate_cse_score_threshold", - default = 0.992d, - min = 0.0d, - max = 1.0d) - - object HssNsfwScoreThreshold - extends FSBoundedParam[Double]( - "hss_predicate_nsfw_score_threshold", - default = 1.5d, - min = -100.0d, - max = 100.0d) - - object HssApiTimeout - extends FSBoundedParam[Duration]( - name = "hss_predicate_timeout_in_millis", - default = 200.millisecond, - min = 1.millisecond, - max = 500.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD deleted file mode 100644 index 0df7b245a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.scala deleted file mode 100644 index 84b8bf7a6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/InvalidRelationshipPredicate.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.HasInvalidRelationshipUserIds -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class InvalidRelationshipPredicate - extends Predicate[(HasInvalidRelationshipUserIds, CandidateUser)] { - - override def apply( - pair: (HasInvalidRelationshipUserIds, CandidateUser) - ): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - targetUser.invalidRelationshipUserIds match { - case Some(users) => - if (!users.contains(candidate.id)) { - InvalidRelationshipPredicate.ValidStitch - } else { - Stitch.value(InvalidRelationshipPredicate.InvalidRelationshipStitch) - } - case None => Stitch.value(PredicateResult.Valid) - } - } -} - -object InvalidRelationshipPredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val InvalidRelationshipStitch: PredicateResult.Invalid = - PredicateResult.Invalid(Set(FilterReason.InvalidRelationship)) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.scala deleted file mode 100644 index 60f0080b8..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/RecentFollowingPredicate.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.stitch.Stitch -import javax.inject.Singleton - -@Singleton -class RecentFollowingPredicate extends Predicate[(HasRecentFollowedUserIds, CandidateUser)] { - - override def apply(pair: (HasRecentFollowedUserIds, CandidateUser)): Stitch[PredicateResult] = { - - val (targetUser, candidate) = pair - targetUser.recentFollowedUserIdsSet match { - case Some(users) => - if (!users.contains(candidate.id)) { - RecentFollowingPredicate.ValidStitch - } else { - RecentFollowingPredicate.AlreadyFollowedStitch - } - case None => RecentFollowingPredicate.ValidStitch - } - } -} - -object RecentFollowingPredicate { - val ValidStitch: Stitch[PredicateResult.Valid.type] = Stitch.value(PredicateResult.Valid) - val AlreadyFollowedStitch: Stitch[PredicateResult.Invalid] = - Stitch.value(PredicateResult.Invalid(Set(FilterReason.AlreadyFollowed))) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.scala deleted file mode 100644 index f661dbbab..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateFSConfig.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SgsPredicateFSConfig @Inject() () extends FeatureSwitchConfig { - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - SgsPredicateParams.SgsRelationshipsPredicateTimeout - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.scala deleted file mode 100644 index dd615c47d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsPredicateParams.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ - -object SgsPredicateParams { - case object SgsRelationshipsPredicateTimeout - extends FSBoundedParam[Duration]( - name = "sgs_predicate_relationships_timeout_in_millis", - default = 300.millisecond, - min = 1.millisecond, - max = 1000.millisecond) - with HasDurationConversion { - override def durationConversion: DurationConversion = DurationConversion.FromMillis - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.scala deleted file mode 100644 index dec936e58..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsByUserIdPredicate.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.google.common.annotations.VisibleForTesting -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason.InvalidRelationshipTypes -import com.twitter.socialgraph.thriftscala.ExistsRequest -import com.twitter.socialgraph.thriftscala.ExistsResult -import com.twitter.socialgraph.thriftscala.LookupContext -import com.twitter.socialgraph.thriftscala.Relationship -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.util.logging.Logging -import javax.inject.Inject -import javax.inject.Singleton - -class SgsRelationshipsByUserIdPredicate( - socialGraph: SocialGraph, - relationshipMappings: Seq[RelationshipMapping], - statsReceiver: StatsReceiver) - extends Predicate[(Option[Long], CandidateUser)] - with Logging { - private val InvalidFromPrimaryCandidateSourceName = "invalid_from_primary_candidate_source" - private val InvalidFromCandidateSourceName = "invalid_from_candidate_source" - private val NoPrimaryCandidateSource = "no_primary_candidate_source" - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getName) - - override def apply( - pair: (Option[Long], CandidateUser) - ): Stitch[PredicateResult] = { - val (idOpt, candidate) = pair - val relationships = relationshipMappings.map { relationshipMapping: RelationshipMapping => - Relationship( - relationshipMapping.relationshipType, - relationshipMapping.includeBasedOnRelationship) - } - idOpt - .map { id: Long => - val existsRequest = ExistsRequest( - id, - candidate.id, - relationships = relationships, - context = SgsRelationshipsByUserIdPredicate.UnionLookupContext - ) - socialGraph - .exists(existsRequest).map { existsResult: ExistsResult => - if (existsResult.exists) { - candidate.getPrimaryCandidateSource match { - case Some(candidateSource) => - stats - .scope(InvalidFromPrimaryCandidateSourceName).counter( - candidateSource.name).incr() - case None => - stats - .scope(InvalidFromPrimaryCandidateSourceName).counter( - NoPrimaryCandidateSource).incr() - } - candidate.getCandidateSources.foreach({ - case (candidateSource, _) => - stats - .scope(InvalidFromCandidateSourceName).counter(candidateSource.name).incr() - }) - PredicateResult.Invalid(Set(InvalidRelationshipTypes(relationshipMappings - .map { relationshipMapping: RelationshipMapping => - relationshipMapping.relationshipType - }.mkString(", ")))) - } else { - PredicateResult.Valid - } - } - } - // if no user id is present, return true by default - .getOrElse(Stitch.value(PredicateResult.Valid)) - } -} - -object SgsRelationshipsByUserIdPredicate { - // OR Operation - @VisibleForTesting - private[follow_recommendations] val UnionLookupContext = Some( - LookupContext(performUnion = Some(true))) -} - -@Singleton -class ExcludeNonFollowersSgsPredicate @Inject() ( - socialGraph: SocialGraph, - statsReceiver: StatsReceiver) - extends SgsRelationshipsByUserIdPredicate( - socialGraph, - Seq(RelationshipMapping(RelationshipType.FollowedBy, includeBasedOnRelationship = false)), - statsReceiver) - -@Singleton -class ExcludeNonFollowingSgsPredicate @Inject() ( - socialGraph: SocialGraph, - statsReceiver: StatsReceiver) - extends SgsRelationshipsByUserIdPredicate( - socialGraph, - Seq(RelationshipMapping(RelationshipType.Following, includeBasedOnRelationship = false)), - statsReceiver) - -@Singleton -class ExcludeFollowingSgsPredicate @Inject() ( - socialGraph: SocialGraph, - statsReceiver: StatsReceiver) - extends SgsRelationshipsByUserIdPredicate( - socialGraph, - Seq(RelationshipMapping(RelationshipType.Following, includeBasedOnRelationship = true)), - statsReceiver) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.scala deleted file mode 100644 index fdb88ad58..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs/SgsRelationshipsPredicate.scala +++ /dev/null @@ -1,146 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.sgs - -import com.google.common.annotations.VisibleForTesting -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasProfileId -import com.twitter.follow_recommendations.common.models.FilterReason.FailOpen -import com.twitter.follow_recommendations.common.models.FilterReason.InvalidRelationshipTypes -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.socialgraph.thriftscala.ExistsRequest -import com.twitter.socialgraph.thriftscala.ExistsResult -import com.twitter.socialgraph.thriftscala.LookupContext -import com.twitter.socialgraph.thriftscala.Relationship -import com.twitter.socialgraph.thriftscala.RelationshipType -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.TimeoutException -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -case class RelationshipMapping( - relationshipType: RelationshipType, - includeBasedOnRelationship: Boolean) - -class SgsRelationshipsPredicate( - socialGraph: SocialGraph, - relationshipMappings: Seq[RelationshipMapping], - statsReceiver: StatsReceiver = NullStatsReceiver) - extends Predicate[(HasClientContext with HasParams, CandidateUser)] - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - override def apply( - pair: (HasClientContext with HasParams, CandidateUser) - ): Stitch[PredicateResult] = { - val (target, candidate) = pair - val timeout = target.params(SgsPredicateParams.SgsRelationshipsPredicateTimeout) - SgsRelationshipsPredicate - .extractUserId(target) - .map { id => - val relationships = relationshipMappings.map { relationshipMapping: RelationshipMapping => - Relationship( - relationshipMapping.relationshipType, - relationshipMapping.includeBasedOnRelationship) - } - val existsRequest = ExistsRequest( - id, - candidate.id, - relationships = relationships, - context = SgsRelationshipsPredicate.UnionLookupContext - ) - socialGraph - .exists(existsRequest).map { existsResult: ExistsResult => - if (existsResult.exists) { - PredicateResult.Invalid(Set(InvalidRelationshipTypes(relationshipMappings - .map { relationshipMapping: RelationshipMapping => - relationshipMapping.relationshipType - }.mkString(", ")))) - } else { - PredicateResult.Valid - } - } - .within(timeout)(com.twitter.finagle.util.DefaultTimer) - } - // if no user id is present, return true by default - .getOrElse(Stitch.value(PredicateResult.Valid)) - .rescue { - case e: TimeoutException => - stats.counter("timeout").incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - case e: Exception => - stats.counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FailOpen))) - } - - } -} - -object SgsRelationshipsPredicate { - // OR Operation - @VisibleForTesting - private[follow_recommendations] val UnionLookupContext = Some( - LookupContext(performUnion = Some(true))) - - private def extractUserId(target: HasClientContext with HasParams): Option[Long] = target match { - case profRequest: HasProfileId => Some(profRequest.profileId) - case userRequest: HasClientContext with HasParams => userRequest.getOptionalUserId - case _ => None - } -} - -@Singleton -class InvalidTargetCandidateRelationshipTypesPredicate @Inject() ( - socialGraph: SocialGraph) - extends SgsRelationshipsPredicate( - socialGraph, - InvalidRelationshipTypesPredicate.InvalidRelationshipTypes) {} - -@Singleton -class NoteworthyAccountsSgsPredicate @Inject() ( - socialGraph: SocialGraph) - extends SgsRelationshipsPredicate( - socialGraph, - InvalidRelationshipTypesPredicate.NoteworthyAccountsInvalidRelationshipTypes) - -object InvalidRelationshipTypesPredicate { - - val InvalidRelationshipTypesExcludeFollowing: Seq[RelationshipMapping] = Seq( - RelationshipMapping(RelationshipType.HideRecommendations, true), - RelationshipMapping(RelationshipType.Blocking, true), - RelationshipMapping(RelationshipType.BlockedBy, true), - RelationshipMapping(RelationshipType.Muting, true), - RelationshipMapping(RelationshipType.MutedBy, true), - RelationshipMapping(RelationshipType.ReportedAsSpam, true), - RelationshipMapping(RelationshipType.ReportedAsSpamBy, true), - RelationshipMapping(RelationshipType.ReportedAsAbuse, true), - RelationshipMapping(RelationshipType.ReportedAsAbuseBy, true) - ) - - val InvalidRelationshipTypes: Seq[RelationshipMapping] = Seq( - RelationshipMapping(RelationshipType.FollowRequestOutgoing, true), - RelationshipMapping(RelationshipType.Following, true), - RelationshipMapping( - RelationshipType.UsedToFollow, - true - ) // this data is accessible for 90 days. - ) ++ InvalidRelationshipTypesExcludeFollowing - - val NoteworthyAccountsInvalidRelationshipTypes: Seq[RelationshipMapping] = Seq( - RelationshipMapping(RelationshipType.Blocking, true), - RelationshipMapping(RelationshipType.BlockedBy, true), - RelationshipMapping(RelationshipType.Muting, true), - RelationshipMapping(RelationshipType.MutedBy, true), - RelationshipMapping(RelationshipType.ReportedAsSpam, true), - RelationshipMapping(RelationshipType.ReportedAsSpamBy, true), - RelationshipMapping(RelationshipType.ReportedAsAbuse, true), - RelationshipMapping(RelationshipType.ReportedAsAbuseBy, true) - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD deleted file mode 100644 index fe3df2d8b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "strato/config/columns/onboarding:onboarding-strato-client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.scala deleted file mode 100644 index e0fd6b42c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicate.scala +++ /dev/null @@ -1,161 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.user_activity - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.Memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.PredicateResult -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.clients.cache.MemcacheClient -import com.twitter.follow_recommendations.common.clients.cache.ThriftEnumOptionBijection -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.UserRecommendabilityWithLongKeysOnUserClientColumn -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -abstract case class UserStateActivityPredicate( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - validCandidateStates: Set[UserState], - client: Client, - statsReceiver: StatsReceiver, - decider: Decider = Decider.False) - extends Predicate[(HasParams with HasClientContext, CandidateUser)] { - - private val stats: StatsReceiver = statsReceiver.scope(this.getClass.getSimpleName) - - // client to memcache cluster - val bijection = new ThriftEnumOptionBijection[UserState](UserState.apply) - val memcacheClient = MemcacheClient[Option[UserState]]( - client = client, - dest = "/s/cache/follow_recos_service:twemcaches", - valueBijection = bijection, - ttl = UserActivityPredicateParams.CacheTTL, - statsReceiver = stats.scope("twemcache") - ) - - override def apply( - targetAndCandidate: (HasParams with HasClientContext, CandidateUser) - ): Stitch[PredicateResult] = { - val userRecommendabilityFetcher = userRecommendabilityClient.fetcher - val (_, candidate) = targetAndCandidate - - val deciderKey: String = DeciderKey.EnableExperimentalCaching.toString - val enableDistributedCaching: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - val userStateStitch: Stitch[Option[UserState]] = - enableDistributedCaching match { - case true => { - memcacheClient.readThrough( - // add a key prefix to address cache key collisions - key = "UserActivityPredicate" + candidate.id.toString, - underlyingCall = () => queryUserRecommendable(candidate.id) - ) - } - case false => queryUserRecommendable(candidate.id) - } - val resultStitch: Stitch[PredicateResult] = - userStateStitch.map { userStateOpt => - userStateOpt match { - case Some(userState) => { - if (validCandidateStates.contains(userState)) { - PredicateResult.Valid - } else { - PredicateResult.Invalid(Set(FilterReason.MinStateNotMet)) - } - } - case None => { - PredicateResult.Invalid(Set(FilterReason.MissingRecommendabilityData)) - } - } - } - - StatsUtil.profileStitch(resultStitch, stats.scope("apply")) - .rescue { - case e: Exception => - stats.scope("rescued").counter(e.getClass.getSimpleName).incr() - Stitch(PredicateResult.Invalid(Set(FilterReason.FailOpen))) - } - } - - def queryUserRecommendable( - userId: Long - ): Stitch[Option[UserState]] = { - val userRecommendabilityFetcher = userRecommendabilityClient.fetcher - userRecommendabilityFetcher.fetch(userId).map { userCandidate => - userCandidate.v.flatMap(_.userState) - } - } -} - -@Singleton -class MinStateUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.Light, - UserState.HeavyNonTweeter, - UserState.MediumNonTweeter, - UserState.HeavyTweeter, - UserState.MediumTweeter - ), - client, - statsReceiver - ) - -@Singleton -class AllTweeterUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.HeavyTweeter, - UserState.MediumTweeter - ), - client, - statsReceiver - ) - -@Singleton -class HeavyTweeterUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.HeavyTweeter - ), - client, - statsReceiver - ) - -@Singleton -class NonNearZeroUserActivityPredicate @Inject() ( - userRecommendabilityClient: UserRecommendabilityWithLongKeysOnUserClientColumn, - client: Client, - statsReceiver: StatsReceiver) - extends UserStateActivityPredicate( - userRecommendabilityClient, - Set( - UserState.New, - UserState.VeryLight, - UserState.Light, - UserState.MediumNonTweeter, - UserState.MediumTweeter, - UserState.HeavyNonTweeter, - UserState.HeavyTweeter - ), - client, - statsReceiver - ) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.scala deleted file mode 100644 index 57e8d958b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity/UserActivityPredicateParams.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.common.predicates.user_activity - -import com.twitter.timelines.configapi.FSParam -import com.twitter.util.Duration - -object UserActivityPredicateParams { - case object HeavyTweeterEnabled - extends FSParam[Boolean]("user_activity_predicate_heavy_tweeter_enabled", false) - val CacheTTL: Duration = Duration.fromHours(6) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.scala deleted file mode 100644 index 23ccd6f4f..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/AdhocScoreModificationType.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.common - -/** - * To manage the extent of adhoc score modifications, we set a hard limit that from each of the - * types below *ONLY ONE* adhoc scorer can be applied to candidates' scores. More details about the - * usage is available in [[AdhocRanker]] - */ - -object AdhocScoreModificationType extends Enumeration { - type AdhocScoreModificationType = Value - - // This type of scorer increases the score of a subset of candidates through various policies. - val BoostingScorer: AdhocScoreModificationType = Value("boosting") - - // This type of scorer shuffles candidates randomly according to some distribution. - val WeightedRandomSamplingScorer: AdhocScoreModificationType = Value("weighted_random_sampling") - - // This is added solely for testing purposes and should not be used in production. - val InvalidAdhocScorer: AdhocScoreModificationType = Value("invalid_adhoc_scorer") -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD deleted file mode 100644 index 77d496dcf..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.scala deleted file mode 100644 index bbbde2b58..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/DedupCandidates.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.common - -import com.twitter.product_mixer.core.model.common.UniversalNoun -import scala.collection.mutable - -object DedupCandidates { - def apply[C <: UniversalNoun[Long]](input: Seq[C]): Seq[C] = { - val seen = mutable.HashSet[Long]() - input.filter { candidate => seen.add(candidate.id) } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.scala deleted file mode 100644 index f6fdb905a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common/RankerId.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.common - -object RankerId extends Enumeration { - type RankerId = Value - - val RandomRanker: RankerId = Value("random") - // The production PostNUX ML warm-start auto-retraining model ranker - val PostNuxProdRanker: RankerId = Value("postnux_prod") - val None: RankerId = Value("none") - - // Sampling from the Placket-Luce distribution. Applied after ranker step. Its ranker id is mainly used for logging. - val PlacketLuceSamplingTransformer: RankerId = Value("placket_luce_sampling_transformer") - - def getRankerByName(name: String): Option[RankerId] = - RankerId.values.toSeq.find(_.equals(Value(name))) - -} - -/** - * ML model based heavy ranker ids. - */ -object ModelBasedHeavyRankerId { - import RankerId._ - val HeavyRankerIds: Set[String] = Set( - PostNuxProdRanker.toString, - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD deleted file mode 100644 index 2fce8d77a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.scala deleted file mode 100644 index 18ac0436b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRanker.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.fatigue_ranker - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasWtfImpressions -import com.twitter.follow_recommendations.common.models.WtfImpression -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Time - -/** - * Ranks candidates based on the given weights for each algorithm while preserving the ranks inside each algorithm. - * Reorders the ranked list based on recent impressions from recentImpressionRepo - * - * Note that the penalty is added to the rank of each candidate. To make producer-side experiments - * with multiple rankers possible, we modify the scores for each candidate and ranker as: - * NewScore(C, R) = -(Rank(C, R) + Impression(C, U) x FatigueFactor), - * where C is a candidate, R a ranker and U the target user. - * Note also that fatigue penalty is independent of any of the rankers. - */ -class ImpressionBasedFatigueRanker[ - Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions -]( - fatigueFactor: Int, - statsReceiver: StatsReceiver) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - val stats = statsReceiver.scope("impression_based_fatigue_ranker") - val droppedStats: MemoizingStatsReceiver = new MemoizingStatsReceiver(stats.scope("hard_drops")) - val impressionStats: StatsReceiver = stats.scope("wtf_impressions") - val noImpressionCounter: Counter = impressionStats.counter("no_impressions") - val oldestImpressionStat: Stat = impressionStats.stat("oldest_sec") - - override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - StatsUtil.profileStitch( - Stitch.value(rankCandidates(target, candidates)), - stats.scope("rank") - ) - } - - private def trackTimeSinceOldestImpression(impressions: Seq[WtfImpression]): Unit = { - val timeSinceOldest = Time.now - impressions.map(_.latestTime).min - oldestImpressionStat.add(timeSinceOldest.inSeconds) - } - - private def rankCandidates( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - target.wtfImpressions - .map { wtfImpressions => - if (wtfImpressions.isEmpty) { - noImpressionCounter.incr() - candidates - } else { - val rankerIds = - candidates.flatMap(_.scores.map(_.scores.flatMap(_.rankerId))).flatten.sorted.distinct - - /** - * In below we create a Map from each CandidateUser's ID to a Map from each Ranker that - * the user has a score for, and candidate's corresponding rank when candidates are sorted - * by that Ranker (Only candidates who have this Ranker are considered for ranking). - */ - val candidateRanks: Map[Long, Map[RankerId, Int]] = rankerIds - .flatMap { rankerId => - // Candidates with no scores from this Ranker is first removed to calculate ranks. - val relatedCandidates = - candidates.filter(_.scores.exists(_.scores.exists(_.rankerId.contains(rankerId)))) - relatedCandidates - .sortBy(-_.scores - .flatMap(_.scores.find(_.rankerId.contains(rankerId)).map(_.value)).getOrElse( - 0.0)).zipWithIndex.map { - case (candidate, rank) => (candidate.id, rankerId, rank) - } - }.groupBy(_._1).map { - case (candidate, ranksForAllRankers) => - ( - candidate, - ranksForAllRankers.map { case (_, rankerId, rank) => (rankerId, rank) }.toMap) - } - - val idFatigueCountMap = - wtfImpressions.groupBy(_.candidateId).mapValues(_.map(_.counts).sum) - trackTimeSinceOldestImpression(wtfImpressions) - val rankedCandidates: Seq[CandidateUser] = candidates - .map { candidate => - val candidateImpressions = idFatigueCountMap.getOrElse(candidate.id, 0) - val fatiguedScores = candidate.scores.map { ss => - ss.copy(scores = ss.scores.map { s => - s.rankerId match { - // We set the new score as -rank after fatigue penalty is applied. - case Some(rankerId) => - // If the candidate's ID is not in the candidate->ranks map, or there is no - // rank for this specific ranker and this candidate, we use maximum possible - // rank instead. Note that this indicates that there is a problem. - s.copy(value = -(candidateRanks - .getOrElse(candidate.id, Map()).getOrElse(rankerId, candidates.length) + - candidateImpressions * fatigueFactor)) - // In case a score exists without a RankerId, we pass on the score as is. - case None => s - } - }) - } - candidate.copy(scores = fatiguedScores) - }.zipWithIndex.map { - // We re-rank candidates with their input ordering (which is done by the request-level - // ranker) and fatigue penalty. - case (candidate, inputRank) => - val candidateImpressions = idFatigueCountMap.getOrElse(candidate.id, 0) - (candidate, inputRank + candidateImpressions * fatigueFactor) - }.sortBy(_._2).map(_._1) - // Only populate ranking info when WTF impression info present - val scribeRankingInfo: Boolean = - target.params(ImpressionBasedFatigueRankerParams.ScribeRankingInfoInFatigueRanker) - if (scribeRankingInfo) Utils.addRankingInfo(rankedCandidates, name) else rankedCandidates - } - }.getOrElse(candidates) // no reranking/filtering when wtf impressions not present - } -} - -object ImpressionBasedFatigueRanker { - val DefaultFatigueFactor = 5 - - def build[ - Target <: HasClientContext with HasDisplayLocation with HasParams with HasWtfImpressions - ]( - baseStatsReceiver: StatsReceiver, - fatigueFactor: Int = DefaultFatigueFactor - ): ImpressionBasedFatigueRanker[Target] = - new ImpressionBasedFatigueRanker(fatigueFactor, baseStatsReceiver) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.scala deleted file mode 100644 index 34fbbeb46..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerFSConfig.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.fatigue_ranker - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ImpressionBasedFatigueRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(ImpressionBasedFatigueRankerParams.ScribeRankingInfoInFatigueRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.scala deleted file mode 100644 index 075d78bf6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker/ImpressionBasedFatigueRankerParams.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.fatigue_ranker - -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object ImpressionBasedFatigueRankerParams { - // Whether to enable hard dropping of impressed candidates - object DropImpressedCandidateEnabled extends Param[Boolean](false) - // At what # of impressions to hard drop candidates. - object DropCandidateImpressionThreshold extends Param[Int](default = 10) - // Whether to scribe candidate ranking/scoring info per ranking stage - object ScribeRankingInfoInFatigueRanker - extends FSParam[Boolean]("fatigue_ranker_scribe_ranking_info", true) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD deleted file mode 100644 index 3de5523b1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.scala deleted file mode 100644 index 8fbeafa25..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRanker.scala +++ /dev/null @@ -1,115 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasQualityFactor -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This class is meant to filter candidates between stages of our ranker by taking the first N - * candidates, merging any candidate source information for candidates with multiple entries. - * To allow us to chain this truncation operation any number of times sequentially within the main - * ranking builder, we abstract the truncation as a separate Ranker - */ -@Singleton -class FirstNRanker[Target <: HasClientContext with HasParams with HasQualityFactor] @Inject() ( - stats: StatsReceiver) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - private val baseStats = stats.scope("first_n_ranker") - val scaledDownByQualityFactorCounter = - baseStats.counter("scaled_down_by_quality_factor") - private val mergeStat = baseStats.scope("merged_candidates") - private val mergeStat2 = mergeStat.counter("2") - private val mergeStat3 = mergeStat.counter("3") - private val mergeStat4 = mergeStat.counter("4+") - private val candidateSizeStats = baseStats.scope("candidate_size") - - private case class CandidateSourceScore( - candidateId: Long, - sourceId: CandidateSourceIdentifier, - score: Option[Double]) - - /** - * Adds the rank of each candidate based on the primary candidate source's score. - * In the event where the provided ordering of candidates do not align with the score, - * we will respect the score, since the ordering might have been mixed up due to other previous - * steps like the shuffleFn in the `WeightedCandidateSourceRanker`. - * @param candidates ordered list of candidates - * @return same ordered list of candidates, but with the rank information appended - */ - def addRank(candidates: Seq[CandidateUser]): Seq[CandidateUser] = { - val candidateSourceRanks = for { - (sourceIdOpt, sourceCandidates) <- candidates.groupBy(_.getPrimaryCandidateSource) - (candidate, rank) <- sourceCandidates.sortBy(-_.score.getOrElse(0.0)).zipWithIndex - } yield { - (candidate, sourceIdOpt) -> rank - } - candidates.map { c => - c.getPrimaryCandidateSource - .map { sourceId => - val sourceRank = candidateSourceRanks((c, c.getPrimaryCandidateSource)) - c.addCandidateSourceRanksMap(Map(sourceId -> sourceRank)) - }.getOrElse(c) - } - } - - override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - - val scaleDownFactor = Math.max( - target.qualityFactor.getOrElse(1.0d), - target.params(FirstNRankerParams.MinNumCandidatesScoredScaleDownFactor) - ) - - if (scaleDownFactor < 1.0d) - scaledDownByQualityFactorCounter.incr() - - val n = (target.params(FirstNRankerParams.CandidatesToRank) * scaleDownFactor).toInt - val scribeRankingInfo: Boolean = - target.params(FirstNRankerParams.ScribeRankingInfoInFirstNRanker) - candidateSizeStats.counter(s"n$n").incr() - val candidatesWithRank = addRank(candidates) - if (target.params(FirstNRankerParams.GroupDuplicateCandidates)) { - val groupedCandidates: Map[Long, Seq[CandidateUser]] = candidatesWithRank.groupBy(_.id) - val topN = candidates - .map { c => - merge(groupedCandidates(c.id)) - }.distinct.take(n) - Stitch.value(if (scribeRankingInfo) Utils.addRankingInfo(topN, name) else topN) - } else { - Stitch.value( - if (scribeRankingInfo) Utils.addRankingInfo(candidatesWithRank, name).take(n) - else candidatesWithRank.take(n)) - } // for efficiency, if don't need to deduplicate - } - - /** - * we use the primary candidate source of the first entry, and aggregate all of the other entries' - * candidate source scores into the first entry's candidateSourceScores - * @param candidates list of candidates with the same id - * @return a single merged candidate - */ - private[first_n_ranker] def merge(candidates: Seq[CandidateUser]): CandidateUser = { - if (candidates.size == 1) { - candidates.head - } else { - candidates.size match { - case 2 => mergeStat2.incr() - case 3 => mergeStat3.incr() - case i if i >= 4 => mergeStat4.incr() - case _ => - } - val allSources = candidates.flatMap(_.getCandidateSources).toMap - val allRanks = candidates.flatMap(_.getCandidateRanks).toMap - candidates.head.addCandidateSourceScoresMap(allSources).addCandidateSourceRanksMap(allRanks) - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.scala deleted file mode 100644 index 484738dc1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFSConfig.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -@Singleton -class FirstNRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(FirstNRankerParams.ScribeRankingInfoInFirstNRanker) - - override val intFSParams: Seq[FSBoundedParam[Int]] = Seq( - FirstNRankerParams.CandidatesToRank - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - FirstNRankerParams.MinNumCandidatesScoredScaleDownFactor - ) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.scala deleted file mode 100644 index 682b60fed..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerFeatureSwitchKeys.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -object FirstNRankerFeatureSwitchKeys { - val CandidatePoolSize = "first_n_ranker_candidate_pool_size" - val ScribeRankingInfo = "first_n_ranker_scribe_ranking_info" - val MinNumCandidatesScoredScaleDownFactor = - "first_n_ranker_min_scale_down_factor" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.scala deleted file mode 100644 index ac65a6dde..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker/FirstNRankerParams.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.first_n_ranker - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object FirstNRankerParams { - case object CandidatesToRank - extends FSBoundedParam[Int]( - FirstNRankerFeatureSwitchKeys.CandidatePoolSize, - default = 100, - min = 50, - max = 600) - - case object GroupDuplicateCandidates extends Param[Boolean](true) - case object ScribeRankingInfoInFirstNRanker - extends FSParam[Boolean](FirstNRankerFeatureSwitchKeys.ScribeRankingInfo, true) - - // the minimum of candidates to score in each request. - object MinNumCandidatesScoredScaleDownFactor - extends FSBoundedParam[Double]( - name = FirstNRankerFeatureSwitchKeys.MinNumCandidatesScoredScaleDownFactor, - default = 0.3, - min = 0.1, - max = 1.0) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD deleted file mode 100644 index e71d8a7ab..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.scala deleted file mode 100644 index 973275f51..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRanker.scala +++ /dev/null @@ -1,204 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.interleave_ranker - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -@Singleton -class InterleaveRanker[Target <: HasParams] @Inject() ( - statsReceiver: StatsReceiver) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - private val stats = statsReceiver.scope("interleave_ranker") - private val inputStats = stats.scope("input") - private val interleavingStats = stats.scope("interleave") - - override def rank( - target: Target, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - StatsUtil.profileStitch( - Stitch.value(rankCandidates(target, candidates)), - stats.scope("rank") - ) - } - - private def rankCandidates( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - - /** - * By this stage, all valid candidates should have: - * 1. Their Scores field populated. - * 2. Their selectedRankerId set. - * 3. Have a score associated to their selectedRankerId. - * If there is any candidate that doesn't meet the conditions above, there is a problem in one - * of the previous rankers. Since no new scoring is done in this ranker, we simply remove them. - */ - val validCandidates = - candidates.filter { c => - c.scores.isDefined && - c.scores.exists(_.selectedRankerId.isDefined) && - getCandidateScoreByRankerId(c, c.scores.flatMap(_.selectedRankerId)).isDefined - } - - // To monitor the percentage of valid candidates, as defined above, we track the following: - inputStats.counter("candidates_with_no_scores").incr(candidates.count(_.scores.isEmpty)) - inputStats - .counter("candidates_with_no_selected_ranker").incr(candidates.count { c => - c.scores.isEmpty || c.scores.exists(_.selectedRankerId.isEmpty) - }) - inputStats - .counter("candidates_with_no_score_for_selected_ranker").incr(candidates.count { c => - c.scores.isEmpty || - c.scores.exists(_.selectedRankerId.isEmpty) || - getCandidateScoreByRankerId(c, c.scores.flatMap(_.selectedRankerId)).isEmpty - }) - inputStats.counter("total_num_candidates").incr(candidates.length) - inputStats.counter("total_valid_candidates").incr(validCandidates.length) - - // We only count rankerIds from those candidates who are valid to exclude those candidates with - // a valid selectedRankerId that don't have an associated score for it. - val rankerIds = validCandidates.flatMap(_.scores.flatMap(_.selectedRankerId)).sorted.distinct - rankerIds.foreach { rankerId => - inputStats - .counter(s"valid_scores_for_${rankerId.toString}").incr( - candidates.count(getCandidateScoreByRankerId(_, Some(rankerId)).isDefined)) - inputStats.counter(s"total_candidates_for_${rankerId.toString}").incr(candidates.length) - } - inputStats.counter(s"num_ranker_ids=${rankerIds.length}").incr() - val scribeRankingInfo: Boolean = - target.params(InterleaveRankerParams.ScribeRankingInfoInInterleaveRanker) - if (rankerIds.length <= 1) - // In the case of "Number of RankerIds = 0", we pass on the candidates even though there is - // a problem in a previous ranker that provided the scores. - if (scribeRankingInfo) Utils.addRankingInfo(candidates, name) else candidates - else - if (scribeRankingInfo) - Utils.addRankingInfo(interleaveCandidates(validCandidates, rankerIds), name) - else interleaveCandidates(validCandidates, rankerIds) - } - - @VisibleForTesting - private[interleave_ranker] def interleaveCandidates( - candidates: Seq[CandidateUser], - rankerIds: Seq[RankerId.RankerId] - ): Seq[CandidateUser] = { - val candidatesWithRank = rankerIds - .flatMap { ranker => - candidates - // We first sort all candidates using this ranker. - .sortBy(-getCandidateScoreByRankerId(_, Some(ranker)).getOrElse(Double.MinValue)) - .zipWithIndex.filter( - // but only hold those candidates whose selected ranker is this ranker. - // These ranks will be forced in the final ordering. - _._1.scores.flatMap(_.selectedRankerId).contains(ranker)) - } - - // Only candidates who have isInProducerScoringExperiment set to true will have their position enforced. We - // separate candidates into two groups: (1) Production and (2) Experiment. - val (expCandidates, prodCandidates) = - candidatesWithRank.partition(_._1.scores.exists(_.isInProducerScoringExperiment)) - - // We resolve (potential) conflicts between the enforced ranks of experimental models. - val expCandidatesFinalPos = resolveConflicts(expCandidates) - - // Retrieve non-occupied positions and assign them to candidates who use production ranker. - val occupiedPos = expCandidatesFinalPos.map(_._2).toSet - val prodCandidatesFinalPos = - prodCandidates - .map(_._1).zip( - candidates.indices.filterNot(occupiedPos.contains).sorted.take(prodCandidates.length)) - - // Merge the two groups and sort them by their corresponding positions. - val finalCandidates = (prodCandidatesFinalPos ++ expCandidatesFinalPos).sortBy(_._2).map(_._1) - - // We count the presence of each ranker in the top-3 final positions. - finalCandidates.zip(0 until 3).foreach { - case (c, r) => - // We only do so for candidates that are in a producer-side experiment. - if (c.scores.exists(_.isInProducerScoringExperiment)) - c.scores.flatMap(_.selectedRankerId).map(_.toString).foreach { rankerName => - interleavingStats - .counter(s"num_final_position_${r}_$rankerName") - .incr() - } - } - - finalCandidates - } - - @VisibleForTesting - private[interleave_ranker] def resolveConflicts( - candidatesWithRank: Seq[(CandidateUser, Int)] - ): Seq[(CandidateUser, Int)] = { - // The following two metrics will allow us to calculate the rate of conflicts occurring. - // Example: If overall there are 10 producers in different bucketing experiments, and 3 of them - // are assigned to the same position. The rate would be 3/10, 30%. - val numCandidatesWithConflicts = interleavingStats.counter("candidates_with_conflict") - val numCandidatesNoConflicts = interleavingStats.counter("candidates_without_conflict") - val candidatesGroupedByRank = candidatesWithRank.groupBy(_._2).toSeq.sortBy(_._1).map { - case (rank, candidatesWithRank) => (rank, candidatesWithRank.map(_._1)) - } - - candidatesGroupedByRank.foldLeft(Seq[(CandidateUser, Int)]()) { (upToHere, nextGroup) => - val (rank, candidates) = nextGroup - if (candidates.length > 1) - numCandidatesWithConflicts.incr(candidates.length) - else - numCandidatesNoConflicts.incr() - - // We use the position after the last-assigned candidate as a starting point, or 0 otherwise. - // If candidates' position is after this "starting point", we enforce that position instead. - val minAvailableIndex = scala.math.max(upToHere.lastOption.map(_._2).getOrElse(-1) + 1, rank) - val enforcedPos = - (minAvailableIndex until minAvailableIndex + candidates.length).toList - val shuffledEnforcedPos = - if (candidates.length > 1) scala.util.Random.shuffle(enforcedPos) else enforcedPos - if (shuffledEnforcedPos.length > 1) { - candidates.zip(shuffledEnforcedPos).sortBy(_._2).map(_._1).zipWithIndex.foreach { - case (c, r) => - c.scores.flatMap(_.selectedRankerId).map(_.toString).foreach { rankerName => - // For each ranker, we count the total number of times it has been in a conflict. - interleavingStats - .counter(s"num_${shuffledEnforcedPos.length}-way_conflicts_$rankerName") - .incr() - // We also count the positions each of the rankers have fallen randomly into. In any - // experiment this should converge to uniform distribution given enough occurrences. - // Note that the position here is relative to the other candidates in the conflict and - // not the overall position of each candidate. - interleavingStats - .counter( - s"num_position_${r}_after_${shuffledEnforcedPos.length}-way_conflict_$rankerName") - .incr() - } - } - } - upToHere ++ candidates.zip(shuffledEnforcedPos).sortBy(_._2) - } - } - - @VisibleForTesting - private[interleave_ranker] def getCandidateScoreByRankerId( - candidate: CandidateUser, - rankerIdOpt: Option[RankerId.RankerId] - ): Option[Double] = { - rankerIdOpt match { - case None => None - case Some(rankerId) => - candidate.scores.flatMap { - _.scores.find(_.rankerId.contains(rankerId)).map(_.value) - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.scala deleted file mode 100644 index 5a8c6de2a..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerFSConfig.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.interleave_ranker - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam - -@Singleton -class InterleaveRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(InterleaveRankerParams.ScribeRankingInfoInInterleaveRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.scala deleted file mode 100644 index 84e6ea314..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker/InterleaveRankerParams.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.interleave_ranker - -import com.twitter.timelines.configapi.FSParam - -object InterleaveRankerParams { - case object ScribeRankingInfoInInterleaveRanker - extends FSParam[Boolean]("interleave_ranker_scribe_ranking_info", true) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD deleted file mode 100644 index 0c277e005..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/BUILD +++ /dev/null @@ -1,37 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "src/java/com/twitter/ml/api:api-base", - "util/util-slf4j-api/src/main/scala", - ], -) - -# This is to import only the params from MlRanker, for instance to get request-level heavy ranker. -scala_library( - name = "ml_ranker_params", - sources = [ - "MlRankerParams.scala", - ], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "timelines/src/main/scala/com/twitter/timelines/config/configapi", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.scala deleted file mode 100644 index 499a14b67..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/HydrateFeaturesTransform.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.base.StatsUtil.profileStitchMapResults -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.feature_hydration.sources.UserScoringFeatureSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasSimilarToContext -import com.twitter.follow_recommendations.common.models.RichDataRecord -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging - -/** - * Hydrate features given target and candidates lists. - * This is a required step before MlRanker. - * If a feature is not hydrated before MlRanker is triggered, a runtime exception will be thrown - */ -@Singleton -class HydrateFeaturesTransform[ - Target <: HasClientContext with HasParams with HasDebugOptions with HasPreFetchedFeature with HasSimilarToContext with HasDisplayLocation] @Inject() ( - userScoringFeatureSource: UserScoringFeatureSource, - stats: StatsReceiver) - extends GatedTransform[Target, CandidateUser] - with Logging { - - private val hydrateFeaturesStats = stats.scope("hydrate_features") - - def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - // get features - val featureMapStitch: Stitch[Map[CandidateUser, DataRecord]] = - profileStitchMapResults( - userScoringFeatureSource.hydrateFeatures(target, candidates), - hydrateFeaturesStats) - - featureMapStitch.map { featureMap => - candidates - .map { candidate => - val dataRecord = featureMap(candidate) - // add debugRecord only when the request parameter is set - val debugDataRecord = if (target.debugOptions.exists(_.fetchDebugInfo)) { - Some(candidate.toDebugDataRecord(dataRecord, userScoringFeatureSource.featureContext)) - } else None - candidate.copy( - dataRecord = Some(RichDataRecord(Some(dataRecord), debugDataRecord)) - ) - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.scala deleted file mode 100644 index 6344c348e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRanker.scala +++ /dev/null @@ -1,219 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.base.StatsUtil.profileSeqResults -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.Scores -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.AdhocScorer -import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.Scorer -import com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring.ScorerFactory -import com.twitter.follow_recommendations.common.utils.CollectionUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params -import com.twitter.util.logging.Logging - -/** - * This class has a rank function that will perform 4 steps: - * - choose which scorer to use for each candidate - * - score candidates given their respective features - * - add scoring information to the candidate - * - sort candidates by their respective scores - * The feature source and scorer will depend on the request's params - */ -@Singleton -class MlRanker[ - Target <: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions] @Inject() ( - scorerFactory: ScorerFactory, - statsReceiver: StatsReceiver) - extends Ranker[Target, CandidateUser] - with Logging { - - private val stats: StatsReceiver = statsReceiver.scope("ml_ranker") - - private val inputStat = stats.scope("1_input") - private val selectScorerStat = stats.scope("2_select_scorer") - private val scoreStat = stats.scope("3_score") - - override def rank( - target: Target, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - profileSeqResults(candidates, inputStat) - val requestRankerId = target.params(MlRankerParams.RequestScorerIdParam) - val rankerIds = chooseRankerByCandidate(candidates, requestRankerId) - - val scoreStitch = score(candidates, rankerIds, requestRankerId).map { scoredCandidates => - { - // sort the candidates by score - val sortedCandidates = sort(target, scoredCandidates) - // add scribe field to candidates (if applicable) and return candidates - scribeCandidates(target, sortedCandidates) - } - } - StatsUtil.profileStitch(scoreStitch, stats.scope("rank")) - } - - /** - * @param target: The WTF request for a given consumer. - * @param candidates A list of candidates considered for recommendation. - * @return A map from each candidate to a tuple that includes: - * (1) The selected scorer that should be used to rank this candidate - * (2) a flag determining whether the candidate is in a producer-side experiment. - */ - private[ranking] def chooseRankerByCandidate( - candidates: Seq[CandidateUser], - requestRankerId: RankerId - ): Map[CandidateUser, RankerId] = { - candidates.map { candidate => - val selectedCandidateRankerId = - if (candidate.params == Params.Invalid || candidate.params == Params.Empty) { - selectScorerStat.counter("candidate_params_empty").incr() - requestRankerId - } else { - val candidateRankerId = candidate.params(MlRankerParams.CandidateScorerIdParam) - if (candidateRankerId == RankerId.None) { - // This candidate is a not part of any producer-side experiment. - selectScorerStat.counter("default_to_request_ranker").incr() - requestRankerId - } else { - // This candidate is in a treatment bucket of a producer-side experiment. - selectScorerStat.counter("use_candidate_ranker").incr() - candidateRankerId - } - } - selectScorerStat.scope("selected").counter(selectedCandidateRankerId.toString).incr() - candidate -> selectedCandidateRankerId - }.toMap - } - - @VisibleForTesting - private[ranking] def score( - candidates: Seq[CandidateUser], - rankerIds: Map[CandidateUser, RankerId], - requestRankerId: RankerId - ): Stitch[Seq[CandidateUser]] = { - val features = candidates.map(_.dataRecord.flatMap(_.dataRecord)) - - require(features.forall(_.nonEmpty), "features are not hydrated for all the candidates") - - val scorers = scorerFactory.getScorers(rankerIds.values.toSeq.sorted.distinct) - - // Scorers are split into ML-based and Adhoc (defined as a scorer that does not need to call an - // ML prediction service and scores candidates using locally-available data). - val (adhocScorers, mlScorers) = scorers.partition { - case _: AdhocScorer => true - case _ => false - } - - // score candidates - val scoresStitch = score(features.map(_.get), mlScorers) - val candidatesWithMlScoresStitch = scoresStitch.map { scoresSeq => - candidates - .zip(scoresSeq).map { // copy datarecord and score into candidate object - case (candidate, scores) => - val selectedRankerId = rankerIds(candidate) - val useRequestRanker = - candidate.params == Params.Invalid || - candidate.params == Params.Empty || - candidate.params(MlRankerParams.CandidateScorerIdParam) == RankerId.None - candidate.copy( - score = scores.scores.find(_.rankerId.contains(requestRankerId)).map(_.value), - scores = if (scores.scores.nonEmpty) { - Some( - scores.copy( - scores = scores.scores, - selectedRankerId = Some(selectedRankerId), - isInProducerScoringExperiment = !useRequestRanker - )) - } else None - ) - } - } - - candidatesWithMlScoresStitch.map { candidates => - // The basis for adhoc scores are the "request-level" ML ranker. We add the base score here - // while adhoc scorers are applied in [[AdhocRanker]]. - addMlBaseScoresForAdhocScorers(candidates, requestRankerId, adhocScorers) - } - } - - @VisibleForTesting - private[ranking] def addMlBaseScoresForAdhocScorers( - candidates: Seq[CandidateUser], - requestRankerId: RankerId, - adhocScorers: Seq[Scorer] - ): Seq[CandidateUser] = { - candidates.map { candidate => - candidate.scores match { - case Some(oldScores) => - // 1. We fetch the ML score that is the basis of adhoc scores: - val baseMlScoreOpt = Utils.getCandidateScoreByRankerId(candidate, requestRankerId) - - // 2. For each adhoc scorer, we copy the ML score object, changing only the ID and type. - val newScores = adhocScorers flatMap { adhocScorer => - baseMlScoreOpt.map( - _.copy(rankerId = Some(adhocScorer.id), scoreType = adhocScorer.scoreType)) - } - - // 3. We add the new adhoc score entries to the candidate. - candidate.copy(scores = Some(oldScores.copy(scores = oldScores.scores ++ newScores))) - case _ => - // Since there is no base ML score, there should be no adhoc score modification as well. - candidate - } - } - } - - private[this] def score( - dataRecords: Seq[DataRecord], - scorers: Seq[Scorer] - ): Stitch[Seq[Scores]] = { - val scoredResponse = scorers.map { scorer => - StatsUtil.profileStitch(scorer.score(dataRecords), scoreStat.scope(scorer.id.toString)) - } - // If we could score a candidate with too many rankers, it is likely to blow up the whole system. - // and fail back to default production model - StatsUtil.profileStitch(Stitch.collect(scoredResponse), scoreStat).map { scoresByScorerId => - CollectionUtil.transposeLazy(scoresByScorerId).map { scoresPerCandidate => - Scores(scoresPerCandidate) - } - } - } - - // sort candidates using score in descending order - private[this] def sort( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - candidates.sortBy(c => -c.score.getOrElse(MlRanker.DefaultScore)) - } - - private[this] def scribeCandidates( - target: Target, - candidates: Seq[CandidateUser] - ): Seq[CandidateUser] = { - val scribeRankingInfo: Boolean = target.params(MlRankerParams.ScribeRankingInfoInMlRanker) - scribeRankingInfo match { - case true => Utils.addRankingInfo(candidates, "MlRanker") - case false => candidates - } - } -} - -object MlRanker { - // this is to ensure candidates with absent scores are ranked the last - val DefaultScore: Double = Double.MinValue -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.scala deleted file mode 100644 index c69a32fc5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerFSConfig.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam - -@Singleton -class MlRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(MlRankerParams.ScribeRankingInfoInMlRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.scala deleted file mode 100644 index 8463963a6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking/MlRankerParams.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking - -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam - -/** - * When adding Producer side experiments, make sure to register the FS Key in [[ProducerFeatureFilter]] - * in [[FeatureSwitchesModule]], otherwise, the FS will not work. - */ -object MlRankerParams { - // which ranker to use by default for the given request - case object RequestScorerIdParam - extends FSEnumParam[RankerId.type]( - name = "post_nux_ml_flow_ml_ranker_id", - default = RankerId.PostNuxProdRanker, - enum = RankerId - ) - - // which ranker to use for the given candidate - case object CandidateScorerIdParam - extends FSEnumParam[RankerId.type]( - name = "post_nux_ml_flow_candidate_user_scorer_id", - default = RankerId.None, - enum = RankerId - ) - - case object ScribeRankingInfoInMlRanker - extends FSParam[Boolean]("post_nux_ml_flow_scribe_ranking_info_in_ml_ranker", true) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.scala deleted file mode 100644 index 39921bb71..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/AdhocScorer.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.follow_recommendations.common.rankers.common.AdhocScoreModificationType.AdhocScoreModificationType -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.ml.api.DataRecord -import com.twitter.stitch.Stitch - -trait AdhocScorer extends Scorer { - - /** - * NOTE: For instances of [[AdhocScorer]] this function SHOULD NOT be used. - * Please use: - * [[score(target: HasClientContext with HasParams, candidates: Seq[CandidateUser])]] - * instead. - */ - @Deprecated - override def score(records: Seq[DataRecord]): Stitch[Seq[Score]] = - throw new UnsupportedOperationException( - "For instances of AdhocScorer this operation is not defined. Please use " + - "`def score(target: HasClientContext with HasParams, candidates: Seq[CandidateUser])` " + - "instead.") - - /** - * This helps us manage the extend of adhoc modification on candidates' score. There is a hard - * limit of applying ONLY ONE scorer of each type to a score. - */ - val scoreModificationType: AdhocScoreModificationType -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD deleted file mode 100644 index bbcd3c708..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking:ml_ranker_params", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/pluck/source/core_workflows/user_model:condensed_user_state-scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.scala deleted file mode 100644 index d27bc6e37..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/DeepbirdScorer.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.cortex.deepbird.thriftjava.ModelSelector -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.prediction_service.{BatchPredictionRequest => JBatchPredictionRequest} -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Future -import com.twitter.util.TimeoutException -import scala.collection.JavaConversions._ -import scala.collection.JavaConverters._ - -/** - * Generic trait that implements the scoring given a deepbirdClient - * To test out a new model, create a scorer extending this trait, override the modelName and inject the scorer - */ -trait DeepbirdScorer extends Scorer { - def modelName: String - def predictionFeature: Feature.Continuous - // Set a default batchSize of 100 when making model prediction calls to the Deepbird V2 prediction server - def batchSize: Int = 100 - def deepbirdClient: DeepbirdPredictionService.ServiceToClient - def baseStats: StatsReceiver - - def modelSelector: ModelSelector = new ModelSelector().setId(modelName) - def stats: StatsReceiver = baseStats.scope(this.getClass.getSimpleName).scope(modelName) - - private def requestCount = stats.counter("requests") - private def emptyRequestCount = stats.counter("empty_requests") - private def successCount = stats.counter("success") - private def failureCount = stats.counter("failures") - private def inputRecordsStat = stats.stat("input_records") - private def outputRecordsStat = stats.stat("output_records") - - // Counters for tracking batch-prediction statistics when making DBv2 prediction calls - // - // numBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers - private def numBatchRequests = stats.counter("batches") - // numEmptyBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers - // that had an empty input DataRecord - private def numEmptyBatchRequests = stats.counter("empty_batches") - // numTimedOutBatchRequests tracks the number of batch prediction requests made to DBv2 prediction servers - // that had timed-out - private def numTimedOutBatchRequests = stats.counter("timeout_batches") - - private def batchPredictionLatency = stats.stat("batch_prediction_latency") - private def predictionLatency = stats.stat("prediction_latency") - - private def numEmptyModelPredictions = stats.counter("empty_model_predictions") - private def numNonEmptyModelPredictions = stats.counter("non_empty_model_predictions") - - private val DefaultPredictionScore = 0.0 - - /** - * NOTE: For instances of [[DeepbirdScorer]] this function SHOULD NOT be used. - * Please use [[score(records: Seq[DataRecord])]] instead. - */ - @Deprecated - def score( - target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions, - candidates: Seq[CandidateUser] - ): Seq[Option[Score]] = - throw new UnsupportedOperationException( - "For instances of DeepbirdScorer this operation is not defined. Please use " + - "`def score(records: Seq[DataRecord]): Stitch[Seq[Score]]` " + - "instead.") - - override def score(records: Seq[DataRecord]): Stitch[Seq[Score]] = { - requestCount.incr() - if (records.isEmpty) { - emptyRequestCount.incr() - Stitch.Nil - } else { - inputRecordsStat.add(records.size) - Stitch.callFuture( - batchPredict(records, batchSize) - .map { recordList => - val scores = recordList.map { record => - Score( - value = record.getOrElse(DefaultPredictionScore), - rankerId = Some(id), - scoreType = scoreType) - } - outputRecordsStat.add(scores.size) - scores - }.onSuccess(_ => successCount.incr()) - .onFailure(_ => failureCount.incr())) - } - } - - def batchPredict( - dataRecords: Seq[DataRecord], - batchSize: Int - ): Future[Seq[Option[Double]]] = { - Stat - .timeFuture(predictionLatency) { - val batchedDataRecords = dataRecords.grouped(batchSize).toSeq - numBatchRequests.incr(batchedDataRecords.size) - Future - .collect(batchedDataRecords.map(batch => predict(batch))) - .map(res => res.reduce(_ ++ _)) - } - } - - def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] = { - Stat - .timeFuture(batchPredictionLatency) { - if (dataRecords.isEmpty) { - numEmptyBatchRequests.incr() - Future.Nil - } else { - deepbirdClient - .batchPredictFromModel(new JBatchPredictionRequest(dataRecords.asJava), modelSelector) - .map { response => - response.predictions.toSeq.map { prediction => - val predictionFeatureOption = Option( - new RichDataRecord(prediction).getFeatureValue(predictionFeature) - ) - predictionFeatureOption match { - case Some(predictionValue) => - numNonEmptyModelPredictions.incr() - Option(predictionValue.toDouble) - case None => - numEmptyModelPredictions.incr() - Option(DefaultPredictionScore) - } - } - } - .rescue { - case e: TimeoutException => // DBv2 prediction calls that timed out - numTimedOutBatchRequests.incr() - stats.counter(e.getClass.getSimpleName).incr() - Future.value(dataRecords.map(_ => Option(DefaultPredictionScore))) - case e: Exception => // other generic DBv2 prediction call failures - stats.counter(e.getClass.getSimpleName).incr() - Future.value(dataRecords.map(_ => Option(DefaultPredictionScore))) - } - } - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.scala deleted file mode 100644 index 861d02fa7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/PostnuxDeepbirdProdScorer.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.ml.api.Feature -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -// This is a standard DeepbirdV2 ML Ranker scoring config that should be extended by all ML scorers -// -// Only modify this trait when adding new fields to DeepbirdV2 scorers which -trait DeepbirdProdScorer extends DeepbirdScorer { - override val batchSize = 20 -} - -// Feature.Continuous("prediction") is specific to ClemNet architecture, we can change it to be more informative in the next iteration -trait PostNuxV1DeepbirdProdScorer extends DeepbirdProdScorer { - override val predictionFeature: Feature.Continuous = - new Feature.Continuous("prediction") -} - -// The current, primary PostNUX DeepbirdV2 scorer used in production -@Singleton -class PostnuxDeepbirdProdScorer @Inject() ( - @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) - override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, - override val baseStats: StatsReceiver) - extends PostNuxV1DeepbirdProdScorer { - override val id = RankerId.PostNuxProdRanker - override val modelName = "PostNUX14531GafClemNetWarmStart" -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.scala deleted file mode 100644 index 92265cc6b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/RandomScorer.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.cortex.deepbird.thriftjava.DeepbirdPredictionService -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/** - * This scorer assigns random values between 0 and 1 to each candidate as scores. - */ - -@Singleton -class RandomScorer @Inject() ( - @Named(GuiceNamedConstants.WTF_PROD_DEEPBIRDV2_CLIENT) - override val deepbirdClient: DeepbirdPredictionService.ServiceToClient, - override val baseStats: StatsReceiver) - extends DeepbirdScorer { - override val id = RankerId.RandomRanker - private val rnd = new scala.util.Random(System.currentTimeMillis()) - - override def predict(dataRecords: Seq[DataRecord]): Future[Seq[Option[Double]]] = { - if (dataRecords.isEmpty) { - Future.Nil - } else { - // All candidates are assigned a random value between 0 and 1 as score. - Future.value(dataRecords.map(_ => Option(rnd.nextDouble()))) - } - } - - override val modelName = "PostNuxRandomRanker" - - // This is not needed since we are overriding the `predict` function, but we have to override - // `predictionFeature` anyway. - override val predictionFeature: Feature.Continuous = - new Feature.Continuous("prediction.pfollow_pengagement") -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.scala deleted file mode 100644 index 2ca611535..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/Scorer.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.follow_recommendations.common.models.ScoreType -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -trait Scorer { - - // unique id of the scorer - def id: RankerId.Value - - // type of the output scores - def scoreType: Option[ScoreType] = None - - // Scoring when an ML model is used. - def score(records: Seq[DataRecord]): Stitch[Seq[Score]] - - /** - * Scoring when a non-ML method is applied. E.g: Boosting, randomized reordering, etc. - * This method assumes that candidates' scores are already retrieved from heavy-ranker models and - * are available for use. - */ - def score( - target: HasClientContext with HasParams with HasDisplayLocation with HasDebugOptions, - candidates: Seq[CandidateUser] - ): Seq[Option[Score]] -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.scala deleted file mode 100644 index a9ea0a21b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring/ScorerFactory.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.ml_ranker.scoring - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScorerFactory @Inject() ( - postnuxProdScorer: PostnuxDeepbirdProdScorer, - randomScorer: RandomScorer, - stats: StatsReceiver) { - - private val scorerFactoryStats = stats.scope("scorer_factory") - private val scorerStat = scorerFactoryStats.scope("scorer") - - def getScorers( - rankerIds: Seq[RankerId] - ): Seq[Scorer] = { - rankerIds.map { scorerId => - val scorer: Scorer = getScorerById(scorerId) - // count # of times a ranker has been requested - scorerStat.counter(scorer.id.toString).incr() - scorer - } - } - - def getScorerById(scorerId: RankerId): Scorer = scorerId match { - case RankerId.PostNuxProdRanker => - postnuxProdScorer - case RankerId.RandomRanker => - randomScorer - case _ => - scorerStat.counter("invalid_scorer_type").incr() - throw new IllegalArgumentException("unknown_scorer_type") - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD deleted file mode 100644 index 82e9fc7f4..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.scala deleted file mode 100644 index 29f00b698..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils/Utils.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.utils - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.follow_recommendations.common.rankers.common.RankerId.RankerId - -object Utils { - - /** - * Add the ranking and scoring info for a list of candidates on a given ranking stage. - * @param candidates A list of CandidateUser - * @param rankingStage Should use `Ranker.name` as the ranking stage. - * @return The list of CandidateUser with ranking/scoring info added. - */ - def addRankingInfo(candidates: Seq[CandidateUser], rankingStage: String): Seq[CandidateUser] = { - candidates.zipWithIndex.map { - case (candidate, rank) => - // 1-based ranking for better readability - candidate.addInfoPerRankingStage(rankingStage, candidate.scores, rank + 1) - } - } - - def getCandidateScoreByRankerId(candidate: CandidateUser, rankerId: RankerId): Option[Score] = - candidate.scores.flatMap { ss => ss.scores.find(_.rankerId.contains(rankerId)) } - - def getAllRankerIds(candidates: Seq[CandidateUser]): Seq[RankerId] = - candidates.flatMap(_.scores.map(_.scores.flatMap(_.rankerId))).flatten.distinct -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD deleted file mode 100644 index 3de5523b1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.scala deleted file mode 100644 index be281a582..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/CandidateShuffle.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.follow_recommendations.common.utils.RandomUtil -import scala.util.Random - -sealed trait CandidateShuffler[T] { - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] -} - -class NoShuffle[T]() extends CandidateShuffler[T] { - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = input -} - -class RandomShuffler[T]() extends CandidateShuffler[T] { - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = { - seed.map(new Random(_)).getOrElse(Random).shuffle(input) - } -} - -trait RankWeightedRandomShuffler[T] extends CandidateShuffler[T] { - - def rankToWeight(rank: Int): Double - def shuffle(seed: Option[Long])(input: Seq[T]): Seq[T] = { - val candWeights = input.zipWithIndex.map { - case (candidate, rank) => (candidate, rankToWeight(rank)) - } - RandomUtil.weightedRandomShuffle(candWeights, seed.map(new Random(_))).unzip._1 - } -} - -class ExponentialShuffler[T]() extends RankWeightedRandomShuffler[T] { - def rankToWeight(rank: Int): Double = { - 1 / math - .pow(rank.toDouble, 2.0) // this function was proved to be effective in previous DDGs - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.scala deleted file mode 100644 index 54e2ad549..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightMethod.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -object WeightMethod extends Enumeration { - type WeightMethod = Value - val WeightedRandomSampling, WeightedRoundRobin = Value -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.scala deleted file mode 100644 index e348560db..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceBaseRanker.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.follow_recommendations.common.utils.RandomUtil -import com.twitter.follow_recommendations.common.utils.MergeUtil -import com.twitter.follow_recommendations.common.utils.Weighted -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightMethod._ -import scala.util.Random - -/** - * This ranker selects the next candidate source to select a candidate from. It supports - * two kinds of algorithm, WeightedRandomSampling or WeightedRoundRobin. WeightedRandomSampling - * pick the next candidate source randomly, WeightedRoundRobin picked the next candidate source - * sequentially based on the weight of the candidate source. It is default to WeightedRandomSampling - * if no weight method is provided. - * - * Example usage of this class: - * - * When use WeightedRandomSampling: - * Input candidate sources and their weights are: {{CS1: 3}, {CS2: 2}, {CS3: 5}} - * Ranked candidates sequence is not determined because of random sampling. - * One possible output candidate sequence is: (CS1_candidate1, CS2_candidate1, CS2_candidate2, - * CS3_candidate1, CS3_candidates2, CS3_candidate3, CS1_candidate2, CS1_candidate3, - * CS3_candidate4, CS3_candidate5, CS1_candidate4, CS1_candidate5, CS2_candidate6, CS2_candidate3,...) - * - * When use WeightedRoundRobin: - * Input candidate sources and their weights are: {{CS1: 3}, {CS2: 2}, {CS3: 5}} - * Output candidate sequence is: (CS1_candidate1, CS1_candidate2, CS1_candidate3, - * CS2_candidate1, CS2_candidates2, CS3_candidate1, CS3_candidate2, CS3_candidate3, - * CS3_candidate4, CS3_candidate5, CS1_candidate4, CS1_candidate5, CS1_candidate6, CS2_candidate3,...) - * - * Note: CS1_candidate1 means the first candidate in CS1 candidate source. - * - * @tparam A candidate source type - * @tparam Rec Recommendation type - * @param candidateSourceWeights relative weights for different candidate sources - */ -class WeightedCandidateSourceBaseRanker[A, Rec]( - candidateSourceWeights: Map[A, Double], - weightMethod: WeightMethod = WeightedRandomSampling, - randomSeed: Option[Long]) { - - /** - * Creates a iterator over algorithms and calls next to return a Stream of candidates - * - * - * @param candidateSources the set of candidate sources that are being sampled - * @param candidateSourceWeights map of candidate source to weight - * @param candidates the map of candidate source to the iterator of its results - * @param weightMethod a enum to indict which weight method to use. Two values are supported - * currently. When WeightedRandomSampling is set, the next candidate is picked from a candidate - * source that is randomly chosen. When WeightedRoundRobin is set, the next candidate is picked - * from current candidate source until the number of candidates reaches to the assigned weight of - * the candidate source. The next call of this function will return a candidate from the next - * candidate source which is after previous candidate source based on the order input - * candidate source sequence. - - * @return stream of candidates - */ - def stream( - candidateSources: Set[A], - candidateSourceWeights: Map[A, Double], - candidates: Map[A, Iterator[Rec]], - weightMethod: WeightMethod = WeightedRandomSampling, - random: Option[Random] = None - ): Stream[Rec] = { - val weightedCandidateSource: Weighted[A] = new Weighted[A] { - override def apply(a: A): Double = candidateSourceWeights.getOrElse(a, 0) - } - - /** - * Generates a stream of candidates. - * - * @param candidateSourceIter an iterator over candidate sources returned by the sampling procedure - * @return stream of candidates - */ - def next(candidateSourceIter: Iterator[A]): Stream[Rec] = { - val source = candidateSourceIter.next() - val it = candidates(source) - if (it.hasNext) { - val currCand = it.next() - currCand #:: next(candidateSourceIter) - } else { - assert(candidateSources.contains(source), "Selected source is not in candidate sources") - // Remove the depleted candidate source and re-sample - stream(candidateSources - source, candidateSourceWeights, candidates, weightMethod, random) - } - } - if (candidateSources.isEmpty) - Stream.empty - else { - val candidateSourceSeq = candidateSources.toSeq - val candidateSourceIter = - if (weightMethod == WeightMethod.WeightedRoundRobin) { - MergeUtil.weightedRoundRobin(candidateSourceSeq)(weightedCandidateSource).iterator - } else { - //default to weighted random sampling if no other weight method is provided - RandomUtil - .weightedRandomSamplingWithReplacement( - candidateSourceSeq, - random - )(weightedCandidateSource).iterator - } - next(candidateSourceIter) - } - } - - def apply(input: Map[A, TraversableOnce[Rec]]): Stream[Rec] = { - stream( - input.keySet, - candidateSourceWeights, - input.map { - case (k, v) => k -> v.toIterator - }, // cannot do mapValues here, as that only returns a view - weightMethod, - randomSeed.map(new Random(_)) - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala deleted file mode 100644 index c6f55adbc..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.common.DedupCandidates -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * Candidate Ranker that mixes and ranks multiple candidate lists from different candidate sources with the - * following steps: - * 1) generate a ranked candidate list of each candidate source by sorting and shuffling the candidate list - * of the algorithm. - * 2) merge the ranked lists generated in 1) into a single list using weighted randomly sampling. - * 3) If dedup is required, dedup the output from 2) by candidate id. - * - * @param basedRanker base ranker - * @param shuffleFn the shuffle function that will be used to shuffle each algorithm's sorted candidate list. - * @param dedup whether to remove duplicated candidates from the final output. - */ -class WeightedCandidateSourceRanker[Target <: HasParams]( - basedRanker: WeightedCandidateSourceBaseRanker[ - CandidateSourceIdentifier, - CandidateUser - ], - shuffleFn: Seq[CandidateUser] => Seq[CandidateUser], - dedup: Boolean) - extends Ranker[Target, CandidateUser] { - - val name: String = this.getClass.getSimpleName - - override def rank(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - val scribeRankingInfo: Boolean = - target.params(WeightedCandidateSourceRankerParams.ScribeRankingInfoInWeightedRanker) - val rankedCands = rankCandidates(group(candidates)) - Stitch.value(if (scribeRankingInfo) Utils.addRankingInfo(rankedCands, name) else rankedCands) - } - - private def group( - candidates: Seq[CandidateUser] - ): Map[CandidateSourceIdentifier, Seq[CandidateUser]] = { - val flattened = for { - candidate <- candidates - identifier <- candidate.getPrimaryCandidateSource - } yield (identifier, candidate) - flattened.groupBy(_._1).mapValues(_.map(_._2)) - } - - private def rankCandidates( - input: Map[CandidateSourceIdentifier, Seq[CandidateUser]] - ): Seq[CandidateUser] = { - // Sort and shuffle candidates per candidate source. - // Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream. - val sortAndShuffledCandidates = input.map { - case (source, candidates) => - // Note 2: toList is required here since candidates is a view, and it will result in infinit loop when used as part of Stream. - // Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources - val sortedCandidates = candidates.toList - source -> shuffleFn(sortedCandidates).iterator - } - val rankedCandidates = basedRanker(sortAndShuffledCandidates) - - if (dedup) DedupCandidates(rankedCandidates) else rankedCandidates - } -} - -object WeightedCandidateSourceRanker { - - def build[Target <: HasParams]( - candidateSourceWeight: Map[CandidateSourceIdentifier, Double], - shuffleFn: Seq[CandidateUser] => Seq[CandidateUser] = identity, - dedup: Boolean = false, - randomSeed: Option[Long] = None - ): WeightedCandidateSourceRanker[Target] = { - new WeightedCandidateSourceRanker( - new WeightedCandidateSourceBaseRanker( - candidateSourceWeight, - WeightMethod.WeightedRandomSampling, - randomSeed = randomSeed), - shuffleFn, - dedup - ) - } -} - -object WeightedCandidateSourceRankerWithoutRandomSampling { - def build[Target <: HasParams]( - candidateSourceWeight: Map[CandidateSourceIdentifier, Double] - ): WeightedCandidateSourceRanker[Target] = { - new WeightedCandidateSourceRanker( - new WeightedCandidateSourceBaseRanker( - candidateSourceWeight, - WeightMethod.WeightedRoundRobin, - randomSeed = None), - identity, - false, - ) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.scala deleted file mode 100644 index 58f0fde3e..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerFSConfig.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSParam - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class WeightedCandidateSourceRankerFSConfig @Inject() extends FeatureSwitchConfig { - override val booleanFSParams: Seq[FSParam[Boolean]] = - Seq(WeightedCandidateSourceRankerParams.ScribeRankingInfoInWeightedRanker) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.scala deleted file mode 100644 index ff4ecae4b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRankerParams.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker - -import com.twitter.timelines.configapi.FSParam - -object WeightedCandidateSourceRankerParams { - case object ScribeRankingInfoInWeightedRanker - extends FSParam[Boolean]("weighted_ranker_scribe_ranking_info", false) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD deleted file mode 100644 index 3b7a46db7..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "strato/config/columns/onboarding/userrecs:userrecs-strato-client", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.scala deleted file mode 100644 index d2f4e035b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/stores/LowTweepCredFollowStore.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.follow_recommendations.common.stores - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.onboarding.userrecs.TweepCredOnUserClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -// Not a candidate source since it's a intermediary. -@Singleton -class LowTweepCredFollowStore @Inject() (tweepCredOnUserClientColumn: TweepCredOnUserClientColumn) { - - def getLowTweepCredUsers(target: HasRecentFollowedUserIds): Stitch[Seq[CandidateUser]] = { - val newFollowings = - target.recentFollowedUserIds.getOrElse(Nil).take(LowTweepCredFollowStore.NumFlockToRetrieve) - - val validTweepScoreUserIdsStitch: Stitch[Seq[Long]] = Stitch - .traverse(newFollowings) { newFollowingUserId => - val tweepCredScoreOptStitch = tweepCredOnUserClientColumn.fetcher - .fetch(newFollowingUserId) - .map(_.v) - tweepCredScoreOptStitch.map(_.flatMap(tweepCred => - if (tweepCred < LowTweepCredFollowStore.TweepCredThreshold) { - Some(newFollowingUserId) - } else { - None - })) - }.map(_.flatten) - - validTweepScoreUserIdsStitch - .map(_.map(CandidateUser(_, Some(CandidateUser.DefaultCandidateScore)))) - } -} - -object LowTweepCredFollowStore { - val NumFlockToRetrieve = 500 - val TweepCredThreshold = 40 -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD deleted file mode 100644 index 35534b064..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.scala deleted file mode 100644 index 64f73d6ae..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup/DedupTransform.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.dedup - -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.stitch.Stitch -import scala.collection.mutable - -class DedupTransform[Request, Candidate <: UniversalNoun[Long]]() - extends Transform[Request, Candidate] { - override def transform(target: Request, candidates: Seq[Candidate]): Stitch[Seq[Candidate]] = { - val seen = mutable.HashSet[Long]() - Stitch.value(candidates.filter(candidate => seen.add(candidate.id))) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD deleted file mode 100644 index 79da9c259..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "configapi/configapi-core", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.scala deleted file mode 100644 index 306578a4d..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/ModifySocialProofTransform.scala +++ /dev/null @@ -1,202 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.modify_social_proof - -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureServiceClient -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FollowProof -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.graph_feature_service.thriftscala.EdgeType -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -object ModifySocialProof { - val GfsLagDuration: Duration = 14.days - val GfsIntersectionIds: Int = 3 - val SgsIntersectionIds: Int = 10 - val LeftEdgeTypes: Set[EdgeType] = Set(EdgeType.Following) - val RightEdgeTypes: Set[EdgeType] = Set(EdgeType.FollowedBy) - - /** - * Given the intersection ID's for a particular candidate, update the candidate's social proof - * @param candidate candidate object - * @param followProof follow proof to be added (includes id's and count) - * @param stats stats for tracking - * @return updated candidate object - */ - def addIntersectionIdsToCandidate( - candidate: CandidateUser, - followProof: FollowProof, - stats: StatsReceiver - ): CandidateUser = { - // create updated set of social proof - val updatedFollowedByOpt = candidate.followedBy match { - case Some(existingFollowedBy) => Some((followProof.followedBy ++ existingFollowedBy).distinct) - case None if followProof.followedBy.nonEmpty => Some(followProof.followedBy.distinct) - case _ => None - } - - val updatedFollowProof = updatedFollowedByOpt.map { updatedFollowedBy => - val updatedCount = followProof.numIds.max(updatedFollowedBy.size) - // track stats - val numSocialProofAdded = updatedFollowedBy.size - candidate.followedBy.size - addCandidatesWithSocialContextCountStat(stats, numSocialProofAdded) - FollowProof(updatedFollowedBy, updatedCount) - } - - candidate.setFollowProof(updatedFollowProof) - } - - private def addCandidatesWithSocialContextCountStat( - statsReceiver: StatsReceiver, - count: Int - ): Unit = { - if (count > 3) { - statsReceiver.counter("4_and_more").incr() - } else { - statsReceiver.counter(count.toString).incr() - } - } -} - -/** - * This class makes a request to gfs/sgs for hydrating additional social proof on each of the - * provided candidates. - */ -@Singleton -class ModifySocialProof @Inject() ( - gfsClient: GraphFeatureServiceClient, - socialGraphClient: SocialGraphClient, - statsReceiver: StatsReceiver, - decider: Decider = Decider.True) - extends Logging { - import ModifySocialProof._ - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val addedStats = stats.scope("num_social_proof_added_per_candidate") - private val gfsStats = stats.scope("graph_feature_service") - private val sgsStats = stats.scope("social_graph_service") - private val previousProofEmptyCounter = stats.counter("previous_proof_empty") - private val emptyFollowProofCounter = stats.counter("empty_followed_proof") - - /** - * For each candidate provided, we get the intersectionIds between the user and the candidate, - * appending the unique results to the social proof (followedBy field) if not already previously - * seen we query GFS for all users, except for cases specified via the mustCallSgs field or for - * very new users, who would not have any data in GFS, due to the lag duration of the service's - * processing. this is determined by GfsLagDuration - * @param userId id of the target user whom we provide recommendations for - * @param candidates list of candidates - * @param intersectionIdsNum if provided, determines the maximum number of accounts we want to be hydrated for social proof - * @param mustCallSgs Determines if we should query SGS regardless of user age or not. - * @return list of candidates updated with additional social proof - */ - def hydrateSocialProof( - userId: Long, - candidates: Seq[CandidateUser], - intersectionIdsNum: Option[Int] = None, - mustCallSgs: Boolean = false, - callSgsCachedColumn: Boolean = false, - gfsLagDuration: Duration = GfsLagDuration, - gfsIntersectionIds: Int = GfsIntersectionIds, - sgsIntersectionIds: Int = SgsIntersectionIds, - ): Stitch[Seq[CandidateUser]] = { - addCandidatesWithSocialContextCountStat( - stats.scope("social_context_count_before_hydration"), - candidates.count(_.followedBy.isDefined) - ) - val candidateIds = candidates.map(_.id) - val userAgeOpt = SnowflakeId.timeFromIdOpt(userId).map(Time.now - _) - - // this decider gate is used to determine what % of requests is allowed to call - // Graph Feature Service. this is useful for ramping down requests to Graph Feature Service - // when necessary - val deciderKey: String = DeciderKey.EnableGraphFeatureServiceRequests.toString - val enableGfsRequests: Boolean = decider.isAvailable(deciderKey, Some(RandomRecipient)) - - // if new query sgs - val (candidateToIntersectionIdsMapFut, isGfs) = - if (!enableGfsRequests || mustCallSgs || userAgeOpt.exists(_ < gfsLagDuration)) { - ( - if (callSgsCachedColumn) - socialGraphClient.getIntersectionsFromCachedColumn( - userId, - candidateIds, - intersectionIdsNum.getOrElse(sgsIntersectionIds) - ) - else - socialGraphClient.getIntersections( - userId, - candidateIds, - intersectionIdsNum.getOrElse(sgsIntersectionIds)), - false) - } else { - ( - gfsClient.getIntersections( - userId, - candidateIds, - intersectionIdsNum.getOrElse(gfsIntersectionIds)), - true) - } - val finalCandidates = candidateToIntersectionIdsMapFut - .map { candidateToIntersectionIdsMap => - { - previousProofEmptyCounter.incr(candidates.count(_.followedBy.exists(_.isEmpty))) - candidates.map { candidate => - addIntersectionIdsToCandidate( - candidate, - candidateToIntersectionIdsMap.getOrElse(candidate.id, FollowProof(Seq.empty, 0)), - addedStats) - } - } - } - .within(250.milliseconds)(DefaultTimer) - .rescue { - case e: Exception => - error(e.getMessage) - if (isGfs) { - gfsStats.scope("rescued").counter(e.getClass.getSimpleName).incr() - } else { - sgsStats.scope("rescued").counter(e.getClass.getSimpleName).incr() - } - Stitch.value(candidates) - } - - finalCandidates.onSuccess { candidatesSeq => - emptyFollowProofCounter.incr(candidatesSeq.count(_.followedBy.exists(_.isEmpty))) - addCandidatesWithSocialContextCountStat( - stats.scope("social_context_count_after_hydration"), - candidatesSeq.count(_.followedBy.isDefined) - ) - } - } -} - -/** - * This transform uses ModifySocialProof (which makes a request to gfs/sgs) for hydrating additional - * social proof on each of the provided candidates. - */ -@Singleton -class ModifySocialProofTransform @Inject() (modifySocialProof: ModifySocialProof) - extends GatedTransform[HasClientContext with HasParams, CandidateUser] - with Logging { - - override def transform( - target: HasClientContext with HasParams, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = - target.getOptionalUserId - .map(modifySocialProof.hydrateSocialProof(_, candidates)).getOrElse(Stitch.value(candidates)) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.scala deleted file mode 100644 index 8face1164..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof/RemoveAccountProofTransform.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.modify_social_proof - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RemoveAccountProofTransform @Inject() (statsReceiver: StatsReceiver) - extends GatedTransform[HasClientContext with HasParams, CandidateUser] { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val removedProofsCounter = stats.counter("num_removed_proofs") - - override def transform( - target: HasClientContext with HasParams, - items: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = - Stitch.value(items.map { candidate => - removedProofsCounter.incr() - candidate.copy(reason = None) - }) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD deleted file mode 100644 index d6dcd8522..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.scala deleted file mode 100644 index 03639da26..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id/RandomRankerIdTransform.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.ranker_id - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams - -/** - * This class appends each candidate's rankerIds with the RandomRankerId. - * This is primarily for determining if a candidate was generated via random shuffling. - */ -@Singleton -class RandomRankerIdTransform @Inject() () extends GatedTransform[HasParams, CandidateUser] { - - override def transform( - target: HasParams, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - Stitch.value(candidates.map(_.addScore(Score.RandomScore))) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.scala deleted file mode 100644 index 87c111a6c..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/AddRecommendationFlowIdentifierTransform.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.recommendation_flow_identifier - -import com.google.inject.Inject -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasRecommendationFlowIdentifier -import com.twitter.stitch.Stitch - -class AddRecommendationFlowIdentifierTransform @Inject() - extends Transform[HasRecommendationFlowIdentifier, CandidateUser] { - - override def transform( - target: HasRecommendationFlowIdentifier, - items: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - Stitch.value(items.map { candidateUser => - candidateUser.copy(recommendationFlowIdentifier = target.recommendationFlowIdentifier) - }) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD deleted file mode 100644 index 820e2df66..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/recommendation_flow_identifier/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD deleted file mode 100644 index d9b257348..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/constants", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.scala deleted file mode 100644 index 5a30c9cb1..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token/TrackingTokenTransform.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.tracking_token - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.Session -import com.twitter.follow_recommendations.common.models.TrackingToken -import com.twitter.hermit.constants.AlgorithmFeedbackTokens.AlgorithmToFeedbackTokenMap -import com.twitter.hermit.model.Algorithm -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.stitch.Stitch -import com.twitter.util.logging.Logging - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * This transform adds the tracking token for all candidates - * Since this happens in the same request, we use the same trace-id for all candidates - * There are no RPC calls in this transform so it's safe to chain it with `andThen` at the end of - * all other product-specific transforms - */ -@Singleton -class TrackingTokenTransform @Inject() (baseStatsReceiver: StatsReceiver) - extends Transform[HasDisplayLocation with HasClientContext, CandidateUser] - with Logging { - - def profileResults( - target: HasDisplayLocation with HasClientContext, - candidates: Seq[CandidateUser] - ) = { - // Metrics to track # results per candidate source - val stats = baseStatsReceiver.scope(target.displayLocation.toString + "/final_results") - stats.stat("total").add(candidates.size) - - stats.counter(target.displayLocation.toString).incr() - - val flattenedCandidates: Seq[(CandidateSourceIdentifier, CandidateUser)] = for { - candidate <- candidates - identifier <- candidate.getPrimaryCandidateSource - } yield (identifier, candidate) - val candidatesGroupedBySource: Map[CandidateSourceIdentifier, Seq[CandidateUser]] = - flattenedCandidates.groupBy(_._1).mapValues(_.map(_._2)) - candidatesGroupedBySource map { - case (source, candidates) => stats.stat(source.name).add(candidates.size) - } - } - - override def transform( - target: HasDisplayLocation with HasClientContext, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - profileResults(target, candidates) - - Stitch.value( - target.getOptionalUserId - .map { _ => - candidates.map { - candidate => - val token = Some(TrackingToken( - sessionId = Session.getSessionId, - displayLocation = Some(target.displayLocation), - controllerData = None, - algorithmId = candidate.userCandidateSourceDetails.flatMap(_.primaryCandidateSource - .flatMap { identifier => - Algorithm.withNameOpt(identifier.name).flatMap(AlgorithmToFeedbackTokenMap.get) - }) - )) - candidate.copy(trackingToken = token) - } - }.getOrElse(candidates)) - - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD deleted file mode 100644 index 606e8edfa..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.scala deleted file mode 100644 index 269a39a48..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransform.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.weighted_sampling -import com.twitter.follow_recommendations.common.base.GatedTransform -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.Score -import com.twitter.follow_recommendations.common.models.Scores -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.utils.Utils -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SamplingTransform @Inject() () - extends GatedTransform[HasClientContext with HasParams with HasDebugOptions, CandidateUser] { - - val name: String = this.getClass.getSimpleName - - /* - Description: This function takes in a set of candidate users and ranks them for a who-to-follow - request by sampling from the Placket-Luce distribution - (https://cran.rstudio.com/web/packages/PlackettLuce/vignettes/Overview.html) with a three - variations. The first variation is that the scores of the candidates are multiplied by - multiplicativeFactor before sampling. The second variation is that the scores are - exponentiated before sampling. The third variation is that depending on how many who-to-follow - positions are being requested, the first k positions are reserved for the candidates with the - highest scores (and they are sorted in decreasing order of score) and the remaining positions - are sampled from a Placket-Luce. We use the efficient algorithm proposed in this blog - https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051 - to sample from a Plackett-Luce. Because of numerical stability reasons, before sampling from this - distribution, (1) we subtract off the maximum score from all the scores and (2) if after - this subtraction and multiplication by the multiplicative factor the resulting score is <= -10, - we force the candidate's transformed score under the above algorithm to be 0 (so r^(1/w) = 0) - where r is a random number and w is the transformed score. - - inputs: - - target: HasClientContext (WTF request) - - candidates: sequence of CandidateUsers (users that need to be ranked from a who-to-follow - request) each of which has a score - - inputs accessed through feature switches, i.e. through target.params (see the following file: - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/ - transforms/weighted_sampling/SamplingTransformParams.scala"): - - topKFixed: the first k positions of the who-to-follow ranking correspond to the users with the k - highest scores and are not sampled from the Placket-Luce distribution - - multiplicativeFactor: multiplicativeFactor is used to transform the scores of each candidate by - multiplying that user's score by multiplicativeFactor - - output: - - Sequence of CandidateUser whose order represents the ranking of users in a who-to-follow request - This ranking is sampled from a Placket-Luce distribution. - */ - override def transform( - target: HasClientContext with HasParams with HasDebugOptions, - candidates: Seq[CandidateUser] - ): Stitch[Seq[CandidateUser]] = { - - // the first k positions of the who-to-follow ranking correspond to the users with the k - // highest scores and are not sampled from the Placket-Luce distribution - val topKFixed = target.params(SamplingTransformParams.TopKFixed) - - // multiplicativeFactor is used to transform the scores of each candidate by - // multiplying that user's score by multiplicativeFactor - val multiplicativeFactor = target.params(SamplingTransformParams.MultiplicativeFactor) - - // sort candidates by their score - val candidatesSorted = candidates.sortBy(-1 * _.score.getOrElse(0.0)) - - // pick the top K candidates by score and the remaining candidates - val (topKFixedCandidates, candidatesOutsideOfTopK) = - candidatesSorted.zipWithIndex.partition { case (value, index) => index < topKFixed } - - val randomNumGenerator = - new scala.util.Random(target.getRandomizationSeed.getOrElse(System.currentTimeMillis)) - - // we need to subtract the maximum score off the scores for numerical stability reasons - // subtracting the max score off does not effect the underlying distribution we are sampling - // the candidates from - // we need the if statement since you cannot take the max of an empty sequence - val maximum_score = if (candidatesOutsideOfTopK.nonEmpty) { - candidatesOutsideOfTopK.map(x => x._1.score.getOrElse(0.0)).max - } else { - 0.0 - } - - // for candidates in candidatesOutsideOfTopK, we transform their score by subtracting off - // maximum_score and then multiply by multiplicativeFactor - val candidatesOutsideOfTopKTransformedScore = candidatesOutsideOfTopK.map(x => - (x._1, multiplicativeFactor * (x._1.score.getOrElse(0.0) - maximum_score))) - - // for each candidate with score transformed and clip score w, sample a random number r, - // create a new score r^(1/w) and sort the candidates to get the final ranking. - // for numerical stability reasons if the score is <=-10, we force r^(1/w) = 0. - // this samples the candidates from the modified Plackett-Luce distribution. See - // https://medium.com/swlh/going-old-school-designing-algorithms-for-fast-weighted-sampling-in-production-c48fc1f40051 - - val candidatesOutsideOfTopKSampled = candidatesOutsideOfTopKTransformedScore - .map(x => - ( - x._1, - if (x._2 <= -10.0) - 0.0 - else - scala.math.pow( - randomNumGenerator.nextFloat(), - 1 / (scala.math - .exp(x._2))))).sortBy(-1 * _._2) - - val topKCandidates: Seq[CandidateUser] = topKFixedCandidates.map(_._1) - - val scribeRankingInfo: Boolean = - target.params(SamplingTransformParams.ScribeRankingInfoInSamplingTransform) - - val transformedCandidates: Seq[CandidateUser] = if (scribeRankingInfo) { - val topKCandidatesWithRankingInfo: Seq[CandidateUser] = - Utils.addRankingInfo(topKCandidates, name) - val candidatesOutsideOfTopKSampledWithRankingInfo: Seq[CandidateUser] = - candidatesOutsideOfTopKSampled.zipWithIndex.map { - case ((candidate, score), rank) => - val newScore = Seq(Score(score, Some(RankerId.PlacketLuceSamplingTransformer))) - val newScores: Option[Scores] = candidate.scores - .map { scores => - scores.copy(scores = scores.scores ++ newScore) - }.orElse(Some(Scores(newScore, Some(RankerId.PlacketLuceSamplingTransformer)))) - val globalRank = rank + topKFixed + 1 - candidate.addInfoPerRankingStage(name, newScores, globalRank) - } - - topKCandidatesWithRankingInfo ++ candidatesOutsideOfTopKSampledWithRankingInfo - } else { - topKCandidates ++ candidatesOutsideOfTopKSampled.map(_._1) - } - - Stitch.value(transformedCandidates) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.scala deleted file mode 100644 index b97251f93..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformFSConfig.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.weighted_sampling - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SamplingTransformFSConfig @Inject() () extends FeatureSwitchConfig { - override val intFSParams: Seq[FSBoundedParam[Int]] = Seq(SamplingTransformParams.TopKFixed) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - SamplingTransformParams.MultiplicativeFactor) - - override val booleanFSParams: Seq[FSParam[Boolean]] = Seq( - SamplingTransformParams.ScribeRankingInfoInSamplingTransform) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala deleted file mode 100644 index 363487a9b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.common.transforms.weighted_sampling - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object SamplingTransformParams { - - case object TopKFixed // indicates how many of the fisrt K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score - extends FSBoundedParam[Int]( - name = "post_nux_ml_flow_weighted_sampling_top_k_fixed", - default = 0, - min = 0, - max = 100) - - case object MultiplicativeFactor // CandidateUser.score gets transformed to multiplicativeFactor*CandidateUser.score before sampling from the Plackett-Luce distribution - extends FSBoundedParam[Double]( - name = "post_nux_ml_flow_weighted_sampling_multiplicative_factor", - default = 1.0, - min = -1000.0, - max = 1000.0) - - case object ScribeRankingInfoInSamplingTransform - extends FSParam[Boolean]("sampling_transform_scribe_ranking_info", false) - -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD deleted file mode 100644 index 7075167e3..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "stitch/stitch-core", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.scala deleted file mode 100644 index db9a1f9f5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/CollectionUtil.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -object CollectionUtil { - - /** - * Transposes a sequence of sequences. As opposed to the Scala collection library version - * of transpose, the sequences do not have to have the same length. - * - * Example: - * transpose(immutable.Seq(immutable.Seq(1,2,3), immutable.Seq(4,5), immutable.Seq(6,7))) - * => immutable.Seq(immutable.Seq(1, 4, 6), immutable.Seq(2, 5, 7), immutable.Seq(3)) - * - * @param seq a sequence of sequences - * @tparam A the type of elements in the seq - * @return the transposed sequence of sequences - */ - def transposeLazy[A](seq: Seq[Seq[A]]): Stream[Seq[A]] = - seq.filter(_.nonEmpty) match { - case Nil => Stream.empty - case ys => ys.map(_.head) #:: transposeLazy(ys.map(_.tail)) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.scala deleted file mode 100644 index 2f6db39b6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/DisplayLocationProductConverterUtil.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Product -import com.twitter.product_mixer.core.model.marshalling.request.Product - -object DisplayLocationProductConverterUtil { - def productToDisplayLocation(product: Product): DisplayLocation = { - product match { - case Product.MagicRecs => DisplayLocation.MagicRecs - case _ => - throw UnconvertibleProductMixerProductException( - s"Cannot convert Product Mixer Product ${product.identifier.name} into a FRS DisplayLocation.") - } - } - - def displayLocationToProduct(displayLocation: DisplayLocation): Product = { - displayLocation match { - case DisplayLocation.MagicRecs => Product.MagicRecs - case _ => - throw UnconvertibleProductMixerProductException( - s"Cannot convert DisplayLocation ${displayLocation.toFsName} into a Product Mixer Product.") - } - } -} - -case class UnconvertibleProductMixerProductException(message: String) extends Exception(message) diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.scala deleted file mode 100644 index 6aaee4c45..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/MergeUtil.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -object MergeUtil { - - /** - * Takes a seq of items which have weights. Returns an infinite stream of each item - * by their weights. All weights need to be greater than or equal to zero. In addition, - * the sum of weights should be greater than zero. - * - * Example usage of this function: - * Input weighted Item {{CS1, 3}, {CS2, 2}, {CS3, 5}} - * Output stream: (CS1, CS1, CS1, CS2, CS2, CS3, CS3, CS3, CS3, CS3, CS1, CS1, CS1, CS2,...} - * - * @param items items - * @param weighted provides weights for items - * @tparam T type of item - * - * @return Stream of Ts - */ - def weightedRoundRobin[T]( - items: Seq[T] - )( - implicit weighted: Weighted[T] - ): Stream[T] = { - if (items.isEmpty) { - Stream.empty - } else { - val weights = items.map { i => weighted(i) } - assert( - weights.forall { - _ >= 0 - }, - "Negative weight exists for sampling") - val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail - assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive") - - var weightIdx = 0 - var weight = 0 - - def next(): Stream[T] = { - val tmpIdx = weightIdx - weight = weight + 1 - weight = if (weight >= weights(weightIdx)) 0 else weight - weightIdx = if (weight == 0) weightIdx + 1 else weightIdx - weightIdx = if (weightIdx == weights.length) 0 else weightIdx - items(tmpIdx) #:: next() - } - next() - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.scala deleted file mode 100644 index 9d66e8deb..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RandomUtil.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.follow_recommendations.common.utils -import scala.util.Random - -object RandomUtil { - - /** - * Takes a seq of items which have weights. Returns an infinite stream that is - * sampled with replacement using the weights for each item. All weights need - * to be greater than or equal to zero. In addition, the sum of weights - * should be greater than zero. - * - * @param items items - * @param weighted provides weights for items - * @tparam T type of item - * @return Stream of Ts - */ - def weightedRandomSamplingWithReplacement[T]( - items: Seq[T], - random: Option[Random] = None - )( - implicit weighted: Weighted[T] - ): Stream[T] = { - if (items.isEmpty) { - Stream.empty - } else { - val weights = items.map { i => weighted(i) } - assert(weights.forall { _ >= 0 }, "Negative weight exists for sampling") - val cumulativeWeight = weights.scanLeft(0.0)(_ + _).tail - assert(cumulativeWeight.last > 0, "Sum of the sampling weights is not positive") - val cumulativeProbability = cumulativeWeight map (_ / cumulativeWeight.last) - def next(): Stream[T] = { - val rand = random.getOrElse(Random).nextDouble() - val idx = cumulativeProbability.indexWhere(_ >= rand) - items(if (idx == -1) items.length - 1 else idx) #:: next() - } - next() - } - } - - /** - * Takes a seq of items and their weights. Returns a lazy weighted shuffle of - * the elements in the list. All weights should be greater than zero. - * - * @param items items - * @param weighted provides weights for items - * @tparam T type of item - * @return Stream of Ts - */ - def weightedRandomShuffle[T]( - items: Seq[T], - random: Option[Random] = None - )( - implicit weighted: Weighted[T] - ): Stream[T] = { - assert(items.forall { i => weighted(i) > 0 }, "Non-positive weight exists for shuffling") - def next(it: Seq[T]): Stream[T] = { - if (it.isEmpty) - Stream.empty - else { - val cumulativeWeight = it.scanLeft(0.0)((acc: Double, curr: T) => acc + weighted(curr)).tail - val cutoff = random.getOrElse(Random).nextDouble() * cumulativeWeight.last - val idx = cumulativeWeight.indexWhere(_ >= cutoff) - val (left, right) = it.splitAt(idx) - it(if (idx == -1) it.size - 1 else idx) #:: next(left ++ right.drop(1)) - } - } - next(items) - } - - /** - * Takes a seq of items and a weight function, returns a lazy weighted shuffle of - * the elements in the list.The weight function is based on the rank of the element - * in the original lst. - * @param items - * @param rankToWeight - * @param random - * @tparam T - * @return - */ - def weightedRandomShuffleByRank[T]( - items: Seq[T], - rankToWeight: Int => Double, - random: Option[Random] = None - ): Stream[T] = { - val candWeights = items.zipWithIndex.map { case (item, rank) => (item, rankToWeight(rank)) } - RandomUtil.weightedRandomShuffle(candWeights, random).map(_._1) - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.scala deleted file mode 100644 index 8275228d6..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/RescueWithStatsUtils.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.stitch.Stitch -import com.twitter.util.Duration -import com.twitter.util.TimeoutException - -object RescueWithStatsUtils { - def rescueWithStats[T]( - s: Stitch[Seq[T]], - stats: StatsReceiver, - source: String - ): Stitch[Seq[T]] = { - StatsUtil.profileStitchSeqResults(s, stats.scope(source)).rescue { - case _: Exception => Stitch.Nil - } - } - - def rescueOptionalWithStats[T]( - s: Stitch[Option[T]], - stats: StatsReceiver, - source: String - ): Stitch[Option[T]] = { - StatsUtil.profileStitchOptionalResults(s, stats.scope(source)).rescue { - case _: Exception => Stitch.None - } - } - - def rescueWithStatsWithin[T]( - s: Stitch[Seq[T]], - stats: StatsReceiver, - source: String, - timeout: Duration - ): Stitch[Seq[T]] = { - val hydratedScopeSource = stats.scope(source) - StatsUtil - .profileStitchSeqResults( - s.within(timeout)(com.twitter.finagle.util.DefaultTimer), - hydratedScopeSource) - .rescue { - case _: TimeoutException => - hydratedScopeSource.counter("timeout").incr() - Stitch.Nil - case _: Exception => - hydratedScopeSource.counter("exception").incr() - Stitch.Nil - } - } -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.scala deleted file mode 100644 index 73d90a85b..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/UserSignupUtil.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.util.Duration -import com.twitter.util.Time - -object UserSignupUtil { - def signupTime(hasClientContext: HasClientContext): Option[Time] = - hasClientContext.clientContext.userId.flatMap(SnowflakeId.timeFromIdOpt) - - def userSignupAge(hasClientContext: HasClientContext): Option[Duration] = - signupTime(hasClientContext).map(Time.now - _) -} diff --git a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.scala b/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.scala deleted file mode 100644 index adb95e5f5..000000000 --- a/follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils/Weighted.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.common.utils - -/** - * Typeclass for any Recommendation type that has a weight - * - */ -trait Weighted[-Rec] { - def apply(rec: Rec): Double -} - -object Weighted { - implicit object WeightedTuple extends Weighted[(_, Double)] { - override def apply(rec: (_, Double)): Double = rec._2 - } - - def fromFunction[Rec](f: Rec => Double): Weighted[Rec] = { - new Weighted[Rec] { - override def apply(rec: Rec): Double = f(rec) - } - } -} diff --git a/follow-recommendations-service/server/src/main/resources/BUILD b/follow-recommendations-service/server/src/main/resources/BUILD deleted file mode 100644 index 610607ec5..000000000 --- a/follow-recommendations-service/server/src/main/resources/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -resources( - sources = [ - "*.tsv", - "*.xml", - "**/*", - "config/*.yml", - ], -) - -# Created for Bazel compatibility. -# In Bazel, loose files must be part of a target to be included into a bundle. -files( - name = "frs_resources", - sources = [ - "*.tsv", - "*.xml", - "*.yml", - "**/*", - ], -) diff --git a/follow-recommendations-service/server/src/main/resources/config/decider.yml b/follow-recommendations-service/server/src/main/resources/config/decider.yml deleted file mode 100644 index a46626094..000000000 --- a/follow-recommendations-service/server/src/main/resources/config/decider.yml +++ /dev/null @@ -1,129 +0,0 @@ -enable_recommendations: - comment: Proportion of requests where we return an actual response as part. Decreasing the value will increase the portion of empty responses (in order to disable the service) as part of the graceful degradation. - default_availability: 10000 -enable_score_user_candidates: - comment: Proportion of requests where score user candidates from the scoreUserCandidates endpoint - default_availability: 10000 -enable_profile_sidebar_product: - comment: Proportion of requests where we return an actual response for profile sidebar product - default_availability: 10000 -enable_magic_recs_product: - comment: Proportion of requests where we return an actual response for magic recs product - default_availability: 10000 -enable_rux_landing_page_product: - comment: Proportion of requests where we return an actual response for rux landing page product - default_availability: 10000 -enable_rux_pymk_product: - comment: Proportion of requests where we return an actual response for rux pymk product - default_availability: 10000 -enable_profile_bonus_follow_product: - comment: Proportion of requests where we return an actual response for profile bonus follow product - default_availability: 10000 -enable_election_explore_wtf_product: - comment: Proportion of requests where we return an actual response for election explore wtf product - default_availability: 10000 -enable_cluster_follow_product: - comment: Proportion of requests where we return an actual response for cluster follow product - default_availability: 10000 -enable_home_timeline_product: - comment: Proportion of requests where we return an actual response for htl wtf product - default_availability: 10000 -enable_htl_bonus_follow_product: - comment: Proportion of requests where we return an actual response for htl bonus follow product - default_availability: 10000 -enable_explore_tab_product: - comment: Proportion of requests where we return an actual response for explore tab product - default_availability: 10000 -enable_sidebar_product: - comment: Proportion of requests where we return an actual response for sidebar product - default_availability: 10000 -enable_campaign_form_product: - comment: Proportion of requests where we return an actual response for campaign form product - default_availability: 10000 -enable_reactive_follow_product: - comment: Proportion of requests where we return an actual response for reactive follow product - default_availability: 10000 -enable_nux_pymk_product: - comment: Proportion of requests where we return an actual response for nux pymk product - default_availability: 10000 -enable_nux_interests_product: - comment: Proportion of requests where we return an actual response for nux interests product - default_availability: 10000 -enable_nux_topic_bonus_follow_product: - comment: Proportion of requests where we return an actual response for nux topic-based bonus follow product - default_availability: 10000 -enable_india_covid19_curated_accounts_wtf_product: - comment: Proportion of requests where we return an actual response for india covid19 curated accounts wtf product - default_availability: 10000 -enable_ab_upload_product: - comment: Proportion of requests where we return an actual response for the address book upload product - default_availability: 10000 -enable_people_plus_plus_product: - comment: Proportion of requests where we return an actual response for the PeoplePlusPlus/Connect Tab product - default_availability: 10000 -enable_tweet_notification_recs_product: - comment: Proportion of requests where we return an actual response for the Tweet Notification Recommendations product - default_availability: 10000 -enable_profile_device_follow_product: - comment: Proportion of requests where we return an actual response for the ProfileDeviceFollow product - default_availability: 10000 -enable_diffy_module_dark_reading: - comment: Percentage of dark read traffic routed to diffy thrift - default_availability: 0 -enable_recos_backfill_product: - comment: Proportion of requests where we return an actual response for the RecosBackfill product - default_availability: 10000 -enable_post_nux_follow_task_product: - comment: Proportion of requests where we return an actual response for post NUX follow task product - default_availability: 10000 -enable_curated_space_hosts_product: - comment: Proportion of requests where we return an actual response for curated space hosts product - default_availability: 10000 -enable_nux_geo_category_product: - comment: Proportion of requests where we return an actual response for nux geo category product - default_availability: 10000 -enable_nux_interests_category_product: - comment: Proportion of requests where we return an actual response for nux interests category product - default_availability: 10000 -enable_nux_pymk_category_product: - comment: Proportion of requests where we return an actual response for nux pymk category product - default_availability: 10000 -enable_home_timeline_tweet_recs_product: - comment: Proportion of requests where we return an actual response for the Home Timeline Tweet Recs product - default_availability: 10000 -enable_htl_bulk_friend_follows_product: - comment: Proportion of requests where we return an actual response for the HTL bulk friend follows product - default_availability: 10000 -enable_nux_auto_follow_product: - comment: Proportion of requests where we return an actual response for the NUX auto follow product - default_availability: 10000 -enable_search_bonus_follow_product: - comment: Proportion of requests where we return an actual response for search bonus follow product - default_availability: 10000 -enable_fetch_user_in_request_builder: - comment: Proportion of requests where we fetch user object from gizmoduck in request builder - default_availability: 0 -enable_product_mixer_magic_recs_product: - comment: Proportion of requests where we enable the product mixer magic recs product - default_availability: 10000 -enable_home_timeline_reverse_chron_product: - comment: Proportion of requests where we return an actual response for Home timeline reverse chron product - default_availability: 10000 -enable_product_mixer_pipeline_magic_recs_dark_read: - comment: Compare product mixer pipeline responses to current FRS pipeline responses for Magic Recs - default_availability: 0 -enable_experimental_caching: - comment: Proportion of requests we use experimental caching for data caching - default_availability: 0 -enable_distributed_caching: - comment: Proportion of requests we use a distributed cache cluster for data caching - default_availability: 10000 -enable_gizmoduck_caching: - comment: Proportion of requests we use a distributed cache cluster for data caching in Gizmoduck - default_availability: 10000 -enable_traffic_dark_reading: - comment: Proportion of requests where we replicate the request for traffic dark reading - default_availability: 0 -enable_graph_feature_service_requests: - comment: Proportion of requests where we allow request calls to Graph Feature Service - default_availability: 10000 diff --git a/follow-recommendations-service/server/src/main/resources/logback.xml b/follow-recommendations-service/server/src/main/resources/logback.xml deleted file mode 100644 index 96348c8e4..000000000 --- a/follow-recommendations-service/server/src/main/resources/logback.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - - - - - - - - - - true - - - - - ${log.service.output} - - ${log.service.output}.%i - 1 - 5 - - - 50MB - - - %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n - - - - - - ${log.access.output} - - ${log.access.output}.%i - 1 - 5 - - - 50MB - - - ${DEFAULT_ACCESS_PATTERN}%n - - - - - - true - ${log.lens.category} - ${log.lens.index} - ${log.lens.tag}/service - - %msg - - - - - - true - ${log.lens.category} - ${log.lens.index} - ${log.lens.tag}/access - - %msg - - - - - - - - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel deleted file mode 100644 index a08d9723c..000000000 --- a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/epModel +++ /dev/null @@ -1,8 +0,0 @@ -# OWNER = jdeng -# Date = 20141223_153423 -# Training Size = 16744473 -# Testing Size = 16767335 -# trained with ElasticNetCV alpha=0.05 cv_folds=5 best_lambda=1.0E-7 -# num base features: 10 -# num nonzero weights: 30 -{bias:-5.67151,featureMetadataMap:["fwd_email":{metadata:{featureWeight:{weight:2.47389}}},"rev_phone":{metadata:{featureWeight:{weight:1.88433}}},"mutual_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:47.0,weight:6.31809},{left:11.0,right:16.0,weight:4.52959},{left:31.0,right:47.0,weight:5.7101},{right:2.0,weight:0.383515},{left:24.0,right:31.0,weight:5.26515},{left:3.0,right:4.0,weight:2.91751},{left:2.0,right:3.0,weight:2.22851},{left:4.0,right:5.0,weight:3.28515},{left:8.0,right:11.0,weight:4.14731},{left:5.0,right:8.0,weight:3.73588},{left:16.0,right:24.0,weight:4.90908}]}}},"fwd_phone":{metadata:{featureWeight:{weight:2.07327}}},"fwd_email_path":{metadata:{featureWeight:{weight:0.961773}}},"rev_phone_path":{metadata:{featureWeight:{weight:0.354484}}},"low_tweepcred_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:4.0,right:5.0,weight:0.177209},{left:7.0,right:8.0,weight:0.12378},{left:3.0,right:4.0,weight:0.197566},{left:5.0,right:6.0,weight:0.15867},{left:2.0,right:3.0,weight:0.196539},{right:2.0,weight:0.1805},{left:75.0,weight:-0.424598},{left:6.0,right:7.0,weight:0.143698},{left:10.0,right:13.0,weight:0.0458502},{left:8.0,right:10.0,weight:0.0919314},{left:13.0,right:75.0,weight:-0.111484}]}}},"rev_email_path":{metadata:{featureWeight:{weight:0.654451}}},"rev_email":{metadata:{featureWeight:{weight:2.33859}}},"fwd_phone_path":{metadata:{featureWeight:{weight:0.210418}}}]} diff --git a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig b/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig deleted file mode 100644 index ab38990dd..000000000 --- a/follow-recommendations-service/server/src/main/resources/quality/stp_models/20141223/trainingConfig +++ /dev/null @@ -1 +0,0 @@ -{input:{context:"discover.prod",startDateTime:"",endDateTime:"",trainingFeatures:["STP_FEATURES":["fwd_email","mutual_follow_path","fwd_email_path","rev_phone_path","low_tweepcred_follow_path","rev_phone","fwd_phone","rev_email_path","rev_email","fwd_phone_path"]],engagementActions:["click","favorite","open_link","open","send_tweet","send_reply","retweet","reply","profile_click","follow"],impressionActions:["discard","results","impression"],dataFormat:1,dataPath:"",isLabeled:0},sample:{positiveSampleRatio:1.0,negativeSampleRatio:1.0,sampleType:1},split:{trainingDataSplitSize:0.5,testingDataSplitSize:0.5,splitType:2},transform:{},filter:{featureOptions:[]},join:{engagementRules:["discover"],contentIdType:"tweet",groupBucketSize:3600000},discretize:{}} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD deleted file mode 100644 index 4dcbaab44..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/BUILD +++ /dev/null @@ -1,48 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-thriftmux/src/main/scala", - "finatra-internal/decider/src/main/scala", - "finatra-internal/international/src/main/scala/com/twitter/finatra/international/modules", - "finatra-internal/mtls-http/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/http-core/src/main/java/com/twitter/finatra/http", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client", - "finatra/jackson/src/main/scala/com/twitter/finatra/jackson/modules", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato", - "follow-recommendations-service/server/src/main/resources", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.scala deleted file mode 100644 index fba889c2f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/FollowRecommendationsServiceThriftServer.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.follow_recommendations - -import com.google.inject.Module -import com.twitter.finagle.ThriftMux -import com.twitter.finatra.decider.modules.DeciderModule -import com.twitter.finatra.http.HttpServer -import com.twitter.finatra.http.routing.HttpRouter -import com.twitter.finatra.international.modules.I18nFactoryModule -import com.twitter.finatra.international.modules.LanguagesModule -import com.twitter.finatra.jackson.modules.ScalaObjectMapperModule -import com.twitter.finatra.mtls.http.{Mtls => HttpMtls} -import com.twitter.finatra.mtls.thriftmux.Mtls -import com.twitter.finatra.thrift.ThriftServer -import com.twitter.finatra.thrift.filters._ -import com.twitter.finagle.thrift.Protocols -import com.twitter.finatra.thrift.routing.ThriftRouter -import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookModule -import com.twitter.follow_recommendations.common.clients.adserver.AdserverModule -import com.twitter.follow_recommendations.common.clients.cache.MemcacheModule -import com.twitter.follow_recommendations.common.clients.deepbirdv2.DeepBirdV2PredictionServiceClientModule -import com.twitter.follow_recommendations.common.clients.email_storage_service.EmailStorageServiceModule -import com.twitter.follow_recommendations.common.clients.geoduck.LocationServiceModule -import com.twitter.follow_recommendations.common.clients.gizmoduck.GizmoduckModule -import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureStoreModule -import com.twitter.follow_recommendations.common.clients.impression_store.ImpressionStoreModule -import com.twitter.follow_recommendations.common.clients.phone_storage_service.PhoneStorageServiceModule -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphModule -import com.twitter.follow_recommendations.common.clients.strato.StratoClientModule -import com.twitter.follow_recommendations.common.constants.ServiceConstants._ -import com.twitter.follow_recommendations.common.feature_hydration.sources.HydrationSourcesModule -import com.twitter.follow_recommendations.controllers.ThriftController -import com.twitter.follow_recommendations.modules._ -import com.twitter.follow_recommendations.service.exceptions.UnknownLoggingExceptionMapper -import com.twitter.follow_recommendations.services.FollowRecommendationsServiceWarmupHandler -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.geoduck.service.common.clientmodules.ReverseGeocoderThriftClientModule -import com.twitter.inject.thrift.filters.DarkTrafficFilter -import com.twitter.inject.thrift.modules.ThriftClientIdModule -import com.twitter.product_mixer.core.controllers.ProductMixerController -import com.twitter.product_mixer.core.module.PipelineExecutionLoggerModule -import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule -import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule -import com.twitter.product_mixer.core.product.guice.ProductScopeModule - -object FollowRecommendationsServiceThriftServerMain extends FollowRecommendationsServiceThriftServer - -class FollowRecommendationsServiceThriftServer - extends ThriftServer - with Mtls - with HttpServer - with HttpMtls { - override val name: String = "follow-recommendations-service-server" - - override val modules: Seq[Module] = - Seq( - ABDeciderModule, - AddressbookModule, - AdserverModule, - ConfigApiModule, - DeciderModule, - DeepBirdV2PredictionServiceClientModule, - DiffyModule, - EmailStorageServiceModule, - FeaturesSwitchesModule, - FlagsModule, - GizmoduckModule, - GraphFeatureStoreModule, - HydrationSourcesModule, - I18nFactoryModule, - ImpressionStoreModule, - LanguagesModule, - LocationServiceModule, - MemcacheModule, - PhoneStorageServiceModule, - PipelineExecutionLoggerModule, - ProductMixerFlagModule, - ProductRegistryModule, - new ProductScopeModule(), - new ProductScopeStringCenterModule(), - new ReverseGeocoderThriftClientModule, - ScalaObjectMapperModule, - ScorerModule, - ScribeModule, - SocialGraphModule, - StratoClientModule, - ThriftClientIdModule, - TimerModule, - ) - - def configureThrift(router: ThriftRouter): Unit = { - router - .filter[LoggingMDCFilter] - .filter[TraceIdMDCFilter] - .filter[ThriftMDCFilter] - .filter[StatsFilter] - .filter[AccessLoggingFilter] - .filter[ExceptionMappingFilter] - .exceptionMapper[UnknownLoggingExceptionMapper] - .filter[DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]] - .add[ThriftController] - } - - override def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server = { - server.withProtocolFactory( - Protocols.binaryFactory( - stringLengthLimit = StringLengthLimit, - containerLengthLimit = ContainerLengthLimit)) - } - - override def configureHttp(router: HttpRouter): Unit = router.add( - ProductMixerController[FollowRecommendationsThriftService.MethodPerEndpoint]( - this.injector, - FollowRecommendationsThriftService.ExecutePipeline)) - - override def warmup(): Unit = { - handle[FollowRecommendationsServiceWarmupHandler]() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.scala deleted file mode 100644 index cd3d88967..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Action.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Action(text: String, actionURL: String) { - lazy val toThrift: t.Action = { - t.Action(text, actionURL) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD deleted file mode 100644 index 65d265480..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "stringcenter/client", - ], - exports = [ - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.scala deleted file mode 100644 index 3346a05e1..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Config.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.stringcenter.client.core.ExternalString - -case class HeaderConfig(title: TitleConfig) -case class TitleConfig(text: ExternalString) -case class FooterConfig(actionConfig: Option[ActionConfig]) -case class ActionConfig(footerText: ExternalString, actionURL: String) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.scala deleted file mode 100644 index 25caa933a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/FeedbackAction.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -trait FeedbackAction { - def toThrift: t.FeedbackAction -} - -case class DismissUserId() extends FeedbackAction { - override lazy val toThrift: t.FeedbackAction = { - t.FeedbackAction.DismissUserId(t.DismissUserId()) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.scala deleted file mode 100644 index f62368431..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Footer.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Footer(action: Option[Action]) { - lazy val toThrift: t.Footer = { - t.Footer(action.map(_.toThrift)) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.scala deleted file mode 100644 index 58c60c789..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Header.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Header(title: Title) { - lazy val toThrift: t.Header = { - t.Header(title.toThrift) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.scala deleted file mode 100644 index f0dc9630a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Layout.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -sealed trait Layout - -case class UserListLayout( - header: Option[HeaderConfig], - userListOptions: UserListOptions, - socialProofs: Option[Seq[SocialProof]], - footer: Option[FooterConfig]) - extends Layout - -case class CarouselLayout( - header: Option[HeaderConfig], - carouselOptions: CarouselOptions, - socialProofs: Option[Seq[SocialProof]]) - extends Layout diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.scala deleted file mode 100644 index 72351e033..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/RecommendationOptions.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -sealed trait RecommendationOptions - -case class UserListOptions( - userBioEnabled: Boolean, - userBioTruncated: Boolean, - userBioMaxLines: Option[Long], -) extends RecommendationOptions - -case class CarouselOptions() extends RecommendationOptions diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.scala deleted file mode 100644 index fd5878af8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/SocialProof.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.stringcenter.client.core.ExternalString - -sealed trait SocialProof - -case class GeoContextProof(popularInCountryText: ExternalString) extends SocialProof -case class FollowedByUsersProof(text1: ExternalString, text2: ExternalString, textN: ExternalString) - extends SocialProof - -sealed trait SocialText { - def text: String -} - -case class GeoSocialText(text: String) extends SocialText -case class FollowedByUsersText(text: String) extends SocialText diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.scala deleted file mode 100644 index 3d128e7c8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/Title.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -case class Title(text: String) { - lazy val toThrift: t.Title = { - t.Title(text) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.scala deleted file mode 100644 index 7cfda1846..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models/WTFPresentation.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.follow_recommendations.assembler.models - -import com.twitter.follow_recommendations.{thriftscala => t} - -trait WTFPresentation { - def toThrift: t.WTFPresentation -} - -case class UserList( - userBioEnabled: Boolean, - userBioTruncated: Boolean, - userBioMaxLines: Option[Long], - feedbackAction: Option[FeedbackAction]) - extends WTFPresentation { - def toThrift: t.WTFPresentation = { - t.WTFPresentation.UserBioList( - t.UserList(userBioEnabled, userBioTruncated, userBioMaxLines, feedbackAction.map(_.toThrift))) - } -} - -object UserList { - def fromUserListOptions( - userListOptions: UserListOptions - ): UserList = { - UserList( - userListOptions.userBioEnabled, - userListOptions.userBioTruncated, - userListOptions.userBioMaxLines, - None) - } -} - -case class Carousel( - feedbackAction: Option[FeedbackAction]) - extends WTFPresentation { - def toThrift: t.WTFPresentation = { - t.WTFPresentation.Carousel(t.Carousel(feedbackAction.map(_.toThrift))) - } -} - -object Carousel { - def fromCarouselOptions( - carouselOptions: CarouselOptions - ): Carousel = { - Carousel(None) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD deleted file mode 100644 index 81d912a99..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.scala deleted file mode 100644 index 8516de53d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders/PromotedAccountsBlender.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.follow_recommendations.blenders - -import com.google.common.annotations.VisibleForTesting -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.AdMetadata -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.inject.Logging -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PromotedAccountsBlender @Inject() (statsReceiver: StatsReceiver) - extends Transform[Int, Recommendation] - with Logging { - - import PromotedAccountsBlender._ - val stats = statsReceiver.scope(Name) - val inputOrganicAccounts = stats.counter(InputOrganic) - val inputPromotedAccounts = stats.counter(InputPromoted) - val outputOrganicAccounts = stats.counter(OutputOrganic) - val outputPromotedAccounts = stats.counter(OutputPromoted) - val promotedAccountsStats = stats.scope(NumPromotedAccounts) - - override def transform( - maxResults: Int, - items: Seq[Recommendation] - ): Stitch[Seq[Recommendation]] = { - val (promoted, organic) = items.partition(_.isPromotedAccount) - val promotedIds = promoted.map(_.id).toSet - val dedupedOrganic = organic.filterNot(u => promotedIds.contains(u.id)) - val blended = blendPromotedAccount(dedupedOrganic, promoted, maxResults) - val (outputPromoted, outputOrganic) = blended.partition(_.isPromotedAccount) - inputOrganicAccounts.incr(dedupedOrganic.size) - inputPromotedAccounts.incr(promoted.size) - outputOrganicAccounts.incr(outputOrganic.size) - val size = outputPromoted.size - outputPromotedAccounts.incr(size) - if (size <= 5) { - promotedAccountsStats.counter(outputPromoted.size.toString).incr() - } else { - promotedAccountsStats.counter(MoreThan5Promoted).incr() - } - Stitch.value(blended) - } - - /** - * Merge Promoted results and organic results. Promoted result dictates the position - * in the merge list. - * - * merge a list of positioned users, aka. promoted, and a list of organic - * users. The positioned promoted users are pre-sorted with regards to their - * position ascendingly. Only requirement about position is to be within the - * range, i.e, can not exceed the combined length if merge is successful, ok - * to be at the last position, but not beyond. - * For more detailed description of location position: - * http://confluence.local.twitter.com/display/ADS/Promoted+Tweets+in+Timeline+Design+Document - */ - @VisibleForTesting - private[blenders] def mergePromotedAccounts( - organicUsers: Seq[Recommendation], - promotedUsers: Seq[Recommendation] - ): Seq[Recommendation] = { - def mergeAccountWithIndex( - organicUsers: Seq[Recommendation], - promotedUsers: Seq[Recommendation], - index: Int - ): Stream[Recommendation] = { - if (promotedUsers.isEmpty) organicUsers.toStream - else { - val promotedHead = promotedUsers.head - val promotedTail = promotedUsers.tail - promotedHead.adMetadata match { - case Some(AdMetadata(position, _)) => - if (position < 0) mergeAccountWithIndex(organicUsers, promotedTail, index) - else if (position == index) - promotedHead #:: mergeAccountWithIndex(organicUsers, promotedTail, index) - else if (organicUsers.isEmpty) organicUsers.toStream - else { - val organicHead = organicUsers.head - val organicTail = organicUsers.tail - organicHead #:: mergeAccountWithIndex(organicTail, promotedUsers, index + 1) - } - case _ => - logger.error("Unknown Candidate type in mergePromotedAccounts") - Stream.empty - } - } - } - - mergeAccountWithIndex(organicUsers, promotedUsers, 0) - } - - private[this] def blendPromotedAccount( - organic: Seq[Recommendation], - promoted: Seq[Recommendation], - maxResults: Int - ): Seq[Recommendation] = { - - val merged = mergePromotedAccounts(organic, promoted) - val mergedServed = merged.take(maxResults) - val promotedServed = promoted.intersect(mergedServed) - - if (isBlendPromotedNeeded( - mergedServed.size - promotedServed.size, - promotedServed.size, - maxResults - )) { - mergedServed - } else { - organic.take(maxResults) - } - } - - @VisibleForTesting - private[blenders] def isBlendPromotedNeeded( - organicSize: Int, - promotedSize: Int, - maxResults: Int - ): Boolean = { - (organicSize > 1) && - (promotedSize > 0) && - (promotedSize < organicSize) && - (promotedSize <= 2) && - (maxResults > 1) - } -} - -object PromotedAccountsBlender { - val Name = "promoted_accounts_blender" - val InputOrganic = "input_organic_accounts" - val InputPromoted = "input_promoted_accounts" - val OutputOrganic = "output_organic_accounts" - val OutputPromoted = "output_promoted_accounts" - val NumPromotedAccounts = "num_promoted_accounts" - val MoreThan5Promoted = "more_than_5" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD deleted file mode 100644 index f64fcda69..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - "configapi/configapi-featureswitches:v2", - "featureswitches/featureswitches-core", - "featureswitches/featureswitches-core:v2", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.scala deleted file mode 100644 index 818f4402c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ConfigBuilder.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.timelines.configapi.CompositeConfig -import com.twitter.timelines.configapi.Config -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ConfigBuilder @Inject() ( - deciderConfigs: DeciderConfigs, - featureSwitchConfigs: FeatureSwitchConfigs) { - // The order of configs added to `CompositeConfig` is important. The config will be matched with - // the first possible rule. So, current setup will give priority to Deciders instead of FS - def build(): Config = - new CompositeConfig(Seq(deciderConfigs.config, featureSwitchConfigs.config)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.scala deleted file mode 100644 index 0154a1703..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/DeciderConfigs.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.decider.Recipient -import com.twitter.decider.SimpleRecipient -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi._ -import com.twitter.timelines.configapi.decider.DeciderSwitchOverrideValue -import com.twitter.timelines.configapi.decider.GuestRecipient -import com.twitter.timelines.configapi.decider.RecipientBuilder -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class DeciderConfigs @Inject() (deciderGateBuilder: DeciderGateBuilder) { - val overrides: Seq[OptionalOverride[_]] = DeciderConfigs.ParamsToDeciderMap.map { - case (params, deciderKey) => - params.optionalOverrideValue( - DeciderSwitchOverrideValue( - feature = deciderGateBuilder.keyToFeature(deciderKey), - enabledValue = true, - recipientBuilder = DeciderConfigs.UserOrGuestOrRequest - ) - ) - }.toSeq - - val config: BaseConfig = BaseConfigBuilder(overrides).build("FollowRecommendationServiceDeciders") -} - -object DeciderConfigs { - val ParamsToDeciderMap = Map( - DeciderParams.EnableRecommendations -> DeciderKey.EnableRecommendations, - DeciderParams.EnableScoreUserCandidates -> DeciderKey.EnableScoreUserCandidates, - HomeTimelineTweetRecsParams.EnableProduct -> DeciderKey.EnableHomeTimelineTweetRecsProduct, - ) - - object UserOrGuestOrRequest extends RecipientBuilder { - - def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match { - case c: WithUserId if c.userId.isDefined => - c.userId.map(SimpleRecipient) - case c: WithGuestId if c.guestId.isDefined => - c.guestId.map(GuestRecipient) - case c: WithGuestId => - RecipientBuilder.Request(c) - case _ => - throw new UndefinedUserIdNorGuestIDException(requestContext) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.scala deleted file mode 100644 index c7f7f6d9e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/FeatureSwitchConfigs.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.candidate_sources.base.SocialProofEnforcedCandidateSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoQualityFollowSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.sims.SimsSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStpSourceFsConfig -import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsFSConfig -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphFSConfig -import com.twitter.follow_recommendations.common.feature_hydration.sources.FeatureHydrationSourcesFSConfig -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRankerFSConfig -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlowFSConfig -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateFSConfig -import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateFSConfig -import com.twitter.follow_recommendations.common.predicates.sgs.SgsPredicateFSConfig -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlowFSConfig -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FeatureSwitchConfigs @Inject() ( - globalFeatureSwitchConfig: GlobalFeatureSwitchConfig, - featureHydrationSourcesFSConfig: FeatureHydrationSourcesFSConfig, - weightedCandidateSourceRankerFSConfig: WeightedCandidateSourceRankerFSConfig, - // Flow related config - contentRecommenderFlowFSConfig: ContentRecommenderFlowFSConfig, - postNuxMlFlowFSConfig: PostNuxMlFlowFSConfig, - // Candidate source related config - crowdSearchAccountsFSConfig: CrowdSearchAccountsFSConfig, - offlineStpSourceFsConfig: OfflineStpSourceFsConfig, - onlineSTPSourceFSConfig: OnlineSTPSourceFSConfig, - popGeoSourceFSConfig: PopGeoSourceFSConfig, - popGeoQualityFollowFSConfig: PopGeoQualityFollowSourceFSConfig, - realGraphOonFSConfig: RealGraphOonFSConfig, - repeatedProfileVisitsFSConfig: RepeatedProfileVisitsFSConfig, - recentEngagementSimilarUsersFSConfig: RecentEngagementSimilarUsersFSConfig, - recentFollowingRecentFollowingExpansionSourceFSConfig: RecentFollowingRecentFollowingExpansionSourceFSConfig, - simsExpansionFSConfig: SimsExpansionFSConfig, - simsSourceFSConfig: SimsSourceFSConfig, - socialProofEnforcedCandidateSourceFSConfig: SocialProofEnforcedCandidateSourceFSConfig, - triangularLoopsFSConfig: TriangularLoopsFSConfig, - userUserGraphFSConfig: UserUserGraphFSConfig, - // Predicate related configs - gizmoduckPredicateFSConfig: GizmoduckPredicateFSConfig, - hssPredicateFSConfig: HssPredicateFSConfig, - sgsPredicateFSConfig: SgsPredicateFSConfig, - ppmiLocaleSourceFSConfig: PPMILocaleFollowSourceFSConfig, - topOrganicFollowsAccountsFSConfig: TopOrganicFollowsAccountsFSConfig, - statsReceiver: StatsReceiver) { - - val logger = Logger(classOf[FeatureSwitchConfigs]) - - val mergedFSConfig = - FeatureSwitchConfig.merge( - Seq( - globalFeatureSwitchConfig, - featureHydrationSourcesFSConfig, - weightedCandidateSourceRankerFSConfig, - // Flow related config - contentRecommenderFlowFSConfig, - postNuxMlFlowFSConfig, - // Candidate source related config - crowdSearchAccountsFSConfig, - offlineStpSourceFsConfig, - onlineSTPSourceFSConfig, - popGeoSourceFSConfig, - popGeoQualityFollowFSConfig, - realGraphOonFSConfig, - repeatedProfileVisitsFSConfig, - recentEngagementSimilarUsersFSConfig, - recentFollowingRecentFollowingExpansionSourceFSConfig, - simsExpansionFSConfig, - simsSourceFSConfig, - socialProofEnforcedCandidateSourceFSConfig, - triangularLoopsFSConfig, - userUserGraphFSConfig, - // Predicate related configs: - gizmoduckPredicateFSConfig, - hssPredicateFSConfig, - sgsPredicateFSConfig, - ppmiLocaleSourceFSConfig, - topOrganicFollowsAccountsFSConfig, - ) - ) - - /** - * enum params have to be listed in this main file together as otherwise we'll have to pass in - * some signature like `Seq[FSEnumParams[_]]` which are generics of generics and won't compile. - * we only have enumFsParams from globalFeatureSwitchConfig at the moment - */ - val enumOverrides = globalFeatureSwitchConfig.enumFsParams.flatMap { enumParam => - FeatureSwitchOverrideUtil.getEnumFSOverrides(statsReceiver, logger, enumParam) - } - - val gatedOverrides = mergedFSConfig.gatedOverridesMap.flatMap { - case (fsName, overrides) => - FeatureSwitchOverrideUtil.gatedOverrides(fsName, overrides: _*) - } - - val enumSeqOverrides = globalFeatureSwitchConfig.enumSeqFsParams.flatMap { enumSeqParam => - FeatureSwitchOverrideUtil.getEnumSeqFSOverrides(statsReceiver, logger, enumSeqParam) - } - - val overrides = - FeatureSwitchOverrideUtil - .getBooleanFSOverrides(mergedFSConfig.booleanFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedIntFSOverrides(mergedFSConfig.intFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedLongFSOverrides(mergedFSConfig.longFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedDoubleFSOverrides(mergedFSConfig.doubleFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getDurationFSOverrides(mergedFSConfig.durationFSParams: _*) ++ - FeatureSwitchOverrideUtil - .getBoundedOptionalDoubleOverrides(mergedFSConfig.optionalDoubleFSParams: _*) ++ - FeatureSwitchOverrideUtil.getStringSeqFSOverrides(mergedFSConfig.stringSeqFSParams: _*) ++ - enumOverrides ++ - gatedOverrides ++ - enumSeqOverrides - - val config = BaseConfigBuilder(overrides).build("FollowRecommendationServiceFeatureSwitches") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.scala deleted file mode 100644 index 537ed660c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/GlobalFeatureSwitchConfig.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.AccountsFilteringAndRankingLogics -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.{ - AccountsFilteringAndRankingLogics => OrganicAccountsFilteringAndRankingLogics -} -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersParams -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionSourceParams -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams.CandidateScorerIdParam -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.configapi.params.GlobalParams.CandidateSourcesToFilter -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableCandidateParamHydrations -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableRecommendationFlowLogs -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts -import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepSocialUserCandidate -import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepUserCandidate -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GlobalFeatureSwitchConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = { - Seq( - EnableCandidateParamHydrations, - KeepUserCandidate, - KeepSocialUserCandidate, - EnableGFSSocialProofTransform, - EnableWhoToFollowProducts, - EnableRecommendationFlowLogs - ) - } - - val enumFsParams = - Seq( - CandidateScorerIdParam, - SimsExpansionSourceParams.Aggregator, - RecentEngagementSimilarUsersParams.Aggregator, - CandidateSourcesToFilter, - ) - - val enumSeqFsParams = - Seq( - AccountsFilteringAndRankingLogics, - OrganicAccountsFilteringAndRankingLogics - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.scala deleted file mode 100644 index 847d69962..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/ParamsFactory.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.timelines.configapi.Config -import com.twitter.timelines.configapi.FeatureValue -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ParamsFactory @Inject() ( - config: Config, - requestContextFactory: RequestContextFactory, - statsReceiver: StatsReceiver) { - - private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi")) - def apply(followRecommendationServiceRequestContext: RequestContext): Params = - config(followRecommendationServiceRequestContext, stats) - - def apply( - clientContext: ClientContext, - displayLocation: DisplayLocation, - featureOverrides: Map[String, FeatureValue] - ): Params = - apply(requestContextFactory(clientContext, displayLocation, featureOverrides)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.scala deleted file mode 100644 index ebc8abf3c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContext.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.twitter.timelines.configapi.BaseRequestContext -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.NullFeatureContext -import com.twitter.timelines.configapi.GuestId -import com.twitter.timelines.configapi.UserId -import com.twitter.timelines.configapi.WithFeatureContext -import com.twitter.timelines.configapi.WithGuestId -import com.twitter.timelines.configapi.WithUserId - -case class RequestContext( - userId: Option[UserId], - guestId: Option[GuestId], - featureContext: FeatureContext = NullFeatureContext) - extends BaseRequestContext - with WithUserId - with WithGuestId - with WithFeatureContext diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.scala deleted file mode 100644 index 89d8617a3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/RequestContextFactory.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.follow_recommendations.configapi - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient} -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.FeatureValue -import com.twitter.timelines.configapi.ForcedFeatureContext -import com.twitter.timelines.configapi.OrElseFeatureContext -import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext -import javax.inject.Singleton - -/* - * Request Context Factory is used to build RequestContext objects which are used - * by the config api to determine the param overrides to apply to the request. - * The param overrides are determined per request by configs which specify which - * FS/Deciders/AB translate to what param overrides. - */ -@Singleton -class RequestContextFactory @Inject() (featureSwitches: FeatureSwitches, decider: Decider) { - def apply( - clientContext: ClientContext, - displayLocation: DisplayLocation, - featureOverrides: Map[String, FeatureValue] - ): RequestContext = { - val featureContext = getFeatureContext(clientContext, displayLocation, featureOverrides) - RequestContext(clientContext.userId, clientContext.guestId, featureContext) - } - - private[configapi] def getFeatureContext( - clientContext: ClientContext, - displayLocation: DisplayLocation, - featureOverrides: Map[String, FeatureValue] - ): FeatureContext = { - val recipient = - getFeatureSwitchRecipient(clientContext) - .withCustomFields("display_location" -> displayLocation.toFsName) - - // userAgeOpt is going to be set to None for logged out users and defaulted to Some(Int.MaxValue) for non-snowflake users - val userAgeOpt = clientContext.userId.map { userId => - SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue) - } - val recipientWithAccountAge = - userAgeOpt - .map(age => recipient.withCustomFields("account_age_in_days" -> age)).getOrElse(recipient) - - val results = featureSwitches.matchRecipient(recipientWithAccountAge) - OrElseFeatureContext( - ForcedFeatureContext(featureOverrides), - new FeatureSwitchResultsFeatureContext(results)) - } - - @VisibleForTesting - private[configapi] def getFeatureSwitchRecipient( - clientContext: ClientContext - ): FeatureSwitchRecipient = { - FeatureSwitchRecipient( - userId = clientContext.userId, - userRoles = clientContext.userRoles, - deviceId = clientContext.deviceId, - guestId = clientContext.guestId, - languageCode = clientContext.languageCode, - countryCode = clientContext.countryCode, - isVerified = None, - clientApplicationId = clientContext.appId, - isTwoffice = clientContext.isTwoffice - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD deleted file mode 100644 index 5470c9bf4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - "configapi/configapi-featureswitches:v2", - "featureswitches/featureswitches-core", - "featureswitches/featureswitches-core:v2", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.scala deleted file mode 100644 index 6b954a911..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContext.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.twitter.timelines.configapi.BaseRequestContext -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.NullFeatureContext -import com.twitter.timelines.configapi.WithFeatureContext -import com.twitter.timelines.configapi.WithUserId - -/** - * represent the context for a recommendation candidate (producer side) - * @param userId id of the recommended user - * @param featureContext feature context - */ -case class CandidateUserContext( - override val userId: Option[Long], - featureContext: FeatureContext = NullFeatureContext) - extends BaseRequestContext - with WithUserId - with WithFeatureContext diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.scala deleted file mode 100644 index 4f30cf17a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserContextFactory.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.google.common.annotations.VisibleForTesting -import com.google.inject.Inject -import com.twitter.decider.Decider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient} -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.timelines.configapi.FeatureContext -import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class CandidateUserContextFactory @Inject() ( - @Named(PRODUCER_SIDE_FEATURE_SWITCHES) featureSwitches: FeatureSwitches, - decider: Decider) { - def apply( - candidateUser: CandidateUser, - displayLocation: DisplayLocation - ): CandidateUserContext = { - val featureContext = getFeatureContext(candidateUser, displayLocation) - - CandidateUserContext(Some(candidateUser.id), featureContext) - } - - private[configapi] def getFeatureContext( - candidateUser: CandidateUser, - displayLocation: DisplayLocation - ): FeatureContext = { - - val recipient = getFeatureSwitchRecipient(candidateUser).withCustomFields( - "display_location" -> displayLocation.toFsName) - new FeatureSwitchResultsFeatureContext(featureSwitches.matchRecipient(recipient)) - } - - @VisibleForTesting - private[configapi] def getFeatureSwitchRecipient( - candidateUser: CandidateUser - ): FeatureSwitchRecipient = { - FeatureSwitchRecipient( - userId = Some(candidateUser.id), - userRoles = None, - deviceId = None, - guestId = None, - languageCode = None, - countryCode = None, - isVerified = None, - clientApplicationId = None, - isTwoffice = None - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.scala deleted file mode 100644 index 5afd09a63..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/CandidateUserParamsFactory.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.servo.util.MemoizingStatsReceiver -import com.twitter.timelines.configapi.Config -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -/** - * CandidateParamsFactory is primarily used for "producer side" experiments, don't use it on consumer side experiments - */ -@Singleton -class CandidateUserParamsFactory[T <: HasParams with HasDisplayLocation] @Inject() ( - config: Config, - candidateContextFactory: CandidateUserContextFactory, - statsReceiver: StatsReceiver) { - private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi_candidate_params")) - def apply(candidateContext: CandidateUser, request: T): CandidateUser = { - if (candidateContext.params == Params.Invalid) { - if (request.params(GlobalParams.EnableCandidateParamHydrations)) { - candidateContext.copy(params = - config(candidateContextFactory(candidateContext, request.displayLocation), stats)) - } else { - candidateContext.copy(params = Params.Empty) - } - } else { - candidateContext - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.scala deleted file mode 100644 index deadd1d70..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates/HydrateCandidateParamsTransform.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.follow_recommendations.configapi.candidates - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.logging.Logging - -@Singleton -class HydrateCandidateParamsTransform[Target <: HasParams with HasDisplayLocation] @Inject() ( - candidateParamsFactory: CandidateUserParamsFactory[Target]) - extends Transform[Target, CandidateUser] - with Logging { - - def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = { - Stitch.value(candidates.map(candidateParamsFactory.apply(_, target))) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD deleted file mode 100644 index 6fee24f89..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.scala deleted file mode 100644 index 798b02670..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common/FeatureSwitchConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.follow_recommendations.configapi.common - -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.DefinedFeatureName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.ValueFeatureName -import com.twitter.timelines.configapi.BoundedParam -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.OptionalOverride -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -trait FeatureSwitchConfig { - def booleanFSParams: Seq[Param[Boolean] with FSName] = Nil - - def intFSParams: Seq[FSBoundedParam[Int]] = Nil - - def longFSParams: Seq[FSBoundedParam[Long]] = Nil - - def doubleFSParams: Seq[FSBoundedParam[Double]] = Nil - - def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Nil - - def optionalDoubleFSParams: Seq[ - (BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName) - ] = Nil - - def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Nil - - /** - * Apply overrides in list when the given FS Key is enabled. - * This override type does NOT work with experiments. Params here will be evaluated for every - * request IMMEDIATELY, not upon param.apply. If you would like to use an experiment pls use - * the primitive type or ENUM overrides. - */ - def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = Map.empty -} - -object FeatureSwitchConfig { - def merge(configs: Seq[FeatureSwitchConfig]): FeatureSwitchConfig = new FeatureSwitchConfig { - override def booleanFSParams: Seq[Param[Boolean] with FSName] = - configs.flatMap(_.booleanFSParams) - override def intFSParams: Seq[FSBoundedParam[Int]] = - configs.flatMap(_.intFSParams) - override def longFSParams: Seq[FSBoundedParam[Long]] = - configs.flatMap(_.longFSParams) - override def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = - configs.flatMap(_.durationFSParams) - override def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = - configs.flatMap(_.gatedOverridesMap).toMap - override def doubleFSParams: Seq[FSBoundedParam[Double]] = - configs.flatMap(_.doubleFSParams) - override def optionalDoubleFSParams: Seq[ - (BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName) - ] = - configs.flatMap(_.optionalDoubleFSParams) - override def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = - configs.flatMap(_.stringSeqFSParams) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD deleted file mode 100644 index e4982dc0f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.scala deleted file mode 100644 index f4c069a63..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderKey.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.follow_recommendations.configapi.deciders - -import com.twitter.servo.decider.DeciderKeyEnum - -object DeciderKey extends DeciderKeyEnum { - val EnableDiffyModuleDarkReading = Value("enable_diffy_module_dark_reading") - val EnableRecommendations = Value("enable_recommendations") - val EnableScoreUserCandidates = Value("enable_score_user_candidates") - val EnableProfileSidebarProduct = Value("enable_profile_sidebar_product") - val EnableMagicRecsProduct = Value("enable_magic_recs_product") - val EnableRuxLandingPageProduct = Value("enable_rux_landing_page_product") - val EnableRuxPymkProduct = Value("enable_rux_pymk_product") - val EnableProfileBonusFollowProduct = Value("enable_profile_bonus_follow_product") - val EnableElectionExploreWtfProduct = Value("enable_election_explore_wtf_product") - val EnableClusterFollowProduct = Value("enable_cluster_follow_product") - val EnableHomeTimelineProduct = Value("enable_home_timeline_product") - val EnableHtlBonusFollowProduct = Value("enable_htl_bonus_follow_product") - val EnableExploreTabProduct = Value("enable_explore_tab_product") - val EnableSidebarProduct = Value("enable_sidebar_product") - val EnableNuxPymkProduct = Value("enable_nux_pymk_product") - val EnableNuxInterestsProduct = Value("enable_nux_interests_product") - val EnableNuxTopicBonusFollowProduct = Value("enable_nux_topic_bonus_follow_product") - val EnableCampaignFormProduct = Value("enable_campaign_form_product") - val EnableReactiveFollowProduct = Value("enable_reactive_follow_product") - val EnableIndiaCovid19CuratedAccountsWtfProduct = Value( - "enable_india_covid19_curated_accounts_wtf_product") - val EnableAbUploadProduct = Value("enable_ab_upload_product") - val EnablePeolePlusPlusProduct = Value("enable_people_plus_plus_product") - val EnableTweetNotificationRecsProduct = Value("enable_tweet_notification_recs_product") - val EnableProfileDeviceFollow = Value("enable_profile_device_follow_product") - val EnableRecosBackfillProduct = Value("enable_recos_backfill_product") - val EnablePostNuxFollowTaskProduct = Value("enable_post_nux_follow_task_product") - val EnableCuratedSpaceHostsProduct = Value("enable_curated_space_hosts_product") - val EnableNuxGeoCategoryProduct = Value("enable_nux_geo_category_product") - val EnableNuxInterestsCategoryProduct = Value("enable_nux_interests_category_product") - val EnableNuxPymkCategoryProduct = Value("enable_nux_pymk_category_product") - val EnableHomeTimelineTweetRecsProduct = Value("enable_home_timeline_tweet_recs_product") - val EnableHtlBulkFriendFollowsProduct = Value("enable_htl_bulk_friend_follows_product") - val EnableNuxAutoFollowProduct = Value("enable_nux_auto_follow_product") - val EnableSearchBonusFollowProduct = Value("enable_search_bonus_follow_product") - val EnableFetchUserInRequestBuilder = Value("enable_fetch_user_in_request_builder") - val EnableProductMixerMagicRecsProduct = Value("enable_product_mixer_magic_recs_product") - val EnableHomeTimelineReverseChronProduct = Value("enable_home_timeline_reverse_chron_product") - val EnableProductMixerPipelineMagicRecsDarkRead = Value( - "enable_product_mixer_pipeline_magic_recs_dark_read") - val EnableExperimentalCaching = Value("enable_experimental_caching") - val EnableDistributedCaching = Value("enable_distributed_caching") - val EnableGizmoduckCaching = Value("enable_gizmoduck_caching") - val EnableTrafficDarkReading = Value("enable_traffic_dark_reading") - val EnableGraphFeatureServiceRequests = Value("enable_graph_feature_service_requests") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.scala deleted file mode 100644 index 07bf3e14d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders/DeciderParams.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.follow_recommendations.configapi.deciders - -import com.twitter.timelines.configapi.Param - -object DeciderParams { - object EnableRecommendations extends Param[Boolean](false) - object EnableScoreUserCandidates extends Param[Boolean](false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD deleted file mode 100644 index 1bb357b2c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "configapi/configapi-core", - "configapi/configapi-decider", - "configapi/configapi-featureswitches:v2", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.scala deleted file mode 100644 index cf1905c41..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params/GlobalParams.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.follow_recommendations.configapi.params - -import com.twitter.follow_recommendations.models.CandidateSourceType -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam - -/** - * When adding Producer side experiments, make sure to register the FS Key in [[ProducerFeatureFilter]] - * in [[FeatureSwitchesModule]], otherwise, the FS will not work. - */ -object GlobalParams { - - object EnableCandidateParamHydrations - extends FSParam[Boolean]("frs_receiver_enable_candidate_params", false) - - object KeepUserCandidate - extends FSParam[Boolean]("frs_receiver_holdback_keep_user_candidate", true) - - object KeepSocialUserCandidate - extends FSParam[Boolean]("frs_receiver_holdback_keep_social_user_candidate", true) - - case object EnableGFSSocialProofTransform - extends FSParam("social_proof_transform_use_graph_feature_service", true) - - case object EnableWhoToFollowProducts extends FSParam("who_to_follow_product_enabled", true) - - case object CandidateSourcesToFilter - extends FSEnumParam[CandidateSourceType.type]( - "candidate_sources_type_filter_id", - CandidateSourceType.None, - CandidateSourceType) - - object EnableRecommendationFlowLogs - extends FSParam[Boolean]("frs_recommendation_flow_logs_enabled", false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD deleted file mode 100644 index 2a625d856..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "decider/src/main/scala", - "finagle/finagle-core/src/main", - "finatra/inject/inject-core/src/main/scala", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/exceptions", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/modules", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/response", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", - "scrooge/scrooge-core/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.scala deleted file mode 100644 index 1695c464d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/CandidateUserDebugParamsBuilder.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.follow_recommendations.common.models._ -import com.twitter.follow_recommendations.configapi.ParamsFactory -import com.twitter.follow_recommendations.models.CandidateUserDebugParams -import com.twitter.follow_recommendations.models.FeatureValue -import com.twitter.follow_recommendations.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CandidateUserDebugParamsBuilder @Inject() (paramsFactory: ParamsFactory) { - def fromThrift(req: t.ScoringUserRequest): CandidateUserDebugParams = { - val clientContext = ClientContextConverter.fromThrift(req.clientContext) - val displayLocation = DisplayLocation.fromThrift(req.displayLocation) - - CandidateUserDebugParams(req.candidates.map { candidate => - candidate.userId -> paramsFactory( - clientContext, - displayLocation, - candidate.featureOverrides - .map(_.mapValues(FeatureValue.fromThrift).toMap).getOrElse(Map.empty)) - }.toMap) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.scala deleted file mode 100644 index bc21fd6a3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RecommendationRequestBuilder.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.DisplayContext -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecommendationRequestBuilder @Inject() ( - requestBuilderUserFetcher: RequestBuilderUserFetcher, - statsReceiver: StatsReceiver) { - private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) - private val isSoftUserCounter = scopedStats.counter("is_soft_user") - - def fromThrift(tRequest: t.RecommendationRequest): Stitch[RecommendationRequest] = { - requestBuilderUserFetcher.fetchUser(tRequest.clientContext.userId).map { userOpt => - val isSoftUser = userOpt.exists(_.userType == UserType.Soft) - if (isSoftUser) isSoftUserCounter.incr() - RecommendationRequest( - clientContext = ClientContextConverter.fromThrift(tRequest.clientContext), - displayLocation = DisplayLocation.fromThrift(tRequest.displayLocation), - displayContext = tRequest.displayContext.map(DisplayContext.fromThrift), - maxResults = tRequest.maxResults, - cursor = tRequest.cursor, - excludedIds = tRequest.excludedIds, - fetchPromotedContent = tRequest.fetchPromotedContent, - debugParams = tRequest.debugParams.map(DebugParams.fromThrift), - userLocationState = tRequest.userLocationState, - isSoftUser = isSoftUser - ) - } - - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.scala deleted file mode 100644 index 3953b5ef3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/RequestBuilderUserFetcher.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.decider.Decider -import com.twitter.decider.SimpleRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.gizmoduck.thriftscala.LookupContext -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RequestBuilderUserFetcher @Inject() ( - gizmoduck: Gizmoduck, - statsReceiver: StatsReceiver, - decider: Decider) { - private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) - - def fetchUser(userIdOpt: Option[Long]): Stitch[Option[User]] = { - userIdOpt match { - case Some(userId) if enableDecider(userId) => - val stitch = gizmoduck - .getUserById( - userId = userId, - context = LookupContext( - forUserId = Some(userId), - includeProtected = true, - includeSoftUsers = true - ) - ).map(user => Some(user)) - StatsUtil - .profileStitch(stitch, scopedStats) - .handle { - case _: Throwable => None - } - case _ => Stitch.None - } - } - - private def enableDecider(userId: Long): Boolean = { - decider.isAvailable( - DeciderKey.EnableFetchUserInRequestBuilder.toString, - Some(SimpleRecipient(userId))) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.scala deleted file mode 100644 index 4a45a19f7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ScoringUserRequestBuilder.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.ScoringUserRequest -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.stitch.Stitch - -@Singleton -class ScoringUserRequestBuilder @Inject() ( - requestBuilderUserFetcher: RequestBuilderUserFetcher, - candidateUserDebugParamsBuilder: CandidateUserDebugParamsBuilder, - statsReceiver: StatsReceiver) { - private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName) - private val isSoftUserCounter = scopedStats.counter("is_soft_user") - - def fromThrift(req: t.ScoringUserRequest): Stitch[ScoringUserRequest] = { - requestBuilderUserFetcher.fetchUser(req.clientContext.userId).map { userOpt => - val isSoftUser = userOpt.exists(_.userType == UserType.Soft) - if (isSoftUser) isSoftUserCounter.incr() - - val candidateUsersParamsMap = candidateUserDebugParamsBuilder.fromThrift(req) - val candidates = req.candidates.map { candidate => - CandidateUser - .fromUserRecommendation(candidate).copy(params = - candidateUsersParamsMap.paramsMap.getOrElse(candidate.userId, Params.Invalid)) - } - - ScoringUserRequest( - clientContext = ClientContextConverter.fromThrift(req.clientContext), - displayLocation = DisplayLocation.fromThrift(req.displayLocation), - params = Params.Empty, - debugOptions = req.debugParams.map(DebugOptions.fromDebugParamsThrift), - recentFollowedUserIds = None, - recentFollowedByUserIds = None, - wtfImpressions = None, - similarToUserIds = Nil, - candidates = candidates, - debugParams = req.debugParams.map(DebugParams.fromThrift), - isSoftUser = isSoftUser - ) - } - } - -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.scala deleted file mode 100644 index f3014982e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers/ThriftController.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.follow_recommendations.controllers - -import com.twitter.finatra.thrift.Controller -import com.twitter.follow_recommendations.configapi.ParamsFactory -import com.twitter.follow_recommendations.services.ProductPipelineSelector -import com.twitter.follow_recommendations.services.UserScoringService -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService._ -import com.twitter.stitch.Stitch -import javax.inject.Inject - -class ThriftController @Inject() ( - userScoringService: UserScoringService, - recommendationRequestBuilder: RecommendationRequestBuilder, - scoringUserRequestBuilder: ScoringUserRequestBuilder, - productPipelineSelector: ProductPipelineSelector, - paramsFactory: ParamsFactory) - extends Controller(FollowRecommendationsThriftService) { - - handle(GetRecommendations) { args: GetRecommendations.Args => - val stitch = recommendationRequestBuilder.fromThrift(args.request).flatMap { request => - val params = paramsFactory( - request.clientContext, - request.displayLocation, - request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty)) - productPipelineSelector.selectPipeline(request, params).map(_.toThrift) - } - Stitch.run(stitch) - } - - handle(ScoreUserCandidates) { args: ScoreUserCandidates.Args => - val stitch = scoringUserRequestBuilder.fromThrift(args.request).flatMap { request => - val params = paramsFactory( - request.clientContext, - request.displayLocation, - request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty)) - userScoringService.get(request.copy(params = params)).map(_.toThrift) - } - Stitch.run(stitch) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD deleted file mode 100644 index aa775c6a7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.scala deleted file mode 100644 index dd9372484..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlow.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource -import com.twitter.follow_recommendations.common.base.IdentityRanker -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.ParamPredicate -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.RecommendationFlow -import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.base.TruePredicate -import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedAccountsCandidateSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate -import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PromotedAccountsFlow @Inject() ( - promotedAccountsCandidateSource: PromotedAccountsCandidateSource, - trackingTokenTransform: TrackingTokenTransform, - baseStatsReceiver: StatsReceiver, - @Flag("fetch_prod_promoted_accounts") fetchProductionPromotedAccounts: Boolean) - extends RecommendationFlow[PromotedAccountsFlowRequest, CandidateUser] { - - protected override def targetEligibility: Predicate[PromotedAccountsFlowRequest] = - new ParamPredicate[PromotedAccountsFlowRequest]( - PromotedAccountsFlowParams.TargetEligibility - ) - - protected override def candidateSources( - target: PromotedAccountsFlowRequest - ): Seq[CandidateSource[PromotedAccountsFlowRequest, CandidateUser]] = { - import EnrichedCandidateSource._ - val candidateSourceStats = statsReceiver.scope("candidate_sources") - val budget: Duration = target.params(PromotedAccountsFlowParams.FetchCandidateSourceBudget) - val candidateSources = Seq( - promotedAccountsCandidateSource - .mapKeys[PromotedAccountsFlowRequest](r => - Seq(r.toAdsRequest(fetchProductionPromotedAccounts))) - .mapValue(PromotedAccountsUtil.toCandidateUser) - ).map { candidateSource => - candidateSource - .failOpenWithin(budget, candidateSourceStats).observe(candidateSourceStats) - } - candidateSources - } - - protected override def preRankerCandidateFilter: Predicate[ - (PromotedAccountsFlowRequest, CandidateUser) - ] = { - val preRankerFilterStats = statsReceiver.scope("pre_ranker") - ExcludedUserIdPredicate.observe(preRankerFilterStats.scope("exclude_user_id_predicate")) - } - - /** - * rank the candidates - */ - protected override def selectRanker( - target: PromotedAccountsFlowRequest - ): Ranker[PromotedAccountsFlowRequest, CandidateUser] = { - new IdentityRanker[PromotedAccountsFlowRequest, CandidateUser] - } - - /** - * transform the candidates after ranking (e.g. dedupping, grouping and etc) - */ - protected override def postRankerTransform: Transform[ - PromotedAccountsFlowRequest, - CandidateUser - ] = { - new IdentityTransform[PromotedAccountsFlowRequest, CandidateUser] - } - - /** - * filter invalid candidates before returning the results. - * - * Some heavy filters e.g. SGS filter could be applied in this step - */ - protected override def validateCandidates: Predicate[ - (PromotedAccountsFlowRequest, CandidateUser) - ] = { - new TruePredicate[(PromotedAccountsFlowRequest, CandidateUser)] - } - - /** - * transform the candidates into results and return - */ - protected override def transformResults: Transform[PromotedAccountsFlowRequest, CandidateUser] = { - trackingTokenTransform - } - - /** - * configuration for recommendation results - */ - protected override def resultsConfig( - target: PromotedAccountsFlowRequest - ): RecommendationResultsConfig = { - RecommendationResultsConfig( - target.params(PromotedAccountsFlowParams.ResultSizeParam), - target.params(PromotedAccountsFlowParams.BatchSizeParam) - ) - } - - override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("promoted_accounts_flow") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.scala deleted file mode 100644 index 010aea509..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowParams.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -abstract class PromotedAccountsFlowParams[A](default: A) extends Param[A](default) { - override val statName: String = "ads/" + this.getClass.getSimpleName -} - -object PromotedAccountsFlowParams { - - // number of total slots returned to the end user, available to put ads - case object TargetEligibility extends PromotedAccountsFlowParams[Boolean](true) - case object ResultSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue) - case object BatchSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue) - case object FetchCandidateSourceBudget - extends PromotedAccountsFlowParams[Duration](1000.millisecond) - -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.scala deleted file mode 100644 index 61afb3049..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsFlowRequest.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads -import com.twitter.follow_recommendations.common.clients.adserver.AdRequest -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasExcludedUserIds -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class PromotedAccountsFlowRequest( - override val clientContext: ClientContext, - override val params: Params, - displayLocation: DisplayLocation, - profileId: Option[Long], - // note we also add userId and profileId to excludeUserIds - excludeIds: Seq[Long]) - extends HasParams - with HasClientContext - with HasExcludedUserIds - with HasDisplayLocation { - def toAdsRequest(fetchProductionPromotedAccounts: Boolean): AdRequest = { - AdRequest( - clientContext = clientContext, - displayLocation = displayLocation, - isTest = Some(!fetchProductionPromotedAccounts), - profileUserId = profileId - ) - } - override val excludedUserIds: Seq[Long] = { - excludeIds ++ clientContext.userId.toSeq ++ profileId.toSeq - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.scala deleted file mode 100644 index 5327911c2..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads/PromotedAccountsUtil.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.flows.ads -import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedCandidateUser -import com.twitter.follow_recommendations.common.models.AccountProof -import com.twitter.follow_recommendations.common.models.AdMetadata -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.Reason -import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails - -object PromotedAccountsUtil { - def toCandidateUser(promotedCandidateUser: PromotedCandidateUser): CandidateUser = { - CandidateUser( - id = promotedCandidateUser.id, - score = None, - adMetadata = - Some(AdMetadata(promotedCandidateUser.position, promotedCandidateUser.adImpression)), - reason = Some( - Reason( - accountProof = Some(AccountProof(followProof = Some(promotedCandidateUser.followProof)))) - ), - userCandidateSourceDetails = Some( - UserCandidateSourceDetails( - promotedCandidateUser.primaryCandidateSource, - Map.empty, - Map.empty, - None)) - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD deleted file mode 100644 index 886c0fe5a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.scala deleted file mode 100644 index 30dfa0d42..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlow.scala +++ /dev/null @@ -1,202 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource -import com.twitter.follow_recommendations.common.base.GatedPredicateBase -import com.twitter.follow_recommendations.common.base.ParamPredicate -import com.twitter.follow_recommendations.common.base.Predicate -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.RecommendationFlow -import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate -import com.twitter.follow_recommendations.common.predicates.InactivePredicate -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker -import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform -import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform -import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil -import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderFlow @Inject() ( - contentRecommenderFlowCandidateSourceRegistry: ContentRecommenderFlowCandidateSourceRegistry, - recentFollowingPredicate: RecentFollowingPredicate, - gizmoduckPredicate: GizmoduckPredicate, - inactivePredicate: InactivePredicate, - sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, - invalidRelationshipPredicate: InvalidRelationshipPredicate, - trackingTokenTransform: TrackingTokenTransform, - baseStatsReceiver: StatsReceiver) - extends RecommendationFlow[ContentRecommenderRequest, CandidateUser] - with RecommendationFlowBaseSideEffectsUtil[ContentRecommenderRequest, CandidateUser] - with CandidateSourceHoldbackUtil { - - override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("content_recommender_flow") - - override val qualityFactorObserver: Option[QualityFactorObserver] = { - val config = LinearLatencyQualityFactorConfig( - qualityFactorBounds = - BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0), - initialDelay = 60.seconds, - targetLatency = 100.milliseconds, - targetLatencyPercentile = 95.0, - delta = 0.001 - ) - val qualityFactor = LinearLatencyQualityFactor(config) - val observer = LinearLatencyQualityFactorObserver(qualityFactor) - statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat) - Some(observer) - } - - protected override def targetEligibility: Predicate[ContentRecommenderRequest] = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.TargetEligibility - ) - - protected override def candidateSources( - target: ContentRecommenderRequest - ): Seq[CandidateSource[ContentRecommenderRequest, CandidateUser]] = { - import EnrichedCandidateSource._ - val identifiers = ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params).keySet - val selected = contentRecommenderFlowCandidateSourceRegistry.select(identifiers) - val budget = - target.params(ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond).millisecond - filterCandidateSources(target, selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq) - } - - protected override val preRankerCandidateFilter: Predicate[ - (ContentRecommenderRequest, CandidateUser) - ] = { - val preRankerFilterStats = statsReceiver.scope("pre_ranker") - val recentFollowingPredicateStats = preRankerFilterStats.scope("recent_following_predicate") - val invalidRelationshipPredicateStats = - preRankerFilterStats.scope("invalid_relationship_predicate") - - object recentFollowingGatedPredicate - extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( - recentFollowingPredicate, - recentFollowingPredicateStats - ) { - override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = - item._1.params(ContentRecommenderParams.EnableRecentFollowingPredicate) - } - - object invalidRelationshipGatedPredicate - extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)]( - invalidRelationshipPredicate, - invalidRelationshipPredicateStats - ) { - override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean = - item._1.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate) - } - - ExcludedUserIdPredicate - .observe(preRankerFilterStats.scope("exclude_user_id_predicate")) - .andThen(recentFollowingGatedPredicate.observe(recentFollowingPredicateStats)) - .andThen(invalidRelationshipGatedPredicate.observe(invalidRelationshipPredicateStats)) - } - - /** - * rank the candidates - */ - protected override def selectRanker( - target: ContentRecommenderRequest - ): Ranker[ContentRecommenderRequest, CandidateUser] = { - val rankersStatsReceiver = statsReceiver.scope("rankers") - WeightedCandidateSourceRanker - .build[ContentRecommenderRequest]( - ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params), - randomSeed = target.getRandomizationSeed - ).observe(rankersStatsReceiver.scope("weighted_candidate_source_ranker")) - } - - /** - * transform the candidates after ranking - */ - protected override def postRankerTransform: Transform[ - ContentRecommenderRequest, - CandidateUser - ] = { - new DedupTransform[ContentRecommenderRequest, CandidateUser] - .observe(statsReceiver.scope("dedupping")) - } - - protected override def validateCandidates: Predicate[ - (ContentRecommenderRequest, CandidateUser) - ] = { - val stats = statsReceiver.scope("validate_candidates") - val gizmoduckPredicateStats = stats.scope("gizmoduck_predicate") - val inactivePredicateStats = stats.scope("inactive_predicate") - val sgsPredicateStats = stats.scope("sgs_predicate") - - val includeGizmoduckPredicate = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.EnableGizmoduckPredicate) - .map[(ContentRecommenderRequest, CandidateUser)] { - case (request: ContentRecommenderRequest, _) => - request - } - - val includeInactivePredicate = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.EnableInactivePredicate) - .map[(ContentRecommenderRequest, CandidateUser)] { - case (request: ContentRecommenderRequest, _) => - request - } - - val includeInvalidTargetCandidateRelationshipTypesPredicate = - new ParamPredicate[ContentRecommenderRequest]( - ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate) - .map[(ContentRecommenderRequest, CandidateUser)] { - case (request: ContentRecommenderRequest, _) => - request - } - - Predicate - .andConcurrently[(ContentRecommenderRequest, CandidateUser)]( - Seq( - gizmoduckPredicate.observe(gizmoduckPredicateStats).gate(includeGizmoduckPredicate), - inactivePredicate.observe(inactivePredicateStats).gate(includeInactivePredicate), - sgsPredicate - .observe(sgsPredicateStats).gate( - includeInvalidTargetCandidateRelationshipTypesPredicate), - ) - ) - } - - /** - * transform the candidates into results and return - */ - protected override def transformResults: Transform[ContentRecommenderRequest, CandidateUser] = { - trackingTokenTransform - } - - /** - * configuration for recommendation results - */ - protected override def resultsConfig( - target: ContentRecommenderRequest - ): RecommendationResultsConfig = { - RecommendationResultsConfig( - target.maxResults.getOrElse(target.params(ContentRecommenderParams.ResultSizeParam)), - target.params(ContentRecommenderParams.BatchSizeParam) - ) - } - -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.scala deleted file mode 100644 index 4a6c61042..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceRegistry.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderFlowCandidateSourceRegistry @Inject() ( - // social based - forwardPhoneBookSource: ForwardPhoneBookSource, - forwardEmailBookSource: ForwardEmailBookSource, - reversePhoneBookSource: ReversePhoneBookSource, - reverseEmailBookSource: ReverseEmailBookSource, - offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, - triangularLoopsSource: TriangularLoopsSource, - userUserGraphCandidateSource: UserUserGraphCandidateSource, - realGraphOonSource: RealGraphOonV2Source, - recentFollowingRecentFollowingExpansionSource: RecentFollowingRecentFollowingExpansionSource, - // activity based - recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, - recentEngagementSimilarUsersSource: RecentEngagementSimilarUsersSource, - repeatedProfileVisitsSource: RepeatedProfileVisitsSource, - // geo based - popCountrySource: PopCountrySource, - popGeohashSource: PopGeohashSource, - popCountryBackFillSource: PopCountryBackFillSource, - crowdSearchAccountsSource: CrowdSearchAccountsSource, - topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, - ppmiLocaleFollowSource: PPMILocaleFollowSource, - baseStatsReceiver: StatsReceiver) - extends CandidateSourceRegistry[ContentRecommenderRequest, CandidateUser] { - - override val statsReceiver = baseStatsReceiver - .scope("content_recommender_flow", "candidate_sources") - - override val sources: Set[CandidateSource[ContentRecommenderRequest, CandidateUser]] = Seq( - forwardPhoneBookSource, - forwardEmailBookSource, - reversePhoneBookSource, - reverseEmailBookSource, - offlineStrongTiePredictionSource, - triangularLoopsSource, - userUserGraphCandidateSource, - realGraphOonSource, - recentFollowingRecentFollowingExpansionSource, - recentFollowingSimilarUsersSource, - recentEngagementSimilarUsersSource, - repeatedProfileVisitsSource, - popCountrySource, - popGeohashSource, - popCountryBackFillSource, - crowdSearchAccountsSource, - topOrganicFollowsAccountsSource, - ppmiLocaleFollowSource, - ).toSet -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.scala deleted file mode 100644 index 845a6ec0a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeights.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.timelines.configapi.Params - -object ContentRecommenderFlowCandidateSourceWeights { - - def getWeights( - params: Params - ): Map[CandidateSourceIdentifier, Double] = { - Map[CandidateSourceIdentifier, Double]( - // Social based - UserUserGraphCandidateSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight), - ForwardPhoneBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight), - ReversePhoneBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight), - ForwardEmailBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight), - ReverseEmailBookSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight), - TriangularLoopsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight), - OfflineStrongTiePredictionSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight), - RecentFollowingRecentFollowingExpansionSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight), - RecentFollowingSimilarUsersSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight), - // Activity based - RealGraphOonV2Source.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight), - RecentEngagementSimilarUsersSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight), - RepeatedProfileVisitsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight), - // Geo based - PopCountrySource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight), - PopGeohashSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight), - PopCountryBackFillSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight), - PPMILocaleFollowSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight), - CrowdSearchAccountsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight), - TopOrganicFollowsAccountsSource.Identifier -> params( - ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight), - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.scala deleted file mode 100644 index 462de260b..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowCandidateSourceWeightsParams.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.timelines.configapi.FSBoundedParam - -object ContentRecommenderFlowCandidateSourceWeightsParams { - // Social based - case object ForwardPhoneBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ForwardPhoneBookSourceWeight, - 1d, - 0d, - 1000d) - case object ForwardEmailBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ForwardEmailBookSourceWeight, - 1d, - 0d, - 1000d) - case object ReversePhoneBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ReversePhoneBookSourceWeight, - 1d, - 0d, - 1000d) - case object ReverseEmailBookSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.ReverseEmailBookSourceWeight, - 1d, - 0d, - 1000d) - case object OfflineStrongTiePredictionSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.OfflineStrongTiePredictionSourceWeight, - 1d, - 0d, - 1000d) - case object TriangularLoopsSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.TriangularLoopsSourceWeight, - 1d, - 0d, - 1000d) - case object UserUserGraphSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.UserUserGraphSourceWeight, - 1d, - 0d, - 1000d) - case object NewFollowingNewFollowingExpansionSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.NewFollowingNewFollowingExpansionSourceWeight, - 1d, - 0d, - 1000d) - // Activity based - case object NewFollowingSimilarUserSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.NewFollowingSimilarUserSourceWeight, - 1d, - 0d, - 1000d) - case object RecentEngagementSimilarUserSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.RecentEngagementSimilarUserSourceWeight, - 1d, - 0d, - 1000d) - case object RepeatedProfileVisitsSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.RepeatedProfileVisitsSourceWeight, - 1d, - 0d, - 1000d) - case object RealGraphOonSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.RealGraphOonSourceWeight, - 1d, - 0d, - 1000d) - // Geo based - case object PopCountrySourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PopCountrySourceWeight, - 1d, - 0d, - 1000d) - case object PopGeohashSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PopGeohashSourceWeight, - 1d, - 0d, - 1000d) - case object PopCountryBackfillSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PopCountryBackfillSourceWeight, - 1d, - 0d, - 1000d) - case object PPMILocaleFollowSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.PPMILocaleFollowSourceWeight, - 1d, - 0d, - 1000d) - case object TopOrganicFollowsAccountsSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.TopOrganicFollowsAccountsSourceWeight, - 1d, - 0d, - 1000d) - case object CrowdSearchAccountSourceWeight - extends FSBoundedParam[Double]( - ContentRecommenderFlowFeatureSwitchKeys.CrowdSearchAccountSourceWeight, - 1d, - 0d, - 1000d) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.scala deleted file mode 100644 index a24032c84..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFSConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderFlowFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq( - ContentRecommenderParams.IncludeActivityBasedCandidateSource, - ContentRecommenderParams.IncludeSocialBasedCandidateSource, - ContentRecommenderParams.IncludeGeoBasedCandidateSource, - ContentRecommenderParams.IncludeHomeTimelineTweetRecsCandidateSource, - ContentRecommenderParams.IncludeSocialProofEnforcedCandidateSource, - ContentRecommenderParams.EnableRecentFollowingPredicate, - ContentRecommenderParams.EnableGizmoduckPredicate, - ContentRecommenderParams.EnableInactivePredicate, - ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate, - ContentRecommenderParams.IncludeNewFollowingNewFollowingExpansionCandidateSource, - ContentRecommenderParams.IncludeMoreGeoBasedCandidateSource, - ContentRecommenderParams.TargetEligibility, - ContentRecommenderParams.GetFollowersFromSgs, - ContentRecommenderParams.EnableInvalidRelationshipPredicate, - ) - - override val intFSParams: Seq[FSBoundedParam[Int]] = - Seq( - ContentRecommenderParams.ResultSizeParam, - ContentRecommenderParams.BatchSizeParam, - ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond, - ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond, - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = - Seq( - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight, - ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight, - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.scala deleted file mode 100644 index ff51dc9f6..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderFlowFeatureSwitchKeys.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -object ContentRecommenderFlowFeatureSwitchKeys { - val TargetUserEligible = "content_recommender_flow_target_eligible" - val ResultSize = "content_recommender_flow_result_size" - val BatchSize = "content_recommender_flow_batch_size" - val RecentFollowingPredicateBudgetInMillisecond = - "content_recommender_flow_recent_following_predicate_budget_in_ms" - val CandidateGenerationBudgetInMillisecond = - "content_recommender_flow_candidate_generation_budget_in_ms" - val EnableRecentFollowingPredicate = "content_recommender_flow_enable_recent_following_predicate" - val EnableGizmoduckPredicate = "content_recommender_flow_enable_gizmoduck_predicate" - val EnableInactivePredicate = "content_recommender_flow_enable_inactive_predicate" - val EnableInvalidTargetCandidateRelationshipPredicate = - "content_recommender_flow_enable_invalid_target_candidate_relationship_predicate" - val IncludeActivityBasedCandidateSource = - "content_recommender_flow_include_activity_based_candidate_source" - val IncludeSocialBasedCandidateSource = - "content_recommender_flow_include_social_based_candidate_source" - val IncludeGeoBasedCandidateSource = - "content_recommender_flow_include_geo_based_candidate_source" - val IncludeHomeTimelineTweetRecsCandidateSource = - "content_recommender_flow_include_home_timeline_tweet_recs_candidate_source" - val IncludeSocialProofEnforcedCandidateSource = - "content_recommender_flow_include_social_proof_enforced_candidate_source" - val IncludeNewFollowingNewFollowingExpansionCandidateSource = - "content_recommender_flow_include_new_following_new_following_expansion_candidate_source" - val IncludeMoreGeoBasedCandidateSource = - "content_recommender_flow_include_more_geo_based_candidate_source" - val GetFollowersFromSgs = "content_recommender_flow_get_followers_from_sgs" - val EnableInvalidRelationshipPredicate = - "content_recommender_flow_enable_invalid_relationship_predicate" - - // Candidate source weight param keys - // Social based - val ForwardPhoneBookSourceWeight = - "content_recommender_flow_candidate_source_weight_forward_phone_book" - val ForwardEmailBookSourceWeight = - "content_recommender_flow_candidate_source_weight_forward_email_book" - val ReversePhoneBookSourceWeight = - "content_recommender_flow_candidate_source_weight_reverse_phone_book" - val ReverseEmailBookSourceWeight = - "content_recommender_flow_candidate_source_weight_reverse_email_book" - val OfflineStrongTiePredictionSourceWeight = - "content_recommender_flow_candidate_source_weight_offline_stp" - val TriangularLoopsSourceWeight = - "content_recommender_flow_candidate_source_weight_triangular_loops" - val UserUserGraphSourceWeight = "content_recommender_flow_candidate_source_weight_user_user_graph" - val NewFollowingNewFollowingExpansionSourceWeight = - "content_recommender_flow_candidate_source_weight_new_following_new_following_expansion" - // Activity based - val NewFollowingSimilarUserSourceWeight = - "content_recommender_flow_candidate_source_weight_new_following_similar_user" - val RecentEngagementSimilarUserSourceWeight = - "content_recommender_flow_candidate_source_weight_recent_engagement_similar_user" - val RepeatedProfileVisitsSourceWeight = - "content_recommender_flow_candidate_source_weight_repeated_profile_visits" - val RealGraphOonSourceWeight = "content_recommender_flow_candidate_source_weight_real_graph_oon" - // Geo based - val PopCountrySourceWeight = "content_recommender_flow_candidate_source_weight_pop_country" - val PopGeohashSourceWeight = "content_recommender_flow_candidate_source_weight_pop_geohash" - val PopCountryBackfillSourceWeight = - "content_recommender_flow_candidate_source_weight_pop_country_backfill" - val PPMILocaleFollowSourceWeight = - "content_recommender_flow_candidate_source_weight_ppmi_locale_follow" - val TopOrganicFollowsAccountsSourceWeight = - "content_recommender_flow_candidate_source_weight_top_organic_follow_account" - val CrowdSearchAccountSourceWeight = - "content_recommender_flow_candidate_source_weight_crowd_search_account" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.scala deleted file mode 100644 index 6b43325af..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderParams.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -abstract class ContentRecommenderParams[A](default: A) extends Param[A](default) { - override val statName: String = "content_recommender/" + this.getClass.getSimpleName -} - -object ContentRecommenderParams { - - case object TargetEligibility - extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.TargetUserEligible, true) - - case object ResultSizeParam - extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.ResultSize, 15, 1, 500) - case object BatchSizeParam - extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.BatchSize, 15, 1, 500) - case object RecentFollowingPredicateBudgetInMillisecond - extends FSBoundedParam[Int]( - ContentRecommenderFlowFeatureSwitchKeys.RecentFollowingPredicateBudgetInMillisecond, - 8, - 1, - 50) - case object FetchCandidateSourceBudgetInMillisecond - extends FSBoundedParam[Int]( - ContentRecommenderFlowFeatureSwitchKeys.CandidateGenerationBudgetInMillisecond, - 60, - 1, - 80) - case object EnableRecentFollowingPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableRecentFollowingPredicate, - true) - case object EnableGizmoduckPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableGizmoduckPredicate, - false) - case object EnableInactivePredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableInactivePredicate, - false) - case object EnableInvalidTargetCandidateRelationshipPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidTargetCandidateRelationshipPredicate, - false) - case object IncludeActivityBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeActivityBasedCandidateSource, - true) - case object IncludeSocialBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialBasedCandidateSource, - true) - case object IncludeGeoBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeGeoBasedCandidateSource, - true) - case object IncludeHomeTimelineTweetRecsCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeHomeTimelineTweetRecsCandidateSource, - false) - case object IncludeSocialProofEnforcedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialProofEnforcedCandidateSource, - false) - case object IncludeNewFollowingNewFollowingExpansionCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeNewFollowingNewFollowingExpansionCandidateSource, - false) - - case object IncludeMoreGeoBasedCandidateSource - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.IncludeMoreGeoBasedCandidateSource, - false) - - case object GetFollowersFromSgs - extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.GetFollowersFromSgs, false) - - case object EnableInvalidRelationshipPredicate - extends FSParam[Boolean]( - ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate, - false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.scala deleted file mode 100644 index 5952314e5..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequest.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode -import com.twitter.follow_recommendations.common.models.HasDebugOptions -import com.twitter.follow_recommendations.common.models.HasDisplayLocation -import com.twitter.follow_recommendations.common.models.HasExcludedUserIds -import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode -import com.twitter.follow_recommendations.common.models.HasInvalidRelationshipUserIds -import com.twitter.follow_recommendations.common.models.HasRecentFollowedByUserIds -import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds -import com.twitter.follow_recommendations.common.models.HasUserState -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class ContentRecommenderRequest( - override val params: Params, - override val clientContext: ClientContext, - inputExcludeUserIds: Seq[Long], - override val recentFollowedUserIds: Option[Seq[Long]], - override val recentFollowedByUserIds: Option[Seq[Long]], - override val invalidRelationshipUserIds: Option[Set[Long]], - override val displayLocation: DisplayLocation, - maxResults: Option[Int] = None, - override val debugOptions: Option[DebugOptions] = None, - override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, - override val userState: Option[UserState] = None) - extends HasParams - with HasClientContext - with HasDisplayLocation - with HasDebugOptions - with HasRecentFollowedUserIds - with HasRecentFollowedByUserIds - with HasInvalidRelationshipUserIds - with HasExcludedUserIds - with HasUserState - with HasGeohashAndCountryCode { - override val excludedUserIds: Seq[Long] = { - inputExcludeUserIds ++ clientContext.userId.toSeq - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.scala deleted file mode 100644 index 769f9ce51..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow/ContentRecommenderRequestBuilder.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.follow_recommendations.flows.content_recommender_flow - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueOptionalWithStats -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStatsWithin -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.stitch.Stitch - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ContentRecommenderRequestBuilder @Inject() ( - socialGraph: SocialGraphClient, - userLocationFetcher: UserLocationFetcher, - userStateClient: UserStateClient, - statsReceiver: StatsReceiver) { - - val stats: StatsReceiver = statsReceiver.scope("content_recommender_request_builder") - val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds") - private val invalidRelationshipUsersMaxSizeCounter = - invalidRelationshipUsersStats.counter("maxSize") - private val invalidRelationshipUsersNotMaxSizeCounter = - invalidRelationshipUsersStats.counter("notMaxSize") - - def build(req: ProductRequest): Stitch[ContentRecommenderRequest] = { - val userStateStitch = Stitch - .collect(req.recommendationRequest.clientContext.userId.map(userId => - userStateClient.getUserState(userId))).map(_.flatten) - val recentFollowedUserIdsStitch = - Stitch - .collect(req.recommendationRequest.clientContext.userId.map { userId => - rescueWithStatsWithin( - socialGraph.getRecentFollowedUserIds(userId), - stats, - "recentFollowedUserIds", - req - .params( - ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond).millisecond - ) - }) - val recentFollowedByUserIdsStitch = - if (req.params(ContentRecommenderParams.GetFollowersFromSgs)) { - Stitch - .collect( - req.recommendationRequest.clientContext.userId.map(userId => - rescueWithStatsWithin( - socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId), - stats, - "recentFollowedByUserIds", - req - .params(ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond) - .millisecond - ))) - } else Stitch.None - val invalidRelationshipUserIdsStitch: Stitch[Option[Seq[Long]]] = - if (req.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate)) { - Stitch - .collect( - req.recommendationRequest.clientContext.userId.map { userId => - rescueWithStats( - socialGraph - .getInvalidRelationshipUserIdsFromCachedColumn(userId) - .onSuccess(ids => - if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) { - invalidRelationshipUsersMaxSizeCounter.incr() - } else { - invalidRelationshipUsersNotMaxSizeCounter.incr() - }), - stats, - "invalidRelationshipUserIds" - ) - } - ) - } else { - Stitch.None - } - val locationStitch = - rescueOptionalWithStats( - userLocationFetcher.getGeohashAndCountryCode( - req.recommendationRequest.clientContext.userId, - req.recommendationRequest.clientContext.ipAddress - ), - stats, - "userLocation" - ) - Stitch - .join( - recentFollowedUserIdsStitch, - recentFollowedByUserIdsStitch, - invalidRelationshipUserIdsStitch, - locationStitch, - userStateStitch) - .map { - case ( - recentFollowedUserIds, - recentFollowedByUserIds, - invalidRelationshipUserIds, - location, - userState) => - ContentRecommenderRequest( - req.params, - req.recommendationRequest.clientContext, - req.recommendationRequest.excludedIds.getOrElse(Nil), - recentFollowedUserIds, - recentFollowedByUserIds, - invalidRelationshipUserIds.map(_.toSet), - req.recommendationRequest.displayLocation, - req.recommendationRequest.maxResults, - req.recommendationRequest.debugParams.flatMap(_.debugOptions), - location, - userState - ) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD deleted file mode 100644 index 9129e17b8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/BUILD +++ /dev/null @@ -1,58 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.scala deleted file mode 100644 index ed15d566c..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceRegistry.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.sims.LinearRegressionFollow2vecNearestNeighborsStore -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceScorer -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PostNuxMlCandidateSourceRegistry @Inject() ( - crowdSearchAccountsCandidateSource: CrowdSearchAccountsSource, - topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource, - linearRegressionfollow2vecNearestNeighborsStore: LinearRegressionFollow2vecNearestNeighborsStore, - forwardEmailBookSource: ForwardEmailBookSource, - forwardPhoneBookSource: ForwardPhoneBookSource, - offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource, - onlineSTPSource: OnlineSTPSourceScorer, - popCountrySource: PopCountrySource, - popCountryBackFillSource: PopCountryBackFillSource, - popGeohashSource: PopGeohashSource, - recentEngagementDirectFollowSimilarUsersSource: RecentEngagementSimilarUsersSource, - recentEngagementNonDirectFollowSource: RecentEngagementNonDirectFollowSource, - recentEngagementDirectFollowSalsaExpansionSource: RecentEngagementDirectFollowSalsaExpansionSource, - recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource, - realGraphOonV2Source: RealGraphOonV2Source, - repeatedProfileVisitSource: RepeatedProfileVisitsSource, - reverseEmailBookSource: ReverseEmailBookSource, - reversePhoneBookSource: ReversePhoneBookSource, - triangularLoopsSource: TriangularLoopsSource, - userUserGraphCandidateSource: UserUserGraphCandidateSource, - ppmiLocaleFollowSource: PPMILocaleFollowSource, - popGeohashQualityFollowSource: PopGeohashQualityFollowSource, - baseStatsReceiver: StatsReceiver, -) extends CandidateSourceRegistry[PostNuxMlRequest, CandidateUser] { - import EnrichedCandidateSource._ - - override val statsReceiver = baseStatsReceiver - .scope("post_nux_ml_flow", "candidate_sources") - - // sources primarily based on social graph signals - private[this] val socialSources = Seq( - linearRegressionfollow2vecNearestNeighborsStore.mapKeys[PostNuxMlRequest]( - _.getOptionalUserId.toSeq), - forwardEmailBookSource, - forwardPhoneBookSource, - offlineStrongTiePredictionSource, - onlineSTPSource, - reverseEmailBookSource, - reversePhoneBookSource, - triangularLoopsSource, - ) - - // sources primarily based on geo signals - private[this] val geoSources = Seq( - popCountrySource, - popCountryBackFillSource, - popGeohashSource, - popGeohashQualityFollowSource, - topOrganicFollowsAccountsSource, - crowdSearchAccountsCandidateSource, - ppmiLocaleFollowSource, - ) - - // sources primarily based on recent activity signals - private[this] val activitySources = Seq( - repeatedProfileVisitSource, - recentEngagementDirectFollowSalsaExpansionSource.mapKeys[PostNuxMlRequest]( - _.getOptionalUserId.toSeq), - recentEngagementDirectFollowSimilarUsersSource, - recentEngagementNonDirectFollowSource.mapKeys[PostNuxMlRequest](_.getOptionalUserId.toSeq), - recentFollowingSimilarUsersSource, - realGraphOonV2Source, - userUserGraphCandidateSource, - ) - - override val sources: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] = ( - geoSources ++ socialSources ++ activitySources - ).toSet -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.scala deleted file mode 100644 index 9492747a4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCandidateSourceWeightParams.scala +++ /dev/null @@ -1,177 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.Param - -abstract class PostNuxMlCandidateSourceWeightParams[A](default: A) extends Param[A](default) { - override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName -} - -object PostNuxMlCandidateSourceWeightParams { - - case object CandidateWeightCrowdSearch - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightCrowdSearch, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightTopOrganicFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTopOrganicFollow, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPPMILocaleFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPPMILocaleFollow, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightForwardEmailBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardEmailBook, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightForwardPhoneBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardPhoneBook, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightOfflineStrongTiePrediction - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOfflineStrongTiePrediction, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightOnlineStp - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOnlineStp, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopCountry - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopCountry, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopGeohash - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohash, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopGeohashQualityFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohashQualityFollow, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightPopGeoBackfill - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeoBackfill, - 1, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentFollowingSimilarUsers - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentFollowingSimilarUsers, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentEngagementDirectFollowSalsaExpansion - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementDirectFollowSalsaExpansion, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentEngagementNonDirectFollow - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementNonDirectFollow, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRecentEngagementSimilarUsers - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementSimilarUsers, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightRepeatedProfileVisits - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRepeatedProfileVisits, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightFollow2vecNearestNeighbors - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightFollow2vecNearestNeighbors, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightReverseEmailBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReverseEmailBook, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightReversePhoneBook - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReversePhoneBook, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightTriangularLoops - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTriangularLoops, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightTwoHopRandomWalk - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTwoHopRandomWalk, - 1.0, - 0.0, - 1000.0 - ) - case object CandidateWeightUserUserGraph - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightUserUserGraph, - 1.0, - 0.0, - 1000.0 - ) - - case object CandidateWeightRealGraphOonV2 - extends FSBoundedParam[Double]( - PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRealGraphOonV2, - 1.0, - 0.0, - 2000.0 - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala deleted file mode 100644 index 14e982a41..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala +++ /dev/null @@ -1,193 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.IdentityRanker -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Ranker -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models._ -import com.twitter.follow_recommendations.common.rankers.common.RankerId -import com.twitter.follow_recommendations.common.rankers.fatigue_ranker.ImpressionBasedFatigueRanker -import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRanker -import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRankerParams -import com.twitter.follow_recommendations.common.rankers.interleave_ranker.InterleaveRanker -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.HydrateFeaturesTransform -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRanker -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker -import com.twitter.follow_recommendations.configapi.candidates.HydrateCandidateParamsTransform -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams - -/** - * Used to build the combined ranker comprising 4 stages of ranking: - * - weighted sampler - * - truncating to the top N merged results for ranking - * - ML ranker - * - Interleaving ranker for producer-side experiments - * - impression-based fatigueing - */ -@Singleton -class PostNuxMlCombinedRankerBuilder[ - T <: HasParams with HasSimilarToContext with HasClientContext with HasExcludedUserIds with HasDisplayLocation with HasDebugOptions with HasPreFetchedFeature with HasDismissedUserIds with HasQualityFactor] @Inject() ( - firstNRanker: FirstNRanker[T], - hydrateFeaturesTransform: HydrateFeaturesTransform[T], - hydrateCandidateParamsTransform: HydrateCandidateParamsTransform[T], - mlRanker: MlRanker[T], - statsReceiver: StatsReceiver) { - private[this] val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_ranker") - - // we construct each ranker independently and chain them together - def build( - request: T, - candidateSourceWeights: Map[CandidateSourceIdentifier, Double] - ): Ranker[T, CandidateUser] = { - val displayLocationStats = stats.scope(request.displayLocation.toString) - val weightedRankerStats: StatsReceiver = - displayLocationStats.scope("weighted_candidate_source_ranker") - val firstNRankerStats: StatsReceiver = - displayLocationStats.scope("first_n_ranker") - val hydrateCandidateParamsStats = - displayLocationStats.scope("hydrate_candidate_params") - val fatigueRankerStats = displayLocationStats.scope("fatigue_ranker") - val interleaveRankerStats = - displayLocationStats.scope("interleave_ranker") - val allRankersStats = displayLocationStats.scope("all_rankers") - - // Checking if the heavy-ranker is an experimental model. - // If it is, InterleaveRanker and candidate parameter hydration are disabled. - // *NOTE* that consumer-side experiments should at any time take a small % of traffic, less - // than 20% for instance, to leave enough room for producer experiments. Increasing bucket - // size for producer experiments lead to other issues and is not a viable option for faster - // experiments. - val requestRankerId = request.params(MlRankerParams.RequestScorerIdParam) - if (requestRankerId != RankerId.PostNuxProdRanker) { - hydrateCandidateParamsStats.counter(s"disabled_by_${requestRankerId.toString}").incr() - interleaveRankerStats.counter(s"disabled_by_${requestRankerId.toString}").incr() - } - - // weighted ranker that samples from the candidate sources - val weightedRanker = WeightedCandidateSourceRanker - .build[T]( - candidateSourceWeights, - request.params(PostNuxMlParams.CandidateShuffler).shuffle(request.getRandomizationSeed), - randomSeed = request.getRandomizationSeed - ).observe(weightedRankerStats) - - // ranker that takes the first n results (ie truncates output) while merging duplicates - val firstNRankerObs = firstNRanker.observe(firstNRankerStats) - // either ML ranker that uses deepbirdv2 to score or no ranking - val mainRanker: Ranker[T, CandidateUser] = - buildMainRanker(request, requestRankerId == RankerId.PostNuxProdRanker, displayLocationStats) - // fatigue ranker that uses wtf impressions to fatigue - val fatigueRanker = buildFatigueRanker(request, fatigueRankerStats).observe(fatigueRankerStats) - - // interleaveRanker combines rankings from several rankers and enforces candidates' ranks in - // experiment buckets according to their assigned ranker model. - val interleaveRanker = - buildInterleaveRanker( - request, - requestRankerId == RankerId.PostNuxProdRanker, - interleaveRankerStats) - .observe(interleaveRankerStats) - - weightedRanker - .andThen(firstNRankerObs) - .andThen(mainRanker) - .andThen(fatigueRanker) - .andThen(interleaveRanker) - .observe(allRankersStats) - } - - def buildMainRanker( - request: T, - isMainRankerPostNuxProd: Boolean, - displayLocationStats: StatsReceiver - ): Ranker[T, CandidateUser] = { - - // note that we may be disabling heavy ranker for users not bucketed - // (due to empty results from the new candidate source) - // need a better solution in the future - val mlRankerStats = displayLocationStats.scope("ml_ranker") - val noMlRankerStats = displayLocationStats.scope("no_ml_ranker") - val hydrateFeaturesStats = - displayLocationStats.scope("hydrate_features") - val hydrateCandidateParamsStats = - displayLocationStats.scope("hydrate_candidate_params") - val notHydrateCandidateParamsStats = - displayLocationStats.scope("not_hydrate_candidate_params") - val rankerStats = displayLocationStats.scope("ranker") - val mlRankerDisabledByExperimentsCounter = - mlRankerStats.counter("disabled_by_experiments") - val mlRankerDisabledByQualityFactorCounter = - mlRankerStats.counter("disabled_by_quality_factor") - - val disabledByQualityFactor = request.qualityFactor - .exists(_ <= request.params(PostNuxMlParams.TurnoffMLScorerQFThreshold)) - - if (disabledByQualityFactor) - mlRankerDisabledByQualityFactorCounter.incr() - - if (request.params(PostNuxMlParams.UseMlRanker) && !disabledByQualityFactor) { - - val hydrateFeatures = hydrateFeaturesTransform - .observe(hydrateFeaturesStats) - - val optionalHydratedParamsTransform: Transform[T, CandidateUser] = { - // We disable candidate parameter hydration for experimental heavy-ranker models. - if (isMainRankerPostNuxProd && - request.params(PostNuxMlParams.EnableCandidateParamHydration)) { - hydrateCandidateParamsTransform - .observe(hydrateCandidateParamsStats) - } else { - new IdentityTransform[T, CandidateUser]() - .observe(notHydrateCandidateParamsStats) - } - } - val candidateSize = request.params(FirstNRankerParams.CandidatesToRank) - Ranker - .chain( - hydrateFeatures.andThen(optionalHydratedParamsTransform), - mlRanker.observe(mlRankerStats), - ) - .within( - request.params(PostNuxMlParams.MlRankerBudget), - rankerStats.scope(s"n$candidateSize")) - } else { - new IdentityRanker[T, CandidateUser].observe(noMlRankerStats) - } - } - - def buildInterleaveRanker( - request: T, - isMainRankerPostNuxProd: Boolean, - interleaveRankerStats: StatsReceiver - ): Ranker[T, CandidateUser] = { - // InterleaveRanker is enabled only for display locations powered by the PostNux heavy-ranker. - if (request.params(PostNuxMlParams.EnableInterleaveRanker) && - // InterleaveRanker is disabled for requests with experimental heavy-rankers. - isMainRankerPostNuxProd) { - new InterleaveRanker[T](interleaveRankerStats) - } else { - new IdentityRanker[T, CandidateUser]() - } - } - - def buildFatigueRanker( - request: T, - fatigueRankerStats: StatsReceiver - ): Ranker[T, CandidateUser] = { - if (request.params(PostNuxMlParams.EnableFatigueRanker)) { - ImpressionBasedFatigueRanker - .build[T]( - fatigueRankerStats - ).within(request.params(PostNuxMlParams.FatigueRankerBudget), fatigueRankerStats) - } else { - new IdentityRanker[T, CandidateUser]() - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.scala deleted file mode 100644 index 092f07100..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlow.scala +++ /dev/null @@ -1,304 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource._ -import com.twitter.follow_recommendations.common.base._ -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.FilterReason -import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicate -import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate -import com.twitter.follow_recommendations.common.transforms.ranker_id.RandomRankerIdTransform -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate -import com.twitter.follow_recommendations.common.predicates.CandidateParamPredicate -import com.twitter.follow_recommendations.common.predicates.CandidateSourceParamPredicate -import com.twitter.follow_recommendations.common.predicates.CuratedCompetitorListPredicate -import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate -import com.twitter.follow_recommendations.common.predicates.InactivePredicate -import com.twitter.follow_recommendations.common.predicates.PreviouslyRecommendedUserIdsPredicate -import com.twitter.follow_recommendations.common.predicates.user_activity.NonNearZeroUserActivityPredicate -import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform -import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProofTransform -import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform -import com.twitter.follow_recommendations.common.transforms.weighted_sampling.SamplingTransform -import com.twitter.follow_recommendations.configapi.candidates.CandidateUserParamsFactory -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform -import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.timelines.configapi.Params -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.predicates.hss.HssPredicate -import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate -import com.twitter.follow_recommendations.common.transforms.modify_social_proof.RemoveAccountProofTransform -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.RecommendationFlowData -import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver -import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver -import com.twitter.stitch.Stitch - -/** - * We use this flow for all post-nux display locations that would use a machine-learning-based-ranker - * eg HTL, Sidebar, etc - * Note that the RankedPostNuxFlow is used primarily for scribing/data collection, and doesn't - * incorporate all of the other components in a flow (candidate source generation, predicates etc) - */ -@Singleton -class PostNuxMlFlow @Inject() ( - postNuxMlCandidateSourceRegistry: PostNuxMlCandidateSourceRegistry, - postNuxMlCombinedRankerBuilder: PostNuxMlCombinedRankerBuilder[PostNuxMlRequest], - curatedCompetitorListPredicate: CuratedCompetitorListPredicate, - gizmoduckPredicate: GizmoduckPredicate, - sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate, - hssPredicate: HssPredicate, - invalidRelationshipPredicate: InvalidRelationshipPredicate, - recentFollowingPredicate: RecentFollowingPredicate, - nonNearZeroUserActivityPredicate: NonNearZeroUserActivityPredicate, - inactivePredicate: InactivePredicate, - dismissedCandidatePredicate: DismissedCandidatePredicate, - previouslyRecommendedUserIdsPredicate: PreviouslyRecommendedUserIdsPredicate, - modifySocialProofTransform: ModifySocialProofTransform, - removeAccountProofTransform: RemoveAccountProofTransform, - trackingTokenTransform: TrackingTokenTransform, - randomRankerIdTransform: RandomRankerIdTransform, - candidateParamsFactory: CandidateUserParamsFactory[PostNuxMlRequest], - samplingTransform: SamplingTransform, - frsLogger: FrsLogger, - baseStatsReceiver: StatsReceiver) - extends RecommendationFlow[PostNuxMlRequest, CandidateUser] - with RecommendationFlowBaseSideEffectsUtil[PostNuxMlRequest, CandidateUser] - with CandidateSourceHoldbackUtil { - override protected val targetEligibility: Predicate[PostNuxMlRequest] = - new ParamPredicate[PostNuxMlRequest](PostNuxMlParams.TargetEligibility) - - override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("post_nux_ml_flow") - - override val qualityFactorObserver: Option[QualityFactorObserver] = { - val config = LinearLatencyQualityFactorConfig( - qualityFactorBounds = - BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0), - initialDelay = 60.seconds, - targetLatency = 700.milliseconds, - targetLatencyPercentile = 95.0, - delta = 0.001 - ) - val qualityFactor = LinearLatencyQualityFactor(config) - val observer = LinearLatencyQualityFactorObserver(qualityFactor) - statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat) - Some(observer) - } - - override protected def updateTarget(request: PostNuxMlRequest): Stitch[PostNuxMlRequest] = { - Stitch.value( - request.copy(qualityFactor = qualityFactorObserver.map(_.qualityFactor.currentValue)) - ) - } - - private[post_nux_ml] def getCandidateSourceIdentifiers( - params: Params - ): Set[CandidateSourceIdentifier] = { - PostNuxMlFlowCandidateSourceWeights.getWeights(params).keySet - } - - override protected def candidateSources( - request: PostNuxMlRequest - ): Seq[CandidateSource[PostNuxMlRequest, CandidateUser]] = { - val identifiers = getCandidateSourceIdentifiers(request.params) - val selected: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] = - postNuxMlCandidateSourceRegistry.select(identifiers) - val budget: Duration = request.params(PostNuxMlParams.FetchCandidateSourceBudget) - filterCandidateSources( - request, - selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq) - } - - override protected val preRankerCandidateFilter: Predicate[(PostNuxMlRequest, CandidateUser)] = { - val stats = statsReceiver.scope("pre_ranker") - - object excludeNearZeroUserPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - nonNearZeroUserActivityPredicate, - stats.scope("exclude_near_zero_predicate") - ) { - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.ExcludeNearZeroCandidates) - } - - object invalidRelationshipGatedPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - invalidRelationshipPredicate, - stats.scope("invalid_relationship_predicate") - ) { - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.EnableInvalidRelationshipPredicate) - } - - ExcludedUserIdPredicate - .observe(stats.scope("exclude_user_id_predicate")) - .andThen( - recentFollowingPredicate.observe(stats.scope("recent_following_predicate")) - ) - .andThen( - dismissedCandidatePredicate.observe(stats.scope("dismissed_candidate_predicate")) - ) - .andThen( - previouslyRecommendedUserIdsPredicate.observe( - stats.scope("previously_recommended_user_ids_predicate")) - ) - .andThen( - invalidRelationshipGatedPredicate.observe(stats.scope("invalid_relationship_predicate")) - ) - .andThen( - excludeNearZeroUserPredicate.observe(stats.scope("exclude_near_zero_user_state")) - ) - .observe(stats.scope("overall_pre_ranker_candidate_filter")) - } - - override protected def selectRanker( - request: PostNuxMlRequest - ): Ranker[PostNuxMlRequest, CandidateUser] = { - postNuxMlCombinedRankerBuilder.build( - request, - PostNuxMlFlowCandidateSourceWeights.getWeights(request.params)) - } - - override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = { - new DedupTransform[PostNuxMlRequest, CandidateUser] - .observe(statsReceiver.scope("dedupping")) - .andThen( - samplingTransform - .gated(PostNuxMlParams.SamplingTransformEnabled) - .observe(statsReceiver.scope("samplingtransform"))) - } - - override protected val validateCandidates: Predicate[(PostNuxMlRequest, CandidateUser)] = { - val stats = statsReceiver.scope("validate_candidates") - val competitorPredicate = - curatedCompetitorListPredicate.map[(PostNuxMlRequest, CandidateUser)](_._2) - - val producerHoldbackPredicate = new CandidateParamPredicate[CandidateUser]( - GlobalParams.KeepUserCandidate, - FilterReason.CandidateSideHoldback - ).map[(PostNuxMlRequest, CandidateUser)] { - case (request, user) => candidateParamsFactory(user, request) - } - val pymkProducerHoldbackPredicate = new CandidateSourceParamPredicate( - GlobalParams.KeepSocialUserCandidate, - FilterReason.CandidateSideHoldback, - CandidateSourceHoldbackUtil.SocialCandidateSourceIds - ).map[(PostNuxMlRequest, CandidateUser)] { - case (request, user) => candidateParamsFactory(user, request) - } - val sgsPredicateStats = stats.scope("sgs_predicate") - object sgsGatedPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - sgsPredicate.observe(sgsPredicateStats), - sgsPredicateStats - ) { - - /** - * When SGS predicate is turned off, only query SGS exists API for (user, candidate, relationship) - * when the user's number of invalid relationships exceeds the threshold during request - * building step. This is to minimize load to SGS and underlying Flock DB. - */ - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.EnableSGSPredicate) || - SocialGraphClient.enablePostRankerSgsPredicate( - item._1.invalidRelationshipUserIds.getOrElse(Set.empty).size) - } - - val hssPredicateStats = stats.scope("hss_predicate") - object hssGatedPredicate - extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)]( - hssPredicate.observe(hssPredicateStats), - hssPredicateStats - ) { - override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean = - item._1.params(PostNuxMlParams.EnableHssPredicate) - } - - Predicate - .andConcurrently[(PostNuxMlRequest, CandidateUser)]( - Seq( - competitorPredicate.observe(stats.scope("curated_competitor_predicate")), - gizmoduckPredicate.observe(stats.scope("gizmoduck_predicate")), - sgsGatedPredicate, - hssGatedPredicate, - inactivePredicate.observe(stats.scope("inactive_predicate")), - ) - ) - // to avoid dilutions, we need to apply the receiver holdback predicates at the very last step - .andThen(pymkProducerHoldbackPredicate.observe(stats.scope("pymk_receiver_side_holdback"))) - .andThen(producerHoldbackPredicate.observe(stats.scope("receiver_side_holdback"))) - .observe(stats.scope("overall_validate_candidates")) - } - - override protected val transformResults: Transform[PostNuxMlRequest, CandidateUser] = { - modifySocialProofTransform - .gated(EnableGFSSocialProofTransform) - .andThen(trackingTokenTransform) - .andThen(randomRankerIdTransform.gated(PostNuxMlParams.LogRandomRankerId)) - .andThen(removeAccountProofTransform.gated(PostNuxMlParams.EnableRemoveAccountProofTransform)) - } - - override protected def resultsConfig(request: PostNuxMlRequest): RecommendationResultsConfig = { - RecommendationResultsConfig( - request.maxResults.getOrElse(request.params(PostNuxMlParams.ResultSizeParam)), - request.params(PostNuxMlParams.BatchSizeParam) - ) - } - - override def applySideEffects( - target: PostNuxMlRequest, - candidateSources: Seq[CandidateSource[PostNuxMlRequest, CandidateUser]], - candidatesFromCandidateSources: Seq[CandidateUser], - mergedCandidates: Seq[CandidateUser], - filteredCandidates: Seq[CandidateUser], - rankedCandidates: Seq[CandidateUser], - transformedCandidates: Seq[CandidateUser], - truncatedCandidates: Seq[CandidateUser], - results: Seq[CandidateUser] - ): Stitch[Unit] = { - frsLogger.logRecommendationFlowData[PostNuxMlRequest]( - target, - RecommendationFlowData[PostNuxMlRequest]( - target, - PostNuxMlFlow.identifier, - candidateSources, - candidatesFromCandidateSources, - mergedCandidates, - filteredCandidates, - rankedCandidates, - transformedCandidates, - truncatedCandidates, - results - ) - ) - super.applySideEffects( - target, - candidateSources, - candidatesFromCandidateSources, - mergedCandidates, - filteredCandidates, - rankedCandidates, - transformedCandidates, - truncatedCandidates, - results - ) - } -} - -object PostNuxMlFlow { - val identifier = RecommendationPipelineIdentifier("PostNuxMlFlow") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.scala deleted file mode 100644 index edb447cba..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeights.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource -import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource -import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource -import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims.Follow2vecNearestNeighborsStore -import com.twitter.follow_recommendations.common.candidate_sources.stp.BaseOnlineSTPSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk.TwoHopRandomWalkSource -import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlCandidateSourceWeightParams._ -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.timelines.configapi.Params - -object PostNuxMlFlowCandidateSourceWeights { - - def getWeights(params: Params): Map[CandidateSourceIdentifier, Double] = { - Map[CandidateSourceIdentifier, Double]( - // Social based - PPMILocaleFollowSource.Identifier -> params(CandidateWeightPPMILocaleFollow), - Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression -> params( - CandidateWeightFollow2vecNearestNeighbors), - RecentFollowingSimilarUsersSource.Identifier -> params( - CandidateWeightRecentFollowingSimilarUsers), - BaseOnlineSTPSource.Identifier -> params(CandidateWeightOnlineStp), - OfflineStrongTiePredictionSource.Identifier -> params( - CandidateWeightOfflineStrongTiePrediction), - ForwardEmailBookSource.Identifier -> params(CandidateWeightForwardEmailBook), - ForwardPhoneBookSource.Identifier -> params(CandidateWeightForwardPhoneBook), - ReverseEmailBookSource.Identifier -> params(CandidateWeightReverseEmailBook), - ReversePhoneBookSource.Identifier -> params(CandidateWeightReversePhoneBook), - TriangularLoopsSource.Identifier -> params(CandidateWeightTriangularLoops), - TwoHopRandomWalkSource.Identifier -> params(CandidateWeightTwoHopRandomWalk), - UserUserGraphCandidateSource.Identifier -> params(CandidateWeightUserUserGraph), - // Geo based - PopCountrySource.Identifier -> params(CandidateWeightPopCountry), - PopCountryBackFillSource.Identifier -> params(CandidateWeightPopGeoBackfill), - PopGeohashSource.Identifier -> params(CandidateWeightPopGeohash), - PopGeohashQualityFollowSource.Identifier -> params(CandidateWeightPopGeohashQualityFollow), - CrowdSearchAccountsSource.Identifier -> params(CandidateWeightCrowdSearch), - TopOrganicFollowsAccountsSource.Identifier -> params(CandidateWeightTopOrganicFollow), - // Engagement based - RealGraphOonV2Source.Identifier -> params(CandidateWeightRealGraphOonV2), - RecentEngagementNonDirectFollowSource.Identifier -> params( - CandidateWeightRecentEngagementNonDirectFollow), - RecentEngagementSimilarUsersSource.Identifier -> params( - CandidateWeightRecentEngagementSimilarUsers), - RepeatedProfileVisitsSource.Identifier -> params(CandidateWeightRepeatedProfileVisits), - RecentEngagementDirectFollowSalsaExpansionSource.Identifier -> params( - CandidateWeightRecentEngagementDirectFollowSalsaExpansion), - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala deleted file mode 100644 index f329cbd13..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -object PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys { - val CandidateWeightCrowdSearch = "post_nux_ml_flow_candidate_source_weights_user_crowd_search" - val CandidateWeightTopOrganicFollow = - "post_nux_ml_flow_candidate_source_weights_top_organic_follow" - val CandidateWeightPPMILocaleFollow = - "post_nux_ml_flow_candidate_source_weights_user_ppmi_locale_follow" - val CandidateWeightForwardEmailBook = - "post_nux_ml_flow_candidate_source_weights_user_forward_email_book" - val CandidateWeightForwardPhoneBook = - "post_nux_ml_flow_candidate_source_weights_user_forward_phone_book" - val CandidateWeightOfflineStrongTiePrediction = - "post_nux_ml_flow_candidate_source_weights_user_offline_strong_tie_prediction" - val CandidateWeightOnlineStp = "post_nux_ml_flow_candidate_source_weights_user_online_stp" - val CandidateWeightPopCountry = "post_nux_ml_flow_candidate_source_weights_user_pop_country" - val CandidateWeightPopGeohash = "post_nux_ml_flow_candidate_source_weights_user_pop_geohash" - val CandidateWeightPopGeohashQualityFollow = - "post_nux_ml_flow_candidate_source_weights_user_pop_geohash_quality_follow" - val CandidateWeightPopGeoBackfill = - "post_nux_ml_flow_candidate_source_weights_user_pop_geo_backfill" - val CandidateWeightRecentFollowingSimilarUsers = - "post_nux_ml_flow_candidate_source_weights_user_recent_following_similar_users" - val CandidateWeightRecentEngagementDirectFollowSalsaExpansion = - "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_direct_follow_salsa_expansion" - val CandidateWeightRecentEngagementNonDirectFollow = - "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_non_direct_follow" - val CandidateWeightRecentEngagementSimilarUsers = - "post_nux_ml_flow_candidate_source_weights_user_recent_engagement_similar_users" - val CandidateWeightRepeatedProfileVisits = - "post_nux_ml_flow_candidate_source_weights_user_repeated_profile_visits" - val CandidateWeightFollow2vecNearestNeighbors = - "post_nux_ml_flow_candidate_source_weights_user_follow2vec_nearest_neighbors" - val CandidateWeightReverseEmailBook = - "post_nux_ml_flow_candidate_source_weights_user_reverse_email_book" - val CandidateWeightReversePhoneBook = - "post_nux_ml_flow_candidate_source_weights_user_reverse_phone_book" - val CandidateWeightTriangularLoops = - "post_nux_ml_flow_candidate_source_weights_user_triangular_loops" - val CandidateWeightTwoHopRandomWalk = - "post_nux_ml_flow_candidate_source_weights_user_two_hop_random_walk" - val CandidateWeightUserUserGraph = - "post_nux_ml_flow_candidate_source_weights_user_user_user_graph" - val CandidateWeightRealGraphOonV2 = - "post_nux_ml_flow_candidate_source_weights_user_real_graph_oon_v2" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.scala deleted file mode 100644 index 0dd059dad..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFSConfig.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.NoShuffle -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.RandomShuffler -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PostNuxMlFlowFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq( - PostNuxMlParams.OnlineSTPEnabled, - PostNuxMlParams.SamplingTransformEnabled, - PostNuxMlParams.Follow2VecLinearRegressionEnabled, - PostNuxMlParams.UseMlRanker, - PostNuxMlParams.EnableCandidateParamHydration, - PostNuxMlParams.EnableInterleaveRanker, - PostNuxMlParams.EnableAdhocRanker, - PostNuxMlParams.ExcludeNearZeroCandidates, - PostNuxMlParams.IncludeRepeatedProfileVisitsCandidateSource, - PostNuxMlParams.EnableInterestsOptOutPredicate, - PostNuxMlParams.EnableSGSPredicate, - PostNuxMlParams.EnableInvalidRelationshipPredicate, - PostNuxMlParams.EnableRemoveAccountProofTransform, - PostNuxMlParams.EnablePPMILocaleFollowSourceInPostNux, - PostNuxMlParams.EnableRealGraphOonV2, - PostNuxMlParams.GetFollowersFromSgs, - PostNuxMlRequestBuilderParams.EnableInvalidRelationshipPredicate - ) - - override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq( - PostNuxMlCandidateSourceWeightParams.CandidateWeightCrowdSearch, - PostNuxMlCandidateSourceWeightParams.CandidateWeightTopOrganicFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPPMILocaleFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardEmailBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardPhoneBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightOfflineStrongTiePrediction, - PostNuxMlCandidateSourceWeightParams.CandidateWeightOnlineStp, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopCountry, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohash, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohashQualityFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeoBackfill, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentFollowingSimilarUsers, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementDirectFollowSalsaExpansion, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementNonDirectFollow, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementSimilarUsers, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRepeatedProfileVisits, - PostNuxMlCandidateSourceWeightParams.CandidateWeightFollow2vecNearestNeighbors, - PostNuxMlCandidateSourceWeightParams.CandidateWeightReverseEmailBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightReversePhoneBook, - PostNuxMlCandidateSourceWeightParams.CandidateWeightTriangularLoops, - PostNuxMlCandidateSourceWeightParams.CandidateWeightTwoHopRandomWalk, - PostNuxMlCandidateSourceWeightParams.CandidateWeightUserUserGraph, - PostNuxMlCandidateSourceWeightParams.CandidateWeightRealGraphOonV2, - PostNuxMlParams.TurnoffMLScorerQFThreshold - ) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - PostNuxMlParams.MlRankerBudget, - PostNuxMlRequestBuilderParams.TopicIdFetchBudget, - PostNuxMlRequestBuilderParams.DismissedIdScanBudget, - PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget - ) - - override val gatedOverridesMap = Map( - PostNuxMlFlowFeatureSwitchKeys.EnableRandomDataCollection -> Seq( - PostNuxMlParams.CandidateShuffler := new RandomShuffler[CandidateUser], - PostNuxMlParams.LogRandomRankerId := true - ), - PostNuxMlFlowFeatureSwitchKeys.EnableNoShuffler -> Seq( - PostNuxMlParams.CandidateShuffler := new NoShuffle[CandidateUser] - ), - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.scala deleted file mode 100644 index 6a44c4bbb..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlFlowFeatureSwitchKeys.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -object PostNuxMlFlowFeatureSwitchKeys { - val UseMlRanker = "post_nux_ml_flow_use_ml_ranker" - val EnableCandidateParamHydration = "post_nux_ml_flow_enable_candidate_param_hydration" - val OnlineSTPEnabled = "post_nux_ml_flow_online_stp_source_enabled" - val Follow2VecLinearRegressionEnabled = "post_nux_ml_flow_follow_to_vec_lr_source_enabled" - val EnableRandomDataCollection = "post_nux_ml_flow_random_data_collection_enabled" - val EnableAdhocRanker = "post_nux_ml_flow_adhoc_ranker_enabled" - val EnableFatigueRanker = "post_nux_ml_flow_fatigue_ranker_enabled" - val EnableInterleaveRanker = "post_nux_ml_flow_interleave_ranker_enabled" - val IncludeRepeatedProfileVisitsCandidateSource = - "post_nux_ml_flow_include_repeated_profile_visits_candidate_source" - val MLRankerBudget = "post_nux_ml_flow_ml_ranker_budget_millis" - val EnableNoShuffler = "post_nux_ml_flow_no_shuffler" - val SamplingTransformEnabled = "post_nux_ml_flow_sampling_transform_enabled" - val ExcludeNearZeroCandidates = "post_nux_ml_flow_exclude_near_zero_candidates" - val EnableInterestsOptOutPredicate = "post_nux_ml_flow_enable_interests_opt_out_predicate" - val EnableRemoveAccountProofTransform = "post_nux_ml_flow_enable_remove_account_proof_transform" - val EnablePPMILocaleFollowSourceInPostNux = "post_nux_ml_flow_enable_ppmilocale_follow_source" - val EnableInvalidRelationshipPredicate = "post_nux_ml_flow_enable_invalid_relationship_predicate" - val EnableRealGraphOonV2 = "post_nux_ml_flow_enable_real_graph_oon_v2" - val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate" - val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate" - val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs" - val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threhsold" -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.scala deleted file mode 100644 index cb5cf3648..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlParams.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.conversions.DurationOps._ -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.CandidateShuffler -import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.ExponentialShuffler -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -abstract class PostNuxMlParams[A](default: A) extends Param[A](default) { - override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName -} - -object PostNuxMlParams { - - // infra params: - case object FetchCandidateSourceBudget extends PostNuxMlParams[Duration](90.millisecond) - - // WTF Impression Store has very high tail latency (p9990 or p9999), but p99 latency is pretty good (~100ms) - // set the time budget for this step to be 200ms to make the performance of service more predictable - case object FatigueRankerBudget extends PostNuxMlParams[Duration](200.millisecond) - - case object MlRankerBudget - extends FSBoundedParam[Duration]( - name = PostNuxMlFlowFeatureSwitchKeys.MLRankerBudget, - default = 400.millisecond, - min = 100.millisecond, - max = 800.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - // product params: - case object TargetEligibility extends PostNuxMlParams[Boolean](true) - - case object ResultSizeParam extends PostNuxMlParams[Int](3) - case object BatchSizeParam extends PostNuxMlParams[Int](12) - - case object CandidateShuffler - extends PostNuxMlParams[CandidateShuffler[CandidateUser]]( - new ExponentialShuffler[CandidateUser]) - case object LogRandomRankerId extends PostNuxMlParams[Boolean](false) - - // whether or not to use the ml ranker at all (feature hydration + ranker) - case object UseMlRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.UseMlRanker, false) - - // whether or not to enable candidate param hydration in postnux_ml_flow - case object EnableCandidateParamHydration - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableCandidateParamHydration, false) - - // Whether or not OnlineSTP candidates are considered in the final pool of candidates. - // If set to `false`, the candidate source will be removed *after* all other considerations. - case object OnlineSTPEnabled - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.OnlineSTPEnabled, false) - - // Whether or not the candidates are sampled from a Plackett-Luce model - case object SamplingTransformEnabled - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.SamplingTransformEnabled, false) - - // Whether or not Follow2Vec candidates are considered in the final pool of candidates. - // If set to `false`, the candidate source will be removed *after* all other considerations. - case object Follow2VecLinearRegressionEnabled - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.Follow2VecLinearRegressionEnabled, - false) - - // Whether or not to enable AdhocRanker to allow adhoc, non-ML, score modifications. - case object EnableAdhocRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableAdhocRanker, false) - - // Whether the impression-based fatigue ranker is enabled or not. - case object EnableFatigueRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableFatigueRanker, true) - - // whether or not to enable InterleaveRanker for producer-side experiments. - case object EnableInterleaveRanker - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterleaveRanker, false) - - // whether to exclude users in near zero user state - case object ExcludeNearZeroCandidates - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.ExcludeNearZeroCandidates, false) - - case object EnablePPMILocaleFollowSourceInPostNux - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.EnablePPMILocaleFollowSourceInPostNux, - false) - - case object EnableInterestsOptOutPredicate - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterestsOptOutPredicate, false) - - case object EnableInvalidRelationshipPredicate - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate, - false) - - // Totally disabling SGS predicate need to disable EnableInvalidRelationshipPredicate as well - case object EnableSGSPredicate - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableSGSPredicate, true) - - case object EnableHssPredicate - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableHssPredicate, true) - - // Whether or not to include RepeatedProfileVisits as one of the candidate sources in the PostNuxMlFlow. If false, - // RepeatedProfileVisitsSource would not be run for the users in candidate_generation. - case object IncludeRepeatedProfileVisitsCandidateSource - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.IncludeRepeatedProfileVisitsCandidateSource, - false) - - case object EnableRealGraphOonV2 - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableRealGraphOonV2, false) - - case object GetFollowersFromSgs - extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.GetFollowersFromSgs, false) - - case object EnableRemoveAccountProofTransform - extends FSParam[Boolean]( - PostNuxMlFlowFeatureSwitchKeys.EnableRemoveAccountProofTransform, - false) - - // quality factor threshold to turn off ML ranker completely - object TurnoffMLScorerQFThreshold - extends FSBoundedParam[Double]( - name = PostNuxMlFlowFeatureSwitchKeys.TurnOffMLScorerQFThreshold, - default = 0.3, - min = 0.1, - max = 1.0) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.scala deleted file mode 100644 index 2cb112638..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequest.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models._ -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class PostNuxMlRequest( - override val params: Params, - override val clientContext: ClientContext, - override val similarToUserIds: Seq[Long], - inputExcludeUserIds: Seq[Long], - override val recentFollowedUserIds: Option[Seq[Long]], - override val invalidRelationshipUserIds: Option[Set[Long]], - override val recentFollowedByUserIds: Option[Seq[Long]], - override val dismissedUserIds: Option[Seq[Long]], - override val displayLocation: DisplayLocation, - maxResults: Option[Int] = None, - override val debugOptions: Option[DebugOptions] = None, - override val wtfImpressions: Option[Seq[WtfImpression]], - override val uttInterestIds: Option[Seq[Long]] = None, - override val customInterests: Option[Seq[String]] = None, - override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None, - inputPreviouslyRecommendedUserIds: Option[Set[Long]] = None, - inputPreviouslyFollowedUserIds: Option[Set[Long]] = None, - override val isSoftUser: Boolean = false, - override val userState: Option[UserState] = None, - override val qualityFactor: Option[Double] = None) - extends HasParams - with HasSimilarToContext - with HasClientContext - with HasExcludedUserIds - with HasDisplayLocation - with HasDebugOptions - with HasGeohashAndCountryCode - with HasPreFetchedFeature - with HasDismissedUserIds - with HasInterestIds - with HasPreviousRecommendationsContext - with HasIsSoftUser - with HasUserState - with HasInvalidRelationshipUserIds - with HasQualityFactor { - override val excludedUserIds: Seq[Long] = { - inputExcludeUserIds ++ clientContext.userId.toSeq ++ similarToUserIds - } - override val previouslyRecommendedUserIDs: Set[Long] = - inputPreviouslyRecommendedUserIds.getOrElse(Set.empty) - override val previouslyFollowedUserIds: Set[Long] = - inputPreviouslyFollowedUserIds.getOrElse(Set.empty) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.scala deleted file mode 100644 index aeb248b7f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilder.scala +++ /dev/null @@ -1,173 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.clients.dismiss_store.DismissStore -import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher -import com.twitter.follow_recommendations.common.clients.impression_store.WtfImpressionStore -import com.twitter.follow_recommendations.common.clients.interests_service.InterestServiceClient -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient -import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicateParams -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils._ -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.DismissedIdScanBudget -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.TopicIdFetchBudget -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.inject.Logging -import com.twitter.stitch.Stitch -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class PostNuxMlRequestBuilder @Inject() ( - socialGraph: SocialGraphClient, - wtfImpressionStore: WtfImpressionStore, - dismissStore: DismissStore, - userLocationFetcher: UserLocationFetcher, - interestServiceClient: InterestServiceClient, - userStateClient: UserStateClient, - statsReceiver: StatsReceiver) - extends Logging { - - val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_request_builder") - val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds") - private val invalidRelationshipUsersMaxSizeCounter = - invalidRelationshipUsersStats.counter("maxSize") - private val invalidRelationshipUsersNotMaxSizeCounter = - invalidRelationshipUsersStats.counter("notMaxSize") - - def build( - req: ProductRequest, - previouslyRecommendedUserIds: Option[Set[Long]] = None, - previouslyFollowedUserIds: Option[Set[Long]] = None - ): Stitch[PostNuxMlRequest] = { - val dl = req.recommendationRequest.displayLocation - val resultsStitch = Stitch.collect( - req.recommendationRequest.clientContext.userId - .map { userId => - val lookBackDuration = req.params(DismissedCandidatePredicateParams.LookBackDuration) - val negativeStartTs = -(Time.now - lookBackDuration).inMillis - val recentFollowedUserIdsStitch = - rescueWithStats( - socialGraph.getRecentFollowedUserIds(userId), - stats, - "recentFollowedUserIds") - val invalidRelationshipUserIdsStitch = - if (req.params(PostNuxMlParams.EnableInvalidRelationshipPredicate)) { - rescueWithStats( - socialGraph - .getInvalidRelationshipUserIds(userId) - .onSuccess(ids => - if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) { - invalidRelationshipUsersMaxSizeCounter.incr() - } else { - invalidRelationshipUsersNotMaxSizeCounter.incr() - }), - stats, - "invalidRelationshipUserIds" - ) - } else { - Stitch.value(Seq.empty) - } - // recentFollowedByUserIds are only used in experiment candidate sources - val recentFollowedByUserIdsStitch = if (req.params(PostNuxMlParams.GetFollowersFromSgs)) { - rescueWithStats( - socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId), - stats, - "recentFollowedByUserIds") - } else Stitch.value(Seq.empty) - val wtfImpressionsStitch = - rescueWithStatsWithin( - wtfImpressionStore.get(userId, dl), - stats, - "wtfImpressions", - req.params(WTFImpressionsScanBudget)) - val dismissedUserIdsStitch = - rescueWithStatsWithin( - dismissStore.get(userId, negativeStartTs, None), - stats, - "dismissedUserIds", - req.params(DismissedIdScanBudget)) - val locationStitch = - rescueOptionalWithStats( - userLocationFetcher.getGeohashAndCountryCode( - Some(userId), - req.recommendationRequest.clientContext.ipAddress), - stats, - "userLocation" - ) - val topicIdsStitch = - rescueWithStatsWithin( - interestServiceClient.fetchUttInterestIds(userId), - stats, - "topicIds", - req.params(TopicIdFetchBudget)) - val userStateStitch = - rescueOptionalWithStats(userStateClient.getUserState(userId), stats, "userState") - Stitch.join( - recentFollowedUserIdsStitch, - invalidRelationshipUserIdsStitch, - recentFollowedByUserIdsStitch, - dismissedUserIdsStitch, - wtfImpressionsStitch, - locationStitch, - topicIdsStitch, - userStateStitch - ) - }) - - resultsStitch.map { - case Some( - ( - recentFollowedUserIds, - invalidRelationshipUserIds, - recentFollowedByUserIds, - dismissedUserIds, - wtfImpressions, - locationInfo, - topicIds, - userState)) => - PostNuxMlRequest( - params = req.params, - clientContext = req.recommendationRequest.clientContext, - similarToUserIds = Nil, - inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil), - recentFollowedUserIds = Some(recentFollowedUserIds), - invalidRelationshipUserIds = Some(invalidRelationshipUserIds.toSet), - recentFollowedByUserIds = Some(recentFollowedByUserIds), - dismissedUserIds = Some(dismissedUserIds), - displayLocation = dl, - maxResults = req.recommendationRequest.maxResults, - debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions), - wtfImpressions = Some(wtfImpressions), - geohashAndCountryCode = locationInfo, - uttInterestIds = Some(topicIds), - inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds, - inputPreviouslyFollowedUserIds = previouslyFollowedUserIds, - isSoftUser = req.recommendationRequest.isSoftUser, - userState = userState - ) - case _ => - PostNuxMlRequest( - params = req.params, - clientContext = req.recommendationRequest.clientContext, - similarToUserIds = Nil, - inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil), - recentFollowedUserIds = None, - invalidRelationshipUserIds = None, - recentFollowedByUserIds = None, - dismissedUserIds = None, - displayLocation = dl, - maxResults = req.recommendationRequest.maxResults, - debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions), - wtfImpressions = None, - geohashAndCountryCode = None, - inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds, - inputPreviouslyFollowedUserIds = previouslyFollowedUserIds, - isSoftUser = req.recommendationRequest.isSoftUser, - userState = None - ) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.scala deleted file mode 100644 index da60f0382..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlRequestBuilderParams.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.follow_recommendations.flows.post_nux_ml - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.util.Duration -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion - -object PostNuxMlRequestBuilderParams { - case object TopicIdFetchBudget - extends FSBoundedParam[Duration]( - name = "post_nux_ml_request_builder_topic_id_fetch_budget_millis", - default = 200.millisecond, - min = 80.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - case object DismissedIdScanBudget - extends FSBoundedParam[Duration]( - name = "post_nux_ml_request_builder_dismissed_id_scan_budget_millis", - default = 200.millisecond, - min = 80.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - case object WTFImpressionsScanBudget - extends FSBoundedParam[Duration]( - name = "post_nux_ml_request_builder_wtf_impressions_scan_budget_millis", - default = 200.millisecond, - min = 80.millisecond, - max = 400.millisecond) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMillis - } - - case object EnableInvalidRelationshipPredicate - extends FSParam[Boolean]( - name = "post_nux_ml_request_builder_enable_invalid_relationship_predicate", - false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD deleted file mode 100644 index a35992e93..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala deleted file mode 100644 index 8b920c556..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala +++ /dev/null @@ -1,164 +0,0 @@ -package com.twitter.follow_recommendations.logging - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.follow_recommendations.common.models.HasIsSoftUser -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.follow_recommendations.logging.thriftscala.RecommendationLog -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.RecommendationFlowData -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.follow_recommendations.models.ScoringUserRequest -import com.twitter.follow_recommendations.models.ScoringUserResponse -import com.twitter.inject.annotations.Flag -import com.twitter.logging.LoggerFactory -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.scribelib.marshallers.ClientDataProvider -import com.twitter.scribelib.marshallers.ExternalRefererDataProvider -import com.twitter.scribelib.marshallers.ScribeSerialization -import com.twitter.timelines.configapi.HasParams -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/** - * This is the standard logging class we use to log data into: - * 1) logs.follow_recommendations_logs - * - * This logger logs data for 2 endpoints: getRecommendations, scoreUserCandidates - * All data scribed via this logger have to be converted into the same thrift type: RecommendationLog - * - * 2) logs.frs_recommendation_flow_logs - * - * This logger logs recommendation flow data for getRecommendations requests - * All data scribed via this logger have to be converted into the same thrift type: FrsRecommendationFlowLog - */ -@Singleton -class FrsLogger @Inject() ( - @Named(GuiceNamedConstants.REQUEST_LOGGER) loggerFactory: LoggerFactory, - @Named(GuiceNamedConstants.FLOW_LOGGER) flowLoggerFactory: LoggerFactory, - stats: StatsReceiver, - @Flag("log_results") serviceShouldLogResults: Boolean) - extends ScribeSerialization { - private val logger = loggerFactory.apply() - private val flowLogger = flowLoggerFactory.apply() - private val logRecommendationCounter = stats.counter("scribe_recommendation") - private val logScoringCounter = stats.counter("scribe_scoring") - private val logRecommendationFlowCounter = stats.counter("scribe_recommendation_flow") - - def logRecommendationResult( - request: RecommendationRequest, - response: RecommendationResponse - ): Unit = { - if (!request.isSoftUser) { - val log = - RecommendationLog(request.toOfflineThrift, response.toOfflineThrift, Time.now.inMillis) - logRecommendationCounter.incr() - logger.info( - serializeThrift( - log, - FrsLogger.LogCategory, - FrsLogger.mkProvider(request.clientContext) - )) - } - } - - def logScoringResult(request: ScoringUserRequest, response: ScoringUserResponse): Unit = { - if (!request.isSoftUser) { - val log = - RecommendationLog( - request.toRecommendationRequest.toOfflineThrift, - response.toRecommendationResponse.toOfflineThrift, - Time.now.inMillis) - logScoringCounter.incr() - logger.info( - serializeThrift( - log, - FrsLogger.LogCategory, - FrsLogger.mkProvider(request.toRecommendationRequest.clientContext) - )) - } - } - - def logRecommendationFlowData[Target <: HasClientContext with HasIsSoftUser with HasParams]( - request: Target, - flowData: RecommendationFlowData[Target] - ): Unit = { - if (!request.isSoftUser && request.params(GlobalParams.EnableRecommendationFlowLogs)) { - val log = flowData.toRecommendationFlowLogOfflineThrift - logRecommendationFlowCounter.incr() - flowLogger.info( - serializeThrift( - log, - FrsLogger.FlowLogCategory, - FrsLogger.mkProvider(request.clientContext) - )) - } - } - - // We prefer the settings given in the user request, and if none provided we default to the - // aurora service configuration. - def shouldLog(debugParamsOpt: Option[DebugParams]): Boolean = - debugParamsOpt match { - case Some(debugParams) => - debugParams.debugOptions match { - case Some(debugOptions) => - !debugOptions.doNotLog - case None => - serviceShouldLogResults - } - case None => - serviceShouldLogResults - } - -} - -object FrsLogger { - val LogCategory = "follow_recommendations_logs" - val FlowLogCategory = "frs_recommendation_flow_logs" - - def mkProvider(clientContext: ClientContext) = new ClientDataProvider { - - /** The id of the current user. When the user is logged out, this method should return None. */ - override val userId: Option[Long] = clientContext.userId - - /** The id of the guest, which is present in logged-in or loged-out states */ - override val guestId: Option[Long] = clientContext.guestId - - /** The personalization id (pid) of the user, used to personalize Twitter services */ - override val personalizationId: Option[String] = None - - /** The id of the individual device the user is currently using. This id will be unique for different users' devices. */ - override val deviceId: Option[String] = clientContext.deviceId - - /** The OAuth application id of the application the user is currently using */ - override val clientApplicationId: Option[Long] = clientContext.appId - - /** The OAuth parent application id of the application the user is currently using */ - override val parentApplicationId: Option[Long] = None - - /** The two-letter, upper-case country code used to designate the country from which the scribe event occurred */ - override val countryCode: Option[String] = clientContext.countryCode - - /** The two-letter, lower-case language code used to designate the probably language spoken by the scribe event initiator */ - override val languageCode: Option[String] = clientContext.languageCode - - /** The user-agent header used to identify the client browser or device that the user is currently active on */ - override val userAgent: Option[String] = clientContext.userAgent - - /** Whether the user is accessing Twitter via a secured connection */ - override val isSsl: Option[Boolean] = Some(true) - - /** The referring URL to the current page for web-based clients, if applicable */ - override val referer: Option[String] = None - - /** - * The external site, partner, or email that lead to the current Twitter application. Returned value consists of a - * tuple including the encrypted referral data and the type of referral - */ - override val externalReferer: Option[ExternalRefererDataProvider] = None - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD deleted file mode 100644 index 597ab76c4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.scala deleted file mode 100644 index 38215c44b..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateSourceType.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.models - -object CandidateSourceType extends Enumeration { - type CandidateSourceType = Value - val Social = Value("social") - val GeoAndInterests = Value("geo_and_interests") - val ActivityContextual = Value("activity_contextual") - val None = Value("none") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.scala deleted file mode 100644 index a5702b2b3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/CandidateUserDebugParams.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.timelines.configapi.Params - -case class CandidateUserDebugParams(paramsMap: Map[Long, Params]) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.scala deleted file mode 100644 index dee7f9b6a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DebugParams.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.common.models.DebugOptions.fromDebugParamsThrift -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.timelines.configapi.{FeatureValue => ConfigApiFeatureValue} - -case class DebugParams( - featureOverrides: Option[Map[String, ConfigApiFeatureValue]], - debugOptions: Option[DebugOptions]) - -object DebugParams { - def fromThrift(thrift: t.DebugParams): DebugParams = DebugParams( - featureOverrides = thrift.featureOverrides.map { map => - map.mapValues(FeatureValue.fromThrift).toMap - }, - debugOptions = Some( - fromDebugParamsThrift(thrift) - ) - ) - def toOfflineThrift(model: DebugParams): offline.OfflineDebugParams = - offline.OfflineDebugParams(randomizationSeed = model.debugOptions.flatMap(_.randomizationSeed)) -} - -trait HasFrsDebugParams { - def frsDebugParams: Option[DebugParams] -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.scala deleted file mode 100644 index 59f0adfd7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/DisplayContext.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.FlowContext -import com.twitter.follow_recommendations.common.models.RecentlyEngagedUserId -import com.twitter.follow_recommendations.logging.thriftscala.OfflineDisplayContext -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} -import scala.reflect.ClassTag -import scala.reflect.classTag - -trait DisplayContext { - def toOfflineThrift: offline.OfflineDisplayContext -} - -object DisplayContext { - case class Profile(profileId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Profile(offline.OfflineProfile(profileId)) - } - case class Search(searchQuery: String) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Search(offline.OfflineSearch(searchQuery)) - } - case class Rux(focalAuthorId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Rux(offline.OfflineRux(focalAuthorId)) - } - - case class Topic(topicId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.Topic(offline.OfflineTopic(topicId)) - } - - case class ReactiveFollow(followedUserIds: Seq[Long]) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.ReactiveFollow(offline.OfflineReactiveFollow(followedUserIds)) - } - - case class NuxInterests(flowContext: Option[FlowContext], uttInterestIds: Option[Seq[Long]]) - extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.NuxInterests( - offline.OfflineNuxInterests(flowContext.map(_.toOfflineThrift))) - } - - case class PostNuxFollowTask(flowContext: Option[FlowContext]) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.PostNuxFollowTask( - offline.OfflinePostNuxFollowTask(flowContext.map(_.toOfflineThrift))) - } - - case class AdCampaignTarget(similarToUserIds: Seq[Long]) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.AdCampaignTarget( - offline.OfflineAdCampaignTarget(similarToUserIds)) - } - - case class ConnectTab( - byfSeedUserIds: Seq[Long], - similarToUserIds: Seq[Long], - engagedUserIds: Seq[RecentlyEngagedUserId]) - extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.ConnectTab( - offline.OfflineConnectTab( - byfSeedUserIds, - similarToUserIds, - engagedUserIds.map(user => user.toOfflineThrift))) - } - - case class SimilarToUser(similarToUserId: Long) extends DisplayContext { - override val toOfflineThrift: OfflineDisplayContext = - offline.OfflineDisplayContext.SimilarToUser(offline.OfflineSimilarToUser(similarToUserId)) - } - - def fromThrift(tDisplayContext: t.DisplayContext): DisplayContext = tDisplayContext match { - case t.DisplayContext.Profile(p) => Profile(p.profileId) - case t.DisplayContext.Search(s) => Search(s.searchQuery) - case t.DisplayContext.Rux(r) => Rux(r.focalAuthorId) - case t.DisplayContext.Topic(t) => Topic(t.topicId) - case t.DisplayContext.ReactiveFollow(f) => ReactiveFollow(f.followedUserIds) - case t.DisplayContext.NuxInterests(n) => - NuxInterests(n.flowContext.map(FlowContext.fromThrift), n.uttInterestIds) - case t.DisplayContext.AdCampaignTarget(a) => - AdCampaignTarget(a.similarToUserIds) - case t.DisplayContext.ConnectTab(connect) => - ConnectTab( - connect.byfSeedUserIds, - connect.similarToUserIds, - connect.recentlyEngagedUserIds.map(RecentlyEngagedUserId.fromThrift)) - case t.DisplayContext.SimilarToUser(r) => - SimilarToUser(r.similarToUserId) - case t.DisplayContext.PostNuxFollowTask(p) => - PostNuxFollowTask(p.flowContext.map(FlowContext.fromThrift)) - case t.DisplayContext.UnknownUnionField(t) => - throw new UnknownDisplayContextException(t.field.name) - } - - def getDisplayContextAs[T <: DisplayContext: ClassTag](displayContext: DisplayContext): T = - displayContext match { - case context: T => context - case _ => - throw new UnexpectedDisplayContextTypeException( - displayContext, - classTag[T].getClass.getSimpleName) - } -} - -class UnknownDisplayContextException(name: String) - extends Exception(s"Unknown DisplayContext in Thrift: ${name}") - -class UnexpectedDisplayContextTypeException(displayContext: DisplayContext, expectedType: String) - extends Exception(s"DisplayContext ${displayContext} not of expected type ${expectedType}") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.scala deleted file mode 100644 index 66f0afafa..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/FeatureValue.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.timelines.configapi._ - -object FeatureValue { - def fromThrift(thriftFeatureValue: t.FeatureValue): FeatureValue = thriftFeatureValue match { - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.BoolValue(bool)) => - BooleanFeatureValue(bool) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.StrValue(string)) => - StringFeatureValue(string) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.IntValue(int)) => - NumberFeatureValue(int) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.LongValue(long)) => - NumberFeatureValue(long) - case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.UnknownUnionField(field)) => - throw new UnknownFeatureValueException(s"Primitive: ${field.field.name}") - case t.FeatureValue.UnknownUnionField(field) => - throw new UnknownFeatureValueException(field.field.name) - } -} - -class UnknownFeatureValueException(fieldName: String) - extends Exception(s"Unknown FeatureValue name in thrift: ${fieldName}") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.scala deleted file mode 100644 index 06b19ac46..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationFlowData.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.HasUserState -import com.twitter.follow_recommendations.common.utils.UserSignupUtil -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.util.Time - -case class RecommendationFlowData[Target <: HasClientContext]( - request: Target, - recommendationFlowIdentifier: RecommendationPipelineIdentifier, - candidateSources: Seq[CandidateSource[Target, CandidateUser]], - candidatesFromCandidateSources: Seq[CandidateUser], - mergedCandidates: Seq[CandidateUser], - filteredCandidates: Seq[CandidateUser], - rankedCandidates: Seq[CandidateUser], - transformedCandidates: Seq[CandidateUser], - truncatedCandidates: Seq[CandidateUser], - results: Seq[CandidateUser]) - extends HasMarshalling { - - import RecommendationFlowData._ - - lazy val toRecommendationFlowLogOfflineThrift: offline.RecommendationFlowLog = { - val userMetadata = userToOfflineRecommendationFlowUserMetadata(request) - val signals = userToOfflineRecommendationFlowSignals(request) - val filteredCandidateSourceCandidates = - candidatesToOfflineRecommendationFlowCandidateSourceCandidates( - candidateSources, - filteredCandidates - ) - val rankedCandidateSourceCandidates = - candidatesToOfflineRecommendationFlowCandidateSourceCandidates( - candidateSources, - rankedCandidates - ) - val truncatedCandidateSourceCandidates = - candidatesToOfflineRecommendationFlowCandidateSourceCandidates( - candidateSources, - truncatedCandidates - ) - - offline.RecommendationFlowLog( - ClientContextConverter.toFRSOfflineClientContextThrift(request.clientContext), - userMetadata, - signals, - Time.now.inMillis, - recommendationFlowIdentifier.name, - Some(filteredCandidateSourceCandidates), - Some(rankedCandidateSourceCandidates), - Some(truncatedCandidateSourceCandidates) - ) - } -} - -object RecommendationFlowData { - def userToOfflineRecommendationFlowUserMetadata[Target <: HasClientContext]( - request: Target - ): Option[offline.OfflineRecommendationFlowUserMetadata] = { - val userSignupAge = UserSignupUtil.userSignupAge(request).map(_.inDays) - val userState = request match { - case req: HasUserState => req.userState.map(_.name) - case _ => None - } - Some(offline.OfflineRecommendationFlowUserMetadata(userSignupAge, userState)) - } - - def userToOfflineRecommendationFlowSignals[Target <: HasClientContext]( - request: Target - ): Option[offline.OfflineRecommendationFlowSignals] = { - val countryCode = request.getCountryCode - Some(offline.OfflineRecommendationFlowSignals(countryCode)) - } - - def candidatesToOfflineRecommendationFlowCandidateSourceCandidates[Target <: HasClientContext]( - candidateSources: Seq[CandidateSource[Target, CandidateUser]], - candidates: Seq[CandidateUser], - ): Seq[offline.OfflineRecommendationFlowCandidateSourceCandidates] = { - val candidatesGroupedByCandidateSources = - candidates.groupBy( - _.getPrimaryCandidateSource.getOrElse(CandidateSourceIdentifier("NoCandidateSource"))) - - candidateSources.map(candidateSource => { - val candidates = - candidatesGroupedByCandidateSources.get(candidateSource.identifier).toSeq.flatten - val candidateUserIds = candidates.map(_.id) - val candidateUserScores = candidates.map(_.score).exists(_.nonEmpty) match { - case true => Some(candidates.map(_.score.getOrElse(-1.0))) - case false => None - } - offline.OfflineRecommendationFlowCandidateSourceCandidates( - candidateSource.identifier.name, - candidateUserIds, - candidateUserScores - ) - }) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.scala deleted file mode 100644 index fa768b536..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationRequest.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.ClientContextConverter -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext - -case class RecommendationRequest( - clientContext: ClientContext, - displayLocation: DisplayLocation, - displayContext: Option[DisplayContext], - maxResults: Option[Int], - cursor: Option[String], - excludedIds: Option[Seq[Long]], - fetchPromotedContent: Option[Boolean], - debugParams: Option[DebugParams] = None, - userLocationState: Option[String] = None, - isSoftUser: Boolean = false) { - def toOfflineThrift: offline.OfflineRecommendationRequest = offline.OfflineRecommendationRequest( - ClientContextConverter.toFRSOfflineClientContextThrift(clientContext), - displayLocation.toOfflineThrift, - displayContext.map(_.toOfflineThrift), - maxResults, - cursor, - excludedIds, - fetchPromotedContent, - debugParams.map(DebugParams.toOfflineThrift) - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.scala deleted file mode 100644 index fadff377b..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/RecommendationResponse.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.{thriftscala => t} -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling - -case class RecommendationResponse(recommendations: Seq[Recommendation]) extends HasMarshalling { - lazy val toThrift: t.RecommendationResponse = - t.RecommendationResponse(recommendations.map(_.toThrift)) - - lazy val toOfflineThrift: offline.OfflineRecommendationResponse = - offline.OfflineRecommendationResponse(recommendations.map(_.toOfflineThrift)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.scala deleted file mode 100644 index a8798bda2..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/Request.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.ProductContext -import com.twitter.product_mixer.core.model.marshalling.request.{Request => ProductMixerRequest} - -case class Request( - override val maxResults: Option[Int], - override val debugParams: Option[request.DebugParams], - override val productContext: Option[ProductContext], - override val product: request.Product, - override val clientContext: ClientContext, - override val serializedRequestCursor: Option[String], - override val frsDebugParams: Option[DebugParams], - displayLocation: DisplayLocation, - excludedIds: Option[Seq[Long]], - fetchPromotedContent: Option[Boolean], - userLocationState: Option[String] = None) - extends ProductMixerRequest - with HasFrsDebugParams {} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.scala deleted file mode 100644 index 84d9d3ee3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserRequest.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature -import com.twitter.follow_recommendations.common.models._ -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.product_mixer.core.model.marshalling.request.ClientContext -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.timelines.configapi.HasParams -import com.twitter.timelines.configapi.Params - -case class ScoringUserRequest( - override val clientContext: ClientContext, - override val displayLocation: DisplayLocation, - override val params: Params, - override val debugOptions: Option[DebugOptions] = None, - override val recentFollowedUserIds: Option[Seq[Long]], - override val recentFollowedByUserIds: Option[Seq[Long]], - override val wtfImpressions: Option[Seq[WtfImpression]], - override val similarToUserIds: Seq[Long], - candidates: Seq[CandidateUser], - debugParams: Option[DebugParams] = None, - isSoftUser: Boolean = false) - extends HasClientContext - with HasDisplayLocation - with HasParams - with HasDebugOptions - with HasPreFetchedFeature - with HasSimilarToContext { - def toOfflineThrift: offline.OfflineScoringUserRequest = offline.OfflineScoringUserRequest( - ClientContextConverter.toFRSOfflineClientContextThrift(clientContext), - displayLocation.toOfflineThrift, - candidates.map(_.toOfflineUserThrift) - ) - def toRecommendationRequest: RecommendationRequest = RecommendationRequest( - clientContext = clientContext, - displayLocation = displayLocation, - displayContext = None, - maxResults = None, - cursor = None, - excludedIds = None, - fetchPromotedContent = None, - debugParams = debugParams, - isSoftUser = isSoftUser - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.scala deleted file mode 100644 index 4611386d3..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/ScoringUserResponse.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.follow_recommendations.models - -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.logging.{thriftscala => offline} -import com.twitter.follow_recommendations.{thriftscala => t} - -case class ScoringUserResponse(candidates: Seq[CandidateUser]) { - lazy val toThrift: t.ScoringUserResponse = - t.ScoringUserResponse(candidates.map(_.toUserThrift)) - - lazy val toRecommendationResponse: RecommendationResponse = RecommendationResponse(candidates) - - lazy val toOfflineThrift: offline.OfflineScoringUserResponse = - offline.OfflineScoringUserResponse(candidates.map(_.toOfflineUserThrift)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD deleted file mode 100644 index 4874d636e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.scala deleted file mode 100644 index 023b4c63e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models/failures/TimeoutPipelineFailure.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.models.failures - -import com.twitter.product_mixer.core.pipeline.pipeline_failure.CandidateSourceTimeout -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure - -object TimeoutPipelineFailure { - def apply(candidateSourceName: String): PipelineFailure = { - PipelineFailure( - CandidateSourceTimeout, - s"Candidate Source $candidateSourceName timed out before returning candidates") - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.scala deleted file mode 100644 index b75b6753e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ABDeciderModule.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.abdecider.ABDeciderFactory -import com.twitter.abdecider.LoggingABDecider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.inject.TwitterModule -import com.twitter.logging.LoggerFactory -import javax.inject.Singleton - -object ABDeciderModule extends TwitterModule { - @Provides - @Singleton - def provideABDecider( - stats: StatsReceiver, - @Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) factory: LoggerFactory - ): LoggingABDecider = { - - val ymlPath = "/usr/local/config/abdecider/abdecider.yml" - - val abDeciderFactory = ABDeciderFactory( - abDeciderYmlPath = ymlPath, - scribeLogger = Some(factory()), - environment = Some("production") - ) - - abDeciderFactory.buildWithLogging() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD deleted file mode 100644 index e7fb68380..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.scala deleted file mode 100644 index ef3865bf2..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ConfigApiModule.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.twitter.decider.Decider -import com.twitter.follow_recommendations.configapi.ConfigBuilder -import com.twitter.inject.TwitterModule -import com.twitter.servo.decider.DeciderGateBuilder -import com.twitter.timelines.configapi.Config -import javax.inject.Singleton - -object ConfigApiModule extends TwitterModule { - @Provides - @Singleton - def providesDeciderGateBuilder(decider: Decider): DeciderGateBuilder = - new DeciderGateBuilder(decider) - - @Provides - @Singleton - def providesConfig(configBuilder: ConfigBuilder): Config = configBuilder.build() -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.scala deleted file mode 100644 index 4ab0e4eba..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/DiffyModule.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.inject.annotations.Flag -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.annotations.DarkTrafficService -import com.twitter.follow_recommendations.configapi.deciders.DeciderKey -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.inject.TwitterModule -import com.twitter.inject.thrift.filters.DarkTrafficFilter -import com.twitter.servo.decider.DeciderGateBuilder - -object DiffyModule extends TwitterModule { - // diffy.dest is defined in the Follow Recommendations Service aurora file - // and points to the Dark Traffic Proxy server - private val destFlag = - flag[String]("diffy.dest", "/$/nil", "Resolvable name of diffy-service or proxy") - - @Provides - @Singleton - @DarkTrafficService - def provideDarkTrafficService( - serviceIdentifier: ServiceIdentifier - ): FollowRecommendationsThriftService.ReqRepServicePerEndpoint = { - ThriftMux.client - .withClientId(ClientId("follow_recos_service_darktraffic_proxy_client")) - .withMutualTls(serviceIdentifier) - .servicePerEndpoint[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]( - dest = destFlag(), - label = "darktrafficproxy" - ) - } - - @Provides - @Singleton - def provideDarkTrafficFilter( - @DarkTrafficService darkService: FollowRecommendationsThriftService.ReqRepServicePerEndpoint, - deciderGateBuilder: DeciderGateBuilder, - statsReceiver: StatsReceiver, - @Flag("environment") env: String - ): DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint] = { - // sampleFunction is used to determine which requests should get replicated - // to the dark traffic proxy server - val sampleFunction: Any => Boolean = { _ => - // check whether the current FRS instance is deployed in production - env match { - case "prod" => - statsReceiver.scope("provideDarkTrafficFilter").counter("prod").incr() - destFlag.isDefined && deciderGateBuilder - .keyToFeature(DeciderKey.EnableTrafficDarkReading).isAvailable(RandomRecipient) - case _ => - statsReceiver.scope("provideDarkTrafficFilter").counter("devel").incr() - // replicate zero requests if in non-production environment - false - } - } - new DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]( - darkService, - sampleFunction, - forwardAfterService = true, - statsReceiver.scope("DarkTrafficFilter"), - lookupByMethod = true - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.scala deleted file mode 100644 index 1600344b6..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FeatureSwitchesModule.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.twitter.abdecider.LoggingABDecider -import com.twitter.featureswitches.v2.Feature -import com.twitter.featureswitches.v2.FeatureFilter -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.v2.builder.FeatureSwitchesBuilder -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES -import com.twitter.inject.TwitterModule -import javax.inject.Named -import javax.inject.Singleton - -object FeaturesSwitchesModule extends TwitterModule { - private val DefaultConfigRepoPath = "/usr/local/config" - private val FeaturesPath = "/features/onboarding/follow-recommendations-service/main" - val isLocal = flag("configrepo.local", false, "Is the server running locally or in a DC") - val localConfigRepoPath = flag( - "local.configrepo", - System.getProperty("user.home") + "/workspace/config", - "Path to your local config repo" - ) - - @Provides - @Singleton - def providesFeatureSwitches( - abDecider: LoggingABDecider, - statsReceiver: StatsReceiver - ): FeatureSwitches = { - val configRepoPath = if (isLocal()) { - localConfigRepoPath() - } else { - DefaultConfigRepoPath - } - - FeatureSwitchesBuilder - .createDefault(FeaturesPath, abDecider, Some(statsReceiver)) - .configRepoAbsPath(configRepoPath) - .serviceDetailsFromAurora() - .build() - } - - @Provides - @Singleton - @Named(PRODUCER_SIDE_FEATURE_SWITCHES) - def providesProducerFeatureSwitches( - abDecider: LoggingABDecider, - statsReceiver: StatsReceiver - ): FeatureSwitches = { - val configRepoPath = if (isLocal()) { - localConfigRepoPath() - } else { - DefaultConfigRepoPath - } - - /** - * Feature Switches evaluate all tied FS Keys on Params construction time, which is very inefficient - * for producer/candidate side holdbacks because we have 100s of candidates, and 100s of FS which result - * in 10,000 FS evaluations when we want 1 per candidate (100 total), so we create a new FS Client - * which has a [[ProducerFeatureFilter]] set for feature filter to reduce the FS Keys we evaluate. - */ - FeatureSwitchesBuilder - .createDefault(FeaturesPath, abDecider, Some(statsReceiver.scope("producer_side_fs"))) - .configRepoAbsPath(configRepoPath) - .serviceDetailsFromAurora() - .addFeatureFilter(ProducerFeatureFilter) - .build() - } -} - -case object ProducerFeatureFilter extends FeatureFilter { - private val AllowedKeys = Set( - "post_nux_ml_flow_candidate_user_scorer_id", - "frs_receiver_holdback_keep_social_user_candidate", - "frs_receiver_holdback_keep_user_candidate") - - override def filter(feature: Feature): Option[Feature] = { - if (AllowedKeys.exists(feature.parameters.contains)) { - Some(feature) - } else { - None - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala deleted file mode 100644 index f8ff5ae94..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.follow_recommendations.modules -import com.twitter.inject.TwitterModule - -object FlagsModule extends TwitterModule { - flag[Boolean]( - name = "fetch_prod_promoted_accounts", - help = "Whether or not to fetch production promoted accounts (true / false)" - ) - flag[Boolean]( - name = "interests_opt_out_prod_enabled", - help = "Whether to fetch intersts opt out data from the prod strato column or not" - ) - flag[Boolean]( - name = "log_results", - default = false, - help = "Whether to log results such that we use them for scoring or metrics" - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.scala deleted file mode 100644 index 218f3b973..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ProductRegistryModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.twitter.follow_recommendations.products.ProdProductRegistry -import com.twitter.follow_recommendations.products.common.ProductRegistry -import com.twitter.inject.TwitterModule -import javax.inject.Singleton - -object ProductRegistryModule extends TwitterModule { - override protected def configure(): Unit = { - bind[ProductRegistry].to[ProdProductRegistry].in[Singleton] - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.scala deleted file mode 100644 index 035cc04bf..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScorerModule.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.inject.TwitterModule -import com.twitter.relevance.ep_model.common.CommonConstants -import com.twitter.relevance.ep_model.scorer.EPScorer -import com.twitter.relevance.ep_model.scorer.EPScorerBuilder -import java.io.File -import java.io.FileOutputStream -import scala.language.postfixOps - -object ScorerModule extends TwitterModule { - private val STPScorerPath = "/quality/stp_models/20141223" - - private def fileFromResource(resource: String): File = { - val inputStream = getClass.getResourceAsStream(resource) - val file = File.createTempFile(resource, "temp") - val fos = new FileOutputStream(file) - Iterator - .continually(inputStream.read) - .takeWhile(-1 !=) - .foreach(fos.write) - file - } - - @Provides - @Singleton - def provideEpScorer: EPScorer = { - val modelPath = - fileFromResource(STPScorerPath + "/" + CommonConstants.EP_MODEL_FILE_NAME).getAbsolutePath - val trainingConfigPath = - fileFromResource(STPScorerPath + "/" + CommonConstants.TRAINING_CONFIG).getAbsolutePath - val epScorer = new EPScorerBuilder - epScorer - .withModelPath(modelPath) - .withTrainingConfig(trainingConfigPath) - .build() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.scala deleted file mode 100644 index 35af77c1a..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/ScribeModule.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants -import com.twitter.inject.TwitterModule -import com.twitter.logging.BareFormatter -import com.twitter.logging.HandlerFactory -import com.twitter.logging.Level -import com.twitter.logging.LoggerFactory -import com.twitter.logging.NullHandler -import com.twitter.logging.QueueingHandler -import com.twitter.logging.ScribeHandler - -object ScribeModule extends TwitterModule { - val useProdLogger = flag( - name = "scribe.use_prod_loggers", - default = false, - help = "whether to use production logging for service" - ) - - @Provides - @Singleton - @Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) - def provideClientEventsLoggerFactory(stats: StatsReceiver): LoggerFactory = { - val loggerCategory = "client_event" - val clientEventsHandler: HandlerFactory = if (useProdLogger()) { - QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = loggerCategory, - formatter = BareFormatter, - level = Some(Level.INFO), - statsReceiver = stats.scope("client_event_scribe") - ) - ) - } else { () => NullHandler } - LoggerFactory( - node = "abdecider", - level = Some(Level.INFO), - useParents = false, - handlers = clientEventsHandler :: Nil - ) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.REQUEST_LOGGER) - def provideFollowRecommendationsLoggerFactory(stats: StatsReceiver): LoggerFactory = { - val loggerCategory = "follow_recommendations_logs" - val handlerFactory: HandlerFactory = if (useProdLogger()) { - QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = loggerCategory, - formatter = BareFormatter, - level = Some(Level.INFO), - statsReceiver = stats.scope("follow_recommendations_logs_scribe") - ) - ) - } else { () => NullHandler } - LoggerFactory( - node = loggerCategory, - level = Some(Level.INFO), - useParents = false, - handlers = handlerFactory :: Nil - ) - } - - @Provides - @Singleton - @Named(GuiceNamedConstants.FLOW_LOGGER) - def provideFrsRecommendationFlowLoggerFactory(stats: StatsReceiver): LoggerFactory = { - val loggerCategory = "frs_recommendation_flow_logs" - val handlerFactory: HandlerFactory = if (useProdLogger()) { - QueueingHandler( - maxQueueSize = 10000, - handler = ScribeHandler( - category = loggerCategory, - formatter = BareFormatter, - level = Some(Level.INFO), - statsReceiver = stats.scope("frs_recommendation_flow_logs_scribe") - ) - ) - } else { () => NullHandler } - LoggerFactory( - node = loggerCategory, - level = Some(Level.INFO), - useParents = false, - handlers = handlerFactory :: Nil - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.scala deleted file mode 100644 index 0572e43bf..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/TimerModule.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.follow_recommendations.modules - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.memcached.ZookeeperStateMonitor.DefaultTimer -import com.twitter.inject.TwitterModule -import com.twitter.util.Timer - -object TimerModule extends TwitterModule { - @Provides - @Singleton - def providesTimer: Timer = DefaultTimer -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD deleted file mode 100644 index 5840c0f2f..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.scala deleted file mode 100644 index 9a0dbb995..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/ProdProductRegistry.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.follow_recommendations.products - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.products.common.ProductRegistry -import com.twitter.follow_recommendations.products.explore_tab.ExploreTabProduct -import com.twitter.follow_recommendations.products.home_timeline.HomeTimelineProduct -import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.HomeTimelineTweetRecsProduct -import com.twitter.follow_recommendations.products.sidebar.SidebarProduct - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ProdProductRegistry @Inject() ( - exploreTabProduct: ExploreTabProduct, - homeTimelineProduct: HomeTimelineProduct, - homeTimelineTweetRecsProduct: HomeTimelineTweetRecsProduct, - sidebarProduct: SidebarProduct, -) extends ProductRegistry { - - override val products: Seq[common.Product] = - Seq( - exploreTabProduct, - homeTimelineProduct, - homeTimelineTweetRecsProduct, - sidebarProduct - ) - - override val displayLocationProductMap: Map[DisplayLocation, common.Product] = - products.groupBy(_.displayLocation).flatMap { - case (loc, products) => - assert(products.size == 1, s"Found more than 1 Product for ${loc}") - products.headOption.map { product => loc -> product } - } - - override def getProductByDisplayLocation(displayLocation: DisplayLocation): common.Product = { - displayLocationProductMap.getOrElse( - displayLocation, - throw new MissingProductException(displayLocation)) - } -} - -class MissingProductException(displayLocation: DisplayLocation) - extends Exception(s"No Product found for ${displayLocation}") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD deleted file mode 100644 index 4b32816e4..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.scala deleted file mode 100644 index c00d8c407..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Exceptions.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.products.common - -abstract class ProductException(message: String) extends Exception(message) - -class MissingFieldException(productRequest: ProductRequest, fieldName: String) - extends ProductException( - s"Missing ${fieldName} field for ${productRequest.recommendationRequest.displayLocation} request") diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.scala deleted file mode 100644 index 28c348204..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/Product.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.follow_recommendations.products.common - -import com.twitter.follow_recommendations.assembler.models.Layout -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.product_mixer.core.model.marshalling.request.{Product => ProductMixerProduct} -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params - -trait Product { - - /** Each product also requires a human-readable name. - * You can change this at any time - */ - def name: String - - /** - * Every product needs a machine-friendly identifier for internal use. - * You should use the same name as the product package name. - * Except dashes are better than underscore - * - * Avoid changing this once it's in production. - */ - def identifier: String - - def displayLocation: DisplayLocation - - def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] - - /** - * Blender is responsible for blending together the candidates generated by different flows used - * in a product. For example, if a product uses two flows, it is blender's responsibility to - * interleave their generated candidates together and make a unified sequence of candidates. - */ - def blender: Transform[ProductRequest, Recommendation] - - /** - * It is resultsTransformer job to do any final transformations needed on the final list of - * candidates generated by a product. For example, if a final quality check on candidates needed, - * resultsTransformer will handle it. - */ - def resultsTransformer(request: ProductRequest): Stitch[Transform[ProductRequest, Recommendation]] - - def enabled(request: ProductRequest): Stitch[Boolean] - - def layout: Option[Layout] = None - - def productMixerProduct: Option[ProductMixerProduct] = None -} - -case class ProductRequest(recommendationRequest: RecommendationRequest, params: Params) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.scala deleted file mode 100644 index fbe486536..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common/ProductRegistry.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.products.common - -import com.twitter.follow_recommendations.common.models.DisplayLocation - -trait ProductRegistry { - def products: Seq[Product] - def displayLocationProductMap: Map[DisplayLocation, Product] - def getProductByDisplayLocation(displayLocation: DisplayLocation): Product -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD deleted file mode 100644 index 2f9412612..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.scala deleted file mode 100644 index a49fccb45..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/ExploreTabProduct.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.products.explore_tab - -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.explore_tab.configapi.ExploreTabParams -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ExploreTabProduct @Inject() ( - postNuxMlFlow: PostNuxMlFlow, - postNuxMlRequestBuilder: PostNuxMlRequestBuilder) - extends Product { - override val name: String = "Explore Tab" - - override val identifier: String = "explore-tab" - - override val displayLocation: DisplayLocation = DisplayLocation.ExploreTab - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => - Seq(postNuxMlFlow.mapKey({ _: ProductRequest => postNuxMlRequest })) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = - new IdentityTransform[ProductRequest, Recommendation] - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = - Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) - - override def enabled(request: ProductRequest): Stitch[Boolean] = { - // Ideally we should hook up is_soft_user as custom FS field and disable the product through FS - val enabledForUserType = !request.recommendationRequest.isSoftUser || request.params( - ExploreTabParams.EnableProductForSoftUser) - Stitch.value(request.params(ExploreTabParams.EnableProduct) && enabledForUserType) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD deleted file mode 100644 index 3bb732e35..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.scala deleted file mode 100644 index 092252aca..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabFSConfig.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.follow_recommendations.products.explore_tab.configapi - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.products.explore_tab.configapi.ExploreTabParams._ -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ExploreTabFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq(EnableProductForSoftUser) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.scala deleted file mode 100644 index b9d9d3b87..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/explore_tab/configapi/ExploreTabParams.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.follow_recommendations.products.explore_tab.configapi - -import com.twitter.timelines.configapi.Param -import com.twitter.timelines.configapi.FSParam - -object ExploreTabParams { - object EnableProduct extends Param[Boolean](false) - object EnableProductForSoftUser - extends FSParam[Boolean]("explore_tab_enable_product_for_soft_user", false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD deleted file mode 100644 index 4b0586ff7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.scala deleted file mode 100644 index a2051c150..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HTLProductMixer.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline - -import com.twitter.product_mixer.core.model.common.identifier.ProductIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product - -case object HTLProductMixer extends Product { - override val identifier: ProductIdentifier = ProductIdentifier("HomeTimeline") - override val stringCenterProject: Option[String] = Some("people-discovery") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.scala deleted file mode 100644 index 590aab182..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineProduct.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline - -import com.twitter.follow_recommendations.assembler.models.ActionConfig -import com.twitter.follow_recommendations.assembler.models.FollowedByUsersProof -import com.twitter.follow_recommendations.assembler.models.FooterConfig -import com.twitter.follow_recommendations.assembler.models.GeoContextProof -import com.twitter.follow_recommendations.assembler.models.HeaderConfig -import com.twitter.follow_recommendations.assembler.models.Layout -import com.twitter.follow_recommendations.assembler.models.TitleConfig -import com.twitter.follow_recommendations.assembler.models.UserListLayout -import com.twitter.follow_recommendations.assembler.models.UserListOptions -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlow -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlowRequest -import com.twitter.follow_recommendations.blenders.PromotedAccountsBlender -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.home_timeline.configapi.HomeTimelineParams._ -import com.twitter.inject.Injector -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.product.guice.ProductScope -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeTimelineProduct @Inject() ( - postNuxMlFlow: PostNuxMlFlow, - postNuxMlRequestBuilder: PostNuxMlRequestBuilder, - promotedAccountsFlow: PromotedAccountsFlow, - promotedAccountsBlender: PromotedAccountsBlender, - productScope: ProductScope, - injector: Injector, -) extends Product { - - override val name: String = "Home Timeline" - - override val identifier: String = "home-timeline" - - override val displayLocation: DisplayLocation = DisplayLocation.HomeTimeline - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => - Seq( - postNuxMlFlow.mapKey({ request: ProductRequest => postNuxMlRequest }), - promotedAccountsFlow.mapKey(mkPromotedAccountsRequest)) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = { - promotedAccountsBlender.mapTarget[ProductRequest](getMaxResults) - } - - private val identityTransform = new IdentityTransform[ProductRequest, Recommendation] - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = Stitch.value(identityTransform) - - override def enabled(request: ProductRequest): Stitch[Boolean] = - Stitch.value(request.params(EnableProduct)) - - override def layout: Option[Layout] = { - productMixerProduct.map { product => - val homeTimelineStrings = productScope.let(product) { - injector.instance[HomeTimelineStrings] - } - UserListLayout( - header = Some(HeaderConfig(TitleConfig(homeTimelineStrings.whoToFollowModuleTitle))), - userListOptions = UserListOptions(userBioEnabled = true, userBioTruncated = true, None), - socialProofs = Some( - Seq( - FollowedByUsersProof( - homeTimelineStrings.whoToFollowFollowedByManyUserSingleString, - homeTimelineStrings.whoToFollowFollowedByManyUserDoubleString, - homeTimelineStrings.whoToFollowFollowedByManyUserMultipleString - ), - GeoContextProof(homeTimelineStrings.whoToFollowPopularInCountryKey) - )), - footer = Some( - FooterConfig( - Some(ActionConfig(homeTimelineStrings.whoToFollowModuleFooter, "http://twitter.com")))) - ) - } - } - - override def productMixerProduct: Option[request.Product] = Some(HTLProductMixer) - - private[home_timeline] def mkPromotedAccountsRequest( - req: ProductRequest - ): PromotedAccountsFlowRequest = { - PromotedAccountsFlowRequest( - req.recommendationRequest.clientContext, - req.params, - req.recommendationRequest.displayLocation, - None, - req.recommendationRequest.excludedIds.getOrElse(Nil) - ) - } - - private[home_timeline] def getMaxResults(req: ProductRequest): Int = { - req.recommendationRequest.maxResults.getOrElse( - req.params(DefaultMaxResults) - ) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.scala deleted file mode 100644 index 75819555e..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/HomeTimelineStrings.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline - -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.ExternalStringRegistry -import com.twitter.stringcenter.client.core.ExternalString -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class HomeTimelineStrings @Inject() ( - @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { - private val externalStringRegistry = externalStringRegistryProvider.get() - val whoToFollowFollowedByManyUserSingleString: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserSingle") - val whoToFollowFollowedByManyUserDoubleString: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserDouble") - val whoToFollowFollowedByManyUserMultipleString: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.followedByManyUserMultiple") - val whoToFollowPopularInCountryKey: ExternalString = - externalStringRegistry.createProdString("WtfRecommendationContext.popularInCountry") - val whoToFollowModuleTitle: ExternalString = - externalStringRegistry.createProdString("WhoToFollowModule.title") - val whoToFollowModuleFooter: ExternalString = - externalStringRegistry.createProdString("WhoToFollowModule.pivot") -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD deleted file mode 100644 index 3bb732e35..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.scala deleted file mode 100644 index 15e97b3a5..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineFSConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline.configapi - -import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig -import com.twitter.follow_recommendations.products.home_timeline.configapi.HomeTimelineParams._ -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeTimelineFSConfig @Inject() () extends FeatureSwitchConfig { - override val booleanFSParams: Seq[Param[Boolean] with FSName] = - Seq(EnableWritingServingHistory) - - override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq( - DurationGuardrailToForceSuggest, - SuggestBasedFatigueDuration - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.scala deleted file mode 100644 index 65ab5ae23..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline/configapi/HomeTimelineParams.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline.configapi - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object HomeTimelineParams { - object EnableProduct extends Param[Boolean](false) - - object DefaultMaxResults extends Param[Int](20) - - object EnableWritingServingHistory - extends FSParam[Boolean]("home_timeline_enable_writing_serving_history", false) - - object DurationGuardrailToForceSuggest - extends FSBoundedParam[Duration]( - name = "home_timeline_duration_guardrail_to_force_suggest_in_hours", - default = 0.hours, - min = 0.hours, - max = 1000.hours) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromHours - } - - object SuggestBasedFatigueDuration - extends FSBoundedParam[Duration]( - name = "home_timeline_suggest_based_fatigue_duration_in_hours", - default = 0.hours, - min = 0.hours, - max = 1000.hours) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromHours - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD deleted file mode 100644 index 140cc928d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.scala deleted file mode 100644 index a5586f296..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/HomeTimelineTweetRecsProduct.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline_tweet_recs - -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlow -import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams._ -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -/* - * This "DisplayLocation" is used to generate user recommendations using the ContentRecommenderFlow. These recommendations are later used downstream - * to generate recommended tweets on Home Timeline. - */ -@Singleton -class HomeTimelineTweetRecsProduct @Inject() ( - contentRecommenderFlow: ContentRecommenderFlow, - contentRecommenderRequestBuilder: ContentRecommenderRequestBuilder) - extends Product { - override val name: String = "Home Timeline Tweet Recs" - - override val identifier: String = "home-timeline-tweet-recs" - - override val displayLocation: DisplayLocation = DisplayLocation.HomeTimelineTweetRecs - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - contentRecommenderRequestBuilder.build(request).map { contentRecommenderRequest => - Seq(contentRecommenderFlow.mapKey({ request: ProductRequest => contentRecommenderRequest })) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = - new IdentityTransform[ProductRequest, Recommendation] - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = - Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) - - override def enabled(request: ProductRequest): Stitch[Boolean] = - Stitch.value(request.params(EnableProduct)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD deleted file mode 100644 index 9be8d9647..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "configapi/configapi-decider", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.scala deleted file mode 100644 index 319a51847..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/home_timeline_tweet_recs/configapi/HomeTimelineTweetRecsParams.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi - -import com.twitter.timelines.configapi.Param - -object HomeTimelineTweetRecsParams { - object EnableProduct extends Param[Boolean](false) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD deleted file mode 100644 index f469a4748..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/blenders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/ads", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.scala deleted file mode 100644 index 29f788011..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/SidebarProduct.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.follow_recommendations.products.sidebar - -import com.twitter.follow_recommendations.common.base.BaseRecommendationFlow -import com.twitter.follow_recommendations.common.base.IdentityTransform -import com.twitter.follow_recommendations.common.base.Transform -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlow -import com.twitter.follow_recommendations.flows.ads.PromotedAccountsFlowRequest -import com.twitter.follow_recommendations.blenders.PromotedAccountsBlender -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlow -import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilder -import com.twitter.follow_recommendations.products.common.Product -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.follow_recommendations.products.sidebar.configapi.SidebarParams -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SidebarProduct @Inject() ( - postNuxMlFlow: PostNuxMlFlow, - postNuxMlRequestBuilder: PostNuxMlRequestBuilder, - promotedAccountsFlow: PromotedAccountsFlow, - promotedAccountsBlender: PromotedAccountsBlender) - extends Product { - override val name: String = "Sidebar" - - override val identifier: String = "sidebar" - - override val displayLocation: DisplayLocation = DisplayLocation.Sidebar - - override def selectWorkflows( - request: ProductRequest - ): Stitch[Seq[BaseRecommendationFlow[ProductRequest, _ <: Recommendation]]] = { - postNuxMlRequestBuilder.build(request).map { postNuxMlRequest => - Seq( - postNuxMlFlow.mapKey({ _: ProductRequest => postNuxMlRequest }), - promotedAccountsFlow.mapKey(mkPromotedAccountsRequest) - ) - } - } - - override val blender: Transform[ProductRequest, Recommendation] = { - promotedAccountsBlender.mapTarget[ProductRequest](getMaxResults) - } - - private[sidebar] def mkPromotedAccountsRequest( - req: ProductRequest - ): PromotedAccountsFlowRequest = { - PromotedAccountsFlowRequest( - req.recommendationRequest.clientContext, - req.params, - req.recommendationRequest.displayLocation, - None, - req.recommendationRequest.excludedIds.getOrElse(Nil) - ) - } - - private[sidebar] def getMaxResults(req: ProductRequest): Int = { - req.recommendationRequest.maxResults.getOrElse( - req.params(SidebarParams.DefaultMaxResults) - ) - } - - override def resultsTransformer( - request: ProductRequest - ): Stitch[Transform[ProductRequest, Recommendation]] = - Stitch.value(new IdentityTransform[ProductRequest, Recommendation]) - - override def enabled(request: ProductRequest): Stitch[Boolean] = - Stitch.value(request.params(SidebarParams.EnableProduct)) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD deleted file mode 100644 index 6fee24f89..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.scala deleted file mode 100644 index bbd026495..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/sidebar/configapi/SidebarParams.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.twitter.follow_recommendations.products.sidebar.configapi - -import com.twitter.timelines.configapi.Param - -object SidebarParams { - object EnableProduct extends Param[Boolean](false) - - object DefaultMaxResults extends Param[Int](20) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD deleted file mode 100644 index fe29c22de..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/utils", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.scala deleted file mode 100644 index 7567fe9ce..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/FollowRecommendationsServiceWarmupHandler.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.thrift.routing.ThriftWarmup -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService.GetRecommendations -import com.twitter.follow_recommendations.thriftscala.ClientContext -import com.twitter.follow_recommendations.thriftscala.DebugParams -import com.twitter.follow_recommendations.thriftscala.DisplayContext -import com.twitter.follow_recommendations.thriftscala.DisplayLocation -import com.twitter.follow_recommendations.thriftscala.Profile -import com.twitter.follow_recommendations.thriftscala.RecommendationRequest -import com.twitter.inject.Logging -import com.twitter.inject.utils.Handler -import com.twitter.scrooge.Request -import com.twitter.scrooge.Response -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowRecommendationsServiceWarmupHandler @Inject() (warmup: ThriftWarmup) - extends Handler - with Logging { - - private val clientId = ClientId("thrift-warmup-client") - - override def handle(): Unit = { - val testIds = Seq(1L) - def warmupQuery(userId: Long, displayLocation: DisplayLocation): RecommendationRequest = { - val clientContext = ClientContext( - userId = Some(userId), - guestId = None, - appId = Some(258901L), - ipAddress = Some("0.0.0.0"), - userAgent = Some("FAKE_USER_AGENT_FOR_WARMUPS"), - countryCode = Some("US"), - languageCode = Some("en"), - isTwoffice = None, - userRoles = None, - deviceId = Some("FAKE_DEVICE_ID_FOR_WARMUPS") - ) - RecommendationRequest( - clientContext = clientContext, - displayLocation = displayLocation, - displayContext = None, - maxResults = Some(3), - fetchPromotedContent = Some(false), - debugParams = Some(DebugParams(doNotLog = Some(true))) - ) - } - - // Add FRS display locations here if they should be targeted for warm-up - // when FRS is starting from a fresh state after a deploy - val displayLocationsToWarmUp: Seq[DisplayLocation] = Seq( - DisplayLocation.HomeTimeline, - DisplayLocation.HomeTimelineReverseChron, - DisplayLocation.ProfileSidebar, - DisplayLocation.NuxInterests, - DisplayLocation.NuxPymk - ) - - try { - clientId.asCurrent { - // Iterate over each user ID created for testing - testIds foreach { id => - // Iterate over each display location targeted for warm-up - displayLocationsToWarmUp foreach { displayLocation => - val warmupReq = warmupQuery(id, displayLocation) - info(s"Sending warm-up request to service with query: $warmupReq") - warmup.sendRequest( - method = GetRecommendations, - req = Request(GetRecommendations.Args(warmupReq)))(assertWarmupResponse) - // send the request one more time so that it goes through cache hits - warmup.sendRequest( - method = GetRecommendations, - req = Request(GetRecommendations.Args(warmupReq)))(assertWarmupResponse) - } - } - } - } catch { - case e: Throwable => - // we don't want a warmup failure to prevent start-up - error(e.getMessage, e) - } - info("Warm-up done.") - } - - /* Private */ - - private def assertWarmupResponse(result: Try[Response[GetRecommendations.SuccessType]]): Unit = { - // we collect and log any exceptions from the result. - result match { - case Return(_) => // ok - case Throw(exception) => - warn() - error(s"Error performing warm-up request: ${exception.getMessage}", exception) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.scala deleted file mode 100644 index daff9040d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductMixerRecommendationService.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.StatsReceiver -import javax.inject.Inject -import javax.inject.Singleton -import com.twitter.timelines.configapi.Params -import com.twitter.follow_recommendations.common.utils.DisplayLocationProductConverterUtil -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.{DebugParams => FrsDebugParams} -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.follow_recommendations.models.Request -import com.twitter.product_mixer.core.model.marshalling.request.{ - DebugParams => ProductMixerDebugParams -} -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest -import com.twitter.stitch.Stitch - -@Singleton -class ProductMixerRecommendationService @Inject() ( - productPipelineRegistry: ProductPipelineRegistry, - resultLogger: FrsLogger, - baseStats: StatsReceiver) { - - private val stats = baseStats.scope("product_mixer_recos_service_stats") - private val loggingStats = stats.scope("logged") - - def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] = { - if (params(DeciderParams.EnableRecommendations)) { - val productMixerRequest = convertToProductMixerRequest(request) - - productPipelineRegistry - .getProductPipeline[Request, RecommendationResponse](productMixerRequest.product) - .process(ProductPipelineRequest(productMixerRequest, params)).onSuccess { response => - if (resultLogger.shouldLog(request.debugParams)) { - loggingStats.counter().incr() - resultLogger.logRecommendationResult(request, response) - } - } - } else { - Stitch.value(RecommendationResponse(Nil)) - } - - } - - def convertToProductMixerRequest(frsRequest: RecommendationRequest): Request = { - Request( - maxResults = frsRequest.maxResults, - debugParams = convertToProductMixerDebugParams(frsRequest.debugParams), - productContext = None, - product = - DisplayLocationProductConverterUtil.displayLocationToProduct(frsRequest.displayLocation), - clientContext = frsRequest.clientContext, - serializedRequestCursor = frsRequest.cursor, - frsDebugParams = frsRequest.debugParams, - displayLocation = frsRequest.displayLocation, - excludedIds = frsRequest.excludedIds, - fetchPromotedContent = frsRequest.fetchPromotedContent, - userLocationState = frsRequest.userLocationState - ) - } - - private def convertToProductMixerDebugParams( - frsDebugParams: Option[FrsDebugParams] - ): Option[ProductMixerDebugParams] = { - frsDebugParams.map { debugParams => - ProductMixerDebugParams(debugParams.featureOverrides, None) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.scala deleted file mode 100644 index 1c949f03d..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelector.scala +++ /dev/null @@ -1,188 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.common.models.DebugOptions -import com.twitter.follow_recommendations.models.DebugParams -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton -import scala.util.Random - -@Singleton -class ProductPipelineSelector @Inject() ( - recommendationsService: RecommendationsService, - productMixerRecommendationService: ProductMixerRecommendationService, - productPipelineSelectorConfig: ProductPipelineSelectorConfig, - baseStats: StatsReceiver) { - - private val frsStats = baseStats.scope("follow_recommendations_service") - private val stats = frsStats.scope("product_pipeline_selector_parity") - - private val readFromProductMixerCounter = stats.counter("select_product_mixer") - private val readFromOldFRSCounter = stats.counter("select_old_frs") - - def selectPipeline( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - productPipelineSelectorConfig - .getDarkReadAndExpParams(request.displayLocation).map { darkReadAndExpParam => - if (params(darkReadAndExpParam.expParam)) { - readFromProductMixerPipeline(request, params) - } else if (params(darkReadAndExpParam.darkReadParam)) { - darkReadAndReturnResult(request, params) - } else { - readFromOldFrsPipeline(request, params) - } - }.getOrElse(readFromOldFrsPipeline(request, params)) - } - - private def readFromProductMixerPipeline( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - readFromProductMixerCounter.incr() - productMixerRecommendationService.get(request, params) - } - - private def readFromOldFrsPipeline( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - readFromOldFRSCounter.incr() - recommendationsService.get(request, params) - } - - private def darkReadAndReturnResult( - request: RecommendationRequest, - params: Params - ): Stitch[RecommendationResponse] = { - val darkReadStats = stats.scope("select_dark_read", request.displayLocation.toFsName) - darkReadStats.counter("count").incr() - - // If no seed is set, create a random one that both requests will use to remove differences - // in randomness for the WeightedCandidateSourceRanker - val randomizationSeed = new Random().nextLong() - - val oldFRSPiplelineRequest = request.copy( - debugParams = Some( - request.debugParams.getOrElse( - DebugParams(None, Some(DebugOptions(randomizationSeed = Some(randomizationSeed)))))) - ) - val productMixerPipelineRequest = request.copy( - debugParams = Some( - request.debugParams.getOrElse( - DebugParams( - None, - Some(DebugOptions(doNotLog = true, randomizationSeed = Some(randomizationSeed)))))) - ) - - StatsUtil - .profileStitch( - readFromOldFrsPipeline(oldFRSPiplelineRequest, params), - darkReadStats.scope("frs_timing")).applyEffect { frsOldPipelineResponse => - Stitch.async( - StatsUtil - .profileStitch( - readFromProductMixerPipeline(productMixerPipelineRequest, params), - darkReadStats.scope("product_mixer_timing")).liftToOption().map { - case Some(frsProductMixerResponse) => - darkReadStats.counter("product_mixer_pipeline_success").incr() - compare(request, frsOldPipelineResponse, frsProductMixerResponse) - case None => - darkReadStats.counter("product_mixer_pipeline_failure").incr() - } - ) - } - } - - def compare( - request: RecommendationRequest, - frsOldPipelineResponse: RecommendationResponse, - frsProductMixerResponse: RecommendationResponse - ): Unit = { - val compareStats = stats.scope("pipeline_comparison", request.displayLocation.toFsName) - compareStats.counter("total-comparisons").incr() - - val oldFrsMap = frsOldPipelineResponse.recommendations.map { user => user.id -> user }.toMap - val productMixerMap = frsProductMixerResponse.recommendations.map { user => - user.id -> user - }.toMap - - compareTopNResults(3, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(5, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(25, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(50, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - compareTopNResults(75, frsOldPipelineResponse, frsProductMixerResponse, compareStats) - - // Compare individual matching candidates - oldFrsMap.keys.foreach(userId => { - if (productMixerMap.contains(userId)) { - (oldFrsMap(userId), productMixerMap(userId)) match { - case (oldFrsUser: CandidateUser, productMixerUser: CandidateUser) => - compareStats.counter("matching-user-count").incr() - compareUser(oldFrsUser, productMixerUser, compareStats) - case _ => - compareStats.counter("unknown-user-type-count").incr() - } - } else { - compareStats.counter("missing-user-count").incr() - } - }) - } - - private def compareTopNResults( - n: Int, - frsOldPipelineResponse: RecommendationResponse, - frsProductMixerResponse: RecommendationResponse, - compareStats: StatsReceiver - ): Unit = { - if (frsOldPipelineResponse.recommendations.size >= n && frsProductMixerResponse.recommendations.size >= n) { - val oldFrsPipelineFirstN = frsOldPipelineResponse.recommendations.take(n).map(_.id) - val productMixerPipelineFirstN = frsProductMixerResponse.recommendations.take(n).map(_.id) - - if (oldFrsPipelineFirstN.sorted == productMixerPipelineFirstN.sorted) - compareStats.counter(s"first-$n-sorted-equal-ids").incr() - if (oldFrsPipelineFirstN == productMixerPipelineFirstN) - compareStats.counter(s"first-$n-unsorted-ids-equal").incr() - else - compareStats.counter(s"first-$n-unsorted-ids-unequal").incr() - } - } - - private def compareUser( - oldFrsUser: CandidateUser, - productMixerUser: CandidateUser, - stats: StatsReceiver - ): Unit = { - val userStats = stats.scope("matching-user") - - if (oldFrsUser.score != productMixerUser.score) - userStats.counter("mismatch-score").incr() - if (oldFrsUser.reason != productMixerUser.reason) - userStats.counter("mismatch-reason").incr() - if (oldFrsUser.userCandidateSourceDetails != productMixerUser.userCandidateSourceDetails) - userStats.counter("mismatch-userCandidateSourceDetails").incr() - if (oldFrsUser.adMetadata != productMixerUser.adMetadata) - userStats.counter("mismatch-adMetadata").incr() - if (oldFrsUser.trackingToken != productMixerUser.trackingToken) - userStats.counter("mismatch-trackingToken").incr() - if (oldFrsUser.dataRecord != productMixerUser.dataRecord) - userStats.counter("mismatch-dataRecord").incr() - if (oldFrsUser.scores != productMixerUser.scores) - userStats.counter("mismatch-scores").incr() - if (oldFrsUser.infoPerRankingStage != productMixerUser.infoPerRankingStage) - userStats.counter("mismatch-infoPerRankingStage").incr() - if (oldFrsUser.params != productMixerUser.params) - userStats.counter("mismatch-params").incr() - if (oldFrsUser.engagements != productMixerUser.engagements) - userStats.counter("mismatch-engagements").incr() - if (oldFrsUser.recommendationFlowIdentifier != productMixerUser.recommendationFlowIdentifier) - userStats.counter("mismatch-recommendationFlowIdentifier").incr() - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.scala deleted file mode 100644 index a1cac3316..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductPipelineSelectorConfig.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.follow_recommendations.common.models.DisplayLocation -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param -import javax.inject.Singleton - -@Singleton -class ProductPipelineSelectorConfig { - private val paramsMap: Map[DisplayLocation, DarkReadAndExpParams] = Map.empty - - def getDarkReadAndExpParams( - displayLocation: DisplayLocation - ): Option[DarkReadAndExpParams] = { - paramsMap.get(displayLocation) - } -} - -case class DarkReadAndExpParams(darkReadParam: Param[Boolean], expParam: FSParam[Boolean]) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.scala deleted file mode 100644 index 967790a08..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/ProductRecommenderService.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil -import com.twitter.follow_recommendations.common.models.Recommendation -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.products.common.ProductRegistry -import com.twitter.follow_recommendations.products.common.ProductRequest -import com.twitter.stitch.Stitch -import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ProductRecommenderService @Inject() ( - productRegistry: ProductRegistry, - statsReceiver: StatsReceiver) { - - private val stats = statsReceiver.scope("ProductRecommenderService") - - def getRecommendations( - request: RecommendationRequest, - params: Params - ): Stitch[Seq[Recommendation]] = { - val displayLocation = request.displayLocation - val displayLocationStatName = displayLocation.toString - val locationStats = stats.scope(displayLocationStatName) - val loggedInOrOutStats = if (request.clientContext.userId.isDefined) { - stats.scope("logged_in").scope(displayLocationStatName) - } else { - stats.scope("logged_out").scope(displayLocationStatName) - } - - loggedInOrOutStats.counter("requests").incr() - val product = productRegistry.getProductByDisplayLocation(displayLocation) - val productRequest = ProductRequest(request, params) - val productEnabledStitch = - StatsUtil.profileStitch(product.enabled(productRequest), locationStats.scope("enabled")) - productEnabledStitch.flatMap { productEnabled => - if (productEnabled && params(EnableWhoToFollowProducts)) { - loggedInOrOutStats.counter("enabled").incr() - val stitch = for { - workflows <- StatsUtil.profileStitch( - product.selectWorkflows(productRequest), - locationStats.scope("select_workflows")) - workflowRecos <- StatsUtil.profileStitch( - Stitch.collect( - workflows.map(_.process(productRequest).map(_.result.getOrElse(Seq.empty)))), - locationStats.scope("execute_workflows") - ) - blendedCandidates <- StatsUtil.profileStitch( - product.blender.transform(productRequest, workflowRecos.flatten), - locationStats.scope("blend_results")) - resultsTransformer <- StatsUtil.profileStitch( - product.resultsTransformer(productRequest), - locationStats.scope("results_transformer")) - transformedCandidates <- StatsUtil.profileStitch( - resultsTransformer.transform(productRequest, blendedCandidates), - locationStats.scope("execute_results_transformer")) - } yield { - transformedCandidates - } - StatsUtil.profileStitchResults[Seq[Recommendation]](stitch, locationStats, _.size) - } else { - loggedInOrOutStats.counter("disabled").incr() - locationStats.counter("disabled_product").incr() - Stitch.Nil - } - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.scala deleted file mode 100644 index e4bc1e3c0..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/RecommendationsService.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.RecommendationRequest -import com.twitter.follow_recommendations.models.RecommendationResponse -import com.twitter.stitch.Stitch -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class RecommendationsService @Inject() ( - productRecommenderService: ProductRecommenderService, - resultLogger: FrsLogger) { - def get(request: RecommendationRequest, params: Params): Stitch[RecommendationResponse] = { - if (params(DeciderParams.EnableRecommendations)) { - productRecommenderService - .getRecommendations(request, params).map(RecommendationResponse).onSuccess { response => - if (resultLogger.shouldLog(request.debugParams)) { - resultLogger.logRecommendationResult(request, response) - } - } - } else { - Stitch.value(RecommendationResponse(Nil)) - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.scala deleted file mode 100644 index b3a8c6664..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/UserScoringService.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.follow_recommendations.services - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.common.base.StatsUtil.profileStitchSeqResults -import com.twitter.follow_recommendations.common.clients.impression_store.WtfImpressionStore -import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.HydrateFeaturesTransform -import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRanker -import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats -import com.twitter.follow_recommendations.configapi.deciders.DeciderParams -import com.twitter.follow_recommendations.logging.FrsLogger -import com.twitter.follow_recommendations.models.ScoringUserRequest -import com.twitter.follow_recommendations.models.ScoringUserResponse -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class UserScoringService @Inject() ( - socialGraph: SocialGraphClient, - wtfImpressionStore: WtfImpressionStore, - hydrateFeaturesTransform: HydrateFeaturesTransform[ScoringUserRequest], - mlRanker: MlRanker[ScoringUserRequest], - resultLogger: FrsLogger, - stats: StatsReceiver) { - - private val scopedStats: StatsReceiver = stats.scope(this.getClass.getSimpleName) - private val disabledCounter: Counter = scopedStats.counter("disabled") - - def get(request: ScoringUserRequest): Stitch[ScoringUserResponse] = { - if (request.params(DeciderParams.EnableScoreUserCandidates)) { - val hydratedRequest = hydrate(request) - val candidatesStitch = hydratedRequest.flatMap { req => - hydrateFeaturesTransform.transform(req, request.candidates).flatMap { - candidateWithFeatures => - mlRanker.rank(req, candidateWithFeatures) - } - } - profileStitchSeqResults(candidatesStitch, scopedStats) - .map(ScoringUserResponse) - .onSuccess { response => - if (resultLogger.shouldLog(request.debugParams)) { - resultLogger.logScoringResult(request, response) - } - } - } else { - disabledCounter.incr() - Stitch.value(ScoringUserResponse(Nil)) - } - } - - private def hydrate(request: ScoringUserRequest): Stitch[ScoringUserRequest] = { - val allStitches = Stitch.collect(request.clientContext.userId.map { userId => - val recentFollowedUserIdsStitch = - rescueWithStats( - socialGraph.getRecentFollowedUserIds(userId), - stats, - "recentFollowedUserIds") - val recentFollowedByUserIdsStitch = - rescueWithStats( - socialGraph.getRecentFollowedByUserIds(userId), - stats, - "recentFollowedByUserIds") - val wtfImpressionsStitch = - rescueWithStats( - wtfImpressionStore.get(userId, request.displayLocation), - stats, - "wtfImpressions") - Stitch.join(recentFollowedUserIdsStitch, recentFollowedByUserIdsStitch, wtfImpressionsStitch) - }) - allStitches.map { - case Some((recentFollowedUserIds, recentFollowedByUserIds, wtfImpressions)) => - request.copy( - recentFollowedUserIds = - if (recentFollowedUserIds.isEmpty) None else Some(recentFollowedUserIds), - recentFollowedByUserIds = - if (recentFollowedByUserIds.isEmpty) None else Some(recentFollowedByUserIds), - wtfImpressions = if (wtfImpressions.isEmpty) None else Some(wtfImpressions) - ) - case _ => request - } - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD deleted file mode 100644 index 8b2c2d041..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["bazel-compatible"], - dependencies = [ - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/exceptions", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/modules", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/response", - "finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.scala deleted file mode 100644 index f3a09a6d7..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions/UnknownExceptionMapper.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.follow_recommendations.service.exceptions - -import com.twitter.finatra.thrift.exceptions.ExceptionMapper -import com.twitter.inject.Logging -import com.twitter.util.Future -import javax.inject.Singleton - -@Singleton -class UnknownLoggingExceptionMapper extends ExceptionMapper[Exception, Throwable] with Logging { - def handleException(throwable: Exception): Future[Throwable] = { - error( - s"Unmapped Exception: ${throwable.getMessage} - ${throwable.getStackTrace.mkString(", \n\t")}", - throwable - ) - - Future.exception(throwable) - } -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD deleted file mode 100644 index e92976ad8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/BUILD +++ /dev/null @@ -1,29 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-app/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-server/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/ppmi_locale_follow", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/socialgraph", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk", - "follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params", - "follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models", - "twitter-server/server/src/main/scala", - "util/util-app/src/main/scala", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.scala deleted file mode 100644 index 60a28f1a8..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/CandidateSourceHoldbackUtil.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.follow_recommendations.utils - -import com.twitter.follow_recommendations.common.candidate_sources.addressbook._ -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSource -import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource -import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource -import com.twitter.follow_recommendations.common.candidate_sources.sims.SwitchingSimsSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentStrongEngagementDirectFollowSimilarUsersSource -import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.MutualFollowStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.BaseOnlineSTPSource -import com.twitter.follow_recommendations.common.candidate_sources.stp.SocialProofEnforcedOfflineStrongTiePredictionSource -import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource -import com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk.TwoHopRandomWalkSource -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.follow_recommendations.configapi.params.GlobalParams -import com.twitter.follow_recommendations.models.CandidateSourceType -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.timelines.configapi.HasParams - -trait CandidateSourceHoldbackUtil { - import CandidateSourceHoldbackUtil._ - def filterCandidateSources[T <: HasParams]( - request: T, - sources: Seq[CandidateSource[T, CandidateUser]] - ): Seq[CandidateSource[T, CandidateUser]] = { - val typeToFilter = request.params(GlobalParams.CandidateSourcesToFilter) - val sourcesToFilter = CandidateSourceTypeToMap.get(typeToFilter).getOrElse(Set.empty) - sources.filterNot { source => sourcesToFilter.contains(source.identifier) } - } -} - -object CandidateSourceHoldbackUtil { - final val ContextualActivityCandidateSourceIds: Set[CandidateSourceIdentifier] = - Set( - RecentFollowingSimilarUsersSource.Identifier, - RecentEngagementNonDirectFollowSource.Identifier, - RecentEngagementSimilarUsersSource.Identifier, - RecentStrongEngagementDirectFollowSimilarUsersSource.Identifier, - SwitchingSimsSource.Identifier, - ) - - final val SocialCandidateSourceIds: Set[CandidateSourceIdentifier] = - Set( - ForwardEmailBookSource.Identifier, - ForwardPhoneBookSource.Identifier, - ReverseEmailBookSource.Identifier, - ReversePhoneBookSource.Identifier, - RecentFollowingRecentFollowingExpansionSource.Identifier, - BaseOnlineSTPSource.Identifier, - MutualFollowStrongTiePredictionSource.Identifier, - OfflineStrongTiePredictionSource.Identifier, - SocialProofEnforcedOfflineStrongTiePredictionSource.Identifier, - TriangularLoopsSource.Identifier, - TwoHopRandomWalkSource.Identifier - ) - - final val GeoCandidateSourceIds: Set[CandidateSourceIdentifier] = - Set( - PPMILocaleFollowSource.Identifier, - PopCountrySource.Identifier, - PopGeohashSource.Identifier, - PopCountryBackFillSource.Identifier, - PopGeoSource.Identifier, - ) - - final val CandidateSourceTypeToMap: Map[CandidateSourceType.Value, Set[ - CandidateSourceIdentifier - ]] = - Map( - CandidateSourceType.Social -> SocialCandidateSourceIds, - CandidateSourceType.ActivityContextual -> ContextualActivityCandidateSourceIds, - CandidateSourceType.GeoAndInterests -> GeoCandidateSourceIds - ) -} diff --git a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.scala b/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.scala deleted file mode 100644 index 9304fb398..000000000 --- a/follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils/RecommendationFlowBaseSideEffectsUtil.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.follow_recommendations.utils - -import com.twitter.follow_recommendations.common.base.RecommendationFlow -import com.twitter.follow_recommendations.common.base.SideEffectsUtil -import com.twitter.follow_recommendations.common.models.CandidateUser -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch - -trait RecommendationFlowBaseSideEffectsUtil[Target <: HasClientContext, Candidate <: CandidateUser] - extends SideEffectsUtil[Target, Candidate] { - recommendationFlow: RecommendationFlow[Target, Candidate] => - - override def applySideEffects( - target: Target, - candidateSources: Seq[CandidateSource[Target, Candidate]], - candidatesFromCandidateSources: Seq[Candidate], - mergedCandidates: Seq[Candidate], - filteredCandidates: Seq[Candidate], - rankedCandidates: Seq[Candidate], - transformedCandidates: Seq[Candidate], - truncatedCandidates: Seq[Candidate], - results: Seq[Candidate] - ): Stitch[Unit] = { - Stitch.async( - Stitch.collect( - Seq( - applySideEffectsCandidateSourceCandidates( - target, - candidateSources, - candidatesFromCandidateSources), - applySideEffectsMergedCandidates(target, mergedCandidates), - applySideEffectsFilteredCandidates(target, filteredCandidates), - applySideEffectsRankedCandidates(target, rankedCandidates), - applySideEffectsTransformedCandidates(target, transformedCandidates), - applySideEffectsTruncatedCandidates(target, truncatedCandidates), - applySideEffectsResults(target, results) - ) - )) - } - - /* - In subclasses, override functions below to apply custom side effects at each step in pipeline. - Call super.applySideEffectsXYZ to scribe basic scribes implemented in this parent class - */ - def applySideEffectsCandidateSourceCandidates( - target: Target, - candidateSources: Seq[CandidateSource[Target, Candidate]], - candidatesFromCandidateSources: Seq[Candidate] - ): Stitch[Unit] = { - val candidatesGroupedByCandidateSources = - candidatesFromCandidateSources.groupBy( - _.getPrimaryCandidateSource.getOrElse(CandidateSourceIdentifier("NoCandidateSource"))) - - target.getOptionalUserId match { - case Some(userId) => - val userAgeOpt = SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays) - userAgeOpt match { - case Some(userAge) if userAge <= 30 => - candidateSources.map { candidateSource => - { - val candidateSourceStats = statsReceiver.scope(candidateSource.identifier.name) - - val isEmpty = - !candidatesGroupedByCandidateSources.keySet.contains(candidateSource.identifier) - - if (userAge <= 1) - candidateSourceStats - .scope("user_age", "1", "empty").counter(isEmpty.toString).incr() - if (userAge <= 7) - candidateSourceStats - .scope("user_age", "7", "empty").counter(isEmpty.toString).incr() - if (userAge <= 30) - candidateSourceStats - .scope("user_age", "30", "empty").counter(isEmpty.toString).incr() - } - } - case _ => Nil - } - case None => Nil - } - Stitch.Unit - } - - def applySideEffectsBaseCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = Stitch.Unit - - def applySideEffectsMergedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsFilteredCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsRankedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsTransformedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsTruncatedCandidates( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) - - def applySideEffectsResults( - target: Target, - candidates: Seq[Candidate] - ): Stitch[Unit] = applySideEffectsBaseCandidates(target, candidates) -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/BUILD b/follow-recommendations-service/thrift/src/main/thrift/BUILD deleted file mode 100644 index e5cbd19cf..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -create_thrift_libraries( - base_name = "thrift", - sources = ["*.thrift"], - platform = "java8", - tags = ["bazel-compatible"], - dependency_roots = [ - "finatra-internal/thrift/src/main/thrift", - "follow-recommendations-service/thrift/src/main/thrift/logging:thrift", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift", - "src/thrift/com/twitter/ads/adserver:adserver_common", - "src/thrift/com/twitter/ml/api:data", - "src/thrift/com/twitter/suggests/controller_data", - ], - generate_languages = [ - "java", - "scala", - "strato", - ], - provides_java_name = "follow-recommendations-java", - provides_scala_name = "follow-recommendations-scala", -) diff --git a/follow-recommendations-service/thrift/src/main/thrift/assembler.thrift b/follow-recommendations-service/thrift/src/main/thrift/assembler.thrift deleted file mode 100644 index eb782d0fe..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/assembler.thrift +++ /dev/null @@ -1,42 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -struct Header { - 1: required Title title -} - -struct Title { - 1: required string text -} - -struct Footer { - 1: optional Action action -} - -struct Action { - 1: required string text - 2: required string actionURL -} - -struct UserList { - 1: required bool userBioEnabled - 2: required bool userBioTruncated - 3: optional i64 userBioMaxLines - 4: optional FeedbackAction feedbackAction -} - -struct Carousel { - 1: optional FeedbackAction feedbackAction -} - -union WTFPresentation { - 1: UserList userBioList - 2: Carousel carousel -} - -struct DismissUserId {} - -union FeedbackAction { - 1: DismissUserId dismissUserId -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/client_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/client_context.thrift deleted file mode 100644 index adbdc407a..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/client_context.thrift +++ /dev/null @@ -1,19 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -// Caller/Client level specific context (e.g, user id/guest id/app id). -struct ClientContext { - 1: optional i64 userId(personalDataType='UserId') - 2: optional i64 guestId(personalDataType='GuestId') - 3: optional i64 appId(personalDataType='AppId') - 4: optional string ipAddress(personalDataType='IpAddress') - 5: optional string userAgent(personalDataType='UserAgent') - 6: optional string countryCode(personalDataType='InferredCountry') - 7: optional string languageCode(personalDataType='InferredLanguage') - 9: optional bool isTwoffice(personalDataType='InferredLocation') - 10: optional set userRoles - 11: optional string deviceId(personalDataType='DeviceId') - 12: optional i64 guestIdAds(personalDataType='GuestId') - 13: optional i64 guestIdMarketing(personalDataType='GuestId') -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/debug.thrift b/follow-recommendations-service/thrift/src/main/thrift/debug.thrift deleted file mode 100644 index a41c59114..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/debug.thrift +++ /dev/null @@ -1,73 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendation - -// These are broken into their own union -// because we can have features that are -// complex flavors of these (such as Seq) -union PrimitiveFeatureValue { - 1: i32 intValue - 2: i64 longValue - 3: string strValue - 4: bool boolValue -} - -union FeatureValue { - 1: PrimitiveFeatureValue primitiveValue -} - -struct DebugParams { - 1: optional map featureOverrides - 2: optional i64 randomizationSeed - 3: optional bool includeDebugInfoInResults - 4: optional bool doNotLog -} - -enum DebugCandidateSourceIdentifier { - UTT_INTERESTS_RELATED_USERS_SOURCE = 0 - UTT_PRODUCER_EXPANSION_SOURCE = 1 - UTT_SEED_ACCOUNT_SOURCE = 2 - BYF_USER_FOLLOW_CLUSTER_SIMS_SOURCE = 3 - BYF_USER_FOLLOW_CLUSTER_SOURCE = 4 - USER_FOLLOW_CLUSTER_SOURCE = 5 - RECENT_SEARCH_BASED_SOURCE = 6 - PEOPLE_ACTIVITY_RECENT_ENGAGEMENT_SOURCE = 7 - PEOPLE_ACTIVITY_RECENT_ENGAGEMENT_SIMS_SOURCE = 8, - REVERSE_PHONE_BOOK_SOURCE = 9, - REVERSE_EMAIL_BOOK_SOURCE = 10, - SIMS_DEBUG_STORE = 11, - UTT_PRODUCER_ONLINE_MBCG_SOURCE = 12, - BONUS_FOLLOW_CONDITIONAL_ENGAGEMENT_STORE = 13, - // 14 (BONUS_FOLLOW_PMI_STORE) was deleted as it's not used anymore - FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 15, - OFFLINE_STP = 16, - OFFLINE_STP_BIG = 17, - OFFLINE_MUTUAL_FOLLOW_EXPANSION = 18, - REPEATED_PROFILE_VISITS = 19, - TIME_DECAY_FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 20, - LINEAR_REGRESSION_FOLLOW2VEC_NEAREST_NEIGHBORS_STORE = 21, - REAL_GRAPH_EXPANSION_SOURCE = 22, - RELATABLE_ACCOUNTS_BY_INTEREST = 23, - EMAIL_TWEET_CLICK = 24, - GOOD_TWEET_CLICK_ENGAGEMENTS = 25, - ENGAGED_FOLLOWER_RATIO = 26, - TWEET_SHARE_ENGAGEMENTS = 27, - BULK_FRIEND_FOLLOWS = 28, - REAL_GRAPH_OON_V2_SOURCE = 30, - CROWD_SEARCH_ACCOUNTS = 31, - POP_GEOHASH = 32, - POP_COUNTRY = 33, - POP_COUNTRY_BACKFILL = 34, - TWEET_SHARER_TO_SHARE_RECIPIENT_ENGAGEMENTS = 35, - TWEET_AUTHOR_TO_SHARE_RECIPIENT_ENGAGEMENTS = 36, - BULK_FRIEND_FOLLOWS_NEW_USER = 37, - ONLINE_STP_EPSCORER = 38, - ORGANIC_FOLLOW_ACCOUNTS = 39, - NUX_LO_HISTORY = 40, - TRAFFIC_ATTRIBUTION_ACCOUNTS = 41, - ONLINE_STP_RAW_ADDRESS_BOOK = 42, - POP_GEOHASH_QUALITY_FOLLOW = 43, - NOTIFICATION_ENGAGEMENT = 44, - EFR_BY_WORLDWIDE_PICTURE_PRODUCER = 45, - POP_GEOHASH_REAL_GRAPH = 46, -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/display_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/display_context.thrift deleted file mode 100644 index cfd613b71..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/display_context.thrift +++ /dev/null @@ -1,62 +0,0 @@ -include "flows.thrift" -include "recently_engaged_user_id.thrift" - -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -struct Profile { - 1: required i64 profileId(personalDataType='UserId') -}(hasPersonalData='true') - -struct Search { - 1: required string searchQuery(personalDataType='SearchQuery') -}(hasPersonalData='true') - -struct Rux { - 1: required i64 focalAuthorId(personalDataType='UserId') -}(hasPersonalData='true') - -struct Topic { - 1: required i64 topicId(personalDataType = 'TopicFollow') -}(hasPersonalData='true') - -struct ReactiveFollow { - 1: required list followedUserIds(personalDataType='UserId') -}(hasPersonalData='true') - -struct NuxInterests { - 1: optional flows.FlowContext flowContext // set for recommendation inside an interactive flow - 2: optional list uttInterestIds // if provided, we use these interestIds for generating candidates instead of for example fetching user selected interests -}(hasPersonalData='true') - -struct AdCampaignTarget { - 1: required list similarToUserIds(personalDataType='UserId') -}(hasPersonalData='true') - -struct ConnectTab { - 1: required list byfSeedUserIds(personalDataType='UserId') - 2: required list similarToUserIds(personalDataType='UserId') - 3: required list recentlyEngagedUserIds -}(hasPersonalData='true') - -struct SimilarToUser { - 1: required i64 similarToUserId(personalDataType='UserId') -}(hasPersonalData='true') - -struct PostNuxFollowTask { - 1: optional flows.FlowContext flowContext // set for recommendation inside an interactive flow -}(hasPersonalData='true') - -union DisplayContext { - 1: Profile profile - 2: Search search - 3: Rux rux - 4: Topic topic - 5: ReactiveFollow reactiveFollow - 6: NuxInterests nuxInterests - 7: AdCampaignTarget adCampaignTarget - 8: ConnectTab connectTab - 9: SimilarToUser similarToUser - 10: PostNuxFollowTask postNuxFollowTask -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/display_location.thrift b/follow-recommendations-service/thrift/src/main/thrift/display_location.thrift deleted file mode 100644 index d94b9842e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/display_location.thrift +++ /dev/null @@ -1,55 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -enum DisplayLocation { - SIDEBAR = 0 - PROFILE_SIDEBAR = 2 - CLUSTER_FOLLOW = 7 - NEW_USER_SARUS_BACKFILL = 12 - PROFILE_DEVICE_FOLLOW = 23 - RECOS_BACKFILL = 32 - HOME_TIMELINE = 39 # HOME_TIMELINE_WTF in Hermit - PROFILE_TOP_FOLLOWING = 42 - PROFILE_TOP_FOLLOWERS = 43 - PEOPLE_PLUS_PLUS = 47 - EXPLORE_TAB = 57 - MagicRecs = 59 # Account recommendation in notification - AB_UPLOAD_INJECTION = 60 - /** - * To prevent setting 2 display locations with the same index in FRS. - * - * The display location should be added to the following files: - * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift - * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift - * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala - */ - CAMPAIGN_FORM = 61 - RUX_LANDING_PAGE = 62 - PROFILE_BONUS_FOLLOW = 63 - ELECTION_EXPLORE_WTF = 64 - HTL_BONUS_FOLLOW = 65 - TOPIC_LANDING_PAGE_HEADER = 66 - NUX_PYMK = 67 - NUX_INTERESTS = 68 - REACTIVE_FOLLOW = 69 - RUX_PYMK = 70 - INDIA_COVID19_CURATED_ACCOUNTS_WTF = 71 - NUX_TOPIC_BONUS_FOLLOW = 72 - TWEET_NOTIFICATION_RECS = 73 - HTL_SPACE_HOSTS = 74 - POST_NUX_FOLLOW_TASK = 75 - TOPIC_LANDING_PAGE = 76 - USER_TYPEAHEAD_PREFETCH = 77 - HOME_TIMELINE_RELATABLE_ACCOUNTS = 78 - NUX_GEO_CATEGORY = 79 - NUX_INTERESTS_CATEGORY = 80 - NUX_PYMK_CATEGORY = 81 - TOP_ARTICLES = 82 - HOME_TIMELINE_TWEET_RECS = 83 - HTL_BULK_FRIEND_FOLLOWS = 84 - NUX_AUTO_FOLLOW = 85 - SEARCH_BONUS_FOLLOW = 86 - CONTENT_RECOMMENDER = 87 - HOME_TIMELINE_REVERSE_CHRON = 88 -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/engagementType.thrift b/follow-recommendations-service/thrift/src/main/thrift/engagementType.thrift deleted file mode 100644 index ef028d008..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/engagementType.thrift +++ /dev/null @@ -1,11 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -enum EngagementType { - Click = 0 - Like = 1 - Mention = 2 - Retweet = 3 - ProfileView = 4 -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/flows.thrift b/follow-recommendations-service/thrift/src/main/thrift/flows.thrift deleted file mode 100644 index 894ebf81e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/flows.thrift +++ /dev/null @@ -1,20 +0,0 @@ -/* - * This file defines additional thrift objects that should be specified in FRS request for context of recommendation, specifically the previous recommendations / new interactions in an interactive flow (series of follow steps). These typically are sent from OCF - */ - -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -struct FlowRecommendation { - 1: required i64 userId(personalDataType='UserId') -}(hasPersonalData='true') - -struct RecommendationStep { - 1: required list recommendations - 2: required set followedUserIds(personalDataType='UserId') -}(hasPersonalData='true') - -struct FlowContext { - 1: required list steps -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.thrift b/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.thrift deleted file mode 100644 index 40aadc0b6..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/follow-recommendations-service.thrift +++ /dev/null @@ -1,100 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "assembler.thrift" -include "client_context.thrift" -include "debug.thrift" -include "display_context.thrift" -include "display_location.thrift" -include "recommendations.thrift" -include "recently_engaged_user_id.thrift" - -include "finatra-thrift/finatra_thrift_exceptions.thrift" -include "com/twitter/product_mixer/core/pipeline_execution_result.thrift" - -struct RecommendationRequest { - 1: required client_context.ClientContext clientContext - 2: required display_location.DisplayLocation displayLocation - 3: optional display_context.DisplayContext displayContext - // Max results to return - 4: optional i32 maxResults - // Cursor to continue returning results if any - 5: optional string cursor - // IDs of Content to exclude from recommendations - 6: optional list excludedIds(personalDataType='UserId') - // Whether to also get promoted content - 7: optional bool fetchPromotedContent - 8: optional debug.DebugParams debugParams - 9: optional string userLocationState(personalDataType='InferredLocation') -}(hasPersonalData='true') - - -struct RecommendationResponse { - 1: required list recommendations -}(hasPersonalData='true') - -// for scoring a list of candidates, while logging hydrated features -struct ScoringUserRequest { - 1: required client_context.ClientContext clientContext - 2: required display_location.DisplayLocation displayLocation - 3: required list candidates - 4: optional debug.DebugParams debugParams -}(hasPersonalData='true') - -struct ScoringUserResponse { - 1: required list candidates // empty for now -}(hasPersonalData='true') - -// for getting the list of candidates generated by a single candidate source -struct DebugCandidateSourceRequest { - 1: required client_context.ClientContext clientContext - 2: required debug.DebugCandidateSourceIdentifier candidateSource - 3: optional list uttInterestIds - 4: optional debug.DebugParams debugParams - 5: optional list recentlyFollowedUserIds - 6: optional list recentlyEngagedUserIds - 7: optional list byfSeedUserIds - 8: optional list similarToUserIds - 9: required bool applySgsPredicate - 10: optional i32 maxResults -}(hasPersonalData='true') - -service FollowRecommendationsThriftService { - RecommendationResponse getRecommendations(1: RecommendationRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - RecommendationDisplayResponse getRecommendationDisplayResponse(1: RecommendationRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - // temporary endpoint for feature hydration and logging for data collection. - ScoringUserResponse scoreUserCandidates(1: ScoringUserRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - // Debug endpoint for getting recommendations of a single candidate source. We can remove this endpoint when ProMix provide this functionality and we integrate with it. - RecommendationResponse debugCandidateSource(1: DebugCandidateSourceRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) - - // Get the full execution log for a pipeline (used by our debugging tools) - pipeline_execution_result.PipelineExecutionResult executePipeline(1: RecommendationRequest request) throws ( - 1: finatra_thrift_exceptions.ServerError serverError, - 2: finatra_thrift_exceptions.UnknownClientIdError unknownClientIdError, - 3: finatra_thrift_exceptions.NoClientIdError noClientIdError - ) -} - -struct RecommendationDisplayResponse { - 1: required list hydratedRecommendation - 2: optional assembler.Header header - 3: optional assembler.Footer footer - 4: optional assembler.WTFPresentation wtfPresentation -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.thrift b/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.thrift deleted file mode 100644 index 404b0ae29..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/follow_recommendations_serving_history.thrift +++ /dev/null @@ -1,9 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -// struct used for storing the history of computing and serving of recommendations to a user -struct FollowRecommendationsServingHistory { - 1: required i64 lastComputationTimeMs (personalDataType = 'PrivateTimestamp') - 2: required i64 lastServingTimeMs (personalDataType = 'PrivateTimestamp') -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD b/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD deleted file mode 100644 index 2f769d498..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -create_thrift_libraries( - base_name = "thrift", - sources = ["*.thrift"], - platform = "java8", - tags = ["bazel-compatible"], - dependency_roots = [ - "src/thrift/com/twitter/ads/adserver:adserver_common", - "src/thrift/com/twitter/ml/api:data", - "src/thrift/com/twitter/suggests/controller_data", - ], - generate_languages = [ - "java", - "scala", - "strato", - ], - provides_java_name = "follow-recommendations-logging-java", - provides_scala_name = "follow-recommendations-logging-scala", -) diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.thrift deleted file mode 100644 index 2b6e454b2..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/client_context.thrift +++ /dev/null @@ -1,14 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -// Offline equal of ClientContext -struct OfflineClientContext { - 1: optional i64 userId(personalDataType='UserId') - 2: optional i64 guestId(personalDataType='GuestId') - 3: optional i64 appId(personalDataType='AppId') - 4: optional string countryCode(personalDataType='InferredCountry') - 5: optional string languageCode(personalDataType='InferredLanguage') - 6: optional i64 guestIdAds(personalDataType='GuestId') - 7: optional i64 guestIdMarketing(personalDataType='GuestId') -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/debug.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/debug.thrift deleted file mode 100644 index 882dca005..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/debug.thrift +++ /dev/null @@ -1,8 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendation.logging - -// subset of DebugParams -struct OfflineDebugParams { - 1: optional i64 randomizationSeed // track if the request was randomly ranked or not -}(persisted='true', hasPersonalData='false') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.thrift deleted file mode 100644 index c38850011..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/display_context.thrift +++ /dev/null @@ -1,66 +0,0 @@ -include "logging/flows.thrift" -include "logging/recently_engaged_user_id.thrift" - -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -// Offline equal of Profile DisplayContext -struct OfflineProfile { - 1: required i64 profileId(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -// Offline equal of Search DisplayContext -struct OfflineSearch { - 1: required string searchQuery(personalDataType='SearchQuery') -}(persisted='true', hasPersonalData='true') - -// Offline equal of Rux Landing Page DisplayContext -struct OfflineRux { - 1: required i64 focalAuthorId(personalDataType="UserId") -}(persisted='true', hasPersonalData='true') - -// Offline equal of Topic DisplayContext -struct OfflineTopic { - 1: required i64 topicId(personalDataType = 'TopicFollow') -}(persisted='true', hasPersonalData='true') - -struct OfflineReactiveFollow { - 1: required list followedUserIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineNuxInterests { - 1: optional flows.OfflineFlowContext flowContext // set for recommendation inside an interactive flow -}(persisted='true', hasPersonalData='true') - -struct OfflineAdCampaignTarget { - 1: required list similarToUserIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineConnectTab { - 1: required list byfSeedUserIds(personalDataType='UserId') - 2: required list similarToUserIds(personalDataType='UserId') - 3: required list recentlyEngagedUserIds -}(persisted='true', hasPersonalData='true') - -struct OfflineSimilarToUser { - 1: required i64 similarToUserId(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflinePostNuxFollowTask { - 1: optional flows.OfflineFlowContext flowContext // set for recommendation inside an interactive flow -}(persisted='true', hasPersonalData='true') - -// Offline equal of DisplayContext -union OfflineDisplayContext { - 1: OfflineProfile profile - 2: OfflineSearch search - 3: OfflineRux rux - 4: OfflineTopic topic - 5: OfflineReactiveFollow reactiveFollow - 6: OfflineNuxInterests nuxInterests - 7: OfflineAdCampaignTarget adCampaignTarget - 8: OfflineConnectTab connectTab - 9: OfflineSimilarToUser similarToUser - 10: OfflinePostNuxFollowTask postNuxFollowTask -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift deleted file mode 100644 index a4dbbecd4..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift +++ /dev/null @@ -1,55 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -/** - * Make sure you add the new DL to the following files and redeploy our attribution jobs - * - follow-recommendations-service/thrift/src/main/thrift/display_location.thrift - * - follow-recommendations-service/thrift/src/main/thrift/logging/display_location.thrift - * - follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models/DisplayLocation.scala - */ - -// Offline equal of DisplayLocation -enum OfflineDisplayLocation { - SIDEBAR = 0 - PROFILE_SIDEBAR = 2 - CLUSTER_FOLLOW = 7 - NEW_USER_SARUS_BACKFILL = 12 - PROFILE_DEVICE_FOLLOW = 23 - RECOS_BACKFILL = 32 - HOME_TIMELINE = 39 - PROFILE_TOP_FOLLOWING = 42 - PROFILE_TOP_FOLLOWERS = 43 - PEOPLE_PLUS_PLUS = 47 - EXPLORE_TAB = 57 - MagicRecs = 59 - AB_UPLOAD_INJECTION = 60 - CAMPAIGN_FORM = 61 - RUX_LANDING_PAGE = 62 - PROFILE_BONUS_FOLLOW = 63 - ELECTION_EXPLORE_WTF = 64 - HTL_BONUS_FOLLOW = 65 - TOPIC_LANDING_PAGE_HEADER = 66 - NUX_PYMK = 67 - NUX_INTERESTS = 68 - REACTIVE_FOLLOW = 69 - RUX_PYMK = 70 - INDIA_COVID19_CURATED_ACCOUNTS_WTF=71 - NUX_TOPIC_BONUS_FOLLOW = 72 - TWEET_NOTIFICATION_RECS = 73 - HTL_SPACE_HOSTS = 74 - POST_NUX_FOLLOW_TASK = 75 - TOPIC_LANDING_PAGE = 76 - USER_TYPEAHEAD_PREFETCH = 77 - HOME_TIMELINE_RELATABLE_ACCOUNTS = 78 - NUX_GEO_CATEGORY = 79 - NUX_INTERESTS_CATEGORY = 80 - NUX_PYMK_CATEGORY = 81 - TOP_ARTICLES = 82 - HOME_TIMELINE_TWEET_RECS = 83 - HTL_BULK_FRIEND_FOLLOWS = 84 - NUX_AUTO_FOLLOW = 85 - SEARCH_BONUS_FOLLOW = 86 - CONTENT_RECOMMENDER = 87 - HOME_TIMELINE_REVERSE_CHRON = 88 -}(persisted='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.thrift deleted file mode 100644 index 75191f16f..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/engagementType.thrift +++ /dev/null @@ -1,11 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -enum EngagementType { - Click = 0 - Like = 1 - Mention = 2 - Retweet = 3 - ProfileView = 4 -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/flows.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/flows.thrift deleted file mode 100644 index 98551c08e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/flows.thrift +++ /dev/null @@ -1,16 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -struct OfflineFlowRecommendation { - 1: required i64 userId(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationStep { - 1: required list recommendations - 2: required set followedUserIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -struct OfflineFlowContext { - 1: required list steps -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/logs.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/logs.thrift deleted file mode 100644 index 33f09cfb9..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/logs.thrift +++ /dev/null @@ -1,72 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "client_context.thrift" -include "debug.thrift" -include "display_context.thrift" -include "display_location.thrift" -include "recommendations.thrift" - -struct OfflineRecommendationRequest { - 1: required client_context.OfflineClientContext clientContext - 2: required display_location.OfflineDisplayLocation displayLocation - 3: optional display_context.OfflineDisplayContext displayContext - 4: optional i32 maxResults - 5: optional string cursor - 6: optional list excludedIds(personalDataType='UserId') - 7: optional bool fetchPromotedContent - 8: optional debug.OfflineDebugParams debugParams -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationResponse { - 1: required list recommendations -}(persisted='true', hasPersonalData='true') - -struct RecommendationLog { - 1: required OfflineRecommendationRequest request - 2: required OfflineRecommendationResponse response - 3: required i64 timestampMs -}(persisted='true', hasPersonalData='true') - -struct OfflineScoringUserRequest { - 1: required client_context.OfflineClientContext clientContext - 2: required display_location.OfflineDisplayLocation displayLocation - 3: required list candidates -}(persisted='true', hasPersonalData='true') - -struct OfflineScoringUserResponse { - 1: required list candidates -}(persisted='true', hasPersonalData='true') - -struct ScoredUsersLog { - 1: required OfflineScoringUserRequest request - 2: required OfflineScoringUserResponse response - 3: required i64 timestampMs -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationFlowUserMetadata { - 1: optional i32 userSignupAge(personalDataType = 'AgeOfAccount') - 2: optional string userState(personalDataType = 'UserState') -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationFlowSignals { - 1: optional string countryCode(personalDataType='InferredCountry') -}(persisted='true', hasPersonalData='true') - -struct OfflineRecommendationFlowCandidateSourceCandidates { - 1: required string candidateSourceName - 2: required list candidateUserIds(personalDataType='UserId') - 3: optional list candidateUserScores -}(persisted='true', hasPersonalData='true') - -struct RecommendationFlowLog { - 1: required client_context.OfflineClientContext clientContext - 2: optional OfflineRecommendationFlowUserMetadata userMetadata - 3: optional OfflineRecommendationFlowSignals signals - 4: required i64 timestampMs - 5: required string recommendationFlowIdentifier - 6: optional list filteredCandidates - 7: optional list rankedCandidates - 8: optional list truncatedCandidates -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.thrift deleted file mode 100644 index 6fc24d919..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/reasons.thrift +++ /dev/null @@ -1,62 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -// Proof based on Follow relationship -struct FollowProof { - 1: required list userIds(personalDataType='UserId') - 2: required i32 numIds(personalDataType='CountOfFollowersAndFollowees') -}(persisted='true', hasPersonalData='true') - -// Similar to userIds in the context (e.g. profileId) -struct SimilarToProof { - 1: required list userIds(personalDataType='UserId') -}(persisted='true', hasPersonalData='true') - -// Proof based on geo location -struct PopularInGeoProof { - 1: required string location(personalDataType='InferredLocation') -}(persisted='true', hasPersonalData='true') - -// Proof based on ttt interest -struct TttInterestProof { - 1: required i64 interestId(personalDataType='ProvidedInterests') - 2: required string interestDisplayName(personalDataType='ProvidedInterests') -}(persisted='true', hasPersonalData='true') - -// Proof based on topics -struct TopicProof { - 1: required i64 topicId(personalDataType='ProvidedInterests') -}(persisted='true', hasPersonalData='true') - -// Proof based on custom interest / search queries -struct CustomInterestProof { - 1: required string customerInterest(personalDataType='SearchQuery') -}(persisted='true', hasPersonalData='true') - -// Proof based on tweet authors -struct TweetsAuthorProof { - 1: required list tweetIds(personalDataType='TweetId') -}(persisted='true', hasPersonalData='true') - -// Proof candidate is of device follow type -struct DeviceFollowProof { - 1: required bool isDeviceFollow(personalDataType='OtherDeviceInfo') -}(persisted='true', hasPersonalData='true') - -// Account level proof that should be attached to each candidate -struct AccountProof { - 1: optional FollowProof followProof - 2: optional SimilarToProof similarToProof - 3: optional PopularInGeoProof popularInGeoProof - 4: optional TttInterestProof tttInterestProof - 5: optional TopicProof topicProof - 6: optional CustomInterestProof customInterestProof - 7: optional TweetsAuthorProof tweetsAuthorProof - 8: optional DeviceFollowProof deviceFollowProof - -}(persisted='true', hasPersonalData='true') - -struct Reason { - 1: optional AccountProof accountProof -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.thrift deleted file mode 100644 index f0af960b9..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/recently_engaged_user_id.thrift +++ /dev/null @@ -1,10 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "engagementType.thrift" - -struct RecentlyEngagedUserId { - 1: required i64 id(personalDataType='UserId') - 2: required engagementType.EngagementType engagementType -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.thrift deleted file mode 100644 index bf94e41b8..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/recommendations.thrift +++ /dev/null @@ -1,26 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "com/twitter/ads/adserver/adserver_common.thrift" -include "reasons.thrift" -include "tracking.thrift" -include "scoring.thrift" - -// Offline equal of UserRecommendation -struct OfflineUserRecommendation { - 1: required i64 userId(personalDataType='UserId') - // reason for this suggestions, eg: social context - 2: optional reasons.Reason reason - // present if it is a promoted account - 3: optional adserver_common.AdImpression adImpression - // tracking token (unserialized) for attribution - 4: optional tracking.TrackingToken trackingToken - // scoring details - 5: optional scoring.ScoringDetails scoringDetails -}(persisted='true', hasPersonalData='true') - -// Offline equal of Recommendation -union OfflineRecommendation { - 1: OfflineUserRecommendation user -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.thrift deleted file mode 100644 index e1524662d..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/scoring.thrift +++ /dev/null @@ -1,38 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "com/twitter/ml/api/data.thrift" - -struct CandidateSourceDetails { - 1: optional map candidateSourceScores - 2: optional i32 primarySource -}(persisted='true', hasPersonalData='false') - -struct Score { - 1: required double value - 2: optional string rankerId - 3: optional string scoreType -}(persisted='true', hasPersonalData='false') // scoring and ranking info per ranking stage - -// Contains (1) the ML-based heavy ranker and score (2) scores and rankers in producer experiment framework -struct Scores { - 1: required list scores - 2: optional string selectedRankerId - 3: required bool isInProducerScoringExperiment -}(persisted='true', hasPersonalData='false') - -struct RankingInfo { - 1: optional Scores scores - 2: optional i32 rank -}(persisted='true', hasPersonalData='false') - -// this encapsulates all information related to the ranking process from generation to scoring -struct ScoringDetails { - 1: optional CandidateSourceDetails candidateSourceDetails - 2: optional double score // The ML-based heavy ranker score - 3: optional data.DataRecord dataRecord - 4: optional list rankerIds // all ranker ids, including (1) ML-based heavy ranker (2) non-ML adhoc rankers - 5: optional map infoPerRankingStage // scoring and ranking info per ranking stage -}(persisted='true', hasPersonalData='true') - diff --git a/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.thrift b/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.thrift deleted file mode 100644 index 067ba1a46..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/logging/tracking.thrift +++ /dev/null @@ -1,16 +0,0 @@ -namespace java com.twitter.follow_recommendations.logging.thriftjava -#@namespace scala com.twitter.follow_recommendations.logging.thriftscala -#@namespace strato com.twitter.follow_recommendations.logging - -include "com/twitter/suggests/controller_data/controller_data.thrift" -include "display_location.thrift" - -struct TrackingToken { - // trace-id of the request - 1: required i64 sessionId (personalDataType='SessionId') - 2: optional display_location.OfflineDisplayLocation displayLocation - // 64-bit encoded binary attributes of our recommendation - 3: optional controller_data.ControllerData controllerData - // WTF Algorithm Id (backward compatibility) - 4: optional i32 algoId -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/reasons.thrift b/follow-recommendations-service/thrift/src/main/thrift/reasons.thrift deleted file mode 100644 index 299e88885..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/reasons.thrift +++ /dev/null @@ -1,61 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -// Proof based on Follow relationship -struct FollowProof { - 1: required list userIds(personalDataType='UserId') - 2: required i32 numIds(personalDataType='CountOfFollowersAndFollowees') -}(hasPersonalData='true') - -// Similar to userIds in the context (e.g. profileId) -struct SimilarToProof { - 1: required list userIds(personalDataType='UserId') -}(hasPersonalData='true') - -// Proof based on geo location -struct PopularInGeoProof { - 1: required string location(personalDataType='InferredLocation') -}(hasPersonalData='true') - -// Proof based on ttt interest -struct TttInterestProof { - 1: required i64 interestId(personalDataType='ProvidedInterests') - 2: required string interestDisplayName(personalDataType='ProvidedInterests') -}(hasPersonalData='true') - -// Proof based on topics -struct TopicProof { - 1: required i64 topicId(personalDataType='ProvidedInterests') -}(hasPersonalData='true') - -// Proof based on custom interest / search queries -struct CustomInterestProof { - 1: required string query(personalDataType='SearchQuery') -}(hasPersonalData='true') - -// Proof based on tweet authors -struct TweetsAuthorProof { - 1: required list tweetIds(personalDataType='TweetId') -}(hasPersonalData='true') - -// Proof candidate is of device follow type -struct DeviceFollowProof { - 1: required bool isDeviceFollow(personalDataType='OtherDeviceInfo') -}(hasPersonalData='true') - -// Account level proof that should be attached to each candidate -struct AccountProof { - 1: optional FollowProof followProof - 2: optional SimilarToProof similarToProof - 3: optional PopularInGeoProof popularInGeoProof - 4: optional TttInterestProof tttInterestProof - 5: optional TopicProof topicProof - 6: optional CustomInterestProof customInterestProof - 7: optional TweetsAuthorProof tweetsAuthorProof - 8: optional DeviceFollowProof deviceFollowProof -}(hasPersonalData='true') - -struct Reason { - 1: optional AccountProof accountProof -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.thrift b/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.thrift deleted file mode 100644 index 6a13bd31e..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/recently_engaged_user_id.thrift +++ /dev/null @@ -1,10 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "engagementType.thrift" - -struct RecentlyEngagedUserId { - 1: required i64 id(personalDataType='UserId') - 2: required engagementType.EngagementType engagementType -}(persisted='true', hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/recommendations.thrift b/follow-recommendations-service/thrift/src/main/thrift/recommendations.thrift deleted file mode 100644 index 1070bb11c..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/recommendations.thrift +++ /dev/null @@ -1,40 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "com/twitter/ads/adserver/adserver_common.thrift" -include "debug.thrift" -include "reasons.thrift" -include "scoring.thrift" - -struct UserRecommendation { - 1: required i64 userId(personalDataType='UserId') - // reason for this suggestions, eg: social context - 2: optional reasons.Reason reason - // present if it is a promoted account - 3: optional adserver_common.AdImpression adImpression - // tracking token for attribution - 4: optional string trackingInfo - // scoring details - 5: optional scoring.ScoringDetails scoringDetails - 6: optional string recommendationFlowIdentifier - // FeatureSwitch overrides for candidates: - 7: optional map featureOverrides -}(hasPersonalData='true') - -union Recommendation { - 1: UserRecommendation user -}(hasPersonalData='true') - -struct HydratedUserRecommendation { - 1: required i64 userId(personalDataType='UserId') - 2: optional string socialProof - // present if it is a promoted account, used by clients for determining ad impression - 3: optional adserver_common.AdImpression adImpression - // tracking token for attribution - 4: optional string trackingInfo -}(hasPersonalData='true') - -union HydratedRecommendation { - 1: HydratedUserRecommendation hydratedUserRecommendation -} diff --git a/follow-recommendations-service/thrift/src/main/thrift/scoring.thrift b/follow-recommendations-service/thrift/src/main/thrift/scoring.thrift deleted file mode 100644 index 33111baf8..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/scoring.thrift +++ /dev/null @@ -1,49 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "com/twitter/ml/api/data.thrift" - -struct CandidateSourceDetails { - 1: optional map candidateSourceScores - 2: optional i32 primarySource - 3: optional map candidateSourceRanks -}(hasPersonalData='false') - -struct Score { - 1: required double value - 2: optional string rankerId - 3: optional string scoreType -}(hasPersonalData='false') - -// Contains (1) the ML-based heavy ranker and score (2) scores and rankers in producer experiment framework -struct Scores { - 1: required list scores - 2: optional string selectedRankerId - 3: required bool isInProducerScoringExperiment -}(hasPersonalData='false') - -struct RankingInfo { - 1: optional Scores scores - 2: optional i32 rank -}(hasPersonalData='false') - -// this encapsulates all information related to the ranking process from generation to scoring -struct ScoringDetails { - 1: optional CandidateSourceDetails candidateSourceDetails - 2: optional double score - 3: optional data.DataRecord dataRecord - 4: optional list rankerIds - 5: optional DebugDataRecord debugDataRecord // this field is not logged as it's only used for debugging - 6: optional map infoPerRankingStage // scoring and ranking info per ranking stage -}(hasPersonalData='true') - -// exactly the same as a data record, except that we store the feature name instead of the id -struct DebugDataRecord { - 1: optional set binaryFeatures; // stores BINARY features - 2: optional map continuousFeatures; // stores CONTINUOUS features - 3: optional map discreteFeatures; // stores DISCRETE features - 4: optional map stringFeatures; // stores STRING features - 5: optional map> sparseBinaryFeatures; // stores sparse BINARY features - 6: optional map> sparseContinuousFeatures; // sparse CONTINUOUS features -}(hasPersonalData='true') diff --git a/follow-recommendations-service/thrift/src/main/thrift/tracking.thrift b/follow-recommendations-service/thrift/src/main/thrift/tracking.thrift deleted file mode 100644 index 81111ead8..000000000 --- a/follow-recommendations-service/thrift/src/main/thrift/tracking.thrift +++ /dev/null @@ -1,17 +0,0 @@ -namespace java com.twitter.follow_recommendations.thriftjava -#@namespace scala com.twitter.follow_recommendations.thriftscala -#@namespace strato com.twitter.follow_recommendations - -include "com/twitter/suggests/controller_data/controller_data.thrift" -include "display_location.thrift" - -// struct used for tracking/attribution purposes in our offline pipelines -struct TrackingToken { - // trace-id of the request - 1: required i64 sessionId (personalDataType='SessionId') - 2: optional display_location.DisplayLocation displayLocation - // 64-bit encoded binary attributes of our recommendation - 3: optional controller_data.ControllerData controllerData - // WTF Algorithm Id (backward compatibility) - 4: optional i32 algoId -}(hasPersonalData='true')