diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/AdsRecommendationsScribeLogger.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/AdsRecommendationsScribeLogger.docx new file mode 100644 index 000000000..2cb7b1560 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/AdsRecommendationsScribeLogger.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/AdsRecommendationsScribeLogger.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/AdsRecommendationsScribeLogger.scala deleted file mode 100644 index f786bd586..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/AdsRecommendationsScribeLogger.scala +++ /dev/null @@ -1,139 +0,0 @@ -package com.twitter.cr_mixer.logging - -import com.twitter.cr_mixer.model.AdsCandidateGeneratorQuery -import com.twitter.cr_mixer.model.InitialAdsCandidate -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.thriftscala.AdsRecommendationTopLevelApiResult -import com.twitter.cr_mixer.thriftscala.AdsRecommendationsResult -import com.twitter.cr_mixer.thriftscala.AdsRequest -import com.twitter.cr_mixer.thriftscala.AdsResponse -import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult -import com.twitter.cr_mixer.thriftscala.GetAdsRecommendationsScribe -import com.twitter.cr_mixer.thriftscala.PerformanceMetrics -import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata -import com.twitter.cr_mixer.util.CandidateGenerationKeyUtil -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.logging.Logger -import com.twitter.simclusters_v2.common.UserId -import com.twitter.util.Future -import com.twitter.util.Stopwatch - -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -case class AdsRecommendationsScribeLogger @Inject() ( - @Named(ModuleNames.AdsRecommendationsLogger) adsRecommendationsScribeLogger: Logger, - decider: CrMixerDecider, - statsReceiver: StatsReceiver) { - - private val scopedStats = statsReceiver.scope(this.getClass.getCanonicalName) - private val upperFunnelsStats = scopedStats.scope("UpperFunnels") - private val topLevelApiStats = scopedStats.scope("TopLevelApi") - - /* - * Scribe first step results after fetching initial ads candidate - * */ - def scribeInitialAdsCandidates( - query: AdsCandidateGeneratorQuery, - getResultFn: => Future[Seq[Seq[InitialAdsCandidate]]], - enableScribe: Boolean // controlled by feature switch so that we can scribe for certain DDG - ): Future[Seq[Seq[InitialAdsCandidate]]] = { - val scribeMetadata = ScribeMetadata.from(query) - val timer = Stopwatch.start() - getResultFn.onSuccess { input => - val latencyMs = timer().inMilliseconds - val result = convertFetchCandidatesResult(input, scribeMetadata.userId) - val traceId = Trace.id.traceId.toLong - val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) - - if (enableScribe && decider.isAvailableForId( - scribeMetadata.userId, - DeciderConstants.adsRecommendationsPerExperimentScribeRate)) { - upperFunnelsStats.counter(scribeMetadata.product.originalName).incr() - scribeResult(scribeMsg) - } - } - } - - /* - * Scribe top level API results - * */ - def scribeGetAdsRecommendations( - request: AdsRequest, - startTime: Long, - scribeMetadata: ScribeMetadata, - getResultFn: => Future[AdsResponse], - enableScribe: Boolean - ): Future[AdsResponse] = { - val timer = Stopwatch.start() - getResultFn.onSuccess { response => - val latencyMs = timer().inMilliseconds - val result = AdsRecommendationsResult.AdsRecommendationTopLevelApiResult( - AdsRecommendationTopLevelApiResult( - timestamp = startTime, - request = request, - response = response - )) - val traceId = Trace.id.traceId.toLong - val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) - - if (enableScribe && decider.isAvailableForId( - scribeMetadata.userId, - DeciderConstants.adsRecommendationsPerExperimentScribeRate)) { - topLevelApiStats.counter(scribeMetadata.product.originalName).incr() - scribeResult(scribeMsg) - } - } - } - - private def convertFetchCandidatesResult( - candidatesSeq: Seq[Seq[InitialAdsCandidate]], - requestUserId: UserId - ): AdsRecommendationsResult = { - val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => - candidates.map { candidate => - TweetCandidateWithMetadata( - tweetId = candidate.tweetId, - candidateGenerationKey = Some( - CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId)), - score = Some(candidate.getSimilarityScore), - numCandidateGenerationKeys = None // not populated yet - ) - } - } - AdsRecommendationsResult.FetchCandidatesResult( - FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) - } - - private def buildScribeMessage( - result: AdsRecommendationsResult, - scribeMetadata: ScribeMetadata, - latencyMs: Long, - traceId: Long - ): GetAdsRecommendationsScribe = { - GetAdsRecommendationsScribe( - uuid = scribeMetadata.requestUUID, - userId = scribeMetadata.userId, - result = result, - traceId = Some(traceId), - performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), - impressedBuckets = getImpressedBuckets(scopedStats) - ) - } - - private def scribeResult( - scribeMsg: GetAdsRecommendationsScribe - ): Unit = { - publish( - logger = adsRecommendationsScribeLogger, - codec = GetAdsRecommendationsScribe, - message = scribeMsg) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/BUILD deleted file mode 100644 index edf0b77f0..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "abdecider/src/main/scala", - "content-recommender/thrift/src/main/thrift:content-recommender-common-scala", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "decider/src/main/scala", - "featureswitches/featureswitches-core/src/main/scala:experimentation-settings", - "finagle/finagle-core/src/main", - "frigate/frigate-common:base", - "frigate/frigate-common:util", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", - "scribelib/validators/src/main/scala/com/twitter/scribelib/validators", - "scrooge/scrooge-serializer/src/main/scala", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "timelines/src/main/scala/com/twitter/timelines/clientevent", - "util-internal/scribe/src/main/scala/com/twitter/logging", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/BUILD.docx new file mode 100644 index 000000000..3a7267188 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/CrMixerScribeLogger.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/CrMixerScribeLogger.docx new file mode 100644 index 000000000..c9e1d28e1 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/CrMixerScribeLogger.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/CrMixerScribeLogger.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/CrMixerScribeLogger.scala deleted file mode 100644 index 024dcf55b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/CrMixerScribeLogger.scala +++ /dev/null @@ -1,489 +0,0 @@ -package com.twitter.cr_mixer.logging - -import com.google.common.base.CaseFormat -import com.twitter.abdecider.ScribingABDeciderUtil -import com.twitter.scribelib.marshallers.ClientDataProvider -import com.twitter.scribelib.marshallers.ScribeSerialization -import com.twitter.timelines.clientevent.MinimalClientDataProvider -import com.twitter.cr_mixer.model.BlendedCandidate -import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery -import com.twitter.cr_mixer.model.InitialCandidate -import com.twitter.cr_mixer.model.RankedCandidate -import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ -import com.twitter.cr_mixer.model.GraphSourceInfo -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.scribe.ScribeCategories -import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest -import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse -import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult -import com.twitter.cr_mixer.thriftscala.FetchSignalSourcesResult -import com.twitter.cr_mixer.thriftscala.GetTweetsRecommendationsScribe -import com.twitter.cr_mixer.thriftscala.InterleaveResult -import com.twitter.cr_mixer.thriftscala.PerformanceMetrics -import com.twitter.cr_mixer.thriftscala.PreRankFilterResult -import com.twitter.cr_mixer.thriftscala.Product -import com.twitter.cr_mixer.thriftscala.RankResult -import com.twitter.cr_mixer.thriftscala.Result -import com.twitter.cr_mixer.thriftscala.SourceSignal -import com.twitter.cr_mixer.thriftscala.TopLevelApiResult -import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata -import com.twitter.cr_mixer.thriftscala.VITTweetCandidateScribe -import com.twitter.cr_mixer.thriftscala.VITTweetCandidatesScribe -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.SourceInfo -import com.twitter.cr_mixer.util.CandidateGenerationKeyUtil -import com.twitter.cr_mixer.util.MetricTagUtil -import com.twitter.decider.SimpleRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.finatra.kafka.producers.KafkaProducerBase -import com.twitter.logging.Logger -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.util.Future -import com.twitter.util.Stopwatch -import com.twitter.util.Time - -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.util.Random - -@Singleton -case class CrMixerScribeLogger @Inject() ( - decider: CrMixerDecider, - statsReceiver: StatsReceiver, - @Named(ModuleNames.TweetRecsLogger) tweetRecsScribeLogger: Logger, - @Named(ModuleNames.BlueVerifiedTweetRecsLogger) blueVerifiedTweetRecsScribeLogger: Logger, - @Named(ModuleNames.TopLevelApiDdgMetricsLogger) ddgMetricsLogger: Logger, - kafkaProducer: KafkaProducerBase[String, GetTweetsRecommendationsScribe]) { - - import CrMixerScribeLogger._ - - private val scopedStats = statsReceiver.scope("CrMixerScribeLogger") - private val topLevelApiStats = scopedStats.scope("TopLevelApi") - private val upperFunnelsStats = scopedStats.scope("UpperFunnels") - private val kafkaMessagesStats = scopedStats.scope("KafkaMessages") - private val topLevelApiDdgMetricsStats = scopedStats.scope("TopLevelApiDdgMetrics") - private val blueVerifiedTweetCandidatesStats = scopedStats.scope("BlueVerifiedTweetCandidates") - - private val serialization = new ScribeSerialization {} - - def scribeSignalSources( - query: CrCandidateGeneratorQuery, - getResultFn: => Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] - ): Future[(Set[SourceInfo], Map[String, Option[GraphSourceInfo]])] = { - scribeResultsAndPerformanceMetrics( - ScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertFetchSignalSourcesResult - ) - } - - def scribeInitialCandidates( - query: CrCandidateGeneratorQuery, - getResultFn: => Future[Seq[Seq[InitialCandidate]]] - ): Future[Seq[Seq[InitialCandidate]]] = { - scribeResultsAndPerformanceMetrics( - ScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertFetchCandidatesResult - ) - } - - def scribePreRankFilterCandidates( - query: CrCandidateGeneratorQuery, - getResultFn: => Future[Seq[Seq[InitialCandidate]]] - ): Future[Seq[Seq[InitialCandidate]]] = { - scribeResultsAndPerformanceMetrics( - ScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertPreRankFilterResult - ) - } - - def scribeInterleaveCandidates( - query: CrCandidateGeneratorQuery, - getResultFn: => Future[Seq[BlendedCandidate]] - ): Future[Seq[BlendedCandidate]] = { - scribeResultsAndPerformanceMetrics( - ScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertInterleaveResult, - enableKafkaScribe = true - ) - } - - def scribeRankedCandidates( - query: CrCandidateGeneratorQuery, - getResultFn: => Future[Seq[RankedCandidate]] - ): Future[Seq[RankedCandidate]] = { - scribeResultsAndPerformanceMetrics( - ScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertRankResult - ) - } - - /** - * Scribe Top Level API Request / Response and performance metrics - * for the getTweetRecommendations() endpoint. - */ - def scribeGetTweetRecommendations( - request: CrMixerTweetRequest, - startTime: Long, - scribeMetadata: ScribeMetadata, - getResultFn: => Future[CrMixerTweetResponse] - ): Future[CrMixerTweetResponse] = { - val timer = Stopwatch.start() - getResultFn.onSuccess { response => - val latencyMs = timer().inMilliseconds - val result = convertTopLevelAPIResult(request, response, startTime) - val traceId = Trace.id.traceId.toLong - val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) - - // We use upperFunnelPerStepScribeRate to cover TopLevelApi scribe logs - if (decider.isAvailableForId( - scribeMetadata.userId, - DeciderConstants.upperFunnelPerStepScribeRate)) { - topLevelApiStats.counter(scribeMetadata.product.originalName).incr() - scribeResult(scribeMsg) - } - if (decider.isAvailableForId( - scribeMetadata.userId, - DeciderConstants.topLevelApiDdgMetricsScribeRate)) { - topLevelApiDdgMetricsStats.counter(scribeMetadata.product.originalName).incr() - val topLevelDdgMetricsMetadata = TopLevelDdgMetricsMetadata.from(request) - publishTopLevelDdgMetrics( - logger = ddgMetricsLogger, - topLevelDdgMetricsMetadata = topLevelDdgMetricsMetadata, - latencyMs = latencyMs, - candidateSize = response.tweets.length) - } - } - } - - /** - * Scribe all of the Blue Verified tweets that are candidates from cr-mixer - * from the getTweetRecommendations() endpoint for stats tracking/debugging purposes. - */ - def scribeGetTweetRecommendationsForBlueVerified( - scribeMetadata: ScribeMetadata, - getResultFn: => Future[Seq[RankedCandidate]] - ): Future[Seq[RankedCandidate]] = { - getResultFn.onSuccess { rankedCandidates => - if (decider.isAvailable(DeciderConstants.enableScribeForBlueVerifiedTweetCandidates)) { - blueVerifiedTweetCandidatesStats.counter("process_request").incr() - - val blueVerifiedTweetCandidates = rankedCandidates.filter { tweet => - tweet.tweetInfo.hasBlueVerifiedAnnotation.contains(true) - } - - val impressedBuckets = getImpressedBuckets(blueVerifiedTweetCandidatesStats).getOrElse(Nil) - - val blueVerifiedCandidateScribes = blueVerifiedTweetCandidates.map { candidate => - blueVerifiedTweetCandidatesStats - .scope(scribeMetadata.product.name).counter( - candidate.tweetInfo.authorId.toString).incr() - VITTweetCandidateScribe( - tweetId = candidate.tweetId, - authorId = candidate.tweetInfo.authorId, - score = candidate.predictionScore, - metricTags = MetricTagUtil.buildMetricTags(candidate) - ) - } - - val blueVerifiedScribe = - VITTweetCandidatesScribe( - uuid = scribeMetadata.requestUUID, - userId = scribeMetadata.userId, - candidates = blueVerifiedCandidateScribes, - product = scribeMetadata.product, - impressedBuckets = impressedBuckets - ) - - publish( - logger = blueVerifiedTweetRecsScribeLogger, - codec = VITTweetCandidatesScribe, - message = blueVerifiedScribe) - } - } - } - - /** - * Scribe Per-step intermediate results and performance metrics - * for each step: fetch signals, fetch candidates, filters, ranker, etc - */ - private[logging] def scribeResultsAndPerformanceMetrics[T]( - scribeMetadata: ScribeMetadata, - getResultFn: => Future[T], - convertToResultFn: (T, UserId) => Result, - enableKafkaScribe: Boolean = false - ): Future[T] = { - val timer = Stopwatch.start() - getResultFn.onSuccess { input => - val latencyMs = timer().inMilliseconds - val result = convertToResultFn(input, scribeMetadata.userId) - val traceId = Trace.id.traceId.toLong - val scribeMsg = buildScribeMessage(result, scribeMetadata, latencyMs, traceId) - - if (decider.isAvailableForId( - scribeMetadata.userId, - DeciderConstants.upperFunnelPerStepScribeRate)) { - upperFunnelsStats.counter(scribeMetadata.product.originalName).incr() - scribeResult(scribeMsg) - } - - // forks the scribe as a Kafka message for async feature hydration - if (enableKafkaScribe && shouldScribeKafkaMessage( - scribeMetadata.userId, - scribeMetadata.product)) { - kafkaMessagesStats.counter(scribeMetadata.product.originalName).incr() - - val batchedKafkaMessages = downsampleKafkaMessage(scribeMsg) - batchedKafkaMessages.foreach { kafkaMessage => - kafkaProducer.send( - topic = ScribeCategories.TweetsRecs.scribeCategory, - key = traceId.toString, - value = kafkaMessage, - timestamp = Time.now.inMilliseconds - ) - } - } - } - } - - private def convertTopLevelAPIResult( - request: CrMixerTweetRequest, - response: CrMixerTweetResponse, - startTime: Long - ): Result = { - Result.TopLevelApiResult( - TopLevelApiResult( - timestamp = startTime, - request = request, - response = response - )) - } - - private def convertFetchSignalSourcesResult( - sourceInfoSetTuple: (Set[SourceInfo], Map[String, Option[GraphSourceInfo]]), - requestUserId: UserId - ): Result = { - val sourceSignals = sourceInfoSetTuple._1.map { sourceInfo => - SourceSignal(id = Some(sourceInfo.internalId)) - } - // For source graphs, we pass in requestUserId as a placeholder - val sourceGraphs = sourceInfoSetTuple._2.map { - case (_, _) => - SourceSignal(id = Some(InternalId.UserId(requestUserId))) - } - Result.FetchSignalSourcesResult( - FetchSignalSourcesResult( - signals = Some(sourceSignals ++ sourceGraphs) - )) - } - - private def convertFetchCandidatesResult( - candidatesSeq: Seq[Seq[InitialCandidate]], - requestUserId: UserId - ): Result = { - val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => - candidates.map { candidate => - TweetCandidateWithMetadata( - tweetId = candidate.tweetId, - candidateGenerationKey = Some( - CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId)), - score = Some(candidate.getSimilarityScore), - numCandidateGenerationKeys = None // not populated yet - ) - } - } - Result.FetchCandidatesResult(FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) - } - - private def convertPreRankFilterResult( - candidatesSeq: Seq[Seq[InitialCandidate]], - requestUserId: UserId - ): Result = { - val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => - candidates.map { candidate => - TweetCandidateWithMetadata( - tweetId = candidate.tweetId, - candidateGenerationKey = Some( - CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId)), - score = Some(candidate.getSimilarityScore), - numCandidateGenerationKeys = None // not populated yet - ) - } - } - Result.PreRankFilterResult(PreRankFilterResult(Some(tweetCandidatesWithMetadata))) - } - - // We take InterleaveResult for Unconstrained dataset ML ranker training - private def convertInterleaveResult( - blendedCandidates: Seq[BlendedCandidate], - requestUserId: UserId - ): Result = { - val tweetCandidatesWithMetadata = blendedCandidates.map { blendedCandidate => - val candidateGenerationKey = - CandidateGenerationKeyUtil.toThrift(blendedCandidate.reasonChosen, requestUserId) - TweetCandidateWithMetadata( - tweetId = blendedCandidate.tweetId, - candidateGenerationKey = Some(candidateGenerationKey), - authorId = Some(blendedCandidate.tweetInfo.authorId), // for ML pipeline training - score = Some(blendedCandidate.getSimilarityScore), - numCandidateGenerationKeys = Some(blendedCandidate.potentialReasons.size) - ) // hydrate fields for light ranking training data - } - Result.InterleaveResult(InterleaveResult(Some(tweetCandidatesWithMetadata))) - } - - private def convertRankResult( - rankedCandidates: Seq[RankedCandidate], - requestUserId: UserId - ): Result = { - val tweetCandidatesWithMetadata = rankedCandidates.map { rankedCandidate => - val candidateGenerationKey = - CandidateGenerationKeyUtil.toThrift(rankedCandidate.reasonChosen, requestUserId) - TweetCandidateWithMetadata( - tweetId = rankedCandidate.tweetId, - candidateGenerationKey = Some(candidateGenerationKey), - score = Some(rankedCandidate.getSimilarityScore), - numCandidateGenerationKeys = Some(rankedCandidate.potentialReasons.size) - ) - } - Result.RankResult(RankResult(Some(tweetCandidatesWithMetadata))) - } - - private def buildScribeMessage( - result: Result, - scribeMetadata: ScribeMetadata, - latencyMs: Long, - traceId: Long - ): GetTweetsRecommendationsScribe = { - GetTweetsRecommendationsScribe( - uuid = scribeMetadata.requestUUID, - userId = scribeMetadata.userId, - result = result, - traceId = Some(traceId), - performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), - impressedBuckets = getImpressedBuckets(scopedStats) - ) - } - - private def scribeResult( - scribeMsg: GetTweetsRecommendationsScribe - ): Unit = { - publish( - logger = tweetRecsScribeLogger, - codec = GetTweetsRecommendationsScribe, - message = scribeMsg) - } - - /** - * Gate for producing messages to Kafka for async feature hydration - */ - private def shouldScribeKafkaMessage( - userId: UserId, - product: Product - ): Boolean = { - val isEligibleUser = decider.isAvailable( - DeciderConstants.kafkaMessageScribeSampleRate, - Some(SimpleRecipient(userId))) - val isHomeProduct = (product == Product.Home) - isEligibleUser && isHomeProduct - } - - /** - * Due to size limits of Strato (see SD-19028), each Kafka message must be downsampled - */ - private[logging] def downsampleKafkaMessage( - scribeMsg: GetTweetsRecommendationsScribe - ): Seq[GetTweetsRecommendationsScribe] = { - val sampledResultSeq: Seq[Result] = scribeMsg.result match { - case Result.InterleaveResult(interleaveResult) => - val sampledTweetsSeq = interleaveResult.tweets - .map { tweets => - Random - .shuffle(tweets).take(KafkaMaxTweetsPerMessage) - .grouped(BatchSize).toSeq - }.getOrElse(Seq.empty) - - sampledTweetsSeq.map { sampledTweets => - Result.InterleaveResult(InterleaveResult(Some(sampledTweets))) - } - - // if it's an unrecognized type, err on the side of sending no candidates - case _ => - kafkaMessagesStats.counter("InvalidKafkaMessageResultType").incr() - Seq(Result.InterleaveResult(InterleaveResult(None))) - } - - sampledResultSeq.map { sampledResult => - GetTweetsRecommendationsScribe( - uuid = scribeMsg.uuid, - userId = scribeMsg.userId, - result = sampledResult, - traceId = scribeMsg.traceId, - performanceMetrics = None, - impressedBuckets = None - ) - } - } - - /** - * Handles client_event serialization to log data into DDG metrics - */ - private[logging] def publishTopLevelDdgMetrics( - logger: Logger, - topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata, - candidateSize: Long, - latencyMs: Long, - ): Unit = { - val data = Map[Any, Any]( - "latency_ms" -> latencyMs, - "event_value" -> candidateSize - ) - val label: (String, String) = ("tweetrec", "") - val namespace = getNamespace(topLevelDdgMetricsMetadata, label) + ("action" -> "candidates") - val message = - serialization - .serializeClientEvent(namespace, getClientData(topLevelDdgMetricsMetadata), data) - logger.info(message) - } - - private def getClientData( - topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata - ): ClientDataProvider = - MinimalClientDataProvider( - userId = topLevelDdgMetricsMetadata.userId, - guestId = None, - clientApplicationId = topLevelDdgMetricsMetadata.clientApplicationId, - countryCode = topLevelDdgMetricsMetadata.countryCode - ) - - private def getNamespace( - topLevelDdgMetricsMetadata: TopLevelDdgMetricsMetadata, - label: (String, String) - ): Map[String, String] = { - val productName = - CaseFormat.UPPER_CAMEL - .to(CaseFormat.LOWER_UNDERSCORE, topLevelDdgMetricsMetadata.product.originalName) - - Map( - "client" -> ScribingABDeciderUtil.clientForAppId( - topLevelDdgMetricsMetadata.clientApplicationId), - "page" -> "cr-mixer", - "section" -> productName, - "component" -> label._1, - "element" -> label._2 - ) - } -} - -object CrMixerScribeLogger { - val KafkaMaxTweetsPerMessage: Int = 200 - val BatchSize: Int = 20 -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/RelatedTweetScribeLogger.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/RelatedTweetScribeLogger.docx new file mode 100644 index 000000000..be9cff23c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/RelatedTweetScribeLogger.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/RelatedTweetScribeLogger.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/RelatedTweetScribeLogger.scala deleted file mode 100644 index b2b36f43c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/RelatedTweetScribeLogger.scala +++ /dev/null @@ -1,193 +0,0 @@ -package com.twitter.cr_mixer.logging - -import com.twitter.cr_mixer.model.RelatedTweetCandidateGeneratorQuery -import com.twitter.cr_mixer.model.InitialCandidate -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult -import com.twitter.cr_mixer.thriftscala.GetRelatedTweetsScribe -import com.twitter.cr_mixer.thriftscala.PerformanceMetrics -import com.twitter.cr_mixer.thriftscala.PreRankFilterResult -import com.twitter.cr_mixer.thriftscala.RelatedTweetRequest -import com.twitter.cr_mixer.thriftscala.RelatedTweetResponse -import com.twitter.cr_mixer.thriftscala.RelatedTweetResult -import com.twitter.cr_mixer.thriftscala.RelatedTweetTopLevelApiResult -import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata -import com.twitter.cr_mixer.util.CandidateGenerationKeyUtil -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.logging.Logger -import com.twitter.simclusters_v2.common.UserId -import com.twitter.util.Future -import com.twitter.util.Stopwatch -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -case class RelatedTweetScribeLogger @Inject() ( - decider: CrMixerDecider, - statsReceiver: StatsReceiver, - @Named(ModuleNames.RelatedTweetsLogger) relatedTweetsScribeLogger: Logger) { - - private val scopedStats = statsReceiver.scope("RelatedTweetsScribeLogger") - private val topLevelApiStats = scopedStats.scope("TopLevelApi") - private val topLevelApiNoUserIdStats = scopedStats.scope("TopLevelApiNoUserId") - private val upperFunnelsStats = scopedStats.scope("UpperFunnels") - private val upperFunnelsNoUserIdStats = scopedStats.scope("UpperFunnelsNoUserId") - - def scribeInitialCandidates( - query: RelatedTweetCandidateGeneratorQuery, - getResultFn: => Future[Seq[Seq[InitialCandidate]]] - ): Future[Seq[Seq[InitialCandidate]]] = { - scribeResultsAndPerformanceMetrics( - RelatedTweetScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertFetchCandidatesResult - ) - } - - def scribePreRankFilterCandidates( - query: RelatedTweetCandidateGeneratorQuery, - getResultFn: => Future[Seq[Seq[InitialCandidate]]] - ): Future[Seq[Seq[InitialCandidate]]] = { - scribeResultsAndPerformanceMetrics( - RelatedTweetScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertPreRankFilterResult - ) - } - - /** - * Scribe Top Level API Request / Response and performance metrics - * for the getRelatedTweets endpoint. - */ - def scribeGetRelatedTweets( - request: RelatedTweetRequest, - startTime: Long, - relatedTweetScribeMetadata: RelatedTweetScribeMetadata, - getResultFn: => Future[RelatedTweetResponse] - ): Future[RelatedTweetResponse] = { - val timer = Stopwatch.start() - getResultFn.onSuccess { response => - relatedTweetScribeMetadata.clientContext.userId match { - case Some(userId) => - if (decider.isAvailableForId(userId, DeciderConstants.upperFunnelPerStepScribeRate)) { - topLevelApiStats.counter(relatedTweetScribeMetadata.product.originalName).incr() - val latencyMs = timer().inMilliseconds - val result = convertTopLevelAPIResult(request, response, startTime) - val traceId = Trace.id.traceId.toLong - val scribeMsg = - buildScribeMessage(result, relatedTweetScribeMetadata, latencyMs, traceId) - - scribeResult(scribeMsg) - } - case _ => - topLevelApiNoUserIdStats.counter(relatedTweetScribeMetadata.product.originalName).incr() - } - } - } - - /** - * Scribe Per-step intermediate results and performance metrics - * for each step: fetch candidates, filters. - */ - private def scribeResultsAndPerformanceMetrics[T]( - relatedTweetScribeMetadata: RelatedTweetScribeMetadata, - getResultFn: => Future[T], - convertToResultFn: (T, UserId) => RelatedTweetResult - ): Future[T] = { - val timer = Stopwatch.start() - getResultFn.onSuccess { input => - relatedTweetScribeMetadata.clientContext.userId match { - case Some(userId) => - if (decider.isAvailableForId(userId, DeciderConstants.upperFunnelPerStepScribeRate)) { - upperFunnelsStats.counter(relatedTweetScribeMetadata.product.originalName).incr() - val latencyMs = timer().inMilliseconds - val result = convertToResultFn(input, userId) - val traceId = Trace.id.traceId.toLong - val scribeMsg = - buildScribeMessage(result, relatedTweetScribeMetadata, latencyMs, traceId) - scribeResult(scribeMsg) - } - case _ => - upperFunnelsNoUserIdStats.counter(relatedTweetScribeMetadata.product.originalName).incr() - } - } - } - - private def convertTopLevelAPIResult( - request: RelatedTweetRequest, - response: RelatedTweetResponse, - startTime: Long - ): RelatedTweetResult = { - RelatedTweetResult.RelatedTweetTopLevelApiResult( - RelatedTweetTopLevelApiResult( - timestamp = startTime, - request = request, - response = response - )) - } - - private def convertFetchCandidatesResult( - candidatesSeq: Seq[Seq[InitialCandidate]], - requestUserId: UserId - ): RelatedTweetResult = { - val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => - candidates.map { candidate => - TweetCandidateWithMetadata( - tweetId = candidate.tweetId, - candidateGenerationKey = None - ) // do not hydrate candidateGenerationKey to save cost - } - } - RelatedTweetResult.FetchCandidatesResult( - FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) - } - - private def convertPreRankFilterResult( - candidatesSeq: Seq[Seq[InitialCandidate]], - requestUserId: UserId - ): RelatedTweetResult = { - val tweetCandidatesWithMetadata = candidatesSeq.flatMap { candidates => - candidates.map { candidate => - val candidateGenerationKey = - CandidateGenerationKeyUtil.toThrift(candidate.candidateGenerationInfo, requestUserId) - TweetCandidateWithMetadata( - tweetId = candidate.tweetId, - candidateGenerationKey = Some(candidateGenerationKey), - authorId = Some(candidate.tweetInfo.authorId), - score = Some(candidate.getSimilarityScore), - numCandidateGenerationKeys = None - ) - } - } - RelatedTweetResult.PreRankFilterResult(PreRankFilterResult(Some(tweetCandidatesWithMetadata))) - } - - private def buildScribeMessage( - relatedTweetResult: RelatedTweetResult, - relatedTweetScribeMetadata: RelatedTweetScribeMetadata, - latencyMs: Long, - traceId: Long - ): GetRelatedTweetsScribe = { - GetRelatedTweetsScribe( - uuid = relatedTweetScribeMetadata.requestUUID, - internalId = relatedTweetScribeMetadata.internalId, - relatedTweetResult = relatedTweetResult, - requesterId = relatedTweetScribeMetadata.clientContext.userId, - guestId = relatedTweetScribeMetadata.clientContext.guestId, - traceId = Some(traceId), - performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), - impressedBuckets = getImpressedBuckets(scopedStats) - ) - } - - private def scribeResult( - scribeMsg: GetRelatedTweetsScribe - ): Unit = { - publish(logger = relatedTweetsScribeLogger, codec = GetRelatedTweetsScribe, message = scribeMsg) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeLoggerUtils.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeLoggerUtils.docx new file mode 100644 index 000000000..963fa6a62 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeLoggerUtils.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeLoggerUtils.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeLoggerUtils.scala deleted file mode 100644 index 3b30c3f10..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeLoggerUtils.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.cr_mixer.logging - -import com.twitter.cr_mixer.featureswitch.CrMixerImpressedBuckets -import com.twitter.cr_mixer.thriftscala.ImpressesedBucketInfo -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.logging.Logger -import com.twitter.scrooge.BinaryThriftStructSerializer -import com.twitter.scrooge.ThriftStruct -import com.twitter.scrooge.ThriftStructCodec - -object ScribeLoggerUtils { - - /** - * Handles base64-encoding, serialization, and publish. - */ - private[logging] def publish[T <: ThriftStruct]( - logger: Logger, - codec: ThriftStructCodec[T], - message: T - ): Unit = { - logger.info(BinaryThriftStructSerializer(codec).toString(message)) - } - - private[logging] def getImpressedBuckets( - scopedStats: StatsReceiver - ): Option[List[ImpressesedBucketInfo]] = { - StatsUtil.trackNonFutureBlockStats(scopedStats.scope("getImpressedBuckets")) { - CrMixerImpressedBuckets.getAllImpressedBuckets.map { listBuckets => - val listBucketsSet = listBuckets.toSet - scopedStats.stat("impressed_buckets").add(listBucketsSet.size) - listBucketsSet.map { bucket => - ImpressesedBucketInfo( - experimentId = bucket.experiment.settings.experimentId.getOrElse(-1L), - bucketName = bucket.name, - version = bucket.experiment.settings.version, - ) - }.toList - } - } - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeMetadata.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeMetadata.docx new file mode 100644 index 000000000..9a68915a8 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeMetadata.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeMetadata.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeMetadata.scala deleted file mode 100644 index 8c0444e38..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/ScribeMetadata.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.cr_mixer.logging - -import com.twitter.cr_mixer.model.AdsCandidateGeneratorQuery -import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery -import com.twitter.cr_mixer.model.RelatedTweetCandidateGeneratorQuery -import com.twitter.cr_mixer.model.UtegTweetCandidateGeneratorQuery -import com.twitter.cr_mixer.thriftscala.Product -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId - -case class ScribeMetadata( - requestUUID: Long, - userId: UserId, - product: Product) - -object ScribeMetadata { - def from(query: CrCandidateGeneratorQuery): ScribeMetadata = { - ScribeMetadata(query.requestUUID, query.userId, query.product) - } - - def from(query: UtegTweetCandidateGeneratorQuery): ScribeMetadata = { - ScribeMetadata(query.requestUUID, query.userId, query.product) - } - - def from(query: AdsCandidateGeneratorQuery): ScribeMetadata = { - ScribeMetadata(query.requestUUID, query.userId, query.product) - } -} - -case class RelatedTweetScribeMetadata( - requestUUID: Long, - internalId: InternalId, - clientContext: ClientContext, - product: Product) - -object RelatedTweetScribeMetadata { - def from(query: RelatedTweetCandidateGeneratorQuery): RelatedTweetScribeMetadata = { - RelatedTweetScribeMetadata( - query.requestUUID, - query.internalId, - query.clientContext, - query.product) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/TopLevelDdgMetricsMetadata.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/TopLevelDdgMetricsMetadata.docx new file mode 100644 index 000000000..b9075ce8d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/TopLevelDdgMetricsMetadata.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/TopLevelDdgMetricsMetadata.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/TopLevelDdgMetricsMetadata.scala deleted file mode 100644 index 3dd07e58e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/TopLevelDdgMetricsMetadata.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.cr_mixer -package logging - -import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest -import com.twitter.cr_mixer.thriftscala.Product - -case class TopLevelDdgMetricsMetadata( - userId: Option[Long], - product: Product, - clientApplicationId: Option[Long], - countryCode: Option[String]) - -object TopLevelDdgMetricsMetadata { - def from(request: CrMixerTweetRequest): TopLevelDdgMetricsMetadata = { - TopLevelDdgMetricsMetadata( - userId = request.clientContext.userId, - product = request.product, - clientApplicationId = request.clientContext.appId, - countryCode = request.clientContext.countryCode - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/UtegTweetScribeLogger.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/UtegTweetScribeLogger.docx new file mode 100644 index 000000000..b7654ec40 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/UtegTweetScribeLogger.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/UtegTweetScribeLogger.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/UtegTweetScribeLogger.scala deleted file mode 100644 index fb01a419b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/logging/UtegTweetScribeLogger.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.twitter.cr_mixer.logging - -import com.twitter.cr_mixer.logging.ScribeLoggerUtils._ -import com.twitter.cr_mixer.model.UtegTweetCandidateGeneratorQuery -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.thriftscala.UtegTweetRequest -import com.twitter.cr_mixer.thriftscala.UtegTweetResponse -import com.twitter.cr_mixer.thriftscala.FetchCandidatesResult -import com.twitter.cr_mixer.thriftscala.GetUtegTweetsScribe -import com.twitter.cr_mixer.thriftscala.PerformanceMetrics -import com.twitter.cr_mixer.thriftscala.UtegTweetResult -import com.twitter.cr_mixer.thriftscala.UtegTweetTopLevelApiResult -import com.twitter.cr_mixer.thriftscala.TweetCandidateWithMetadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.logging.Logger -import com.twitter.simclusters_v2.common.UserId -import com.twitter.util.Future -import com.twitter.util.Stopwatch -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -case class UtegTweetScribeLogger @Inject() ( - decider: CrMixerDecider, - statsReceiver: StatsReceiver, - @Named(ModuleNames.UtegTweetsLogger) utegTweetScribeLogger: Logger) { - - private val scopedStats = statsReceiver.scope("UtegTweetScribeLogger") - private val topLevelApiStats = scopedStats.scope("TopLevelApi") - private val upperFunnelsStats = scopedStats.scope("UpperFunnels") - - def scribeInitialCandidates( - query: UtegTweetCandidateGeneratorQuery, - getResultFn: => Future[Seq[TweetWithScoreAndSocialProof]] - ): Future[Seq[TweetWithScoreAndSocialProof]] = { - scribeResultsAndPerformanceMetrics( - ScribeMetadata.from(query), - getResultFn, - convertToResultFn = convertFetchCandidatesResult - ) - } - - /** - * Scribe Top Level API Request / Response and performance metrics - * for the GetUtegTweetRecommendations() endpoint. - */ - def scribeGetUtegTweetRecommendations( - request: UtegTweetRequest, - startTime: Long, - scribeMetadata: ScribeMetadata, - getResultFn: => Future[UtegTweetResponse] - ): Future[UtegTweetResponse] = { - val timer = Stopwatch.start() - getResultFn.onSuccess { response => - if (decider.isAvailableForId( - scribeMetadata.userId, - DeciderConstants.upperFunnelPerStepScribeRate)) { - topLevelApiStats.counter(scribeMetadata.product.originalName).incr() - val latencyMs = timer().inMilliseconds - val result = convertTopLevelAPIResult(request, response, startTime) - val traceId = Trace.id.traceId.toLong - val scribeMsg = - buildScribeMessage(result, scribeMetadata, latencyMs, traceId) - - scribeResult(scribeMsg) - } - } - } - - private def convertTopLevelAPIResult( - request: UtegTweetRequest, - response: UtegTweetResponse, - startTime: Long - ): UtegTweetResult = { - UtegTweetResult.UtegTweetTopLevelApiResult( - UtegTweetTopLevelApiResult( - timestamp = startTime, - request = request, - response = response - )) - } - - private def buildScribeMessage( - utegTweetResult: UtegTweetResult, - scribeMetadata: ScribeMetadata, - latencyMs: Long, - traceId: Long - ): GetUtegTweetsScribe = { - GetUtegTweetsScribe( - uuid = scribeMetadata.requestUUID, - userId = scribeMetadata.userId, - utegTweetResult = utegTweetResult, - traceId = Some(traceId), - performanceMetrics = Some(PerformanceMetrics(Some(latencyMs))), - impressedBuckets = getImpressedBuckets(scopedStats) - ) - } - - private def scribeResult( - scribeMsg: GetUtegTweetsScribe - ): Unit = { - publish(logger = utegTweetScribeLogger, codec = GetUtegTweetsScribe, message = scribeMsg) - } - - private def convertFetchCandidatesResult( - candidates: Seq[TweetWithScoreAndSocialProof], - requestUserId: UserId - ): UtegTweetResult = { - val tweetCandidatesWithMetadata = candidates.map { candidate => - TweetCandidateWithMetadata( - tweetId = candidate.tweetId, - candidateGenerationKey = None - ) // do not hydrate candidateGenerationKey to save cost - } - UtegTweetResult.FetchCandidatesResult(FetchCandidatesResult(Some(tweetCandidatesWithMetadata))) - } - - /** - * Scribe Per-step intermediate results and performance metrics - * for each step: fetch candidates, filters. - */ - private def scribeResultsAndPerformanceMetrics[T]( - scribeMetadata: ScribeMetadata, - getResultFn: => Future[T], - convertToResultFn: (T, UserId) => UtegTweetResult - ): Future[T] = { - val timer = Stopwatch.start() - getResultFn.onSuccess { input => - if (decider.isAvailableForId( - scribeMetadata.userId, - DeciderConstants.upperFunnelPerStepScribeRate)) { - upperFunnelsStats.counter(scribeMetadata.product.originalName).incr() - val latencyMs = timer().inMilliseconds - val result = convertToResultFn(input, scribeMetadata.userId) - val traceId = Trace.id.traceId.toLong - val scribeMsg = - buildScribeMessage(result, scribeMetadata, latencyMs, traceId) - scribeResult(scribeMsg) - } - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/BUILD deleted file mode 100644 index 87c714254..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "configapi/configapi-core", - "content-recommender/thrift/src/main/thrift:thrift-scala", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/recos:recos-common-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/BUILD.docx new file mode 100644 index 000000000..1838ccdbb Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/Candidate.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/Candidate.docx new file mode 100644 index 000000000..b5f5e8787 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/Candidate.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/Candidate.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/Candidate.scala deleted file mode 100644 index c357c9472..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/Candidate.scala +++ /dev/null @@ -1,200 +0,0 @@ -package com.twitter.cr_mixer.model - -import com.twitter.contentrecommender.thriftscala.TweetInfo -import com.twitter.cr_mixer.thriftscala.LineItemInfo -import com.twitter.simclusters_v2.common.TweetId - -sealed trait Candidate { - val tweetId: TweetId - - override def hashCode: Int = tweetId.toInt -} - -case class TweetWithCandidateGenerationInfo( - tweetId: TweetId, - candidateGenerationInfo: CandidateGenerationInfo) - extends Candidate { - - def getSimilarityScore: Double = - candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) -} - -case class InitialCandidate( - tweetId: TweetId, - tweetInfo: TweetInfo, - candidateGenerationInfo: CandidateGenerationInfo) - extends Candidate { - - /** * - * Get the Similarity Score of a Tweet from its CG Info. For instance, - * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score - * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score - */ - def getSimilarityScore: Double = - candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) - - /** - * The same candidate can be generated by multiple algorithms. - * During blending, candidate deduping happens. In order to retain the candidateGenerationInfo - * from different algorithms, we attach them to a list of potentialReasons. - */ - def toBlendedCandidate( - potentialReasons: Seq[CandidateGenerationInfo], - ): BlendedCandidate = { - BlendedCandidate( - tweetId, - tweetInfo, - candidateGenerationInfo, - potentialReasons, - ) - } - - // for experimental purposes only when bypassing interleave / ranking - def toRankedCandidate(): RankedCandidate = { - RankedCandidate( - tweetId, - tweetInfo, - 0.0, // prediction score is default to 0.0 to help differentiate that it is a no-op - candidateGenerationInfo, - Seq(candidateGenerationInfo) - ) - } -} - -case class InitialAdsCandidate( - tweetId: TweetId, - lineItemInfo: Seq[LineItemInfo], - candidateGenerationInfo: CandidateGenerationInfo) - extends Candidate { - - /** * - * Get the Similarity Score of a Tweet from its CG Info. For instance, - * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score - * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score - */ - def getSimilarityScore: Double = - candidateGenerationInfo.similarityEngineInfo.score.getOrElse(0.0) - - /** - * The same candidate can be generated by multiple algorithms. - * During blending, candidate deduping happens. In order to retain the candidateGenerationInfo - * from different algorithms, we attach them to a list of potentialReasons. - */ - def toBlendedAdsCandidate( - potentialReasons: Seq[CandidateGenerationInfo], - ): BlendedAdsCandidate = { - BlendedAdsCandidate( - tweetId, - lineItemInfo, - candidateGenerationInfo, - potentialReasons, - ) - } - - // for experimental purposes only when bypassing interleave / ranking - def toRankedAdsCandidate(): RankedAdsCandidate = { - RankedAdsCandidate( - tweetId, - lineItemInfo, - 0.0, // prediction score is default to 0.0 to help differentiate that it is a no-op - candidateGenerationInfo, - Seq(candidateGenerationInfo) - ) - } -} - -case class BlendedCandidate( - tweetId: TweetId, - tweetInfo: TweetInfo, - reasonChosen: CandidateGenerationInfo, - potentialReasons: Seq[CandidateGenerationInfo]) - extends Candidate { - - /** * - * Get the Similarity Score of a Tweet from its CG Info. For instance, - * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score - * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score - */ - def getSimilarityScore: Double = - reasonChosen.similarityEngineInfo.score.getOrElse(0.0) - - assert(potentialReasons.contains(reasonChosen)) - - def toRankedCandidate(predictionScore: Double): RankedCandidate = { - RankedCandidate( - tweetId, - tweetInfo, - predictionScore, - reasonChosen, - potentialReasons - ) - } -} - -case class BlendedAdsCandidate( - tweetId: TweetId, - lineItemInfo: Seq[LineItemInfo], - reasonChosen: CandidateGenerationInfo, - potentialReasons: Seq[CandidateGenerationInfo]) - extends Candidate { - - /** * - * Get the Similarity Score of a Tweet from its CG Info. For instance, - * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score - * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score - */ - def getSimilarityScore: Double = - reasonChosen.similarityEngineInfo.score.getOrElse(0.0) - - assert(potentialReasons.contains(reasonChosen)) - - def toRankedAdsCandidate(predictionScore: Double): RankedAdsCandidate = { - RankedAdsCandidate( - tweetId, - lineItemInfo, - predictionScore, - reasonChosen, - potentialReasons - ) - } -} - -case class RankedCandidate( - tweetId: TweetId, - tweetInfo: TweetInfo, - predictionScore: Double, - reasonChosen: CandidateGenerationInfo, - potentialReasons: Seq[CandidateGenerationInfo]) - extends Candidate { - - /** * - * Get the Similarity Score of a Tweet from its CG Info. For instance, - * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score - * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score - */ - def getSimilarityScore: Double = - reasonChosen.similarityEngineInfo.score.getOrElse(0.0) - - assert(potentialReasons.contains(reasonChosen)) -} - -case class RankedAdsCandidate( - tweetId: TweetId, - lineItemInfo: Seq[LineItemInfo], - predictionScore: Double, - reasonChosen: CandidateGenerationInfo, - potentialReasons: Seq[CandidateGenerationInfo]) - extends Candidate { - - /** * - * Get the Similarity Score of a Tweet from its CG Info. For instance, - * If it is from a UnifiedTweetBasedSimilarityEngine, the score will be the weighted combined score - * And if it is from a SimClustersANNSimilarityEngine, the score will be the SANN score - */ - def getSimilarityScore: Double = - reasonChosen.similarityEngineInfo.score.getOrElse(0.0) - - assert(potentialReasons.contains(reasonChosen)) -} - -case class TripTweetWithScore(tweetId: TweetId, score: Double) extends Candidate diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.docx new file mode 100644 index 000000000..a88b6aa05 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.scala deleted file mode 100644 index 879c96b66..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.cr_mixer.model - -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.cr_mixer.thriftscala.SourceType -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.util.Time - -/*** - * Tweet-level attributes. Represents the source used in candidate generation - * Due to legacy reason, SourceType used to represent both SourceType and SimilarityEngineType - * Moving forward, SourceType will be used for SourceType ONLY. eg., TweetFavorite, UserFollow, TwiceUserId - * At the same time, We create a new SimilarityEngineType to separate them. eg., SimClustersANN - * - * Currently, one special case is that we have TwiceUserId as a source, which is not necessarily a "signal" - * @param sourceType, e.g., SourceType.TweetFavorite, SourceType.UserFollow, SourceType.TwiceUserId - * @param internalId, e.g., UserId(0L), TweetId(0L) - */ -case class SourceInfo( - sourceType: SourceType, - internalId: InternalId, - sourceEventTime: Option[Time]) - -/*** - * Tweet-level attributes. Represents the source User Graph used in candidate generation - * It is an intermediate product, and will not be stored, unlike SourceInfo. - * Essentially, CrMixer queries a graph, and the graph returns a list of users to be used as sources. - * For instance, RealGraph, EarlyBird, FRS, Stp, etc. The underlying similarity engines such as - * UTG or UTEG will leverage these sources to build candidates. - * - * We extended the definition of SourceType to cover both "Source Signal" and "Source Graph" - * See [CrMixer] Graph Based Source Fetcher Abstraction Proposal: - * - * consider making both SourceInfo and GraphSourceInfo extends the same trait to - * have a unified interface. - */ -case class GraphSourceInfo( - sourceType: SourceType, - seedWithScores: Map[UserId, Double]) - -/*** - * Tweet-level attributes. Represents the similarity engine (the algorithm) used for - * candidate generation along with their metadata. - * @param similarityEngineType, e.g., SimClustersANN, UserTweetGraph - * @param modelId. e.g., UserTweetGraphConsumerEmbedding_ALL_20210708 - * @param score - a score generated by this sim engine - */ -case class SimilarityEngineInfo( - similarityEngineType: SimilarityEngineType, - modelId: Option[String], // ModelId can be a None. e.g., UTEG, UnifiedTweetBasedSE. etc - score: Option[Double]) - -/**** - * Tweet-level attributes. A combination for both SourceInfo and SimilarityEngineInfo - * SimilarityEngine is a composition, and it can be composed by many leaf Similarity Engines. - * For instance, the TweetBasedUnified SE could be a composition of both UserTweetGraph SE, SimClustersANN SE. - * Note that a SimilarityEngine (Composite) may call other SimilarityEngines (Atomic, Contributing) - * to contribute to its final candidate list. We track these Contributing SEs in the contributingSimilarityEngines list - * - * @param sourceInfoOpt - this is optional as many consumerBased CG does not have a source - * @param similarityEngineInfo - the similarity engine used in Candidate Generation (eg., TweetBasedUnifiedSE). It can be an atomic SE or an composite SE - * @param contributingSimilarityEngines - only composite SE will have it (e.g., SANNN, UTG). Otherwise it is an empty Seq. All contributing SEs mst be atomic - */ -case class CandidateGenerationInfo( - sourceInfoOpt: Option[SourceInfo], - similarityEngineInfo: SimilarityEngineInfo, - contributingSimilarityEngines: Seq[SimilarityEngineInfo]) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGeneratorQuery.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGeneratorQuery.docx new file mode 100644 index 000000000..07ecc0e5f Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGeneratorQuery.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGeneratorQuery.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGeneratorQuery.scala deleted file mode 100644 index 084cbb042..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGeneratorQuery.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.cr_mixer.model - -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.cr_mixer.thriftscala.Product -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.timelines.configapi.Params - -sealed trait CandidateGeneratorQuery { - val product: Product - val maxNumResults: Int - val impressedTweetList: Set[TweetId] - val params: Params - val requestUUID: Long -} - -sealed trait HasUserId { - val userId: UserId -} - -case class CrCandidateGeneratorQuery( - userId: UserId, - product: Product, - userState: UserState, - maxNumResults: Int, - impressedTweetList: Set[TweetId], - params: Params, - requestUUID: Long, - languageCode: Option[String] = None) - extends CandidateGeneratorQuery - with HasUserId - -case class UtegTweetCandidateGeneratorQuery( - userId: UserId, - product: Product, - userState: UserState, - maxNumResults: Int, - impressedTweetList: Set[TweetId], - params: Params, - requestUUID: Long) - extends CandidateGeneratorQuery - with HasUserId - -case class RelatedTweetCandidateGeneratorQuery( - internalId: InternalId, - clientContext: ClientContext, // To scribe LogIn/LogOut requests - product: Product, - maxNumResults: Int, - impressedTweetList: Set[TweetId], - params: Params, - requestUUID: Long) - extends CandidateGeneratorQuery - -case class RelatedVideoTweetCandidateGeneratorQuery( - internalId: InternalId, - clientContext: ClientContext, // To scribe LogIn/LogOut requests - product: Product, - maxNumResults: Int, - impressedTweetList: Set[TweetId], - params: Params, - requestUUID: Long) - extends CandidateGeneratorQuery - -case class FrsTweetCandidateGeneratorQuery( - userId: UserId, - product: Product, - maxNumResults: Int, - impressedUserList: Set[UserId], - impressedTweetList: Set[TweetId], - params: Params, - languageCodeOpt: Option[String] = None, - countryCodeOpt: Option[String] = None, - requestUUID: Long) - extends CandidateGeneratorQuery - -case class AdsCandidateGeneratorQuery( - userId: UserId, - product: Product, - userState: UserState, - maxNumResults: Int, - params: Params, - requestUUID: Long) - -case class TopicTweetCandidateGeneratorQuery( - userId: UserId, - topicIds: Set[TopicId], - product: Product, - maxNumResults: Int, - impressedTweetList: Set[TweetId], - params: Params, - requestUUID: Long, - isVideoOnly: Boolean) - extends CandidateGeneratorQuery diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/EarlybirdSimilarityEngineType.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/EarlybirdSimilarityEngineType.docx new file mode 100644 index 000000000..35fcb0a03 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/EarlybirdSimilarityEngineType.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/EarlybirdSimilarityEngineType.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/EarlybirdSimilarityEngineType.scala deleted file mode 100644 index aa3040373..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/EarlybirdSimilarityEngineType.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.cr_mixer.model - -sealed trait EarlybirdSimilarityEngineType -object EarlybirdSimilarityEngineType_RecencyBased extends EarlybirdSimilarityEngineType -object EarlybirdSimilarityEngineType_ModelBased extends EarlybirdSimilarityEngineType -object EarlybirdSimilarityEngineType_TensorflowBased extends EarlybirdSimilarityEngineType diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/HealthThreshold.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/HealthThreshold.docx new file mode 100644 index 000000000..6320ac425 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/HealthThreshold.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/HealthThreshold.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/HealthThreshold.scala deleted file mode 100644 index 0249798bd..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/HealthThreshold.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.cr_mixer.model - -object HealthThreshold { - object Enum extends Enumeration { - val Off: Value = Value(1) - val Moderate: Value = Value(2) - val Strict: Value = Value(3) - val Stricter: Value = Value(4) - val StricterPlus: Value = Value(5) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.docx new file mode 100644 index 000000000..078b7be30 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.scala deleted file mode 100644 index 26db7898b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.cr_mixer.model - -/** - * A Configuration class for all Model Based Candidate Sources. - * - * The Model Name Guideline. Please your modelId as "Algorithm_Product_Date" - * If your model is used for multiple product surfaces, name it as all - * Don't name your algorithm as MBCG. All the algorithms here are MBCG =.= - * - * Don't forgot to add your new models into allHnswANNSimilarityEngineModelIds list. - */ -object ModelConfig { - // Offline SimClusters CG Experiment related Model Ids - val OfflineInterestedInFromKnownFor2020: String = "OfflineIIKF_ALL_20220414" - val OfflineInterestedInFromKnownFor2020Hl0El15: String = "OfflineIIKF_ALL_20220414_Hl0_El15" - val OfflineInterestedInFromKnownFor2020Hl2El15: String = "OfflineIIKF_ALL_20220414_Hl2_El15" - val OfflineInterestedInFromKnownFor2020Hl2El50: String = "OfflineIIKF_ALL_20220414_Hl2_El50" - val OfflineInterestedInFromKnownFor2020Hl8El50: String = "OfflineIIKF_ALL_20220414_Hl8_El50" - val OfflineMTSConsumerEmbeddingsFav90P20M: String = - "OfflineMTSConsumerEmbeddingsFav90P20M_ALL_20220414" - - // Twhin Model Ids - val ConsumerBasedTwHINRegularUpdateAll20221024: String = - "ConsumerBasedTwHINRegularUpdate_All_20221024" - - // Averaged Twhin Model Ids - val TweetBasedTwHINRegularUpdateAll20221024: String = - "TweetBasedTwHINRegularUpdate_All_20221024" - - // Collaborative Filtering Twhin Model Ids - val TwhinCollabFilterForFollow: String = - "TwhinCollabFilterForFollow" - val TwhinCollabFilterForEngagement: String = - "TwhinCollabFilterForEngagement" - val TwhinMultiClusterForFollow: String = - "TwhinMultiClusterForFollow" - val TwhinMultiClusterForEngagement: String = - "TwhinMultiClusterForEngagement" - - // Two Tower model Ids - val TwoTowerFavALL20220808: String = - "TwoTowerFav_ALL_20220808" - - // Debugger Demo-Only Model Ids - val DebuggerDemo: String = "DebuggerDemo" - - // ColdStartLookalike - this is not really a model name, it is as a placeholder to - // indicate ColdStartLookalike candidate source, which is currently being pluged into - // CustomizedRetrievalCandidateGeneration temporarily. - val ColdStartLookalikeModelName: String = "ConsumersBasedUtgColdStartLookalike20220707" - - // consumersBasedUTG-RealGraphOon Model Id - val ConsumersBasedUtgRealGraphOon20220705: String = "ConsumersBasedUtgRealGraphOon_All_20220705" - // consumersBasedUAG-RealGraphOon Model Id - val ConsumersBasedUagRealGraphOon20221205: String = "ConsumersBasedUagRealGraphOon_All_20221205" - - // FTR - val OfflineFavDecayedSum: String = "OfflineFavDecayedSum" - val OfflineFtrAt5Pop1000RnkDcy11: String = "OfflineFtrAt5Pop1000RnkDcy11" - val OfflineFtrAt5Pop10000RnkDcy11: String = "OfflineFtrAt5Pop10000RnkDcy11" - - // All Model Ids of HnswANNSimilarityEngines - val allHnswANNSimilarityEngineModelIds = Seq( - ConsumerBasedTwHINRegularUpdateAll20221024, - TwoTowerFavALL20220808, - DebuggerDemo - ) - - val ConsumerLogFavBasedInterestedInEmbedding: String = - "ConsumerLogFavBasedInterestedIn_ALL_20221228" - val ConsumerFollowBasedInterestedInEmbedding: String = - "ConsumerFollowBasedInterestedIn_ALL_20221228" - - val RetweetBasedDiffusion: String = - "RetweetBasedDiffusion" - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModuleNames.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModuleNames.docx new file mode 100644 index 000000000..09c50bd36 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModuleNames.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModuleNames.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModuleNames.scala deleted file mode 100644 index 6aec7b052..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModuleNames.scala +++ /dev/null @@ -1,122 +0,0 @@ -package com.twitter.cr_mixer.model - -/** - * Define name annotated module names here - */ -object ModuleNames { - - final val FrsStore = "FrsStore" - final val UssStore = "UssStore" - final val UssStratoColumn = "UssStratoColumn" - final val RsxStore = "RsxStore" - final val RmsTweetLogFavLongestL2EmbeddingStore = "RmsTweetLogFavLongestL2EmbeddingStore" - final val RmsUserFavBasedProducerEmbeddingStore = "RmsUserFavBasedProducerEmbeddingStore" - final val RmsUserLogFavInterestedInEmbeddingStore = "RmsUserLogFavInterestedInEmbeddingStore" - final val RmsUserFollowInterestedInEmbeddingStore = "RmsUserFollowInterestedInEmbeddingStore" - final val StpStore = "StpStore" - final val TwiceClustersMembersStore = "TwiceClustersMembersStore" - final val TripCandidateStore = "TripCandidateStore" - - final val ConsumerEmbeddingBasedTripSimilarityEngine = - "ConsumerEmbeddingBasedTripSimilarityEngine" - final val ConsumerEmbeddingBasedTwHINANNSimilarityEngine = - "ConsumerEmbeddingBasedTwHINANNSimilarityEngine" - final val ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine = - "ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine" - final val ConsumersBasedUserAdGraphSimilarityEngine = - "ConsumersBasedUserAdGraphSimilarityEngine" - final val ConsumersBasedUserVideoGraphSimilarityEngine = - "ConsumersBasedUserVideoGraphSimilarityEngine" - - final val ConsumerBasedWalsSimilarityEngine = "ConsumerBasedWalsSimilarityEngine" - - final val TweetBasedTwHINANNSimilarityEngine = "TweetBasedTwHINANNSimilarityEngine" - - final val SimClustersANNSimilarityEngine = "SimClustersANNSimilarityEngine" - - final val ProdSimClustersANNServiceClientName = "ProdSimClustersANNServiceClient" - final val ExperimentalSimClustersANNServiceClientName = "ExperimentalSimClustersANNServiceClient" - final val SimClustersANNServiceClientName1 = "SimClustersANNServiceClient1" - final val SimClustersANNServiceClientName2 = "SimClustersANNServiceClient2" - final val SimClustersANNServiceClientName3 = "SimClustersANNServiceClient3" - final val SimClustersANNServiceClientName5 = "SimClustersANNServiceClient5" - final val SimClustersANNServiceClientName4 = "SimClustersANNServiceClient4" - final val UnifiedCache = "unifiedCache" - final val MLScoreCache = "mlScoreCache" - final val TweetRecommendationResultsCache = "tweetRecommendationResultsCache" - final val EarlybirdTweetsCache = "earlybirdTweetsCache" - final val EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache = - "earlybirdTweetsWithoutRetweetsRepliesCacheStore" - final val EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache = - "earlybirdTweetsWithRetweetsRepliesCacheStore" - - final val AbDeciderLogger = "abDeciderLogger" - final val TopLevelApiDdgMetricsLogger = "topLevelApiDdgMetricsLogger" - final val TweetRecsLogger = "tweetRecsLogger" - final val BlueVerifiedTweetRecsLogger = "blueVerifiedTweetRecsLogger" - final val RelatedTweetsLogger = "relatedTweetsLogger" - final val UtegTweetsLogger = "utegTweetsLogger" - final val AdsRecommendationsLogger = "adsRecommendationLogger" - - final val OfflineSimClustersANNInterestedInSimilarityEngine = - "OfflineSimClustersANNInterestedInSimilarityEngine" - - final val RealGraphOonStore = "RealGraphOonStore" - final val RealGraphInStore = "RealGraphInStore" - - final val OfflineTweet2020CandidateStore = "OfflineTweet2020CandidateStore" - final val OfflineTweet2020Hl0El15CandidateStore = "OfflineTweet2020Hl0El15CandidateStore" - final val OfflineTweet2020Hl2El15CandidateStore = "OfflineTweet2020Hl2El15CandidateStore" - final val OfflineTweet2020Hl2El50CandidateStore = "OfflineTweet2020Hl2El50CandidateStore" - final val OfflineTweet2020Hl8El50CandidateStore = "OfflineTweet2020Hl8El50CandidateStore" - final val OfflineTweetMTSCandidateStore = "OfflineTweetMTSCandidateStore" - - final val OfflineFavDecayedSumCandidateStore = "OfflineFavDecayedSumCandidateStore" - final val OfflineFtrAt5Pop1000RankDecay11CandidateStore = - "OfflineFtrAt5Pop1000RankDecay11CandidateStore" - final val OfflineFtrAt5Pop10000RankDecay11CandidateStore = - "OfflineFtrAt5Pop10000RankDecay11CandidateStore" - - final val TwhinCollabFilterStratoStoreForFollow = "TwhinCollabFilterStratoStoreForFollow" - final val TwhinCollabFilterStratoStoreForEngagement = "TwhinCollabFilterStratoStoreForEngagement" - final val TwhinMultiClusterStratoStoreForFollow = "TwhinMultiClusterStratoStoreForFollow" - final val TwhinMultiClusterStratoStoreForEngagement = "TwhinMultiClusterStratoStoreForEngagement" - - final val ProducerBasedUserAdGraphSimilarityEngine = - "ProducerBasedUserAdGraphSimilarityEngine" - final val ProducerBasedUserTweetGraphSimilarityEngine = - "ProducerBasedUserTweetGraphSimilarityEngine" - final val ProducerBasedUnifiedSimilarityEngine = "ProducerBasedUnifiedSimilarityEngine" - - final val TweetBasedUserAdGraphSimilarityEngine = "TweetBasedUserAdGraphSimilarityEngine" - final val TweetBasedUserTweetGraphSimilarityEngine = "TweetBasedUserTweetGraphSimilarityEngine" - final val TweetBasedUserVideoGraphSimilarityEngine = "TweetBasedUserVideoGraphSimilarityEngine" - final val TweetBasedQigSimilarityEngine = "TweetBasedQigSimilarityEngine" - final val TweetBasedUnifiedSimilarityEngine = "TweetBasedUnifiedSimilarityEngine" - - final val TwhinCollabFilterSimilarityEngine = "TwhinCollabFilterSimilarityEngine" - - final val ConsumerBasedUserTweetGraphStore = "ConsumerBasedUserTweetGraphStore" - final val ConsumerBasedUserVideoGraphStore = "ConsumerBasedUserVideoGraphStore" - final val ConsumerBasedUserAdGraphStore = "ConsumerBasedUserAdGraphStore" - - final val UserTweetEntityGraphSimilarityEngine = - "UserTweetEntityGraphSimilarityEngine" - - final val CertoTopicTweetSimilarityEngine = "CertoTopicTweetSimilarityEngine" - final val CertoStratoStoreName = "CertoStratoStore" - - final val SkitTopicTweetSimilarityEngine = "SkitTopicTweetSimilarityEngine" - final val SkitHighPrecisionTopicTweetSimilarityEngine = - "SkitHighPrecisionTopicTweetSimilarityEngine" - final val SkitStratoStoreName = "SkitStratoStore" - - final val HomeNaviGRPCClient = "HomeNaviGRPCClient" - final val AdsFavedNaviGRPCClient = "AdsFavedNaviGRPCClient" - final val AdsMonetizableNaviGRPCClient = "AdsMonetizableNaviGRPCClient" - - final val RetweetBasedDiffusionRecsMhStore = "RetweetBasedDiffusionRecsMhStore" - final val DiffusionBasedSimilarityEngine = "DiffusionBasedSimilarityEngine" - - final val BlueVerifiedAnnotationStore = "BlueVerifiedAnnotationStore" -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TopicTweetWithScore.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TopicTweetWithScore.docx new file mode 100644 index 000000000..16c446288 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TopicTweetWithScore.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TopicTweetWithScore.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TopicTweetWithScore.scala deleted file mode 100644 index e9a0cf173..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TopicTweetWithScore.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.cr_mixer.model - -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.simclusters_v2.common.TweetId - -/*** - * Bind a tweetId with a raw score generated from one single Similarity Engine - * @param similarityEngineType, which underlying topic source the topic tweet is from - */ -case class TopicTweetWithScore( - tweetId: TweetId, - score: Double, - similarityEngineType: SimilarityEngineType) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithAuthor.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithAuthor.docx new file mode 100644 index 000000000..280818c9e Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithAuthor.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithAuthor.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithAuthor.scala deleted file mode 100644 index 16a506a4c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithAuthor.scala +++ /dev/null @@ -1,6 +0,0 @@ -package com.twitter.cr_mixer.model - -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId - -case class TweetWithAuthor(tweetId: TweetId, authorId: UserId) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScore.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScore.docx new file mode 100644 index 000000000..746c51c42 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScore.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScore.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScore.scala deleted file mode 100644 index ad8866912..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScore.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.cr_mixer.model - -import com.twitter.simclusters_v2.common.TweetId - -/*** - * Bind a tweetId with a raw score generated from one single Similarity Engine - */ -case class TweetWithScore(tweetId: TweetId, score: Double) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScoreAndSocialProof.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScoreAndSocialProof.docx new file mode 100644 index 000000000..7e313da09 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScoreAndSocialProof.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScoreAndSocialProof.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScoreAndSocialProof.scala deleted file mode 100644 index 94e430d8e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/TweetWithScoreAndSocialProof.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.cr_mixer.model - -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.recos.recos_common.thriftscala.SocialProofType - -/*** - * Bind a tweetId with a raw score and social proofs by type - */ -case class TweetWithScoreAndSocialProof( - tweetId: TweetId, - score: Double, - socialProofByType: Map[SocialProofType, Seq[Long]]) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ActivePromotedTweetStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ActivePromotedTweetStoreModule.docx new file mode 100644 index 000000000..2381673a1 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ActivePromotedTweetStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ActivePromotedTweetStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ActivePromotedTweetStoreModule.scala deleted file mode 100644 index d6529531a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ActivePromotedTweetStoreModule.scala +++ /dev/null @@ -1,135 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.bijection.thrift.CompactThriftCodec -import com.twitter.ads.entities.db.thriftscala.LineItemObjective -import com.twitter.bijection.Injection -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.thriftscala.LineItemInfo -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hermit.store.common.ObservedCachedReadableStore -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataType -import com.twitter.ml.api.Feature -import com.twitter.ml.api.GeneralTensor -import com.twitter.ml.api.RichDataRecord -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.manhattan.Revenue -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.storehaus_internal.util.HDFSPath -import com.twitter.util.Future -import javax.inject.Named -import scala.collection.JavaConverters._ - -object ActivePromotedTweetStoreModule extends TwitterModule { - - case class ActivePromotedTweetStore( - activePromotedTweetMHStore: ReadableStore[String, DataRecord], - statsReceiver: StatsReceiver) - extends ReadableStore[TweetId, Seq[LineItemInfo]] { - override def get(tweetId: TweetId): Future[Option[Seq[LineItemInfo]]] = { - activePromotedTweetMHStore.get(tweetId.toString).map { - _.map { dataRecord => - val richDataRecord = new RichDataRecord(dataRecord) - val lineItemIdsFeature: Feature[GeneralTensor] = - new Feature.Tensor("active_promoted_tweets.line_item_ids", DataType.INT64) - - val lineItemObjectivesFeature: Feature[GeneralTensor] = - new Feature.Tensor("active_promoted_tweets.line_item_objectives", DataType.INT64) - - val lineItemIdsTensor: GeneralTensor = richDataRecord.getFeatureValue(lineItemIdsFeature) - val lineItemObjectivesTensor: GeneralTensor = - richDataRecord.getFeatureValue(lineItemObjectivesFeature) - - val lineItemIds: Seq[Long] = - if (lineItemIdsTensor.getSetField == GeneralTensor._Fields.INT64_TENSOR && lineItemIdsTensor.getInt64Tensor.isSetLongs) { - lineItemIdsTensor.getInt64Tensor.getLongs.asScala.map(_.toLong) - } else Seq.empty - - val lineItemObjectives: Seq[LineItemObjective] = - if (lineItemObjectivesTensor.getSetField == GeneralTensor._Fields.INT64_TENSOR && lineItemObjectivesTensor.getInt64Tensor.isSetLongs) { - lineItemObjectivesTensor.getInt64Tensor.getLongs.asScala.map(objective => - LineItemObjective(objective.toInt)) - } else Seq.empty - - val lineItemInfo = - if (lineItemIds.size == lineItemObjectives.size) { - lineItemIds.zipWithIndex.map { - case (lineItemId, index) => - LineItemInfo( - lineItemId = lineItemId, - lineItemObjective = lineItemObjectives(index) - ) - } - } else Seq.empty - - lineItemInfo - } - } - } - } - - @Provides - @Singleton - def providesActivePromotedTweetStore( - manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - crMixerStatsReceiver: StatsReceiver - ): ReadableStore[TweetId, Seq[LineItemInfo]] = { - - val mhConfig = new ManhattanROConfig { - val hdfsPath = HDFSPath("") - val applicationID = ApplicationID("ads_bigquery_features") - val datasetName = DatasetName("active_promoted_tweets") - val cluster = Revenue - - override def statsReceiver: StatsReceiver = - crMixerStatsReceiver.scope("active_promoted_tweets_mh") - } - val mhStore: ReadableStore[String, DataRecord] = - ManhattanRO - .getReadableStoreWithMtls[String, DataRecord]( - mhConfig, - manhattanKVClientMtlsParams - )( - implicitly[Injection[String, Array[Byte]]], - CompactThriftCodec[DataRecord] - ) - - val underlyingStore = - ActivePromotedTweetStore(mhStore, crMixerStatsReceiver.scope("ActivePromotedTweetStore")) - val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 60.minutes, - asyncUpdate = false - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[LineItemInfo]()), - statsReceiver = crMixerStatsReceiver.scope("memCachedActivePromotedTweetStore"), - keyToString = { k: TweetId => s"apt/$k" } - ) - - ObservedCachedReadableStore.from( - memcachedStore, - ttl = 30.minutes, - maxKeys = 250000, // size of promoted tweet is around 200,000 - windowSize = 10000L, - cacheName = "active_promoted_tweet_cache", - maxMultiGetSize = 20 - )(crMixerStatsReceiver.scope("inMemoryCachedActivePromotedTweetStore")) - - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BUILD.bazel b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BUILD.bazel deleted file mode 100644 index 6773b526c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BUILD.bazel +++ /dev/null @@ -1,130 +0,0 @@ -scala_library( - sources = [ - "*.scala", - "core/*.scala", - "grpc_client/*.scala", - "similarity_engine/*.scala", - "source_signal/*.scala", - "thrift_client/*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:core", - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/com/twitter/storehaus:memcache", - "3rdparty/jvm/io/grpc:grpc-api", - "3rdparty/jvm/io/grpc:grpc-auth", - "3rdparty/jvm/io/grpc:grpc-core", - "3rdparty/jvm/io/grpc:grpc-netty", - "3rdparty/jvm/io/grpc:grpc-protobuf", - "3rdparty/jvm/io/grpc:grpc-stub", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/org/scalanlp:breeze", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "abdecider/src/main/scala", - "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", - "configapi/configapi-abdecider", - "configapi/configapi-core", - "configapi/configapi-featureswitches:v2", - "content-recommender/server/src/main/scala/com/twitter/contentrecommender:cr-mixer-deps", - "content-recommender/thrift/src/main/thrift:thrift-scala", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/featureswitch", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "decider/src/main/scala", - "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", - "featureswitches/featureswitches-core", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "finagle-internal/finagle-grpc/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-modules/src/main/scala", - "finatra/inject/inject-thrift-client", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "frigate/frigate-common:util", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/health", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/interests", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "hydra/partition/thrift/src/main/thrift:thrift-scala", - "hydra/root/thrift/src/main/thrift:thrift-scala", - "mediaservices/commons/src/main/scala:futuretracker", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "qig-ranker/thrift/src/main/thrift:thrift-scala", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/health_store", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", - "relevance-platform/thrift/src/main/thrift:thrift-scala", - "representation-manager/client/src/main/scala/com/twitter/representation_manager", - "representation-manager/client/src/main/scala/com/twitter/representation_manager/config", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/migration", - "representation-manager/server/src/main/thrift:thrift-scala", - "representation-scorer/server/src/main/thrift:thrift-scala", - "servo/decider", - "servo/util/src/main/scala", - "simclusters-ann/thrift/src/main/thrift:thrift-scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/scala/com/twitter/algebird_internal/injection", - "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", - "src/scala/com/twitter/ml/api/embedding", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/scalding_internal/multiformat/format", - "src/scala/com/twitter/simclusters_v2/candidate_source", - "src/scala/com/twitter/simclusters_v2/common", - "src/scala/com/twitter/storehaus_internal/manhattan", - "src/scala/com/twitter/storehaus_internal/manhattan/config", - "src/scala/com/twitter/storehaus_internal/memcache", - "src/scala/com/twitter/storehaus_internal/memcache/config", - "src/scala/com/twitter/storehaus_internal/offline", - "src/scala/com/twitter/storehaus_internal/util", - "src/scala/com/twitter/topic_recos/stores", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/frigate:frigate-common-thrift-scala", - "src/thrift/com/twitter/frigate:frigate-thrift-scala", - "src/thrift/com/twitter/frigate/data_pipeline/scalding:blue_verified_annotations-scala", - "src/thrift/com/twitter/hermit/stp:hermit-stp-scala", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:embedding-scala", - "src/thrift/com/twitter/ml/featurestore:ml-feature-store-embedding-scala", - "src/thrift/com/twitter/onboarding/relevance/coldstart_lookalike:coldstartlookalike-thrift-scala", - "src/thrift/com/twitter/recos:recos-common-scala", - "src/thrift/com/twitter/recos/user_ad_graph:user_ad_graph-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", - "src/thrift/com/twitter/recos/user_tweet_graph_plus:user_tweet_graph_plus-scala", - "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/topic_recos:topic_recos-thrift-scala", - "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/twistly:twistly-scala", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "stitch/stitch-storehaus", - "stitch/stitch-tweetypie/src/main/scala", - "strato/src/main/scala/com/twitter/strato/client", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - "util-internal/scribe/src/main/scala/com/twitter/logging", - "util/util-hashing", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BUILD.docx new file mode 100644 index 000000000..c2a096861 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BlueVerifiedAnnotationStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BlueVerifiedAnnotationStoreModule.docx new file mode 100644 index 000000000..97d390099 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BlueVerifiedAnnotationStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BlueVerifiedAnnotationStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BlueVerifiedAnnotationStoreModule.scala deleted file mode 100644 index 21769d3fa..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/BlueVerifiedAnnotationStoreModule.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.inject.TwitterModule -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.data_pipeline.scalding.thriftscala.BlueVerifiedAnnotationsV2 -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Athena -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.storehaus_internal.util.HDFSPath -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.hermit.store.common.ObservedCachedReadableStore - -object BlueVerifiedAnnotationStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.BlueVerifiedAnnotationStore) - def providesBlueVerifiedAnnotationStore( - statsReceiver: StatsReceiver, - manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, - ): ReadableStore[String, BlueVerifiedAnnotationsV2] = { - - implicit val valueCodec = new BinaryScalaCodec(BlueVerifiedAnnotationsV2) - - val underlyingStore = ManhattanRO - .getReadableStoreWithMtls[String, BlueVerifiedAnnotationsV2]( - ManhattanROConfig( - HDFSPath(""), - ApplicationID("content_recommender_athena"), - DatasetName("blue_verified_annotations"), - Athena), - manhattanKVClientMtlsParams - ) - - ObservedCachedReadableStore.from( - underlyingStore, - ttl = 24.hours, - maxKeys = 100000, - windowSize = 10000L, - cacheName = "blue_verified_annotation_cache" - )(statsReceiver.scope("inMemoryCachedBlueVerifiedAnnotationStore")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CertoStratoStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CertoStratoStoreModule.docx new file mode 100644 index 000000000..bfe5a5430 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CertoStratoStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CertoStratoStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CertoStratoStoreModule.scala deleted file mode 100644 index 9908aa702..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CertoStratoStoreModule.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hermit.store.common.ObservedCachedReadableStore -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.topic_recos.stores.CertoTopicTopKTweetsStore -import com.twitter.topic_recos.thriftscala.TweetWithScores - -object CertoStratoStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.CertoStratoStoreName) - def providesCertoStratoStore( - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - stratoClient: Client, - statsReceiver: StatsReceiver - ): ReadableStore[TopicId, Seq[TweetWithScores]] = { - val certoStore = ObservedReadableStore(CertoTopicTopKTweetsStore.prodStore(stratoClient))( - statsReceiver.scope(ModuleNames.CertoStratoStoreName)).mapValues { topKTweetsWithScores => - topKTweetsWithScores.topTweetsByFollowerL2NormalizedCosineSimilarityScore - } - - val memCachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = certoStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScores]()), - statsReceiver = statsReceiver.scope("memcached_certo_store"), - keyToString = { k => s"certo:${keyHasher.hashKey(k.toString.getBytes)}" } - ) - - ObservedCachedReadableStore.from[TopicId, Seq[TweetWithScores]]( - memCachedStore, - ttl = 5.minutes, - maxKeys = 100000, // ~150MB max - cacheName = "certo_in_memory_cache", - windowSize = 10000L - )(statsReceiver.scope("certo_in_memory_cache")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserAdGraphStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserAdGraphStoreModule.docx new file mode 100644 index 000000000..59297ad48 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserAdGraphStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserAdGraphStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserAdGraphStoreModule.scala deleted file mode 100644 index 33a0d33fc..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserAdGraphStoreModule.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest -import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse -import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Named -import javax.inject.Singleton - -object ConsumersBasedUserAdGraphStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ConsumerBasedUserAdGraphStore) - def providesConsumerBasedUserAdGraphStore( - userAdGraphService: UserAdGraph.MethodPerEndpoint - ): ReadableStore[ConsumersBasedRelatedAdRequest, RelatedAdResponse] = { - new ReadableStore[ConsumersBasedRelatedAdRequest, RelatedAdResponse] { - override def get( - k: ConsumersBasedRelatedAdRequest - ): Future[Option[RelatedAdResponse]] = { - userAdGraphService.consumersBasedRelatedAds(k).map(Some(_)) - } - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserTweetGraphStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserTweetGraphStoreModule.docx new file mode 100644 index 000000000..c928f5423 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserTweetGraphStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserTweetGraphStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserTweetGraphStoreModule.scala deleted file mode 100644 index ab027744a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserTweetGraphStoreModule.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_tweet_graph.thriftscala.ConsumersBasedRelatedTweetRequest -import com.twitter.recos.user_tweet_graph.thriftscala.RelatedTweetResponse -import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Named -import javax.inject.Singleton - -object ConsumersBasedUserTweetGraphStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ConsumerBasedUserTweetGraphStore) - def providesConsumerBasedUserTweetGraphStore( - userTweetGraphService: UserTweetGraph.MethodPerEndpoint - ): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] = { - new ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] { - override def get( - k: ConsumersBasedRelatedTweetRequest - ): Future[Option[RelatedTweetResponse]] = { - userTweetGraphService.consumersBasedRelatedTweets(k).map(Some(_)) - } - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserVideoGraphStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserVideoGraphStoreModule.docx new file mode 100644 index 000000000..ecc39a60f Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserVideoGraphStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserVideoGraphStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserVideoGraphStoreModule.scala deleted file mode 100644 index 05cf496d8..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/ConsumersBasedUserVideoGraphStoreModule.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest -import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse -import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Named -import javax.inject.Singleton - -object ConsumersBasedUserVideoGraphStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ConsumerBasedUserVideoGraphStore) - def providesConsumerBasedUserVideoGraphStore( - userVideoGraphService: UserVideoGraph.MethodPerEndpoint - ): ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] = { - new ReadableStore[ConsumersBasedRelatedTweetRequest, RelatedTweetResponse] { - override def get( - k: ConsumersBasedRelatedTweetRequest - ): Future[Option[RelatedTweetResponse]] = { - userVideoGraphService.consumersBasedRelatedTweets(k).map(Some(_)) - } - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CrMixerParamConfigModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CrMixerParamConfigModule.docx new file mode 100644 index 000000000..2b5035f64 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CrMixerParamConfigModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CrMixerParamConfigModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CrMixerParamConfigModule.scala deleted file mode 100644 index baece7947..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/CrMixerParamConfigModule.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.timelines.configapi.Config -import com.twitter.cr_mixer.param.CrMixerParamConfig -import com.twitter.inject.TwitterModule -import javax.inject.Singleton - -object CrMixerParamConfigModule extends TwitterModule { - - @Provides - @Singleton - def provideConfig(): Config = { - CrMixerParamConfig.config - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/DiffusionStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/DiffusionStoreModule.docx new file mode 100644 index 000000000..8551be622 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/DiffusionStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/DiffusionStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/DiffusionStoreModule.scala deleted file mode 100644 index dc95f07f5..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/DiffusionStoreModule.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.thriftscala.TweetsWithScore -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.storehaus_internal.util.HDFSPath -import javax.inject.Named -import javax.inject.Singleton - -object DiffusionStoreModule extends TwitterModule { - type UserId = Long - implicit val longCodec = implicitly[Injection[Long, Array[Byte]]] - implicit val tweetRecsInjection: Injection[TweetsWithScore, Array[Byte]] = - BinaryScalaCodec(TweetsWithScore) - - @Provides - @Singleton - @Named(ModuleNames.RetweetBasedDiffusionRecsMhStore) - def retweetBasedDiffusionRecsMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[Long, TweetsWithScore] = { - val manhattanROConfig = ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("cr_mixer_apollo"), - DatasetName("diffusion_retweet_tweet_recs"), - Apollo - ) - - buildTweetRecsStore(serviceIdentifier, manhattanROConfig) - } - - private def buildTweetRecsStore( - serviceIdentifier: ServiceIdentifier, - manhattanROConfig: ManhattanROConfig - ): ReadableStore[Long, TweetsWithScore] = { - - ManhattanRO - .getReadableStoreWithMtls[Long, TweetsWithScore]( - manhattanROConfig, - ManhattanKVClientMtlsParams(serviceIdentifier) - )(longCodec, tweetRecsInjection) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EarlybirdRecencyBasedCandidateStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EarlybirdRecencyBasedCandidateStoreModule.docx new file mode 100644 index 000000000..e9ed4a767 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EarlybirdRecencyBasedCandidateStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EarlybirdRecencyBasedCandidateStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EarlybirdRecencyBasedCandidateStoreModule.scala deleted file mode 100644 index c0fe025f0..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EarlybirdRecencyBasedCandidateStoreModule.scala +++ /dev/null @@ -1,189 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.EarlybirdClientId -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.FacetsToFetch -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetCollectorTerminationParams -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetEarlybirdQuery -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.MetadataOptions -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.SeqLongInjection -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams -import com.twitter.search.earlybird.thriftscala.EarlybirdRequest -import com.twitter.search.earlybird.thriftscala.EarlybirdResponseCode -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.search.earlybird.thriftscala.ThriftSearchQuery -import com.twitter.search.earlybird.thriftscala.ThriftSearchRankingMode -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Duration -import com.twitter.util.Future -import javax.inject.Named - -object EarlybirdRecencyBasedCandidateStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache) - def providesEarlybirdRecencyBasedWithoutRetweetsRepliesCandidateStore( - statsReceiver: StatsReceiver, - earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, - @Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient, - timeoutConfig: TimeoutConfig - ): ReadableStore[UserId, Seq[TweetId]] = { - val stats = statsReceiver.scope("EarlybirdRecencyBasedWithoutRetweetsRepliesCandidateStore") - val underlyingStore = new ReadableStore[UserId, Seq[TweetId]] { - override def get(userId: UserId): Future[Option[Seq[TweetId]]] = { - // Home based EB filters out retweets and replies - val earlybirdRequest = - buildEarlybirdRequest( - userId, - FilterOutRetweetsAndReplies, - DefaultMaxNumTweetPerUser, - timeoutConfig.earlybirdServerTimeout) - getEarlybirdSearchResult(earlybirdSearchClient, earlybirdRequest, stats) - } - } - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlyingStore, - cacheClient = earlybirdRecencyBasedTweetsCache, - ttl = MemcacheKeyTimeToLiveDuration, - asyncUpdate = true - )( - valueInjection = SeqLongInjection, - statsReceiver = statsReceiver.scope("earlybird_recency_based_tweets_home_memcache"), - keyToString = { k => - f"uEBRBHM:${keyHasher.hashKey(k.toString.getBytes)}%X" // prefix = EarlyBirdRecencyBasedHoMe - } - ) - } - - @Provides - @Singleton - @Named(ModuleNames.EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache) - def providesEarlybirdRecencyBasedWithRetweetsRepliesCandidateStore( - statsReceiver: StatsReceiver, - earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, - @Named(ModuleNames.EarlybirdTweetsCache) earlybirdRecencyBasedTweetsCache: MemcachedClient, - timeoutConfig: TimeoutConfig - ): ReadableStore[UserId, Seq[TweetId]] = { - val stats = statsReceiver.scope("EarlybirdRecencyBasedWithRetweetsRepliesCandidateStore") - val underlyingStore = new ReadableStore[UserId, Seq[TweetId]] { - override def get(userId: UserId): Future[Option[Seq[TweetId]]] = { - val earlybirdRequest = buildEarlybirdRequest( - userId, - // Notifications based EB keeps retweets and replies - NotFilterOutRetweetsAndReplies, - DefaultMaxNumTweetPerUser, - processingTimeout = timeoutConfig.earlybirdServerTimeout - ) - getEarlybirdSearchResult(earlybirdSearchClient, earlybirdRequest, stats) - } - } - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlyingStore, - cacheClient = earlybirdRecencyBasedTweetsCache, - ttl = MemcacheKeyTimeToLiveDuration, - asyncUpdate = true - )( - valueInjection = SeqLongInjection, - statsReceiver = statsReceiver.scope("earlybird_recency_based_tweets_notifications_memcache"), - keyToString = { k => - f"uEBRBN:${keyHasher.hashKey(k.toString.getBytes)}%X" // prefix = EarlyBirdRecencyBasedNotifications - } - ) - } - - private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - - /** - * Note the DefaultMaxNumTweetPerUser is used to adjust the result size per cache entry. - * If the value changes, it will increase the size of the memcache. - */ - private val DefaultMaxNumTweetPerUser: Int = 100 - private val FilterOutRetweetsAndReplies = true - private val NotFilterOutRetweetsAndReplies = false - private val MemcacheKeyTimeToLiveDuration: Duration = Duration.fromMinutes(15) - - private def buildEarlybirdRequest( - seedUserId: UserId, - filterOutRetweetsAndReplies: Boolean, - maxNumTweetsPerSeedUser: Int, - processingTimeout: Duration - ): EarlybirdRequest = - EarlybirdRequest( - searchQuery = getThriftSearchQuery( - seedUserId = seedUserId, - filterOutRetweetsAndReplies = filterOutRetweetsAndReplies, - maxNumTweetsPerSeedUser = maxNumTweetsPerSeedUser, - processingTimeout = processingTimeout - ), - clientId = Some(EarlybirdClientId), - timeoutMs = processingTimeout.inMilliseconds.intValue(), - getOlderResults = Some(false), - adjustedProtectedRequestParams = None, - adjustedFullArchiveRequestParams = None, - getProtectedTweetsOnly = Some(false), - skipVeryRecentTweets = true, - ) - - private def getThriftSearchQuery( - seedUserId: UserId, - filterOutRetweetsAndReplies: Boolean, - maxNumTweetsPerSeedUser: Int, - processingTimeout: Duration - ): ThriftSearchQuery = ThriftSearchQuery( - serializedQuery = GetEarlybirdQuery( - None, - None, - Set.empty, - filterOutRetweetsAndReplies - ).map(_.serialize), - fromUserIDFilter64 = Some(Seq(seedUserId)), - numResults = maxNumTweetsPerSeedUser, - rankingMode = ThriftSearchRankingMode.Recency, - collectorParams = Some( - CollectorParams( - // numResultsToReturn defines how many results each EB shard will return to search root - numResultsToReturn = maxNumTweetsPerSeedUser, - // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. - terminationParams = - GetCollectorTerminationParams(maxNumTweetsPerSeedUser, processingTimeout) - )), - facetFieldNames = Some(FacetsToFetch), - resultMetadataOptions = Some(MetadataOptions), - searchStatusIds = None - ) - - private def getEarlybirdSearchResult( - earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, - request: EarlybirdRequest, - statsReceiver: StatsReceiver - ): Future[Option[Seq[TweetId]]] = earlybirdSearchClient - .search(request) - .map { response => - response.responseCode match { - case EarlybirdResponseCode.Success => - val earlybirdSearchResult = - response.searchResults - .map { - _.results - .map(searchResult => searchResult.id) - } - statsReceiver.scope("result").stat("size").add(earlybirdSearchResult.size) - earlybirdSearchResult - case e => - statsReceiver.scope("failures").counter(e.getClass.getSimpleName).incr() - Some(Seq.empty) - } - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EmbeddingStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EmbeddingStoreModule.docx new file mode 100644 index 000000000..85f016cc2 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EmbeddingStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EmbeddingStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EmbeddingStoreModule.scala deleted file mode 100644 index 26d9f8ad1..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/EmbeddingStoreModule.scala +++ /dev/null @@ -1,195 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.inject.TwitterModule -import com.twitter.ml.api.{thriftscala => api} -import com.twitter.simclusters_v2.thriftscala.CandidateTweetsList -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.storehaus_internal.util.HDFSPath -import javax.inject.Named -import javax.inject.Singleton - -object EmbeddingStoreModule extends TwitterModule { - type UserId = Long - implicit val mbcgUserEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = - CompactScalaCodec(api.Embedding) - implicit val tweetCandidatesInjection: Injection[CandidateTweetsList, Array[Byte]] = - CompactScalaCodec(CandidateTweetsList) - - final val TwHINEmbeddingRegularUpdateMhStoreName = "TwHINEmbeddingRegularUpdateMhStore" - @Provides - @Singleton - @Named(TwHINEmbeddingRegularUpdateMhStoreName) - def twHINEmbeddingRegularUpdateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[InternalId, api.Embedding] = { - val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = - BinaryScalaCodec(api.Embedding) - - val longCodec = implicitly[Injection[Long, Array[Byte]]] - - ManhattanRO - .getReadableStoreWithMtls[TweetId, api.Embedding]( - ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("cr_mixer_apollo"), - DatasetName("twhin_regular_update_tweet_embedding_apollo"), - Apollo - ), - ManhattanKVClientMtlsParams(serviceIdentifier) - )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { - case InternalId.TweetId(tweetId) => - tweetId - case _ => - throw new UnsupportedOperationException("Invalid Internal Id") - } - } - - final val ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName = - "ConsumerBasedTwHINEmbeddingRegularUpdateMhStore" - @Provides - @Singleton - @Named(ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName) - def consumerBasedTwHINEmbeddingRegularUpdateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[InternalId, api.Embedding] = { - val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = - BinaryScalaCodec(api.Embedding) - - val longCodec = implicitly[Injection[Long, Array[Byte]]] - - ManhattanRO - .getReadableStoreWithMtls[UserId, api.Embedding]( - ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("cr_mixer_apollo"), - DatasetName("twhin_user_embedding_regular_update_apollo"), - Apollo - ), - ManhattanKVClientMtlsParams(serviceIdentifier) - )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { - case InternalId.UserId(userId) => - userId - case _ => - throw new UnsupportedOperationException("Invalid Internal Id") - } - } - - final val TwoTowerFavConsumerEmbeddingMhStoreName = "TwoTowerFavConsumerEmbeddingMhStore" - @Provides - @Singleton - @Named(TwoTowerFavConsumerEmbeddingMhStoreName) - def twoTowerFavConsumerEmbeddingMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[InternalId, api.Embedding] = { - val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = - BinaryScalaCodec(api.Embedding) - - val longCodec = implicitly[Injection[Long, Array[Byte]]] - - ManhattanRO - .getReadableStoreWithMtls[UserId, api.Embedding]( - ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("cr_mixer_apollo"), - DatasetName("two_tower_fav_user_embedding_apollo"), - Apollo - ), - ManhattanKVClientMtlsParams(serviceIdentifier) - )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { - case InternalId.UserId(userId) => - userId - case _ => - throw new UnsupportedOperationException("Invalid Internal Id") - } - } - - final val DebuggerDemoUserEmbeddingMhStoreName = "DebuggerDemoUserEmbeddingMhStoreName" - @Provides - @Singleton - @Named(DebuggerDemoUserEmbeddingMhStoreName) - def debuggerDemoUserEmbeddingStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[InternalId, api.Embedding] = { - // This dataset is from src/scala/com/twitter/wtf/beam/bq_embedding_export/sql/MlfExperimentalUserEmbeddingScalaDataset.sql - // Change the above sql if you want to use a diff embedding - val manhattanROConfig = ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("cr_mixer_apollo"), - DatasetName("experimental_user_embedding"), - Apollo - ) - buildUserEmbeddingStore(serviceIdentifier, manhattanROConfig) - } - - final val DebuggerDemoTweetEmbeddingMhStoreName = "DebuggerDemoTweetEmbeddingMhStore" - @Provides - @Singleton - @Named(DebuggerDemoTweetEmbeddingMhStoreName) - def debuggerDemoTweetEmbeddingStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[InternalId, api.Embedding] = { - // This dataset is from src/scala/com/twitter/wtf/beam/bq_embedding_export/sql/MlfExperimentalTweetEmbeddingScalaDataset.sql - // Change the above sql if you want to use a diff embedding - val manhattanROConfig = ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("cr_mixer_apollo"), - DatasetName("experimental_tweet_embedding"), - Apollo - ) - buildTweetEmbeddingStore(serviceIdentifier, manhattanROConfig) - } - - private def buildUserEmbeddingStore( - serviceIdentifier: ServiceIdentifier, - manhattanROConfig: ManhattanROConfig - ): ReadableStore[InternalId, api.Embedding] = { - val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = - BinaryScalaCodec(api.Embedding) - - val longCodec = implicitly[Injection[Long, Array[Byte]]] - ManhattanRO - .getReadableStoreWithMtls[UserId, api.Embedding]( - manhattanROConfig, - ManhattanKVClientMtlsParams(serviceIdentifier) - )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { - case InternalId.UserId(userId) => - userId - case _ => - throw new UnsupportedOperationException("Invalid Internal Id") - } - } - - private def buildTweetEmbeddingStore( - serviceIdentifier: ServiceIdentifier, - manhattanROConfig: ManhattanROConfig - ): ReadableStore[InternalId, api.Embedding] = { - val binaryEmbeddingInjection: Injection[api.Embedding, Array[Byte]] = - BinaryScalaCodec(api.Embedding) - - val longCodec = implicitly[Injection[Long, Array[Byte]]] - - ManhattanRO - .getReadableStoreWithMtls[TweetId, api.Embedding]( - manhattanROConfig, - ManhattanKVClientMtlsParams(serviceIdentifier) - )(longCodec, binaryEmbeddingInjection).composeKeyMapping[InternalId] { - case InternalId.TweetId(tweetId) => - tweetId - case _ => - throw new UnsupportedOperationException("Invalid Internal Id") - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/FrsStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/FrsStoreModule.docx new file mode 100644 index 000000000..a56187e40 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/FrsStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/FrsStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/FrsStoreModule.scala deleted file mode 100644 index cfe044afd..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/FrsStoreModule.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.source_signal.FrsStore -import com.twitter.cr_mixer.source_signal.FrsStore.FrsQueryResult -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.storehaus.ReadableStore -import javax.inject.Named - -object FrsStoreModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.FrsStore) - def providesFrsStore( - frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): ReadableStore[FrsStore.Query, Seq[FrsQueryResult]] = { - ObservedReadableStore(FrsStore(frsClient, statsReceiver, decider))( - statsReceiver.scope("follow_recommendations_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/MHMtlsParamsModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/MHMtlsParamsModule.docx new file mode 100644 index 000000000..a630d5877 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/MHMtlsParamsModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/MHMtlsParamsModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/MHMtlsParamsModule.scala deleted file mode 100644 index 339d0330a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/MHMtlsParamsModule.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.inject.TwitterModule -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import javax.inject.Singleton - -object MHMtlsParamsModule extends TwitterModule { - @Singleton - @Provides - def providesManhattanMtlsParams( - serviceIdentifier: ServiceIdentifier - ): ManhattanKVClientMtlsParams = { - ManhattanKVClientMtlsParams(serviceIdentifier) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/OfflineCandidateStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/OfflineCandidateStoreModule.docx new file mode 100644 index 000000000..c6b2d0254 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/OfflineCandidateStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/OfflineCandidateStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/OfflineCandidateStoreModule.scala deleted file mode 100644 index db4a3fa5e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/OfflineCandidateStoreModule.scala +++ /dev/null @@ -1,150 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.CompactScalaCodec -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.thriftscala.CandidateTweetsList -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.storehaus_internal.util.HDFSPath -import javax.inject.Named -import javax.inject.Singleton - -object OfflineCandidateStoreModule extends TwitterModule { - type UserId = Long - implicit val tweetCandidatesInjection: Injection[CandidateTweetsList, Array[Byte]] = - CompactScalaCodec(CandidateTweetsList) - - @Provides - @Singleton - @Named(ModuleNames.OfflineTweet2020CandidateStore) - def offlineTweet2020CandidateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_interestedin_2020" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineTweet2020Hl0El15CandidateStore) - def offlineTweet2020Hl0El15CandidateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_0_el_15" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineTweet2020Hl2El15CandidateStore) - def offlineTweet2020Hl2El15CandidateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_2_el_15" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineTweet2020Hl2El50CandidateStore) - def offlineTweet2020Hl2El50CandidateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_2_el_50" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineTweet2020Hl8El50CandidateStore) - def offlineTweet2020Hl8El50CandidateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_interestedin_2020_hl_8_el_50" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineTweetMTSCandidateStore) - def offlineTweetMTSCandidateMhStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_mts_consumer_embeddings" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineFavDecayedSumCandidateStore) - def offlineFavDecayedSumCandidateStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_decayed_sum" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineFtrAt5Pop1000RankDecay11CandidateStore) - def offlineFtrAt5Pop1000RankDecay11CandidateStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_ftrat5_pop1000_rank_decay_1_1" - ) - } - - @Provides - @Singleton - @Named(ModuleNames.OfflineFtrAt5Pop10000RankDecay11CandidateStore) - def offlineFtrAt5Pop10000RankDecay11CandidateStore( - serviceIdentifier: ServiceIdentifier - ): ReadableStore[UserId, CandidateTweetsList] = { - buildOfflineCandidateStore( - serviceIdentifier, - datasetName = "offline_tweet_recommendations_from_ftrat5_pop10000_rank_decay_1_1" - ) - } - - private def buildOfflineCandidateStore( - serviceIdentifier: ServiceIdentifier, - datasetName: String - ): ReadableStore[UserId, CandidateTweetsList] = { - ManhattanRO - .getReadableStoreWithMtls[Long, CandidateTweetsList]( - ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("multi_type_simclusters"), - DatasetName(datasetName), - Apollo - ), - ManhattanKVClientMtlsParams(serviceIdentifier) - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphOonStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphOonStoreModule.docx new file mode 100644 index 000000000..549ff526d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphOonStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphOonStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphOonStoreModule.scala deleted file mode 100644 index 3d9a71a1c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphOonStoreModule.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.app.Flag -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import javax.inject.Singleton -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.wtf.candidate.thriftscala.CandidateSeq - -object RealGraphOonStoreModule extends TwitterModule { - - private val userRealGraphOonColumnPath: Flag[String] = flag[String]( - name = "crMixer.userRealGraphOonColumnPath", - default = "recommendations/twistly/userRealgraphOon", - help = "Strato column path for user real graph OON Store" - ) - - @Provides - @Singleton - @Named(ModuleNames.RealGraphOonStore) - def providesRealGraphOonStore( - stratoClient: StratoClient, - statsReceiver: StatsReceiver - ): ReadableStore[UserId, CandidateSeq] = { - val realGraphOonStratoFetchableStore = StratoFetchableStore - .withUnitView[UserId, CandidateSeq](stratoClient, userRealGraphOonColumnPath()) - - ObservedReadableStore( - realGraphOonStratoFetchableStore - )(statsReceiver.scope("user_real_graph_oon_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphStoreMhModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphStoreMhModule.docx new file mode 100644 index 000000000..5be60149c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphStoreMhModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphStoreMhModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphStoreMhModule.scala deleted file mode 100644 index 0cd1a3ad7..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RealGraphStoreMhModule.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.UserId -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.storehaus_internal.util.HDFSPath -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.cr_mixer.param.decider.DeciderKey -import com.twitter.hermit.store.common.DeciderableReadableStore -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.wtf.candidate.thriftscala.CandidateSeq - -object RealGraphStoreMhModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.RealGraphInStore) - def providesRealGraphStoreMh( - decider: CrMixerDecider, - statsReceiver: StatsReceiver, - manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - ): ReadableStore[UserId, CandidateSeq] = { - - implicit val valueCodec = new BinaryScalaCodec(CandidateSeq) - val underlyingStore = ManhattanRO - .getReadableStoreWithMtls[UserId, CandidateSeq]( - ManhattanROConfig( - HDFSPath(""), - ApplicationID("cr_mixer_apollo"), - DatasetName("real_graph_scores_apollo"), - Apollo), - manhattanKVClientMtlsParams - ) - - val memCachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 24.hours, - )( - valueInjection = valueCodec, - statsReceiver = statsReceiver.scope("memCachedUserRealGraphMh"), - keyToString = { k: UserId => s"uRGraph/$k" } - ) - - DeciderableReadableStore( - memCachedStore, - decider.deciderGateBuilder.idGate(DeciderKey.enableRealGraphMhStoreDeciderKey), - statsReceiver.scope("RealGraphMh") - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationManagerModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationManagerModule.docx new file mode 100644 index 000000000..5ef151b58 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationManagerModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationManagerModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationManagerModule.scala deleted file mode 100644 index 227e5fff3..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationManagerModule.scala +++ /dev/null @@ -1,107 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.google.inject.Provides -import com.google.inject.Singleton -import javax.inject.Named -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} - -object RepresentationManagerModule extends TwitterModule { - private val ColPathPrefix = "recommendations/representation_manager/" - private val SimclustersTweetColPath = ColPathPrefix + "simClustersEmbedding.Tweet" - private val SimclustersUserColPath = ColPathPrefix + "simClustersEmbedding.User" - - @Provides - @Singleton - @Named(ModuleNames.RmsTweetLogFavLongestL2EmbeddingStore) - def providesRepresentationManagerTweetStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[TweetId, SimClustersEmbedding] = { - ObservedReadableStore( - StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersTweetColPath, - SimClustersEmbeddingView( - EmbeddingType.LogFavLongestL2EmbeddingTweet, - ModelVersion.Model20m145k2020)) - .mapValues(SimClustersEmbedding(_)))( - statsReceiver.scope("rms_tweet_log_fav_longest_l2_store")) - } - - @Provides - @Singleton - @Named(ModuleNames.RmsUserFavBasedProducerEmbeddingStore) - def providesRepresentationManagerUserFavBasedProducerEmbeddingStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[UserId, SimClustersEmbedding] = { - ObservedReadableStore( - StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersUserColPath, - SimClustersEmbeddingView( - EmbeddingType.FavBasedProducer, - ModelVersion.Model20m145k2020 - ) - ) - .mapValues(SimClustersEmbedding(_)))( - statsReceiver.scope("rms_user_fav_based_producer_store")) - } - - @Provides - @Singleton - @Named(ModuleNames.RmsUserLogFavInterestedInEmbeddingStore) - def providesRepresentationManagerUserLogFavConsumerEmbeddingStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[UserId, SimClustersEmbedding] = { - ObservedReadableStore( - StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersUserColPath, - SimClustersEmbeddingView( - EmbeddingType.LogFavBasedUserInterestedIn, - ModelVersion.Model20m145k2020 - ) - ) - .mapValues(SimClustersEmbedding(_)))( - statsReceiver.scope("rms_user_log_fav_interestedin_store")) - } - - @Provides - @Singleton - @Named(ModuleNames.RmsUserFollowInterestedInEmbeddingStore) - def providesRepresentationManagerUserFollowInterestedInEmbeddingStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[UserId, SimClustersEmbedding] = { - ObservedReadableStore( - StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersUserColPath, - SimClustersEmbeddingView( - EmbeddingType.FollowBasedUserInterestedIn, - ModelVersion.Model20m145k2020 - ) - ) - .mapValues(SimClustersEmbedding(_)))( - statsReceiver.scope("rms_user_follow_interestedin_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationScorerModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationScorerModule.docx new file mode 100644 index 000000000..2e00f18bf Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationScorerModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationScorerModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationScorerModule.scala deleted file mode 100644 index 7db6474cc..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/RepresentationScorerModule.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.storehaus.ReadableStore -import com.twitter.simclusters_v2.thriftscala.ScoringAlgorithm -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.hermit.store.common.ObservedReadableStore -import javax.inject.Named -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.representationscorer.thriftscala.ListScoreId - -object RepresentationScorerModule extends TwitterModule { - - private val rsxColumnPath = "recommendations/representation_scorer/listScore" - - private final val SimClusterModelVersion = ModelVersion.Model20m145k2020 - private final val TweetEmbeddingType = EmbeddingType.LogFavBasedTweet - - @Provides - @Singleton - @Named(ModuleNames.RsxStore) - def providesRepresentationScorerStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[(UserId, TweetId), Double] = { - ObservedReadableStore( - StratoFetchableStore - .withUnitView[ListScoreId, Double](stratoClient, rsxColumnPath).composeKeyMapping[( - UserId, - TweetId - )] { key => - representationScorerStoreKeyMapping(key._1, key._2) - } - )(statsReceiver.scope("rsx_store")) - } - - private def representationScorerStoreKeyMapping(t1: TweetId, t2: TweetId): ListScoreId = { - ListScoreId( - algorithm = ScoringAlgorithm.PairEmbeddingLogCosineSimilarity, - modelVersion = SimClusterModelVersion, - targetEmbeddingType = TweetEmbeddingType, - targetId = InternalId.TweetId(t1), - candidateEmbeddingType = TweetEmbeddingType, - candidateIds = Seq(InternalId.TweetId(t2)) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SampleSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SampleSimilarityEngineModule.docx new file mode 100644 index 000000000..fd9f3bab4 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SampleSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SampleSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SampleSimilarityEngineModule.scala deleted file mode 100644 index 98c3f4af6..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SampleSimilarityEngineModule.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.LookupSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import javax.inject.Singleton - -/** - * In this example we build a [[StandardSimilarityEngine]] to wrap a dummy store - */ -object SimpleSimilarityEngineModule extends TwitterModule { - @Provides - @Singleton - def providesSimpleSimilarityEngine( - timeoutConfig: TimeoutConfig, - globalStats: StatsReceiver - ): StandardSimilarityEngine[UserId, (TweetId, Double)] = { - // Inject your readableStore implementation here - val dummyStore = ReadableStore.fromMap( - Map( - 1L -> Seq((100L, 1.0), (101L, 1.0)), - 2L -> Seq((200L, 2.0), (201L, 2.0)), - 3L -> Seq((300L, 3.0), (301L, 3.0)) - )) - - new StandardSimilarityEngine[UserId, (TweetId, Double)]( - implementingStore = dummyStore, - identifier = SimilarityEngineType.EnumUnknownSimilarityEngineType(9997), - globalStats = globalStats, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} - -/** - * In this example we build a [[LookupSimilarityEngine]] to wrap a dummy store with 2 versions - */ -object LookupSimilarityEngineModule extends TwitterModule { - @Provides - @Singleton - def providesLookupSimilarityEngine( - timeoutConfig: TimeoutConfig, - globalStats: StatsReceiver - ): LookupSimilarityEngine[UserId, (TweetId, Double)] = { - // Inject your readableStore implementation here - val dummyStoreV1 = ReadableStore.fromMap( - Map( - 1L -> Seq((100L, 1.0), (101L, 1.0)), - 2L -> Seq((200L, 2.0), (201L, 2.0)), - )) - - val dummyStoreV2 = ReadableStore.fromMap( - Map( - 1L -> Seq((100L, 1.0), (101L, 1.0)), - 2L -> Seq((200L, 2.0), (201L, 2.0)), - )) - - new LookupSimilarityEngine[UserId, (TweetId, Double)]( - versionedStoreMap = Map( - "V1" -> dummyStoreV1, - "V2" -> dummyStoreV2 - ), - identifier = SimilarityEngineType.EnumUnknownSimilarityEngineType(9998), - globalStats = globalStats, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SimClustersANNServiceNameToClientMapper.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SimClustersANNServiceNameToClientMapper.docx new file mode 100644 index 000000000..ce48cee41 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SimClustersANNServiceNameToClientMapper.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SimClustersANNServiceNameToClientMapper.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SimClustersANNServiceNameToClientMapper.scala deleted file mode 100644 index 305839816..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SimClustersANNServiceNameToClientMapper.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.inject.TwitterModule -import com.twitter.simclustersann.thriftscala.SimClustersANNService -import javax.inject.Named - -object SimClustersANNServiceNameToClientMapper extends TwitterModule { - - @Provides - @Singleton - def providesSimClustersANNServiceNameToClientMapping( - @Named(ModuleNames.ProdSimClustersANNServiceClientName) simClustersANNServiceProd: SimClustersANNService.MethodPerEndpoint, - @Named(ModuleNames.ExperimentalSimClustersANNServiceClientName) simClustersANNServiceExperimental: SimClustersANNService.MethodPerEndpoint, - @Named(ModuleNames.SimClustersANNServiceClientName1) simClustersANNService1: SimClustersANNService.MethodPerEndpoint, - @Named(ModuleNames.SimClustersANNServiceClientName2) simClustersANNService2: SimClustersANNService.MethodPerEndpoint, - @Named(ModuleNames.SimClustersANNServiceClientName3) simClustersANNService3: SimClustersANNService.MethodPerEndpoint, - @Named(ModuleNames.SimClustersANNServiceClientName5) simClustersANNService5: SimClustersANNService.MethodPerEndpoint, - @Named(ModuleNames.SimClustersANNServiceClientName4) simClustersANNService4: SimClustersANNService.MethodPerEndpoint - ): Map[String, SimClustersANNService.MethodPerEndpoint] = { - Map[String, SimClustersANNService.MethodPerEndpoint]( - "simclusters-ann" -> simClustersANNServiceProd, - "simclusters-ann-experimental" -> simClustersANNServiceExperimental, - "simclusters-ann-1" -> simClustersANNService1, - "simclusters-ann-2" -> simClustersANNService2, - "simclusters-ann-3" -> simClustersANNService3, - "simclusters-ann-5" -> simClustersANNService5, - "simclusters-ann-4" -> simClustersANNService4 - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SkitStratoStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SkitStratoStoreModule.docx new file mode 100644 index 000000000..35713ec92 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SkitStratoStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SkitStratoStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SkitStratoStoreModule.scala deleted file mode 100644 index 318c2ed00..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/SkitStratoStoreModule.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedCachedReadableStore -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.topic_recos.thriftscala.TopicTopTweets -import com.twitter.topic_recos.thriftscala.TopicTweet -import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey - -/** - * Strato store that wraps the topic top tweets pipeline indexed from a Summingbird job - */ -object SkitStratoStoreModule extends TwitterModule { - - val column = "recommendations/topic_recos/topicTopTweets" - - @Provides - @Singleton - @Named(ModuleNames.SkitStratoStoreName) - def providesSkitStratoStore( - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - stratoClient: Client, - statsReceiver: StatsReceiver - ): ReadableStore[TopicTweetPartitionFlatKey, Seq[TopicTweet]] = { - val skitStore = ObservedReadableStore( - StratoFetchableStore - .withUnitView[TopicTweetPartitionFlatKey, TopicTopTweets](stratoClient, column))( - statsReceiver.scope(ModuleNames.SkitStratoStoreName)).mapValues { topicTopTweets => - topicTopTweets.topTweets - } - - val memCachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = skitStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[TopicTweet]()), - statsReceiver = statsReceiver.scope("memcached_skit_store"), - keyToString = { k => s"skit:${keyHasher.hashKey(k.toString.getBytes)}" } - ) - - ObservedCachedReadableStore.from[TopicTweetPartitionFlatKey, Seq[TopicTweet]]( - memCachedStore, - ttl = 5.minutes, - maxKeys = 100000, // ~150MB max - cacheName = "skit_in_memory_cache", - windowSize = 10000L - )(statsReceiver.scope("skit_in_memory_cache")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/StrongTiePredictionStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/StrongTiePredictionStoreModule.docx new file mode 100644 index 000000000..435d5011d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/StrongTiePredictionStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/StrongTiePredictionStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/StrongTiePredictionStoreModule.scala deleted file mode 100644 index 51d556077..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/StrongTiePredictionStoreModule.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.app.Flag -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.UserId -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import javax.inject.Named - -object StrongTiePredictionStoreModule extends TwitterModule { - - private val strongTiePredictionColumnPath: Flag[String] = flag[String]( - name = "crMixer.strongTiePredictionColumnPath", - default = "onboarding/userrecs/strong_tie_prediction_big", - help = "Strato column path for StrongTiePredictionStore" - ) - - @Provides - @Singleton - @Named(ModuleNames.StpStore) - def providesStrongTiePredictionStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[UserId, STPResult] = { - val strongTiePredictionStratoFetchableStore = StratoFetchableStore - .withUnitView[UserId, STPResult](stratoClient, strongTiePredictionColumnPath()) - - ObservedReadableStore( - strongTiePredictionStratoFetchableStore - )(statsReceiver.scope("strong_tie_prediction_big_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TripCandidateStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TripCandidateStoreModule.docx new file mode 100644 index 000000000..323770d8a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TripCandidateStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TripCandidateStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TripCandidateStoreModule.scala deleted file mode 100644 index 802d5c986..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TripCandidateStoreModule.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweets -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import javax.inject.Named - -object TripCandidateStoreModule extends TwitterModule { - private val stratoColumn = "trends/trip/tripTweetsDataflowProd" - - @Provides - @Named(ModuleNames.TripCandidateStore) - def providesSimClustersTripCandidateStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient - ): ReadableStore[TripDomain, Seq[TripTweet]] = { - val tripCandidateStratoFetchableStore = - StratoFetchableStore - .withUnitView[TripDomain, TripTweets](stratoClient, stratoColumn) - .mapValues(_.tweets) - - ObservedReadableStore( - tripCandidateStratoFetchableStore - )(statsReceiver.scope("simclusters_trip_candidate_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetInfoStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetInfoStoreModule.docx new file mode 100644 index 000000000..beb1c621d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetInfoStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetInfoStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetInfoStoreModule.scala deleted file mode 100644 index a3a794e8e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetInfoStoreModule.scala +++ /dev/null @@ -1,205 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Module -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.contentrecommender.thriftscala.TweetInfo -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.frigate.common.store.health.TweetHealthModelStore -import com.twitter.frigate.common.store.health.TweetHealthModelStore.TweetHealthModelStoreConfig -import com.twitter.frigate.common.store.health.UserHealthModelStore -import com.twitter.frigate.thriftscala.TweetHealthScores -import com.twitter.frigate.thriftscala.UserAgathaScores -import com.twitter.hermit.store.common.DeciderableReadableStore -import com.twitter.hermit.store.common.ObservedCachedReadableStore -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.contentrecommender.store.TweetInfoStore -import com.twitter.contentrecommender.store.TweetyPieFieldsStore -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderKey -import com.twitter.frigate.data_pipeline.scalding.thriftscala.BlueVerifiedAnnotationsV2 -import com.twitter.recos.user_tweet_graph_plus.thriftscala.UserTweetGraphPlus -import com.twitter.recos.user_tweet_graph_plus.thriftscala.TweetEngagementScores -import com.twitter.relevance_platform.common.health_store.UserMediaRepresentationHealthStore -import com.twitter.relevance_platform.common.health_store.MagicRecsRealTimeAggregatesStore -import com.twitter.relevance_platform.thriftscala.MagicRecsRealTimeAggregatesScores -import com.twitter.relevance_platform.thriftscala.UserMediaRepresentationScores -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.tweetypie.thriftscala.TweetService -import com.twitter.util.Future -import com.twitter.util.JavaTimer -import com.twitter.util.Timer - -import javax.inject.Named - -object TweetInfoStoreModule extends TwitterModule { - implicit val timer: Timer = new JavaTimer(true) - override def modules: Seq[Module] = Seq(UnifiedCacheClient) - - @Provides - @Singleton - def providesTweetInfoStore( - statsReceiver: StatsReceiver, - serviceIdentifier: ServiceIdentifier, - stratoClient: StratoClient, - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, - tweetyPieService: TweetService.MethodPerEndpoint, - userTweetGraphPlusService: UserTweetGraphPlus.MethodPerEndpoint, - @Named(ModuleNames.BlueVerifiedAnnotationStore) blueVerifiedAnnotationStore: ReadableStore[ - String, - BlueVerifiedAnnotationsV2 - ], - decider: CrMixerDecider - ): ReadableStore[TweetId, TweetInfo] = { - - val tweetEngagementScoreStore: ReadableStore[TweetId, TweetEngagementScores] = { - val underlyingStore = - ObservedReadableStore(new ReadableStore[TweetId, TweetEngagementScores] { - override def get( - k: TweetId - ): Future[Option[TweetEngagementScores]] = { - userTweetGraphPlusService.tweetEngagementScore(k).map { - Some(_) - } - } - })(statsReceiver.scope("UserTweetGraphTweetEngagementScoreStore")) - - DeciderableReadableStore( - underlyingStore, - decider.deciderGateBuilder.idGate( - DeciderKey.enableUtgRealTimeTweetEngagementScoreDeciderKey), - statsReceiver.scope("UserTweetGraphTweetEngagementScoreStore") - ) - - } - - val tweetHealthModelStore: ReadableStore[TweetId, TweetHealthScores] = { - val underlyingStore = TweetHealthModelStore.buildReadableStore( - stratoClient, - Some( - TweetHealthModelStoreConfig( - enablePBlock = true, - enableToxicity = true, - enablePSpammy = true, - enablePReported = true, - enableSpammyTweetContent = true, - enablePNegMultimodal = true, - )) - )(statsReceiver.scope("UnderlyingTweetHealthModelStore")) - - DeciderableReadableStore( - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 2.hours - )( - valueInjection = BinaryScalaCodec(TweetHealthScores), - statsReceiver = statsReceiver.scope("memCachedTweetHealthModelStore"), - keyToString = { k: TweetId => s"tHMS/$k" } - ), - decider.deciderGateBuilder.idGate(DeciderKey.enableHealthSignalsScoreDeciderKey), - statsReceiver.scope("TweetHealthModelStore") - ) // use s"tHMS/$k" instead of s"tweetHealthModelStore/$k" to differentiate from CR cache - } - - val userHealthModelStore: ReadableStore[UserId, UserAgathaScores] = { - val underlyingStore = UserHealthModelStore.buildReadableStore(stratoClient)( - statsReceiver.scope("UnderlyingUserHealthModelStore")) - DeciderableReadableStore( - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 18.hours - )( - valueInjection = BinaryScalaCodec(UserAgathaScores), - statsReceiver = statsReceiver.scope("memCachedUserHealthModelStore"), - keyToString = { k: UserId => s"uHMS/$k" } - ), - decider.deciderGateBuilder.idGate(DeciderKey.enableUserAgathaScoreDeciderKey), - statsReceiver.scope("UserHealthModelStore") - ) - } - - val userMediaRepresentationHealthStore: ReadableStore[UserId, UserMediaRepresentationScores] = { - val underlyingStore = - UserMediaRepresentationHealthStore.buildReadableStore( - manhattanKVClientMtlsParams, - statsReceiver.scope("UnderlyingUserMediaRepresentationHealthStore") - ) - DeciderableReadableStore( - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 12.hours - )( - valueInjection = BinaryScalaCodec(UserMediaRepresentationScores), - statsReceiver = statsReceiver.scope("memCacheUserMediaRepresentationHealthStore"), - keyToString = { k: UserId => s"uMRHS/$k" } - ), - decider.deciderGateBuilder.idGate(DeciderKey.enableUserMediaRepresentationStoreDeciderKey), - statsReceiver.scope("UserMediaRepresentationHealthStore") - ) - } - - val magicRecsRealTimeAggregatesStore: ReadableStore[ - TweetId, - MagicRecsRealTimeAggregatesScores - ] = { - val underlyingStore = - MagicRecsRealTimeAggregatesStore.buildReadableStore( - serviceIdentifier, - statsReceiver.scope("UnderlyingMagicRecsRealTimeAggregatesScores") - ) - DeciderableReadableStore( - underlyingStore, - decider.deciderGateBuilder.idGate(DeciderKey.enableMagicRecsRealTimeAggregatesStore), - statsReceiver.scope("MagicRecsRealTimeAggregatesStore") - ) - } - - val tweetInfoStore: ReadableStore[TweetId, TweetInfo] = { - val underlyingStore = TweetInfoStore( - TweetyPieFieldsStore.getStoreFromTweetyPie(tweetyPieService), - userMediaRepresentationHealthStore, - magicRecsRealTimeAggregatesStore, - tweetEngagementScoreStore, - blueVerifiedAnnotationStore - )(statsReceiver.scope("tweetInfoStore")) - - val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 15.minutes, - // Hydrating tweetInfo is now a required step for all candidates, - // hence we needed to tune these thresholds. - asyncUpdate = serviceIdentifier.environment == "prod" - )( - valueInjection = BinaryScalaCodec(TweetInfo), - statsReceiver = statsReceiver.scope("memCachedTweetInfoStore"), - keyToString = { k: TweetId => s"tIS/$k" } - ) - - ObservedCachedReadableStore.from( - memcachedStore, - ttl = 15.minutes, - maxKeys = 8388607, // Check TweetInfo definition. size~92b. Around 736 MB - windowSize = 10000L, - cacheName = "tweet_info_cache", - maxMultiGetSize = 20 - )(statsReceiver.scope("inMemoryCachedTweetInfoStore")) - } - tweetInfoStore - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecentEngagedUserStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecentEngagedUserStoreModule.docx new file mode 100644 index 000000000..bcc8fd7df Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecentEngagedUserStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecentEngagedUserStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecentEngagedUserStoreModule.scala deleted file mode 100644 index 2e379e545..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecentEngagedUserStoreModule.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.app.Flag -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers - -object TweetRecentEngagedUserStoreModule extends TwitterModule { - - private val tweetRecentEngagedUsersStoreDefaultVersion = - 0 // DefaultVersion for tweetEngagedUsersStore, whose key = (tweetId, DefaultVersion) - private val tweetRecentEngagedUsersColumnPath: Flag[String] = flag[String]( - name = "crMixer.tweetRecentEngagedUsersColumnPath", - default = "recommendations/twistly/tweetRecentEngagedUsers", - help = "Strato column path for TweetRecentEngagedUsersStore" - ) - private type Version = Long - - @Provides - @Singleton - def providesTweetRecentEngagedUserStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[TweetId, TweetRecentEngagedUsers] = { - val tweetRecentEngagedUsersStratoFetchableStore = StratoFetchableStore - .withUnitView[(TweetId, Version), TweetRecentEngagedUsers]( - stratoClient, - tweetRecentEngagedUsersColumnPath()).composeKeyMapping[TweetId](tweetId => - (tweetId, tweetRecentEngagedUsersStoreDefaultVersion)) - - ObservedReadableStore( - tweetRecentEngagedUsersStratoFetchableStore - )(statsReceiver.scope("tweet_recent_engaged_users_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecommendationResultsStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecommendationResultsStoreModule.docx new file mode 100644 index 000000000..e776452b8 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecommendationResultsStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecommendationResultsStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecommendationResultsStoreModule.scala deleted file mode 100644 index 04c03eda6..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TweetRecommendationResultsStoreModule.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.hermit.store.common.ReadableWritableStore -import com.twitter.hermit.store.common.ObservedReadableWritableMemcacheStore -import com.twitter.simclusters_v2.common.UserId -import javax.inject.Named - -object TweetRecommendationResultsStoreModule extends TwitterModule { - @Provides - @Singleton - def providesTweetRecommendationResultsStore( - @Named(ModuleNames.TweetRecommendationResultsCache) tweetRecommendationResultsCacheClient: MemcachedClient, - statsReceiver: StatsReceiver - ): ReadableWritableStore[UserId, CrMixerTweetResponse] = { - ObservedReadableWritableMemcacheStore.fromCacheClient( - cacheClient = tweetRecommendationResultsCacheClient, - ttl = 24.hours)( - valueInjection = BinaryScalaCodec(CrMixerTweetResponse), - statsReceiver = statsReceiver.scope("TweetRecommendationResultsMemcacheStore"), - keyToString = { k: UserId => k.toString } - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwhinCollabFilterStratoStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwhinCollabFilterStratoStoreModule.docx new file mode 100644 index 000000000..bbf1535e7 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwhinCollabFilterStratoStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwhinCollabFilterStratoStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwhinCollabFilterStratoStoreModule.scala deleted file mode 100644 index 4275ad2a8..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwhinCollabFilterStratoStoreModule.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.inject.TwitterModule -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.cr_mixer.similarity_engine.TwhinCollabFilterSimilarityEngine.TwhinCollabFilterView -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.storehaus.ReadableStore -import javax.inject.Named - -object TwhinCollabFilterStratoStoreModule extends TwitterModule { - - val stratoColumnPath: String = "cuad/twhin/getCollabFilterTweetCandidatesProd.User" - - @Provides - @Singleton - @Named(ModuleNames.TwhinCollabFilterStratoStoreForFollow) - def providesTwhinCollabFilterStratoStoreForFollow( - stratoClient: StratoClient - ): ReadableStore[Long, Seq[TweetId]] = { - StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( - stratoClient, - column = stratoColumnPath, - view = TwhinCollabFilterView("follow_2022_03_10_c_500K") - ) - } - - @Provides - @Singleton - @Named(ModuleNames.TwhinCollabFilterStratoStoreForEngagement) - def providesTwhinCollabFilterStratoStoreForEngagement( - stratoClient: StratoClient - ): ReadableStore[Long, Seq[TweetId]] = { - StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( - stratoClient, - column = stratoColumnPath, - view = TwhinCollabFilterView("engagement_2022_04_10_c_500K")) - } - - @Provides - @Singleton - @Named(ModuleNames.TwhinMultiClusterStratoStoreForFollow) - def providesTwhinMultiClusterStratoStoreForFollow( - stratoClient: StratoClient - ): ReadableStore[Long, Seq[TweetId]] = { - StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( - stratoClient, - column = stratoColumnPath, - view = TwhinCollabFilterView("multiclusterFollow20220921") - ) - } - - @Provides - @Singleton - @Named(ModuleNames.TwhinMultiClusterStratoStoreForEngagement) - def providesTwhinMultiClusterStratoStoreForEngagement( - stratoClient: StratoClient - ): ReadableStore[Long, Seq[TweetId]] = { - StratoFetchableStore.withView[Long, TwhinCollabFilterView, Seq[TweetId]]( - stratoClient, - column = stratoColumnPath, - view = TwhinCollabFilterView("multiclusterEng20220921")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwiceClustersMembersStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwiceClustersMembersStoreModule.docx new file mode 100644 index 000000000..b5d94325b Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwiceClustersMembersStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwiceClustersMembersStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwiceClustersMembersStoreModule.scala deleted file mode 100644 index a15e2549a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/TwiceClustersMembersStoreModule.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.app.Flag -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.simclusters_v2.thriftscala.OrderedClustersAndMembers -import javax.inject.Named - -object TwiceClustersMembersStoreModule extends TwitterModule { - - private val twiceClustersMembersColumnPath: Flag[String] = flag[String]( - name = "crMixer.twiceClustersMembersColumnPath", - default = - "recommendations/simclusters_v2/embeddings/TwiceClustersMembersLargestDimApeSimilarity", - help = "Strato column path for TweetRecentEngagedUsersStore" - ) - - @Provides - @Singleton - @Named(ModuleNames.TwiceClustersMembersStore) - def providesTweetRecentEngagedUserStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[UserId, OrderedClustersAndMembers] = { - val twiceClustersMembersStratoFetchableStore = StratoFetchableStore - .withUnitView[UserId, OrderedClustersAndMembers]( - stratoClient, - twiceClustersMembersColumnPath()) - - ObservedReadableStore( - twiceClustersMembersStratoFetchableStore - )(statsReceiver.scope("twice_clusters_members_largestDimApe_similarity_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UnifiedCacheClient.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UnifiedCacheClient.docx new file mode 100644 index 000000000..122fb666e Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UnifiedCacheClient.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UnifiedCacheClient.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UnifiedCacheClient.scala deleted file mode 100644 index 3b48f4c02..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UnifiedCacheClient.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.app.Flag -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus_internal.memcache.MemcacheStore -import com.twitter.storehaus_internal.util.ClientName -import com.twitter.storehaus_internal.util.ZkEndPoint -import javax.inject.Named - -object UnifiedCacheClient extends TwitterModule { - - private val TIME_OUT = 20.milliseconds - - val crMixerUnifiedCacheDest: Flag[String] = flag[String]( - name = "crMixer.unifiedCacheDest", - default = "/s/cache/content_recommender_unified_v2", - help = "Wily path to Content Recommender unified cache" - ) - - val tweetRecommendationResultsCacheDest: Flag[String] = flag[String]( - name = "tweetRecommendationResults.CacheDest", - default = "/s/cache/tweet_recommendation_results", - help = "Wily path to CrMixer getTweetRecommendations() results cache" - ) - - val earlybirdTweetsCacheDest: Flag[String] = flag[String]( - name = "earlybirdTweets.CacheDest", - default = "/s/cache/crmixer_earlybird_tweets", - help = "Wily path to CrMixer Earlybird Recency Based Similarity Engine result cache" - ) - - @Provides - @Singleton - @Named(ModuleNames.UnifiedCache) - def provideUnifiedCacheClient( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver, - ): Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache-content-recommender-unified"), - dest = ZkEndPoint(crMixerUnifiedCacheDest()), - statsReceiver = statsReceiver.scope("cache_client"), - serviceIdentifier = serviceIdentifier, - timeout = TIME_OUT - ) - - @Provides - @Singleton - @Named(ModuleNames.TweetRecommendationResultsCache) - def providesTweetRecommendationResultsCache( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver, - ): Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache-tweet-recommendation-results"), - dest = ZkEndPoint(tweetRecommendationResultsCacheDest()), - statsReceiver = statsReceiver.scope("cache_client"), - serviceIdentifier = serviceIdentifier, - timeout = TIME_OUT - ) - - @Provides - @Singleton - @Named(ModuleNames.EarlybirdTweetsCache) - def providesEarlybirdTweetsCache( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver, - ): Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache-crmixer-earlybird-tweets"), - dest = ZkEndPoint(earlybirdTweetsCacheDest()), - statsReceiver = statsReceiver.scope("cache_client"), - serviceIdentifier = serviceIdentifier, - timeout = TIME_OUT - ) -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceColumnModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceColumnModule.docx new file mode 100644 index 000000000..265f1c17e Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceColumnModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceColumnModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceColumnModule.scala deleted file mode 100644 index b15ebe0fe..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceColumnModule.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.cr_mixer.module -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.usersignalservice.thriftscala.BatchSignalRequest -import com.twitter.usersignalservice.thriftscala.BatchSignalResponse -import javax.inject.Named - -object UserSignalServiceColumnModule extends TwitterModule { - private val UssColumnPath = "recommendations/user-signal-service/signals" - - @Provides - @Singleton - @Named(ModuleNames.UssStratoColumn) - def providesUserSignalServiceStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[BatchSignalRequest, BatchSignalResponse] = { - ObservedReadableStore( - StratoFetchableStore - .withUnitView[BatchSignalRequest, BatchSignalResponse](stratoClient, UssColumnPath))( - statsReceiver.scope("user_signal_service_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceStoreModule.docx new file mode 100644 index 000000000..ce2ad0e31 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceStoreModule.scala deleted file mode 100644 index cc55f0e9a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserSignalServiceStoreModule.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.source_signal.UssStore -import com.twitter.cr_mixer.source_signal.UssStore.Query -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.usersignalservice.thriftscala.BatchSignalRequest -import com.twitter.usersignalservice.thriftscala.BatchSignalResponse -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.usersignalservice.thriftscala.{Signal => UssSignal} -import javax.inject.Named - -object UserSignalServiceStoreModule extends TwitterModule { - - private val UssColumnPath = "recommendations/user-signal-service/signals" - - @Provides - @Singleton - @Named(ModuleNames.UssStore) - def providesUserSignalServiceStore( - statsReceiver: StatsReceiver, - stratoClient: StratoClient, - ): ReadableStore[Query, Seq[(SignalType, Seq[UssSignal])]] = { - ObservedReadableStore( - UssStore( - StratoFetchableStore - .withUnitView[BatchSignalRequest, BatchSignalResponse](stratoClient, UssColumnPath), - statsReceiver))(statsReceiver.scope("user_signal_service_store")) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserStateStoreModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserStateStoreModule.docx new file mode 100644 index 000000000..0d4e2af96 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserStateStoreModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserStateStoreModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserStateStoreModule.scala deleted file mode 100644 index 6db2c38fd..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/UserStateStoreModule.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.cr_mixer.module - -import com.google.inject.Provides -import com.google.inject.Singleton -import com.twitter.bijection.Bufferable -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.UserId -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.HDFSPath -import com.twitter.core_workflows.user_model.thriftscala.UserState -import com.twitter.core_workflows.user_model.thriftscala.CondensedUserState -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderKey -import com.twitter.hermit.store.common.DeciderableReadableStore -import com.twitter.storehaus_internal.manhattan.Apollo -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.JavaTimer -import com.twitter.util.Time -import com.twitter.util.TimeoutException -import com.twitter.util.Timer -import javax.inject.Named - -object UserStateStoreModule extends TwitterModule { - implicit val timer: Timer = new JavaTimer(true) - final val NewUserCreateDaysThreshold = 7 - final val DefaultUnknownUserStateValue = 100 - - // Convert CondensedUserState to UserState Enum - // If CondensedUserState is None, back fill by checking whether the user is new user - class UserStateStore( - userStateStore: ReadableStore[UserId, CondensedUserState], - timeout: Duration, - statsReceiver: StatsReceiver) - extends ReadableStore[UserId, UserState] { - override def get(userId: UserId): Future[Option[UserState]] = { - userStateStore - .get(userId).map(_.flatMap(_.userState)).map { - case Some(userState) => Some(userState) - case None => - val isNewUser = SnowflakeId.timeFromIdOpt(userId).exists { userCreateTime => - Time.now - userCreateTime < Duration.fromDays(NewUserCreateDaysThreshold) - } - if (isNewUser) Some(UserState.New) - else Some(UserState.EnumUnknownUserState(DefaultUnknownUserStateValue)) - - }.raiseWithin(timeout)(timer).rescue { - case _: TimeoutException => - statsReceiver.counter("TimeoutException").incr() - Future.None - } - } - } - - @Provides - @Singleton - def providesUserStateStore( - crMixerDecider: CrMixerDecider, - statsReceiver: StatsReceiver, - manhattanKVClientMtlsParams: ManhattanKVClientMtlsParams, - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig - ): ReadableStore[UserId, UserState] = { - - val underlyingStore = new UserStateStore( - ManhattanRO - .getReadableStoreWithMtls[UserId, CondensedUserState]( - ManhattanROConfig( - HDFSPath(""), - ApplicationID("cr_mixer_apollo"), - DatasetName("condensed_user_state"), - Apollo), - manhattanKVClientMtlsParams - )( - implicitly[Injection[Long, Array[Byte]]], - BinaryScalaCodec(CondensedUserState) - ), - timeoutConfig.userStateStoreTimeout, - statsReceiver.scope("UserStateStore") - ).mapValues(_.value) // Read the value of Enum so that we only caches the Int - - val memCachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 24.hours, - )( - valueInjection = Bufferable.injectionOf[Int], // Cache Value is Enum Value for UserState - statsReceiver = statsReceiver.scope("memCachedUserStateStore"), - keyToString = { k: UserId => s"uState/$k" } - ).mapValues(value => UserState.getOrUnknown(value)) - - DeciderableReadableStore( - memCachedStore, - crMixerDecider.deciderGateBuilder.idGate(DeciderKey.enableUserStateStoreDeciderKey), - statsReceiver.scope("UserStateStore") - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/ABDeciderModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/ABDeciderModule.docx new file mode 100644 index 000000000..594d6e10d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/ABDeciderModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/ABDeciderModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/ABDeciderModule.scala deleted file mode 100644 index 9d981f4f3..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/ABDeciderModule.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.google.inject.Provides -import com.google.inject.name.Named -import com.twitter.abdecider.ABDeciderFactory -import com.twitter.abdecider.LoggingABDecider -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.logging.Logger -import javax.inject.Singleton - -object ABDeciderModule extends TwitterModule { - - flag( - name = "abdecider.path", - default = "/usr/local/config/abdecider/abdecider.yml", - help = "path to the abdecider Yml file location" - ) - - @Provides - @Singleton - def provideABDecider( - @Flag("abdecider.path") abDeciderYmlPath: String, - @Named(ModuleNames.AbDeciderLogger) scribeLogger: Logger - ): LoggingABDecider = { - ABDeciderFactory( - abDeciderYmlPath = abDeciderYmlPath, - scribeLogger = Some(scribeLogger), - environment = Some("production") - ).buildWithLogging() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerFlagModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerFlagModule.docx new file mode 100644 index 000000000..31a5eeedc Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerFlagModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerFlagModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerFlagModule.scala deleted file mode 100644 index 9e7b9938a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerFlagModule.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.twitter.inject.TwitterModule - -object CrMixerFlagName { - val SERVICE_FLAG = "cr_mixer.flag" - val DarkTrafficFilterDeciderKey = "thrift.dark.traffic.filter.decider_key" -} - -object CrMixerFlagModule extends TwitterModule { - import CrMixerFlagName._ - - flag[Boolean](name = SERVICE_FLAG, default = false, help = "This is a CR Mixer flag") - - flag[String]( - name = DarkTrafficFilterDeciderKey, - default = "dark_traffic_filter", - help = "Dark traffic filter decider key" - ) -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerLoggingABDeciderModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerLoggingABDeciderModule.docx new file mode 100644 index 000000000..79dbae907 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerLoggingABDeciderModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerLoggingABDeciderModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerLoggingABDeciderModule.scala deleted file mode 100644 index 6b674495f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/CrMixerLoggingABDeciderModule.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.google.inject.Provides -import com.twitter.abdecider.LoggingABDecider -import com.twitter.cr_mixer.featureswitch.CrMixerLoggingABDecider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import javax.inject.Singleton - -object CrMixerLoggingABDeciderModule extends TwitterModule { - - @Provides - @Singleton - def provideABDecider( - loggingABDecider: LoggingABDecider, - statsReceiver: StatsReceiver - ): CrMixerLoggingABDecider = { - CrMixerLoggingABDecider(loggingABDecider, statsReceiver) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureContextBuilderModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureContextBuilderModule.docx new file mode 100644 index 000000000..fddab1af1 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureContextBuilderModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureContextBuilderModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureContextBuilderModule.scala deleted file mode 100644 index 18d262c54..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureContextBuilderModule.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.google.inject.Provides -import com.twitter.discovery.common.configapi.FeatureContextBuilder -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.inject.TwitterModule -import javax.inject.Singleton - -object FeatureContextBuilderModule extends TwitterModule { - - @Provides - @Singleton - def providesFeatureContextBuilder(featureSwitches: FeatureSwitches): FeatureContextBuilder = { - FeatureContextBuilder(featureSwitches) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureSwitchesModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureSwitchesModule.docx new file mode 100644 index 000000000..d3206f574 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureSwitchesModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureSwitchesModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureSwitchesModule.scala deleted file mode 100644 index a87d1f54b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/FeatureSwitchesModule.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.google.inject.Provides -import com.twitter.cr_mixer.featureswitch.CrMixerLoggingABDecider -import com.twitter.featureswitches.v2.FeatureSwitches -import com.twitter.featureswitches.v2.builder.FeatureSwitchesBuilder -import com.twitter.featureswitches.v2.experimentation.NullBucketImpressor -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.inject.annotations.Flag -import com.twitter.util.Duration -import javax.inject.Singleton - -object FeatureSwitchesModule extends TwitterModule { - - flag( - name = "featureswitches.path", - default = "/features/cr-mixer/main", - help = "path to the featureswitch configuration directory" - ) - flag( - "use_config_repo_mirror.bool", - false, - "If true, read config from a different directory, to facilitate testing.") - - val DefaultFastRefresh: Boolean = false - val AddServiceDetailsFromAurora: Boolean = true - val ImpressExperiments: Boolean = true - - @Provides - @Singleton - def providesFeatureSwitches( - @Flag("featureswitches.path") featureSwitchDirectory: String, - @Flag("use_config_repo_mirror.bool") useConfigRepoMirrorFlag: Boolean, - abDecider: CrMixerLoggingABDecider, - statsReceiver: StatsReceiver - ): FeatureSwitches = { - val configRepoAbsPath = - getConfigRepoAbsPath(useConfigRepoMirrorFlag) - val fastRefresh = - shouldFastRefresh(useConfigRepoMirrorFlag) - - val featureSwitches = FeatureSwitchesBuilder() - .abDecider(abDecider) - .statsReceiver(statsReceiver.scope("featureswitches-v2")) - .configRepoAbsPath(configRepoAbsPath) - .featuresDirectory(featureSwitchDirectory) - .limitToReferencedExperiments(shouldLimit = true) - .experimentImpressionStatsEnabled(true) - - if (!ImpressExperiments) featureSwitches.experimentBucketImpressor(NullBucketImpressor) - if (AddServiceDetailsFromAurora) featureSwitches.serviceDetailsFromAurora() - if (fastRefresh) featureSwitches.refreshPeriod(Duration.fromSeconds(10)) - - featureSwitches.build() - } - - private def getConfigRepoAbsPath( - useConfigRepoMirrorFlag: Boolean - ): String = { - if (useConfigRepoMirrorFlag) - "config_repo_mirror/" - else "/usr/local/config" - } - - private def shouldFastRefresh( - useConfigRepoMirrorFlag: Boolean - ): Boolean = { - if (useConfigRepoMirrorFlag) - true - else DefaultFastRefresh - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/KafkaProducerModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/KafkaProducerModule.docx new file mode 100644 index 000000000..931bf7229 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/KafkaProducerModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/KafkaProducerModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/KafkaProducerModule.scala deleted file mode 100644 index 770ad1e7e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/KafkaProducerModule.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.google.inject.Provides -import com.twitter.cr_mixer.thriftscala.GetTweetsRecommendationsScribe -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder -import com.twitter.finatra.kafka.producers.KafkaProducerBase -import com.twitter.finatra.kafka.producers.NullKafkaProducer -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.inject.TwitterModule -import javax.inject.Singleton -import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.common.config.SaslConfigs -import org.apache.kafka.common.config.SslConfigs -import org.apache.kafka.common.record.CompressionType -import org.apache.kafka.common.security.auth.SecurityProtocol -import org.apache.kafka.common.serialization.Serdes - -object KafkaProducerModule extends TwitterModule { - - @Provides - @Singleton - def provideTweetRecsLoggerFactory( - serviceIdentifier: ServiceIdentifier, - ): KafkaProducerBase[String, GetTweetsRecommendationsScribe] = { - KafkaProducerFactory.getKafkaProducer(serviceIdentifier.environment) - } -} - -object KafkaProducerFactory { - private val jaasConfig = - """com.sun.security.auth.module.Krb5LoginModule - |required - |principal="cr-mixer@TWITTER.BIZ" - |debug=true - |useKeyTab=true - |storeKey=true - |keyTab="/var/lib/tss/keys/fluffy/keytabs/client/cr-mixer.keytab" - |doNotPrompt=true; - """.stripMargin.replaceAll("\n", " ") - - private val trustStoreLocation = "/etc/tw_truststore/messaging/kafka/client.truststore.jks" - - def getKafkaProducer( - environment: String - ): KafkaProducerBase[String, GetTweetsRecommendationsScribe] = { - if (environment == "prod") { - FinagleKafkaProducerBuilder() - .dest("/s/kafka/recommendations:kafka-tls") - // kerberos params - .withConfig(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig) - .withConfig( - CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, - SecurityProtocol.SASL_SSL.toString) - .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, trustStoreLocation) - .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) - .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") - .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") - // Kafka params - .keySerializer(Serdes.String.serializer) - .valueSerializer(ScalaSerdes.CompactThrift[GetTweetsRecommendationsScribe].serializer()) - .clientId("cr-mixer") - .enableIdempotence(true) - .compressionType(CompressionType.LZ4) - .build() - } else { - new NullKafkaProducer[String, GetTweetsRecommendationsScribe] - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/LoggerFactoryModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/LoggerFactoryModule.docx new file mode 100644 index 000000000..239c2a910 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/LoggerFactoryModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/LoggerFactoryModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/LoggerFactoryModule.scala deleted file mode 100644 index 877ed4bb2..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/LoggerFactoryModule.scala +++ /dev/null @@ -1,155 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.scribe.ScribeCategories -import com.twitter.cr_mixer.scribe.ScribeCategory -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.logging.BareFormatter -import com.twitter.logging.Level -import com.twitter.logging.Logger -import com.twitter.logging.NullHandler -import com.twitter.logging.QueueingHandler -import com.twitter.logging.ScribeHandler -import com.twitter.logging.{LoggerFactory => TwitterLoggerFactory} -import javax.inject.Named -import javax.inject.Singleton - -object LoggerFactoryModule extends TwitterModule { - - private val DefaultQueueSize = 10000 - - @Provides - @Singleton - @Named(ModuleNames.AbDeciderLogger) - def provideAbDeciderLogger( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Logger = { - buildLoggerFactory( - ScribeCategories.AbDecider, - serviceIdentifier.environment, - statsReceiver.scope("ScribeLogger")) - .apply() - } - - @Provides - @Singleton - @Named(ModuleNames.TopLevelApiDdgMetricsLogger) - def provideTopLevelApiDdgMetricsLogger( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Logger = { - buildLoggerFactory( - ScribeCategories.TopLevelApiDdgMetrics, - serviceIdentifier.environment, - statsReceiver.scope("ScribeLogger")) - .apply() - } - - @Provides - @Singleton - @Named(ModuleNames.TweetRecsLogger) - def provideTweetRecsLogger( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Logger = { - buildLoggerFactory( - ScribeCategories.TweetsRecs, - serviceIdentifier.environment, - statsReceiver.scope("ScribeLogger")) - .apply() - } - - @Provides - @Singleton - @Named(ModuleNames.BlueVerifiedTweetRecsLogger) - def provideVITTweetRecsLogger( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Logger = { - buildLoggerFactory( - ScribeCategories.VITTweetsRecs, - serviceIdentifier.environment, - statsReceiver.scope("ScribeLogger")) - .apply() - } - - @Provides - @Singleton - @Named(ModuleNames.RelatedTweetsLogger) - def provideRelatedTweetsLogger( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Logger = { - buildLoggerFactory( - ScribeCategories.RelatedTweets, - serviceIdentifier.environment, - statsReceiver.scope("ScribeLogger")) - .apply() - } - - @Provides - @Singleton - @Named(ModuleNames.UtegTweetsLogger) - def provideUtegTweetsLogger( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Logger = { - buildLoggerFactory( - ScribeCategories.UtegTweets, - serviceIdentifier.environment, - statsReceiver.scope("ScribeLogger")) - .apply() - } - - @Provides - @Singleton - @Named(ModuleNames.AdsRecommendationsLogger) - def provideAdsRecommendationsLogger( - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): Logger = { - buildLoggerFactory( - ScribeCategories.AdsRecommendations, - serviceIdentifier.environment, - statsReceiver.scope("ScribeLogger")) - .apply() - } - - private def buildLoggerFactory( - category: ScribeCategory, - environment: String, - statsReceiver: StatsReceiver - ): TwitterLoggerFactory = { - environment match { - case "prod" => - TwitterLoggerFactory( - node = category.getProdLoggerFactoryNode, - level = Some(Level.INFO), - useParents = false, - handlers = List( - QueueingHandler( - maxQueueSize = DefaultQueueSize, - handler = ScribeHandler( - category = category.scribeCategory, - formatter = BareFormatter, - statsReceiver = statsReceiver.scope(category.getProdLoggerFactoryNode) - ) - ) - ) - ) - case _ => - TwitterLoggerFactory( - node = category.getStagingLoggerFactoryNode, - level = Some(Level.DEBUG), - useParents = false, - handlers = List( - { () => NullHandler } - ) - ) - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/MemoizingStatsReceiverModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/MemoizingStatsReceiverModule.docx new file mode 100644 index 000000000..06d732e71 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/MemoizingStatsReceiverModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/MemoizingStatsReceiverModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/MemoizingStatsReceiverModule.scala deleted file mode 100644 index ee94cf166..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/MemoizingStatsReceiverModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.twitter.finagle.stats.LoadedStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.servo.util.MemoizingStatsReceiver - -object MemoizingStatsReceiverModule extends TwitterModule { - override def configure(): Unit = { - bind[StatsReceiver].toInstance(new MemoizingStatsReceiver(LoadedStatsReceiver)) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/TimeoutConfigModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/TimeoutConfigModule.docx new file mode 100644 index 000000000..aa64aac4a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/TimeoutConfigModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/TimeoutConfigModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/TimeoutConfigModule.scala deleted file mode 100644 index 1b6200812..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/core/TimeoutConfigModule.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.cr_mixer.module.core - -import com.twitter.inject.TwitterModule -import com.google.inject.Provides -import javax.inject.Singleton -import com.twitter.util.Duration -import com.twitter.app.Flag -import com.twitter.cr_mixer.config.TimeoutConfig - -/** - * All timeout settings in CrMixer. - * Timeout numbers are defined in source/cr-mixer/server/config/deploy.aurora - */ -object TimeoutConfigModule extends TwitterModule { - - /** - * Flag names for client timeout - * These are used in modules extending ThriftMethodBuilderClientModule - * which cannot accept injection of TimeoutConfig - */ - val EarlybirdClientTimeoutFlagName = "earlybird.client.timeout" - val FrsClientTimeoutFlagName = "frsSignalFetch.client.timeout" - val QigRankerClientTimeoutFlagName = "qigRanker.client.timeout" - val TweetypieClientTimeoutFlagName = "tweetypie.client.timeout" - val UserTweetGraphClientTimeoutFlagName = "userTweetGraph.client.timeout" - val UserTweetGraphPlusClientTimeoutFlagName = "userTweetGraphPlus.client.timeout" - val UserAdGraphClientTimeoutFlagName = "userAdGraph.client.timeout" - val UserVideoGraphClientTimeoutFlagName = "userVideoGraph.client.timeout" - val UtegClientTimeoutFlagName = "uteg.client.timeout" - val NaviRequestTimeoutFlagName = "navi.client.request.timeout" - - /** - * Flags for timeouts - * These are defined and initialized only in this file - */ - // timeout for the service - private val serviceTimeout: Flag[Duration] = - flag("service.timeout", "service total timeout") - - // timeout for signal fetch - private val signalFetchTimeout: Flag[Duration] = - flag[Duration]("signalFetch.timeout", "signal fetch timeout") - - // timeout for similarity engine - private val similarityEngineTimeout: Flag[Duration] = - flag[Duration]("similarityEngine.timeout", "similarity engine timeout") - private val annServiceClientTimeout: Flag[Duration] = - flag[Duration]("annService.client.timeout", "annQueryService client timeout") - - // timeout for user affinities fetcher - private val userStateUnderlyingStoreTimeout: Flag[Duration] = - flag[Duration]("userStateUnderlyingStore.timeout", "user state underlying store timeout") - - private val userStateStoreTimeout: Flag[Duration] = - flag[Duration]("userStateStore.timeout", "user state store timeout") - - private val utegSimilarityEngineTimeout: Flag[Duration] = - flag[Duration]("uteg.similarityEngine.timeout", "uteg similarity engine timeout") - - private val earlybirdServerTimeout: Flag[Duration] = - flag[Duration]("earlybird.server.timeout", "earlybird server timeout") - - private val earlybirdSimilarityEngineTimeout: Flag[Duration] = - flag[Duration]("earlybird.similarityEngine.timeout", "Earlybird similarity engine timeout") - - private val frsBasedTweetEndpointTimeout: Flag[Duration] = - flag[Duration]( - "frsBasedTweet.endpoint.timeout", - "frsBasedTweet endpoint timeout" - ) - - private val topicTweetEndpointTimeout: Flag[Duration] = - flag[Duration]( - "topicTweet.endpoint.timeout", - "topicTweet endpoint timeout" - ) - - // timeout for Navi client - private val naviRequestTimeout: Flag[Duration] = - flag[Duration]( - NaviRequestTimeoutFlagName, - Duration.fromMilliseconds(2000), - "Request timeout for a single RPC Call", - ) - - @Provides - @Singleton - def provideTimeoutBudget(): TimeoutConfig = - TimeoutConfig( - serviceTimeout = serviceTimeout(), - signalFetchTimeout = signalFetchTimeout(), - similarityEngineTimeout = similarityEngineTimeout(), - annServiceClientTimeout = annServiceClientTimeout(), - utegSimilarityEngineTimeout = utegSimilarityEngineTimeout(), - userStateUnderlyingStoreTimeout = userStateUnderlyingStoreTimeout(), - userStateStoreTimeout = userStateStoreTimeout(), - earlybirdServerTimeout = earlybirdServerTimeout(), - earlybirdSimilarityEngineTimeout = earlybirdSimilarityEngineTimeout(), - frsBasedTweetEndpointTimeout = frsBasedTweetEndpointTimeout(), - topicTweetEndpointTimeout = topicTweetEndpointTimeout(), - naviRequestTimeout = naviRequestTimeout() - ) - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/grpc_client/NaviGRPCClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/grpc_client/NaviGRPCClientModule.docx new file mode 100644 index 000000000..89d56e80c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/grpc_client/NaviGRPCClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/grpc_client/NaviGRPCClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/grpc_client/NaviGRPCClientModule.scala deleted file mode 100644 index 418f44747..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/grpc_client/NaviGRPCClientModule.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.cr_mixer.module.grpc_client - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.finagle.Http -import com.twitter.finagle.grpc.FinagleChannelBuilder -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsStackClientSyntax -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.util.Duration -import io.grpc.ManagedChannel -import javax.inject.Named -import javax.inject.Singleton - -object NaviGRPCClientModule extends TwitterModule { - - val maxRetryAttempts = 3 - - @Provides - @Singleton - @Named(ModuleNames.HomeNaviGRPCClient) - def providesHomeNaviGRPCClient( - serviceIdentifier: ServiceIdentifier, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): ManagedChannel = { - val label = "navi-wals-recommended-tweets-home-client" - val dest = "/s/ads-prediction/navi-wals-recommended-tweets-home" - buildClient(serviceIdentifier, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.AdsFavedNaviGRPCClient) - def providesAdsFavedNaviGRPCClient( - serviceIdentifier: ServiceIdentifier, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): ManagedChannel = { - val label = "navi-wals-ads-faved-tweets" - val dest = "/s/ads-prediction/navi-wals-ads-faved-tweets" - buildClient(serviceIdentifier, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.AdsMonetizableNaviGRPCClient) - def providesAdsMonetizableNaviGRPCClient( - serviceIdentifier: ServiceIdentifier, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): ManagedChannel = { - val label = "navi-wals-ads-monetizable-tweets" - val dest = "/s/ads-prediction/navi-wals-ads-monetizable-tweets" - buildClient(serviceIdentifier, timeoutConfig, statsReceiver, dest, label) - } - - private def buildClient( - serviceIdentifier: ServiceIdentifier, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - dest: String, - label: String - ): ManagedChannel = { - - val stats = statsReceiver.scope("clnt").scope(label) - - val client = Http.client - .withLabel(label) - .withMutualTls(serviceIdentifier) - .withRequestTimeout(timeoutConfig.naviRequestTimeout) - .withTransport.connectTimeout(Duration.fromMilliseconds(10000)) - .withSession.acquisitionTimeout(Duration.fromMilliseconds(20000)) - .withStatsReceiver(stats) - .withHttpStats - - FinagleChannelBuilder - .forTarget(dest) - .overrideAuthority("rustserving") - .maxRetryAttempts(maxRetryAttempts) - .enableRetryForStatus(io.grpc.Status.RESOURCE_EXHAUSTED) - .enableRetryForStatus(io.grpc.Status.UNKNOWN) - .enableUnsafeFullyBufferingMode() - .httpClient(client) - .build() - - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/CertoTopicTweetSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/CertoTopicTweetSimilarityEngineModule.docx new file mode 100644 index 000000000..4ff991dd3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/CertoTopicTweetSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/CertoTopicTweetSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/CertoTopicTweetSimilarityEngineModule.scala deleted file mode 100644 index 6c82329b0..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/CertoTopicTweetSimilarityEngineModule.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TopicTweetWithScore -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.CertoTopicTweetSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.CertoTopicTweetSimilarityEngine.Query -import com.twitter.cr_mixer.similarity_engine.EngineQuery -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.storehaus.ReadableStore -import com.twitter.topic_recos.thriftscala.TweetWithScores -import javax.inject.Named -import javax.inject.Singleton - -object CertoTopicTweetSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.CertoTopicTweetSimilarityEngine) - def providesCertoTopicTweetSimilarityEngine( - @Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[ - TopicId, - Seq[TweetWithScores] - ], - timeoutConfig: TimeoutConfig, - decider: CrMixerDecider, - statsReceiver: StatsReceiver - ): StandardSimilarityEngine[ - EngineQuery[Query], - TopicTweetWithScore - ] = { - new StandardSimilarityEngine[EngineQuery[Query], TopicTweetWithScore]( - implementingStore = CertoTopicTweetSimilarityEngine(certoStratoStore, statsReceiver), - identifier = SimilarityEngineType.CertoTopicTweet, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.topicTweetEndpointTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableTopicTweetTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ) - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerBasedWalsSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerBasedWalsSimilarityEngineModule.docx new file mode 100644 index 000000000..1bcc3785d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerBasedWalsSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerBasedWalsSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerBasedWalsSimilarityEngineModule.scala deleted file mode 100644 index e09f8b639..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerBasedWalsSimilarityEngineModule.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.similarity_engine.ConsumerBasedWalsSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import io.grpc.ManagedChannel -import javax.inject.Named - -object ConsumerBasedWalsSimilarityEngineModule extends TwitterModule { - @Provides - @Named(ModuleNames.ConsumerBasedWalsSimilarityEngine) - def providesConsumerBasedWalsSimilarityEngine( - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - @Named(ModuleNames.HomeNaviGRPCClient) homeNaviGRPCClient: ManagedChannel, - @Named(ModuleNames.AdsFavedNaviGRPCClient) adsFavedNaviGRPCClient: ManagedChannel, - @Named(ModuleNames.AdsMonetizableNaviGRPCClient) adsMonetizableNaviGRPCClient: ManagedChannel, - ): StandardSimilarityEngine[ - ConsumerBasedWalsSimilarityEngine.Query, - TweetWithScore - ] = { - - val underlyingStore = new ConsumerBasedWalsSimilarityEngine( - homeNaviGRPCClient, - adsFavedNaviGRPCClient, - adsMonetizableNaviGRPCClient, - statsReceiver - ) - - new StandardSimilarityEngine[ - ConsumerBasedWalsSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = underlyingStore, - identifier = SimilarityEngineType.ConsumerBasedWalsANN, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngineModule.docx new file mode 100644 index 000000000..534c82bff Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngineModule.scala deleted file mode 100644 index 8d209798b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngineModule.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TripTweetWithScore -import com.twitter.cr_mixer.similarity_engine.ConsumerEmbeddingBasedTripSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TripEngineQuery -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import javax.inject.Named - -object ConsumerEmbeddingBasedTripSimilarityEngineModule extends TwitterModule { - @Provides - @Named(ModuleNames.ConsumerEmbeddingBasedTripSimilarityEngine) - def providesConsumerEmbeddingBasedTripSimilarityEngineModule( - @Named(ModuleNames.RmsUserLogFavInterestedInEmbeddingStore) - userLogFavInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding], - @Named(ModuleNames.RmsUserFollowInterestedInEmbeddingStore) - userFollowInterestedInEmbeddingStore: ReadableStore[UserId, SimClustersEmbedding], - @Named(ModuleNames.TripCandidateStore) - tripCandidateStore: ReadableStore[TripDomain, Seq[TripTweet]], - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): StandardSimilarityEngine[TripEngineQuery, TripTweetWithScore] = { - val underlyingStore = ObservedReadableStore( - ConsumerEmbeddingBasedTripSimilarityEngine( - embeddingStoreLookUpMap = Map( - ModelConfig.ConsumerLogFavBasedInterestedInEmbedding -> userLogFavInterestedInEmbeddingStore, - ModelConfig.ConsumerFollowBasedInterestedInEmbedding -> userFollowInterestedInEmbeddingStore, - ), - tripCandidateSource = tripCandidateStore, - statsReceiver - ))(statsReceiver.scope("TripSimilarityEngine")) - - new StandardSimilarityEngine[TripEngineQuery, TripTweetWithScore]( - implementingStore = underlyingStore, - identifier = SimilarityEngineType.ExploreTripOfflineSimClustersTweets, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngineModule.docx new file mode 100644 index 000000000..b039903e0 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngineModule.scala deleted file mode 100644 index 289d052b4..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngineModule.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.ann.common.thriftscala.AnnQueryService -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.cr_mixer.module.EmbeddingStoreModule -import com.twitter.cr_mixer.module.thrift_client.AnnQueryServiceClientModule -import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import com.twitter.ml.api.{thriftscala => api} -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType - -object ConsumerEmbeddingBasedTwHINSimilarityEngineModule extends TwitterModule { - @Provides - @Named(ModuleNames.ConsumerEmbeddingBasedTwHINANNSimilarityEngine) - def providesConsumerEmbeddingBasedTwHINANNSimilarityEngine( - // MH stores - @Named(EmbeddingStoreModule.ConsumerBasedTwHINEmbeddingRegularUpdateMhStoreName) - consumerBasedTwHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding], - @Named(EmbeddingStoreModule.DebuggerDemoUserEmbeddingMhStoreName) - debuggerDemoUserEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], - @Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName) - twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint, - @Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName) - debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint, - // Other configs - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver - ): HnswANNSimilarityEngine = { - new HnswANNSimilarityEngine( - embeddingStoreLookUpMap = Map( - ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024 -> consumerBasedTwHINEmbeddingRegularUpdateMhStore, - ModelConfig.DebuggerDemo -> debuggerDemoUserEmbeddingMhStore, - ), - annServiceLookUpMap = Map( - ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024 -> twHINRegularUpdateAnnService, - ModelConfig.DebuggerDemo -> debuggerDemoAnnService, - ), - globalStats = statsReceiver, - identifier = SimilarityEngineType.ConsumerEmbeddingBasedTwHINANN, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.docx new file mode 100644 index 000000000..538029a86 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.scala deleted file mode 100644 index 704093e36..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.cr_mixer.module -package similarity_engine - -import com.google.inject.Provides -import com.twitter.ann.common.thriftscala.AnnQueryService -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.cr_mixer.module.EmbeddingStoreModule -import com.twitter.cr_mixer.module.thrift_client.AnnQueryServiceClientModule -import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import com.twitter.ml.api.{thriftscala => api} -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType - -object ConsumerEmbeddingBasedTwoTowerSimilarityEngineModule extends TwitterModule { - @Provides - @Named(ModuleNames.ConsumerEmbeddingBasedTwoTowerANNSimilarityEngine) - def providesConsumerEmbeddingBasedTwoTowerANNSimilarityEngine( - @Named(EmbeddingStoreModule.TwoTowerFavConsumerEmbeddingMhStoreName) - twoTowerFavConsumerEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], - @Named(AnnQueryServiceClientModule.TwoTowerFavAnnServiceClientName) - twoTowerFavAnnService: AnnQueryService.MethodPerEndpoint, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver - ): HnswANNSimilarityEngine = { - new HnswANNSimilarityEngine( - embeddingStoreLookUpMap = Map( - ModelConfig.TwoTowerFavALL20220808 -> twoTowerFavConsumerEmbeddingMhStore, - ), - annServiceLookUpMap = Map( - ModelConfig.TwoTowerFavALL20220808 -> twoTowerFavAnnService, - ), - globalStats = statsReceiver, - identifier = SimilarityEngineType.ConsumerEmbeddingBasedTwoTowerANN, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..9e95ca3b7 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngineModule.scala deleted file mode 100644 index e66a48a87..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.ConsumersBasedUserAdGraphSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest -import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import javax.inject.Singleton - -object ConsumersBasedUserAdGraphSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ConsumersBasedUserAdGraphSimilarityEngine) - def providesConsumersBasedUserAdGraphSimilarityEngine( - @Named(ModuleNames.ConsumerBasedUserAdGraphStore) - consumersBasedUserAdGraphStore: ReadableStore[ - ConsumersBasedRelatedAdRequest, - RelatedAdResponse - ], - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - ConsumersBasedUserAdGraphSimilarityEngine.Query, - TweetWithScore - ] = { - - new StandardSimilarityEngine[ - ConsumersBasedUserAdGraphSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = - ConsumersBasedUserAdGraphSimilarityEngine(consumersBasedUserAdGraphStore, statsReceiver), - identifier = SimilarityEngineType.ConsumersBasedUserTweetGraph, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableUserTweetGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ), - memCacheConfig = None - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..8f1d8fe63 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngineModule.scala deleted file mode 100644 index 977a90f25..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.ConsumersBasedUserVideoGraphSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest -import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import javax.inject.Singleton - -object ConsumersBasedUserVideoGraphSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ConsumersBasedUserVideoGraphSimilarityEngine) - def providesConsumersBasedUserVideoGraphSimilarityEngine( - @Named(ModuleNames.ConsumerBasedUserVideoGraphStore) - consumersBasedUserVideoGraphStore: ReadableStore[ - ConsumersBasedRelatedTweetRequest, - RelatedTweetResponse - ], - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - ConsumersBasedUserVideoGraphSimilarityEngine.Query, - TweetWithScore - ] = { - - new StandardSimilarityEngine[ - ConsumersBasedUserVideoGraphSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = ConsumersBasedUserVideoGraphSimilarityEngine( - consumersBasedUserVideoGraphStore, - statsReceiver), - identifier = SimilarityEngineType.ConsumersBasedUserVideoGraph, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableUserVideoGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ), - memCacheConfig = None - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/DiffusionBasedSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/DiffusionBasedSimilarityEngineModule.docx new file mode 100644 index 000000000..755cc2c90 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/DiffusionBasedSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/DiffusionBasedSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/DiffusionBasedSimilarityEngineModule.scala deleted file mode 100644 index f48521085..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/DiffusionBasedSimilarityEngineModule.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.cr_mixer.module -package similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.simclusters_v2.thriftscala.TweetsWithScore -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.similarity_engine.DiffusionBasedSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.DiffusionBasedSimilarityEngine.Query -import com.twitter.cr_mixer.similarity_engine.LookupSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import javax.inject.Singleton - -object DiffusionBasedSimilarityEngineModule extends TwitterModule { - @Provides - @Singleton - @Named(ModuleNames.DiffusionBasedSimilarityEngine) - def providesDiffusionBasedSimilarityEngineModule( - @Named(ModuleNames.RetweetBasedDiffusionRecsMhStore) - retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore], - timeoutConfig: TimeoutConfig, - globalStats: StatsReceiver - ): LookupSimilarityEngine[Query, TweetWithScore] = { - - val versionedStoreMap = Map( - ModelConfig.RetweetBasedDiffusion -> DiffusionBasedSimilarityEngine( - retweetBasedDiffusionRecsMhStore, - globalStats), - ) - - new LookupSimilarityEngine[Query, TweetWithScore]( - versionedStoreMap = versionedStoreMap, - identifier = SimilarityEngineType.DiffusionBasedTweet, - globalStats = globalStats, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/EarlybirdSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/EarlybirdSimilarityEngineModule.docx new file mode 100644 index 000000000..31ce4e747 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/EarlybirdSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/EarlybirdSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/EarlybirdSimilarityEngineModule.scala deleted file mode 100644 index 6cdabfce4..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/EarlybirdSimilarityEngineModule.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.EarlybirdModelBasedSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.EarlybirdRecencyBasedSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.EarlybirdSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.EarlybirdTensorflowBasedSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import javax.inject.Singleton - -object EarlybirdSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - def providesRecencyBasedEarlybirdSimilarityEngine( - earlybirdRecencyBasedSimilarityEngine: EarlybirdRecencyBasedSimilarityEngine, - timeoutConfig: TimeoutConfig, - decider: CrMixerDecider, - statsReceiver: StatsReceiver - ): EarlybirdSimilarityEngine[ - EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, - EarlybirdRecencyBasedSimilarityEngine - ] = { - new EarlybirdSimilarityEngine[ - EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, - EarlybirdRecencyBasedSimilarityEngine - ]( - implementingStore = earlybirdRecencyBasedSimilarityEngine, - identifier = SimilarityEngineType.EarlybirdRecencyBasedSimilarityEngine, - globalStats = - statsReceiver.scope(SimilarityEngineType.EarlybirdRecencyBasedSimilarityEngine.name), - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.earlybirdSimilarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = Some( - DeciderConfig( - decider = decider, - deciderString = DeciderConstants.enableEarlybirdTrafficDeciderKey - )), - enableFeatureSwitch = None - ) - ) - ) - } - - @Provides - @Singleton - def providesModelBasedEarlybirdSimilarityEngine( - earlybirdModelBasedSimilarityEngine: EarlybirdModelBasedSimilarityEngine, - timeoutConfig: TimeoutConfig, - decider: CrMixerDecider, - statsReceiver: StatsReceiver - ): EarlybirdSimilarityEngine[ - EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, - EarlybirdModelBasedSimilarityEngine - ] = { - new EarlybirdSimilarityEngine[ - EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, - EarlybirdModelBasedSimilarityEngine - ]( - implementingStore = earlybirdModelBasedSimilarityEngine, - identifier = SimilarityEngineType.EarlybirdModelBasedSimilarityEngine, - globalStats = - statsReceiver.scope(SimilarityEngineType.EarlybirdModelBasedSimilarityEngine.name), - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.earlybirdSimilarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = Some( - DeciderConfig( - decider = decider, - deciderString = DeciderConstants.enableEarlybirdTrafficDeciderKey - )), - enableFeatureSwitch = None - ) - ) - ) - } - - @Provides - @Singleton - def providesTensorflowBasedEarlybirdSimilarityEngine( - earlybirdTensorflowBasedSimilarityEngine: EarlybirdTensorflowBasedSimilarityEngine, - timeoutConfig: TimeoutConfig, - decider: CrMixerDecider, - statsReceiver: StatsReceiver - ): EarlybirdSimilarityEngine[ - EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, - EarlybirdTensorflowBasedSimilarityEngine - ] = { - new EarlybirdSimilarityEngine[ - EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, - EarlybirdTensorflowBasedSimilarityEngine - ]( - implementingStore = earlybirdTensorflowBasedSimilarityEngine, - identifier = SimilarityEngineType.EarlybirdTensorflowBasedSimilarityEngine, - globalStats = - statsReceiver.scope(SimilarityEngineType.EarlybirdTensorflowBasedSimilarityEngine.name), - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.earlybirdSimilarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = Some( - DeciderConfig( - decider = decider, - deciderString = DeciderConstants.enableEarlybirdTrafficDeciderKey - )), - enableFeatureSwitch = None - ) - ) - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUnifiedSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUnifiedSimilarityEngineModule.docx new file mode 100644 index 000000000..d6a35c69c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUnifiedSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUnifiedSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUnifiedSimilarityEngineModule.scala deleted file mode 100644 index b16d59924..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUnifiedSimilarityEngineModule.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.ProducerBasedUserTweetGraphSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.ProducerBasedUnifiedSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import javax.inject.Singleton - -object ProducerBasedUnifiedSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ProducerBasedUnifiedSimilarityEngine) - def providesProducerBasedUnifiedSimilarityEngine( - @Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) - producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ - ProducerBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ], - @Named(ModuleNames.SimClustersANNSimilarityEngine) - simClustersANNSimilarityEngine: StandardSimilarityEngine[ - SimClustersANNSimilarityEngine.Query, - TweetWithScore - ], - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): StandardSimilarityEngine[ - ProducerBasedUnifiedSimilarityEngine.Query, - TweetWithCandidateGenerationInfo - ] = { - - val underlyingStore: ReadableStore[ProducerBasedUnifiedSimilarityEngine.Query, Seq[ - TweetWithCandidateGenerationInfo - ]] = ProducerBasedUnifiedSimilarityEngine( - producerBasedUserTweetGraphSimilarityEngine, - simClustersANNSimilarityEngine, - statsReceiver - ) - - new StandardSimilarityEngine[ - ProducerBasedUnifiedSimilarityEngine.Query, - TweetWithCandidateGenerationInfo - ]( - implementingStore = underlyingStore, - identifier = SimilarityEngineType.ProducerBasedUnifiedSimilarityEngine, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserAdGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserAdGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..d18cd17a0 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserAdGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserAdGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserAdGraphSimilarityEngineModule.scala deleted file mode 100644 index d221a58a9..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserAdGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.ProducerBasedUserAdGraphSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine._ -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph -import javax.inject.Named -import javax.inject.Singleton - -object ProducerBasedUserAdGraphSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ProducerBasedUserAdGraphSimilarityEngine) - def providesProducerBasedUserAdGraphSimilarityEngine( - userAdGraphService: UserAdGraph.MethodPerEndpoint, - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - ProducerBasedUserAdGraphSimilarityEngine.Query, - TweetWithScore - ] = { - new StandardSimilarityEngine[ - ProducerBasedUserAdGraphSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = - ProducerBasedUserAdGraphSimilarityEngine(userAdGraphService, statsReceiver), - identifier = SimilarityEngineType.ProducerBasedUserAdGraph, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableUserAdGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ), - memCacheConfig = Some( - MemCacheConfig( - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes, - keyToString = { k => - //Example Query CRMixer:ProducerBasedUTG:1234567890ABCDEF - f"ProducerBasedUTG:${keyHasher.hashKey(k.toString.getBytes)}%X" - } - )) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..355807735 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngineModule.scala deleted file mode 100644 index a5821d01c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.ProducerBasedUserTweetGraphSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine._ -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph -import javax.inject.Named -import javax.inject.Singleton - -object ProducerBasedUserTweetGraphSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) - def providesProducerBasedUserTweetGraphSimilarityEngine( - userTweetGraphService: UserTweetGraph.MethodPerEndpoint, - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - ProducerBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ] = { - new StandardSimilarityEngine[ - ProducerBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = - ProducerBasedUserTweetGraphSimilarityEngine(userTweetGraphService, statsReceiver), - identifier = SimilarityEngineType.ProducerBasedUserTweetGraph, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableUserTweetGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ), - memCacheConfig = Some( - MemCacheConfig( - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes, - keyToString = { k => - //Example Query CRMixer:ProducerBasedUTG:1234567890ABCDEF - f"ProducerBasedUTG:${keyHasher.hashKey(k.toString.getBytes)}%X" - } - )) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SimClustersANNSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SimClustersANNSimilarityEngineModule.docx new file mode 100644 index 000000000..47b447692 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SimClustersANNSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SimClustersANNSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SimClustersANNSimilarityEngineModule.scala deleted file mode 100644 index 7af68327d..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SimClustersANNSimilarityEngineModule.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine.Query -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.simclusters_v2.candidate_source.SimClustersANNCandidateSource.CacheableShortTTLEmbeddingTypes -import com.twitter.simclustersann.thriftscala.SimClustersANNService -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Named -import javax.inject.Singleton - -object SimClustersANNSimilarityEngineModule extends TwitterModule { - - private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - - @Provides - @Singleton - @Named(ModuleNames.SimClustersANNSimilarityEngine) - def providesProdSimClustersANNSimilarityEngine( - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver - ): StandardSimilarityEngine[Query, TweetWithScore] = { - - val underlyingStore = - SimClustersANNSimilarityEngine(simClustersANNServiceNameToClientMapper, statsReceiver) - - val observedReadableStore = - ObservedReadableStore(underlyingStore)(statsReceiver.scope("SimClustersANNServiceStore")) - - val memCachedStore: ReadableStore[Query, Seq[TweetWithScore]] = - ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = observedReadableStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), - statsReceiver = statsReceiver.scope("simclusters_ann_store_memcache"), - keyToString = { k => - //Example Query CRMixer:SCANN:1:2:1234567890ABCDEF:1234567890ABCDEF - f"CRMixer:SCANN:${k.simClustersANNQuery.sourceEmbeddingId.embeddingType.getValue()}%X" + - f":${k.simClustersANNQuery.sourceEmbeddingId.modelVersion.getValue()}%X" + - f":${keyHasher.hashKey(k.simClustersANNQuery.sourceEmbeddingId.internalId.toString.getBytes)}%X" + - f":${keyHasher.hashKey(k.simClustersANNQuery.config.toString.getBytes)}%X" - } - ) - - // Only cache the candidates if it's not Consumer-source. For example, TweetSource, - // ProducerSource, TopicSource - val wrapperStats = statsReceiver.scope("SimClustersANNWrapperStore") - - val wrapperStore: ReadableStore[Query, Seq[TweetWithScore]] = - buildWrapperStore(memCachedStore, observedReadableStore, wrapperStats) - - new StandardSimilarityEngine[ - Query, - TweetWithScore - ]( - implementingStore = wrapperStore, - identifier = SimilarityEngineType.SimClustersANN, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } - - def buildWrapperStore( - memCachedStore: ReadableStore[Query, Seq[TweetWithScore]], - underlyingStore: ReadableStore[Query, Seq[TweetWithScore]], - wrapperStats: StatsReceiver - ): ReadableStore[Query, Seq[TweetWithScore]] = { - - // Only cache the candidates if it's not Consumer-source. For example, TweetSource, - // ProducerSource, TopicSource - val wrapperStore: ReadableStore[Query, Seq[TweetWithScore]] = - new ReadableStore[Query, Seq[TweetWithScore]] { - - override def multiGet[K1 <: Query]( - queries: Set[K1] - ): Map[K1, Future[Option[Seq[TweetWithScore]]]] = { - val (cacheableQueries, nonCacheableQueries) = - queries.partition { query => - CacheableShortTTLEmbeddingTypes.contains( - query.simClustersANNQuery.sourceEmbeddingId.embeddingType) - } - memCachedStore.multiGet(cacheableQueries) ++ - underlyingStore.multiGet(nonCacheableQueries) - } - } - wrapperStore - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SkitTopicTweetSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SkitTopicTweetSimilarityEngineModule.docx new file mode 100644 index 000000000..96fb614e4 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SkitTopicTweetSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SkitTopicTweetSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SkitTopicTweetSimilarityEngineModule.scala deleted file mode 100644 index 4de20fcfe..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/SkitTopicTweetSimilarityEngineModule.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TopicTweetWithScore -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.EngineQuery -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.SkitHighPrecisionTopicTweetSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine.Query -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import com.twitter.topic_recos.thriftscala.TopicTweet -import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey -import javax.inject.Named -import javax.inject.Singleton - -object SkitTopicTweetSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.SkitHighPrecisionTopicTweetSimilarityEngine) - def providesSkitHighPrecisionTopicTweetSimilarityEngine( - @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ - TopicTweetPartitionFlatKey, - Seq[TopicTweet] - ], - timeoutConfig: TimeoutConfig, - decider: CrMixerDecider, - statsReceiver: StatsReceiver - ): StandardSimilarityEngine[ - EngineQuery[Query], - TopicTweetWithScore - ] = { - new StandardSimilarityEngine[EngineQuery[Query], TopicTweetWithScore]( - implementingStore = - SkitHighPrecisionTopicTweetSimilarityEngine(skitStratoStore, statsReceiver), - identifier = SimilarityEngineType.SkitHighPrecisionTopicTweet, - globalStats = statsReceiver.scope(SimilarityEngineType.SkitHighPrecisionTopicTweet.name), - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.topicTweetEndpointTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableTopicTweetTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ) - ) - } - @Provides - @Singleton - @Named(ModuleNames.SkitTopicTweetSimilarityEngine) - def providesSkitTfgTopicTweetSimilarityEngine( - @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ - TopicTweetPartitionFlatKey, - Seq[TopicTweet] - ], - timeoutConfig: TimeoutConfig, - decider: CrMixerDecider, - statsReceiver: StatsReceiver - ): StandardSimilarityEngine[ - EngineQuery[Query], - TopicTweetWithScore - ] = { - new StandardSimilarityEngine[EngineQuery[Query], TopicTweetWithScore]( - implementingStore = SkitTopicTweetSimilarityEngine(skitStratoStore, statsReceiver), - identifier = SimilarityEngineType.SkitTfgTopicTweet, - globalStats = statsReceiver.scope(SimilarityEngineType.SkitTfgTopicTweet.name), - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.topicTweetEndpointTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableTopicTweetTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ) - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedQigSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedQigSimilarityEngineModule.docx new file mode 100644 index 000000000..0b7ebfea6 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedQigSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedQigSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedQigSimilarityEngineModule.scala deleted file mode 100644 index 06d9a2186..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedQigSimilarityEngineModule.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine._ -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.keyHasher -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedQigSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.qig_ranker.thriftscala.QigRanker -import javax.inject.Named -import javax.inject.Singleton - -object TweetBasedQigSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.TweetBasedQigSimilarityEngine) - def providesTweetBasedQigSimilarTweetsCandidateSource( - qigRanker: QigRanker.MethodPerEndpoint, - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - TweetBasedQigSimilarityEngine.Query, - TweetWithScore - ] = { - new StandardSimilarityEngine[ - TweetBasedQigSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = TweetBasedQigSimilarityEngine(qigRanker, statsReceiver), - identifier = SimilarityEngineType.Qig, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableQigSimilarTweetsTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ), - memCacheConfig = Some( - MemCacheConfig( - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes, - keyToString = { k => - f"TweetBasedQIGRanker:${keyHasher.hashKey(k.sourceId.toString.getBytes)}%X" - } - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedTwHINSimlarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedTwHINSimlarityEngineModule.docx new file mode 100644 index 000000000..d1aae1c81 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedTwHINSimlarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedTwHINSimlarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedTwHINSimlarityEngineModule.scala deleted file mode 100644 index cc9da4772..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedTwHINSimlarityEngineModule.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine -import com.google.inject.Provides -import com.twitter.ann.common.thriftscala.AnnQueryService -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.cr_mixer.module.EmbeddingStoreModule -import com.twitter.cr_mixer.module.thrift_client.AnnQueryServiceClientModule -import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import com.twitter.ml.api.{thriftscala => api} -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.HnswANNEngineQuery -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} - -object TweetBasedTwHINSimlarityEngineModule extends TwitterModule { - @Provides - @Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) - def providesTweetBasedTwHINANNSimilarityEngine( - // MH stores - @Named(EmbeddingStoreModule.TwHINEmbeddingRegularUpdateMhStoreName) - twHINEmbeddingRegularUpdateMhStore: ReadableStore[InternalId, api.Embedding], - @Named(EmbeddingStoreModule.DebuggerDemoTweetEmbeddingMhStoreName) - debuggerDemoTweetEmbeddingMhStore: ReadableStore[InternalId, api.Embedding], - // ANN clients - @Named(AnnQueryServiceClientModule.TwHINRegularUpdateAnnServiceClientName) - twHINRegularUpdateAnnService: AnnQueryService.MethodPerEndpoint, - @Named(AnnQueryServiceClientModule.DebuggerDemoAnnServiceClientName) - debuggerDemoAnnService: AnnQueryService.MethodPerEndpoint, - // Other configs - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver - ): HnswANNSimilarityEngine = { - new HnswANNSimilarityEngine( - embeddingStoreLookUpMap = Map( - ModelConfig.TweetBasedTwHINRegularUpdateAll20221024 -> twHINEmbeddingRegularUpdateMhStore, - ModelConfig.DebuggerDemo -> debuggerDemoTweetEmbeddingMhStore, - ), - annServiceLookUpMap = Map( - ModelConfig.TweetBasedTwHINRegularUpdateAll20221024 -> twHINRegularUpdateAnnService, - ModelConfig.DebuggerDemo -> debuggerDemoAnnService, - ), - globalStats = statsReceiver, - identifier = SimilarityEngineType.TweetBasedTwHINANN, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ), - memCacheConfigOpt = Some( - SimilarityEngine.MemCacheConfig[HnswANNEngineQuery]( - cacheClient = crMixerUnifiedCacheClient, - ttl = 30.minutes, - keyToString = (query: HnswANNEngineQuery) => - SimilarityEngine.keyHasher.hashKey(query.cacheKey.getBytes).toString - )) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUnifiedSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUnifiedSimilarityEngineModule.docx new file mode 100644 index 000000000..370c997b7 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUnifiedSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUnifiedSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUnifiedSimilarityEngineModule.scala deleted file mode 100644 index aa54bf071..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUnifiedSimilarityEngineModule.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.HnswANNSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimClustersANNSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedQigSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedUnifiedSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedUserTweetGraphSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedUserVideoGraphSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import javax.inject.Singleton - -object TweetBasedUnifiedSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.TweetBasedUnifiedSimilarityEngine) - def providesTweetBasedUnifiedSimilarityEngine( - @Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ - TweetBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ], - @Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ - TweetBasedUserVideoGraphSimilarityEngine.Query, - TweetWithScore - ], - @Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) - tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, - @Named(ModuleNames.TweetBasedQigSimilarityEngine) tweetBasedQigSimilarityEngine: StandardSimilarityEngine[ - TweetBasedQigSimilarityEngine.Query, - TweetWithScore - ], - @Named(ModuleNames.SimClustersANNSimilarityEngine) - simClustersANNSimilarityEngine: StandardSimilarityEngine[ - SimClustersANNSimilarityEngine.Query, - TweetWithScore - ], - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): StandardSimilarityEngine[ - TweetBasedUnifiedSimilarityEngine.Query, - TweetWithCandidateGenerationInfo - ] = { - - val underlyingStore: ReadableStore[TweetBasedUnifiedSimilarityEngine.Query, Seq[ - TweetWithCandidateGenerationInfo - ]] = TweetBasedUnifiedSimilarityEngine( - tweetBasedUserTweetGraphSimilarityEngine, - tweetBasedUserVideoGraphSimilarityEngine, - simClustersANNSimilarityEngine, - tweetBasedQigSimilarityEngine, - tweetBasedTwHINANNSimilarityEngine, - statsReceiver - ) - - new StandardSimilarityEngine[ - TweetBasedUnifiedSimilarityEngine.Query, - TweetWithCandidateGenerationInfo - ]( - implementingStore = underlyingStore, - identifier = SimilarityEngineType.TweetBasedUnifiedSimilarityEngine, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserAdGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserAdGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..efa83197d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserAdGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserAdGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserAdGraphSimilarityEngineModule.scala deleted file mode 100644 index 7288e603f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserAdGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedUserAdGraphSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.storehaus.ReadableStore -import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers -import javax.inject.Named -import javax.inject.Singleton - -object TweetBasedUserAdGraphSimilarityEngineModule extends TwitterModule { - - private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - - @Provides - @Singleton - @Named(ModuleNames.TweetBasedUserAdGraphSimilarityEngine) - def providesTweetBasedUserAdGraphSimilarityEngine( - userAdGraphService: UserAdGraph.MethodPerEndpoint, - tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - TweetBasedUserAdGraphSimilarityEngine.Query, - TweetWithScore - ] = { - - val underlyingStore = TweetBasedUserAdGraphSimilarityEngine( - userAdGraphService, - tweetRecentEngagedUserStore, - statsReceiver) - - val memCachedStore: ReadableStore[ - TweetBasedUserAdGraphSimilarityEngine.Query, - Seq[ - TweetWithScore - ] - ] = - ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), - statsReceiver = statsReceiver.scope("tweet_based_user_ad_graph_store_memcache"), - keyToString = { k => - //Example Query CRMixer:TweetBasedUTG:1234567890ABCDEF - f"CRMixer:TweetBasedUAG:${keyHasher.hashKey(k.toString.getBytes)}%X" - } - ) - - new StandardSimilarityEngine[ - TweetBasedUserAdGraphSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = memCachedStore, - identifier = SimilarityEngineType.TweetBasedUserAdGraph, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableUserAdGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserTweetGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserTweetGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..51d010a0d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserTweetGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserTweetGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserTweetGraphSimilarityEngineModule.scala deleted file mode 100644 index a7a388199..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserTweetGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.cr_mixer.module -package similarity_engine - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedUserTweetGraphSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.storehaus.ReadableStore -import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers -import javax.inject.Named -import javax.inject.Singleton - -object TweetBasedUserTweetGraphSimilarityEngineModule extends TwitterModule { - - private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - - @Provides - @Singleton - @Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) - def providesTweetBasedUserTweetGraphSimilarityEngine( - userTweetGraphService: UserTweetGraph.MethodPerEndpoint, - tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - TweetBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ] = { - - val underlyingStore = TweetBasedUserTweetGraphSimilarityEngine( - userTweetGraphService, - tweetRecentEngagedUserStore, - statsReceiver) - - val memCachedStore: ReadableStore[ - TweetBasedUserTweetGraphSimilarityEngine.Query, - Seq[ - TweetWithScore - ] - ] = - ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), - statsReceiver = statsReceiver.scope("tweet_based_user_tweet_graph_store_memcache"), - keyToString = { k => - //Example Query CRMixer:TweetBasedUTG:1234567890ABCDEF - f"CRMixer:TweetBasedUTG:${keyHasher.hashKey(k.toString.getBytes)}%X" - } - ) - - new StandardSimilarityEngine[ - TweetBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = memCachedStore, - identifier = SimilarityEngineType.TweetBasedUserTweetGraph, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableUserTweetGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserVideoGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserVideoGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..e4f929be9 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserVideoGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserVideoGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserVideoGraphSimilarityEngineModule.scala deleted file mode 100644 index efc354d21..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TweetBasedUserVideoGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TweetBasedUserVideoGraphSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.storehaus.ReadableStore -import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers -import javax.inject.Named -import javax.inject.Singleton - -object TweetBasedUserVideoGraphSimilarityEngineModule extends TwitterModule { - - private val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - - @Provides - @Singleton - @Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) - def providesTweetBasedUserVideoGraphSimilarityEngine( - userVideoGraphService: UserVideoGraph.MethodPerEndpoint, - tweetRecentEngagedUserStore: ReadableStore[TweetId, TweetRecentEngagedUsers], - @Named(ModuleNames.UnifiedCache) crMixerUnifiedCacheClient: MemcachedClient, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - TweetBasedUserVideoGraphSimilarityEngine.Query, - TweetWithScore - ] = { - - val underlyingStore = - TweetBasedUserVideoGraphSimilarityEngine( - userVideoGraphService, - tweetRecentEngagedUserStore, - statsReceiver) - - val memCachedStore: ReadableStore[ - TweetBasedUserVideoGraphSimilarityEngine.Query, - Seq[ - TweetWithScore - ] - ] = - ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = crMixerUnifiedCacheClient, - ttl = 10.minutes - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[TweetWithScore]()), - statsReceiver = statsReceiver.scope("tweet_based_user_video_graph_store_memcache"), - keyToString = { k => - //Example Query CRMixer:TweetBasedUVG:1234567890ABCDEF - f"CRMixer:TweetBasedUVG:${keyHasher.hashKey(k.toString.getBytes)}%X" - } - ) - - new StandardSimilarityEngine[ - TweetBasedUserVideoGraphSimilarityEngine.Query, - TweetWithScore - ]( - implementingStore = memCachedStore, - identifier = SimilarityEngineType.TweetBasedUserVideoGraph, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = - Some(DeciderConfig(decider, DeciderConstants.enableUserVideoGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TwhinCollabFilterLookupSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TwhinCollabFilterLookupSimilarityEngineModule.docx new file mode 100644 index 000000000..214728fe3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TwhinCollabFilterLookupSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TwhinCollabFilterLookupSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TwhinCollabFilterLookupSimilarityEngineModule.scala deleted file mode 100644 index 4f7c909e3..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/TwhinCollabFilterLookupSimilarityEngineModule.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.cr_mixer.module -package similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.similarity_engine.LookupSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.TwhinCollabFilterSimilarityEngine.Query -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.TwhinCollabFilterSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.storehaus.ReadableStore -import javax.inject.Named -import javax.inject.Singleton - -/** - * TwhinCandidatesLookupSimilarityEngineModule routes the request to the corresponding - * twhin based candidate store which follow the same pattern as TwHIN Collaborative Filtering. - */ - -object TwhinCollabFilterLookupSimilarityEngineModule extends TwitterModule { - @Provides - @Singleton - @Named(ModuleNames.TwhinCollabFilterSimilarityEngine) - def providesTwhinCollabFilterLookupSimilarityEngineModule( - @Named(ModuleNames.TwhinCollabFilterStratoStoreForFollow) - twhinCollabFilterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]], - @Named(ModuleNames.TwhinCollabFilterStratoStoreForEngagement) - twhinCollabFilterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]], - @Named(ModuleNames.TwhinMultiClusterStratoStoreForFollow) - twhinMultiClusterStratoStoreForFollow: ReadableStore[Long, Seq[TweetId]], - @Named(ModuleNames.TwhinMultiClusterStratoStoreForEngagement) - twhinMultiClusterStratoStoreForEngagement: ReadableStore[Long, Seq[TweetId]], - timeoutConfig: TimeoutConfig, - globalStats: StatsReceiver - ): LookupSimilarityEngine[Query, TweetWithScore] = { - val versionedStoreMap = Map( - ModelConfig.TwhinCollabFilterForFollow -> TwhinCollabFilterSimilarityEngine( - twhinCollabFilterStratoStoreForFollow, - globalStats), - ModelConfig.TwhinCollabFilterForEngagement -> TwhinCollabFilterSimilarityEngine( - twhinCollabFilterStratoStoreForEngagement, - globalStats), - ModelConfig.TwhinMultiClusterForFollow -> TwhinCollabFilterSimilarityEngine( - twhinMultiClusterStratoStoreForFollow, - globalStats), - ModelConfig.TwhinMultiClusterForEngagement -> TwhinCollabFilterSimilarityEngine( - twhinMultiClusterStratoStoreForEngagement, - globalStats), - ) - - new LookupSimilarityEngine[Query, TweetWithScore]( - versionedStoreMap = versionedStoreMap, - identifier = SimilarityEngineType.TwhinCollabFilter, - globalStats = globalStats, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.similarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = None, - enableFeatureSwitch = None - ) - ) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/UserTweetEntityGraphSimilarityEngineModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/UserTweetEntityGraphSimilarityEngineModule.docx new file mode 100644 index 000000000..39b87230a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/UserTweetEntityGraphSimilarityEngineModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/UserTweetEntityGraphSimilarityEngineModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/UserTweetEntityGraphSimilarityEngineModule.scala deleted file mode 100644 index cf2093208..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/similarity_engine/UserTweetEntityGraphSimilarityEngineModule.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.cr_mixer.module.similarity_engine - -import com.google.inject.Provides -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.param.decider.DeciderConstants -import com.twitter.cr_mixer.similarity_engine.UserTweetEntityGraphSimilarityEngine -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.DeciderConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.GatingConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.similarity_engine.StandardSimilarityEngine -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph -import javax.inject.Named -import javax.inject.Singleton - -object UserTweetEntityGraphSimilarityEngineModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.UserTweetEntityGraphSimilarityEngine) - def providesUserTweetEntityGraphSimilarityEngine( - userTweetEntityGraphService: UserTweetEntityGraph.MethodPerEndpoint, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - decider: CrMixerDecider - ): StandardSimilarityEngine[ - UserTweetEntityGraphSimilarityEngine.Query, - TweetWithScoreAndSocialProof - ] = { - new StandardSimilarityEngine[ - UserTweetEntityGraphSimilarityEngine.Query, - TweetWithScoreAndSocialProof - ]( - implementingStore = - UserTweetEntityGraphSimilarityEngine(userTweetEntityGraphService, statsReceiver), - identifier = SimilarityEngineType.Uteg, - globalStats = statsReceiver, - engineConfig = SimilarityEngineConfig( - timeout = timeoutConfig.utegSimilarityEngineTimeout, - gatingConfig = GatingConfig( - deciderConfig = Some( - DeciderConfig(decider, DeciderConstants.enableUserTweetEntityGraphTrafficDeciderKey)), - enableFeatureSwitch = None - ) - ), - // We cannot use the key to cache anything in UTEG because the key contains a long list of userIds - memCacheConfig = None - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/AnnQueryServiceClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/AnnQueryServiceClientModule.docx new file mode 100644 index 000000000..d63a26732 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/AnnQueryServiceClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/AnnQueryServiceClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/AnnQueryServiceClientModule.scala deleted file mode 100644 index 17dbfcae5..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/AnnQueryServiceClientModule.scala +++ /dev/null @@ -1,107 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.google.inject.Provides -import com.twitter.ann.common.thriftscala.AnnQueryService -import com.twitter.conversions.DurationOps._ -import com.twitter.conversions.PercentOps._ -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.inject.TwitterModule -import javax.inject.Named -import javax.inject.Singleton - -object AnnQueryServiceClientModule extends TwitterModule { - final val DebuggerDemoAnnServiceClientName = "DebuggerDemoAnnServiceClient" - - @Provides - @Singleton - @Named(DebuggerDemoAnnServiceClientName) - def debuggerDemoAnnServiceClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - statsReceiver: StatsReceiver, - timeoutConfig: TimeoutConfig, - ): AnnQueryService.MethodPerEndpoint = { - // This ANN is built from the embeddings in src/scala/com/twitter/wtf/beam/bq_embedding_export/sql/MlfExperimentalTweetEmbeddingScalaDataset.sql - // Change the above sql if you want to build the index from a diff embedding - val dest = "/s/cassowary/mlf-experimental-ann-service" - val label = "experimental-ann" - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - final val TwHINUuaAnnServiceClientName = "TwHINUuaAnnServiceClient" - @Provides - @Singleton - @Named(TwHINUuaAnnServiceClientName) - def twhinUuaAnnServiceClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - statsReceiver: StatsReceiver, - timeoutConfig: TimeoutConfig, - ): AnnQueryService.MethodPerEndpoint = { - val dest = "/s/cassowary/twhin-uua-ann-service" - val label = "twhin_uua_ann" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - final val TwHINRegularUpdateAnnServiceClientName = "TwHINRegularUpdateAnnServiceClient" - @Provides - @Singleton - @Named(TwHINRegularUpdateAnnServiceClientName) - def twHINRegularUpdateAnnServiceClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - statsReceiver: StatsReceiver, - timeoutConfig: TimeoutConfig, - ): AnnQueryService.MethodPerEndpoint = { - val dest = "/s/cassowary/twhin-regular-update-ann-service" - val label = "twhin_regular_update" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - final val TwoTowerFavAnnServiceClientName = "TwoTowerFavAnnServiceClient" - @Provides - @Singleton - @Named(TwoTowerFavAnnServiceClientName) - def twoTowerFavAnnServiceClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - statsReceiver: StatsReceiver, - timeoutConfig: TimeoutConfig, - ): AnnQueryService.MethodPerEndpoint = { - val dest = "/s/cassowary/tweet-rec-two-tower-fav-ann" - val label = "tweet_rec_two_tower_fav_ann" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - private def buildClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - dest: String, - label: String - ): AnnQueryService.MethodPerEndpoint = { - val thriftClient = ThriftMux.client - .withMutualTls(serviceIdentifier) - .withClientId(clientId) - .withLabel(label) - .withStatsReceiver(statsReceiver) - .withTransport.connectTimeout(500.milliseconds) - .withSession.acquisitionTimeout(500.milliseconds) - .methodBuilder(dest) - .withTimeoutPerRequest(timeoutConfig.annServiceClientTimeout) - .withRetryDisabled - .idempotent(5.percent) - .servicePerEndpoint[AnnQueryService.ServicePerEndpoint] - - ThriftMux.Client.methodPerEndpoint(thriftClient) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/EarlybirdSearchClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/EarlybirdSearchClientModule.docx new file mode 100644 index 000000000..4968a55b6 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/EarlybirdSearchClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/EarlybirdSearchClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/EarlybirdSearchClientModule.scala deleted file mode 100644 index c399a5a37..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/EarlybirdSearchClientModule.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client -import com.twitter.app.Flag -import com.twitter.finagle.ThriftMux -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.inject.Injector -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.EarlybirdClientTimeoutFlagName -import com.twitter.finagle.service.RetryBudget -import com.twitter.util.Duration -import org.apache.thrift.protocol.TCompactProtocol - -object EarlybirdSearchClientModule - extends ThriftMethodBuilderClientModule[ - EarlybirdService.ServicePerEndpoint, - EarlybirdService.MethodPerEndpoint - ] - with MtlsClient { - - override def label: String = "earlybird" - override def dest: String = "/s/earlybird-root-superroot/root-superroot" - private val requestTimeoutFlag: Flag[Duration] = - flag[Duration](EarlybirdClientTimeoutFlagName, "Earlybird client timeout") - override protected def requestTimeout: Duration = requestTimeoutFlag() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = { - super - .configureThriftMuxClient(injector, client) - .withProtocolFactory(new TCompactProtocol.Factory()) - .withSessionQualifier - .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/FrsClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/FrsClientModule.docx new file mode 100644 index 000000000..9206a1b01 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/FrsClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/FrsClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/FrsClientModule.scala deleted file mode 100644 index 1084f2c1a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/FrsClientModule.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.app.Flag -import com.twitter.finagle.ThriftMux -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.FrsClientTimeoutFlagName -import com.twitter.finagle.service.RetryBudget -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.util.Duration - -object FrsClientModule - extends ThriftMethodBuilderClientModule[ - FollowRecommendationsThriftService.ServicePerEndpoint, - FollowRecommendationsThriftService.MethodPerEndpoint - ] - with MtlsClient { - - override def label: String = "follow-recommendations-service" - override def dest: String = "/s/follow-recommendations/follow-recos-service" - - private val frsSignalFetchTimeout: Flag[Duration] = - flag[Duration](FrsClientTimeoutFlagName, "FRS signal fetch client timeout") - override def requestTimeout: Duration = frsSignalFetchTimeout() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = { - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withSessionQualifier - .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraPartitionClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraPartitionClientModule.docx new file mode 100644 index 000000000..ed3d81039 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraPartitionClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraPartitionClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraPartitionClientModule.scala deleted file mode 100644 index c208e111c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraPartitionClientModule.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.hydra.partition.{thriftscala => ht} - -object HydraPartitionClientModule - extends ThriftMethodBuilderClientModule[ - ht.HydraPartition.ServicePerEndpoint, - ht.HydraPartition.MethodPerEndpoint - ] - with MtlsClient { - override def label: String = "hydra-partition" - - override def dest: String = "/s/hydra/hydra-partition" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = methodBuilder.withTimeoutTotal(500.milliseconds) - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraRootClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraRootClientModule.docx new file mode 100644 index 000000000..25924602c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraRootClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraRootClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraRootClientModule.scala deleted file mode 100644 index 28d5b1767..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/HydraRootClientModule.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.thriftmux.MethodBuilder -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.hydra.root.{thriftscala => ht} -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule - -object HydraRootClientModule - extends ThriftMethodBuilderClientModule[ - ht.HydraRoot.ServicePerEndpoint, - ht.HydraRoot.MethodPerEndpoint - ] - with MtlsClient { - override def label: String = "hydra-root" - - override def dest: String = "/s/hydra/hydra-root" - - override protected def configureMethodBuilder( - injector: Injector, - methodBuilder: MethodBuilder - ): MethodBuilder = methodBuilder.withTimeoutTotal(500.milliseconds) - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/QigServiceClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/QigServiceClientModule.docx new file mode 100644 index 000000000..1bae5f66b Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/QigServiceClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/QigServiceClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/QigServiceClientModule.scala deleted file mode 100644 index 86675e349..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/QigServiceClientModule.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.app.Flag -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.QigRankerClientTimeoutFlagName -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.qig_ranker.thriftscala.QigRanker -import com.twitter.util.Duration -import com.twitter.util.Throw - -object QigServiceClientModule - extends ThriftMethodBuilderClientModule[ - QigRanker.ServicePerEndpoint, - QigRanker.MethodPerEndpoint - ] - with MtlsClient { - override val label: String = "qig-ranker" - override val dest: String = "/s/qig-shared/qig-ranker" - private val qigRankerClientTimeout: Flag[Duration] = - flag[Duration](QigRankerClientTimeoutFlagName, "ranking timeout") - - override def requestTimeout: Duration = qigRankerClientTimeout() - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/SimClustersAnnServiceClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/SimClustersAnnServiceClientModule.docx new file mode 100644 index 000000000..a94778ebe Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/SimClustersAnnServiceClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala deleted file mode 100644 index 7504ab6c3..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/SimClustersAnnServiceClientModule.scala +++ /dev/null @@ -1,147 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.google.inject.Provides -import com.twitter.conversions.PercentOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.inject.TwitterModule -import com.twitter.simclustersann.{thriftscala => t} -import javax.inject.Named -import javax.inject.Singleton - -object SimClustersAnnServiceClientModule extends TwitterModule { - - @Provides - @Singleton - @Named(ModuleNames.ProdSimClustersANNServiceClientName) - def providesProdSimClustersANNServiceClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): t.SimClustersANNService.MethodPerEndpoint = { - val label = "simclusters-ann-server" - val dest = "/s/simclusters-ann/simclusters-ann" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.ExperimentalSimClustersANNServiceClientName) - def providesExperimentalSimClustersANNServiceClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): t.SimClustersANNService.MethodPerEndpoint = { - val label = "simclusters-ann-experimental-server" - val dest = "/s/simclusters-ann/simclusters-ann-experimental" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.SimClustersANNServiceClientName1) - def providesSimClustersANNServiceClient1( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): t.SimClustersANNService.MethodPerEndpoint = { - val label = "simclusters-ann-server-1" - val dest = "/s/simclusters-ann/simclusters-ann-1" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.SimClustersANNServiceClientName2) - def providesSimClustersANNServiceClient2( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): t.SimClustersANNService.MethodPerEndpoint = { - val label = "simclusters-ann-server-2" - val dest = "/s/simclusters-ann/simclusters-ann-2" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.SimClustersANNServiceClientName3) - def providesSimClustersANNServiceClient3( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): t.SimClustersANNService.MethodPerEndpoint = { - val label = "simclusters-ann-server-3" - val dest = "/s/simclusters-ann/simclusters-ann-3" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.SimClustersANNServiceClientName5) - def providesSimClustersANNServiceClient5( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): t.SimClustersANNService.MethodPerEndpoint = { - val label = "simclusters-ann-server-5" - val dest = "/s/simclusters-ann/simclusters-ann-5" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - - @Provides - @Singleton - @Named(ModuleNames.SimClustersANNServiceClientName4) - def providesSimClustersANNServiceClient4( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - ): t.SimClustersANNService.MethodPerEndpoint = { - val label = "simclusters-ann-server-4" - val dest = "/s/simclusters-ann/simclusters-ann-4" - - buildClient(serviceIdentifier, clientId, timeoutConfig, statsReceiver, dest, label) - } - private def buildClient( - serviceIdentifier: ServiceIdentifier, - clientId: ClientId, - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver, - dest: String, - label: String - ): t.SimClustersANNService.MethodPerEndpoint = { - val stats = statsReceiver.scope("clnt") - - val thriftClient = ThriftMux.client - .withMutualTls(serviceIdentifier) - .withClientId(clientId) - .withLabel(label) - .withStatsReceiver(stats) - .methodBuilder(dest) - .idempotent(5.percent) - .withTimeoutPerRequest(timeoutConfig.annServiceClientTimeout) - .withRetryDisabled - .servicePerEndpoint[t.SimClustersANNService.ServicePerEndpoint] - - ThriftMux.Client.methodPerEndpoint(thriftClient) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/TweetyPieClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/TweetyPieClientModule.docx new file mode 100644 index 000000000..8c905f637 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/TweetyPieClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/TweetyPieClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/TweetyPieClientModule.scala deleted file mode 100644 index 610ccc95a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/TweetyPieClientModule.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.google.inject.Provides -import com.twitter.app.Flag -import com.twitter.conversions.DurationOps.richDurationFromInt -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.TweetypieClientTimeoutFlagName -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.service.RetryBudget -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.stitch.tweetypie.{TweetyPie => STweetyPie} -import com.twitter.tweetypie.thriftscala.TweetService -import com.twitter.util.Duration -import com.twitter.util.Throw -import javax.inject.Singleton - -object TweetyPieClientModule - extends ThriftMethodBuilderClientModule[ - TweetService.ServicePerEndpoint, - TweetService.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "tweetypie" - override val dest = "/s/tweetypie/tweetypie" - - private val tweetypieClientTimeout: Flag[Duration] = - flag[Duration](TweetypieClientTimeoutFlagName, "tweetypie client timeout") - override def requestTimeout: Duration = tweetypieClientTimeout() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - // We bump the success rate from the default of 0.8 to 0.9 since we're dropping the - // consecutive failures part of the default policy. - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withSessionQualifier - .successRateFailureAccrual(successRate = 0.9, window = 30.seconds) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } - - @Provides - @Singleton - def providesTweetyPie( - tweetyPieService: TweetService.MethodPerEndpoint - ): STweetyPie = { - STweetyPie(tweetyPieService) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserAdGraphClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserAdGraphClientModule.docx new file mode 100644 index 000000000..297369fd1 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserAdGraphClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserAdGraphClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserAdGraphClientModule.scala deleted file mode 100644 index 4c1f337ab..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserAdGraphClientModule.scala +++ /dev/null @@ -1,47 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.app.Flag -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserAdGraphClientTimeoutFlagName -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.service.RetryBudget -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph -import com.twitter.util.Duration -import com.twitter.util.Throw - -object UserAdGraphClientModule - extends ThriftMethodBuilderClientModule[ - UserAdGraph.ServicePerEndpoint, - UserAdGraph.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "user-ad-graph" - override val dest = "/s/user-tweet-graph/user-ad-graph" - private val userAdGraphClientTimeout: Flag[Duration] = - flag[Duration](UserAdGraphClientTimeoutFlagName, "userAdGraph client timeout") - override def requestTimeout: Duration = userAdGraphClientTimeout() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withMutualTls(injector.instance[ServiceIdentifier]) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetEntityGraphClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetEntityGraphClientModule.docx new file mode 100644 index 000000000..f4b254b8e Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetEntityGraphClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetEntityGraphClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetEntityGraphClientModule.scala deleted file mode 100644 index 337f943f7..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetEntityGraphClientModule.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.app.Flag -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UtegClientTimeoutFlagName -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.service.RetryBudget -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph -import com.twitter.util.Duration -import com.twitter.util.Throw - -object UserTweetEntityGraphClientModule - extends ThriftMethodBuilderClientModule[ - UserTweetEntityGraph.ServicePerEndpoint, - UserTweetEntityGraph.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "user-tweet-entity-graph" - override val dest = "/s/cassowary/user_tweet_entity_graph" - private val userTweetEntityGraphClientTimeout: Flag[Duration] = - flag[Duration](UtegClientTimeoutFlagName, "user tweet entity graph client timeout") - override def requestTimeout: Duration = userTweetEntityGraphClientTimeout() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphClientModule.docx new file mode 100644 index 000000000..df880db1c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphClientModule.scala deleted file mode 100644 index 572786fd1..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphClientModule.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.app.Flag -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph -import com.twitter.util.Duration -import com.twitter.util.Throw -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserTweetGraphClientTimeoutFlagName -import com.twitter.finagle.service.RetryBudget - -object UserTweetGraphClientModule - extends ThriftMethodBuilderClientModule[ - UserTweetGraph.ServicePerEndpoint, - UserTweetGraph.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "user-tweet-graph" - override val dest = "/s/user-tweet-graph/user-tweet-graph" - private val userTweetGraphClientTimeout: Flag[Duration] = - flag[Duration](UserTweetGraphClientTimeoutFlagName, "userTweetGraph client timeout") - override def requestTimeout: Duration = userTweetGraphClientTimeout() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphPlusClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphPlusClientModule.docx new file mode 100644 index 000000000..6bb845ebd Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphPlusClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphPlusClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphPlusClientModule.scala deleted file mode 100644 index 41ae96e53..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserTweetGraphPlusClientModule.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.app.Flag -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserTweetGraphPlusClientTimeoutFlagName -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.service.RetryBudget -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.recos.user_tweet_graph_plus.thriftscala.UserTweetGraphPlus -import com.twitter.util.Duration -import com.twitter.util.Throw - -object UserTweetGraphPlusClientModule - extends ThriftMethodBuilderClientModule[ - UserTweetGraphPlus.ServicePerEndpoint, - UserTweetGraphPlus.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "user-tweet-graph-plus" - override val dest = "/s/user-tweet-graph/user-tweet-graph-plus" - private val userTweetGraphPlusClientTimeout: Flag[Duration] = - flag[Duration]( - UserTweetGraphPlusClientTimeoutFlagName, - "userTweetGraphPlus client timeout" - ) - override def requestTimeout: Duration = userTweetGraphPlusClientTimeout() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserVideoGraphClientModule.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserVideoGraphClientModule.docx new file mode 100644 index 000000000..990dd0280 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserVideoGraphClientModule.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserVideoGraphClientModule.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserVideoGraphClientModule.scala deleted file mode 100644 index 7c311cbfa..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/module/thrift_client/UserVideoGraphClientModule.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.cr_mixer.module.thrift_client - -import com.twitter.app.Flag -import com.twitter.cr_mixer.module.core.TimeoutConfigModule.UserVideoGraphClientTimeoutFlagName -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.service.RetryBudget -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.mtls.thriftmux.modules.MtlsClient -import com.twitter.inject.Injector -import com.twitter.inject.thrift.modules.ThriftMethodBuilderClientModule -import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph -import com.twitter.util.Duration -import com.twitter.util.Throw - -object UserVideoGraphClientModule - extends ThriftMethodBuilderClientModule[ - UserVideoGraph.ServicePerEndpoint, - UserVideoGraph.MethodPerEndpoint - ] - with MtlsClient { - - override val label = "user-video-graph" - override val dest = "/s/user-tweet-graph/user-video-graph" - private val userVideoGraphClientTimeout: Flag[Duration] = - flag[Duration]( - UserVideoGraphClientTimeoutFlagName, - "userVideoGraph client timeout" - ) - override def requestTimeout: Duration = userVideoGraphClientTimeout() - - override def retryBudget: RetryBudget = RetryBudget.Empty - - override def configureThriftMuxClient( - injector: Injector, - client: ThriftMux.Client - ): ThriftMux.Client = - super - .configureThriftMuxClient(injector, client) - .withStatsReceiver(injector.instance[StatsReceiver].scope("clnt")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/AdsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/AdsParams.docx new file mode 100644 index 000000000..53a8a9939 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/AdsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/AdsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/AdsParams.scala deleted file mode 100644 index 880f1b27c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/AdsParams.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object AdsParams { - object AdsCandidateGenerationMaxCandidatesNumParam - extends FSBoundedParam[Int]( - name = "ads_candidate_generation_max_candidates_num", - default = 400, - min = 0, - max = 2000 - ) - - object EnableScoreBoost - extends FSParam[Boolean]( - name = "ads_candidate_generation_enable_score_boost", - default = false - ) - - object AdsCandidateGenerationScoreBoostFactor - extends FSBoundedParam[Double]( - name = "ads_candidate_generation_score_boost_factor", - default = 10000.0, - min = 1.0, - max = 100000.0 - ) - - object EnableScribe - extends FSParam[Boolean]( - name = "ads_candidate_generation_enable_scribe", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - AdsCandidateGenerationMaxCandidatesNumParam, - EnableScoreBoost, - AdsCandidateGenerationScoreBoostFactor - ) - - lazy val config: BaseConfig = { - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - AdsCandidateGenerationMaxCandidatesNumParam) - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableScoreBoost, - EnableScribe - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(AdsCandidateGenerationScoreBoostFactor) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BUILD deleted file mode 100644 index b24a21394..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "abdecider/src/main/scala", - "configapi/configapi-abdecider", - "configapi/configapi-core", - "configapi/configapi-featureswitches:v2", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "decider/src/main/scala", - "discovery-common/src/main/scala/com/twitter/discovery/common/configapi", - "featureswitches/featureswitches-core", - "featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BUILD.docx new file mode 100644 index 000000000..746e4458d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BlenderParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BlenderParams.docx new file mode 100644 index 000000000..5725ae353 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BlenderParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BlenderParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BlenderParams.scala deleted file mode 100644 index 185fc4440..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BlenderParams.scala +++ /dev/null @@ -1,152 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object BlenderParams { - object BlendingAlgorithmEnum extends Enumeration { - val RoundRobin: Value = Value - val SourceTypeBackFill: Value = Value - val SourceSignalSorting: Value = Value - } - object ContentBasedSortingAlgorithmEnum extends Enumeration { - val FavoriteCount: Value = Value - val SourceSignalRecency: Value = Value - val RandomSorting: Value = Value - val SimilarityToSignalSorting: Value = Value - val CandidateRecency: Value = Value - } - - object BlendingAlgorithmParam - extends FSEnumParam[BlendingAlgorithmEnum.type]( - name = "blending_algorithm_id", - default = BlendingAlgorithmEnum.RoundRobin, - enum = BlendingAlgorithmEnum - ) - - object RankingInterleaveWeightShrinkageParam - extends FSBoundedParam[Double]( - name = "blending_enable_ml_ranking_interleave_weights_shrinkage", - default = 1.0, - min = 0.0, - max = 1.0 - ) - - object RankingInterleaveMaxWeightAdjustments - extends FSBoundedParam[Int]( - name = "blending_interleave_max_weighted_adjustments", - default = 3000, - min = 0, - max = 9999 - ) - - object SignalTypeSortingAlgorithmParam - extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type]( - name = "blending_algorithm_inner_signal_sorting_id", - default = ContentBasedSortingAlgorithmEnum.SourceSignalRecency, - enum = ContentBasedSortingAlgorithmEnum - ) - - object ContentBlenderTypeSortingAlgorithmParam - extends FSEnumParam[ContentBasedSortingAlgorithmEnum.type]( - name = "blending_algorithm_content_blender_sorting_id", - default = ContentBasedSortingAlgorithmEnum.FavoriteCount, - enum = ContentBasedSortingAlgorithmEnum - ) - - //UserAffinities Algo Param: whether to distributed the source type weights - object EnableDistributedSourceTypeWeightsParam - extends FSParam[Boolean]( - name = "blending_algorithm_enable_distributed_source_type_weights", - default = false - ) - - object BlendGroupingMethodEnum extends Enumeration { - val SourceKeyDefault: Value = Value("SourceKey") - val SourceTypeSimilarityEngine: Value = Value("SourceTypeSimilarityEngine") - val AuthorId: Value = Value("AuthorId") - } - - object BlendGroupingMethodParam - extends FSEnumParam[BlendGroupingMethodEnum.type]( - name = "blending_grouping_method_id", - default = BlendGroupingMethodEnum.SourceKeyDefault, - enum = BlendGroupingMethodEnum - ) - - object RecencyBasedRandomSamplingHalfLifeInDays - extends FSBoundedParam[Int]( - name = "blending_interleave_random_sampling_recency_based_half_life_in_days", - default = 7, - min = 1, - max = 28 - ) - - object RecencyBasedRandomSamplingDefaultWeight - extends FSBoundedParam[Double]( - name = "blending_interleave_random_sampling_recency_based_default_weight", - default = 1.0, - min = 0.1, - max = 2.0 - ) - - object SourceTypeBackFillEnableVideoBackFill - extends FSParam[Boolean]( - name = "blending_enable_video_backfill", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - BlendingAlgorithmParam, - RankingInterleaveWeightShrinkageParam, - RankingInterleaveMaxWeightAdjustments, - EnableDistributedSourceTypeWeightsParam, - BlendGroupingMethodParam, - RecencyBasedRandomSamplingHalfLifeInDays, - RecencyBasedRandomSamplingDefaultWeight, - SourceTypeBackFillEnableVideoBackFill, - SignalTypeSortingAlgorithmParam, - ContentBlenderTypeSortingAlgorithmParam, - ) - - lazy val config: BaseConfig = { - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - BlendingAlgorithmParam, - BlendGroupingMethodParam, - SignalTypeSortingAlgorithmParam, - ContentBlenderTypeSortingAlgorithmParam - ) - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableDistributedSourceTypeWeightsParam, - SourceTypeBackFillEnableVideoBackFill - ) - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - RankingInterleaveMaxWeightAdjustments, - RecencyBasedRandomSamplingHalfLifeInDays - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - RankingInterleaveWeightShrinkageParam, - RecencyBasedRandomSamplingDefaultWeight - ) - - BaseConfigBuilder() - .set(enumOverrides: _*) - .set(booleanOverrides: _*) - .set(intOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BypassInterleaveAndRankParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BypassInterleaveAndRankParams.docx new file mode 100644 index 000000000..85ef718ab Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BypassInterleaveAndRankParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BypassInterleaveAndRankParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BypassInterleaveAndRankParams.scala deleted file mode 100644 index 20cbc369a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/BypassInterleaveAndRankParams.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object BypassInterleaveAndRankParams { - object EnableTwhinCollabFilterBypassParam - extends FSParam[Boolean]( - name = "bypass_interleave_and_rank_twhin_collab_filter", - default = false - ) - - object EnableTwoTowerBypassParam - extends FSParam[Boolean]( - name = "bypass_interleave_and_rank_two_tower", - default = false - ) - - object EnableConsumerBasedTwhinBypassParam - extends FSParam[Boolean]( - name = "bypass_interleave_and_rank_consumer_based_twhin", - default = false - ) - - object EnableConsumerBasedWalsBypassParam - extends FSParam[Boolean]( - name = "bypass_interleave_and_rank_consumer_based_wals", - default = false - ) - - object TwhinCollabFilterBypassPercentageParam - extends FSBoundedParam[Double]( - name = "bypass_interleave_and_rank_twhin_collab_filter_percentage", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - object TwoTowerBypassPercentageParam - extends FSBoundedParam[Double]( - name = "bypass_interleave_and_rank_two_tower_percentage", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - object ConsumerBasedTwhinBypassPercentageParam - extends FSBoundedParam[Double]( - name = "bypass_interleave_and_rank_consumer_based_twhin_percentage", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - object ConsumerBasedWalsBypassPercentageParam - extends FSBoundedParam[Double]( - name = "bypass_interleave_and_rank_consumer_based_wals_percentage", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableTwhinCollabFilterBypassParam, - EnableTwoTowerBypassParam, - EnableConsumerBasedTwhinBypassParam, - EnableConsumerBasedWalsBypassParam, - TwhinCollabFilterBypassPercentageParam, - TwoTowerBypassPercentageParam, - ConsumerBasedTwhinBypassPercentageParam, - ConsumerBasedWalsBypassPercentageParam, - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableTwhinCollabFilterBypassParam, - EnableTwoTowerBypassParam, - EnableConsumerBasedTwhinBypassParam, - EnableConsumerBasedWalsBypassParam, - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - TwhinCollabFilterBypassPercentageParam, - TwoTowerBypassPercentageParam, - ConsumerBasedTwhinBypassPercentageParam, - ConsumerBasedWalsBypassPercentageParam, - ) - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerBasedWalsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerBasedWalsParams.docx new file mode 100644 index 000000000..ec1680608 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerBasedWalsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerBasedWalsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerBasedWalsParams.scala deleted file mode 100644 index 15f4d36ab..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerBasedWalsParams.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.conversions.DurationOps.richDurationFromInt -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object ConsumerBasedWalsParams { - - object EnableSourceParam - extends FSParam[Boolean]( - name = "consumer_based_wals_enable_source", - default = false - ) - - object ModelNameParam - extends FSParam[String]( - name = "consumer_based_wals_model_name", - default = "model_0" - ) - - object WilyNsNameParam - extends FSParam[String]( - name = "consumer_based_wals_wily_ns_name", - default = "" - ) - - object ModelInputNameParam - extends FSParam[String]( - name = "consumer_based_wals_model_input_name", - default = "examples" - ) - - object ModelOutputNameParam - extends FSParam[String]( - name = "consumer_based_wals_model_output_name", - default = "all_tweet_ids" - ) - - object ModelSignatureNameParam - extends FSParam[String]( - name = "consumer_based_wals_model_signature_name", - default = "serving_default" - ) - - object MaxTweetSignalAgeHoursParam - extends FSBoundedParam[Duration]( - name = "consumer_based_wals_max_tweet_signal_age_hours", - default = 72.hours, - min = 1.hours, - max = 720.hours - ) - with HasDurationConversion { - - override val durationConversion: DurationConversion = DurationConversion.FromHours - } - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - ModelNameParam, - ModelInputNameParam, - ModelOutputNameParam, - ModelSignatureNameParam, - MaxTweetSignalAgeHoursParam, - WilyNsNameParam, - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - ) - val stringOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( - ModelNameParam, - ModelInputNameParam, - ModelOutputNameParam, - ModelSignatureNameParam, - WilyNsNameParam - ) - - val boundedDurationFSOverrides = - FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides(MaxTweetSignalAgeHoursParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(stringOverrides: _*) - .set(boundedDurationFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedCandidateGenerationParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedCandidateGenerationParams.docx new file mode 100644 index 000000000..28c1b0120 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedCandidateGenerationParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedCandidateGenerationParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedCandidateGenerationParams.scala deleted file mode 100644 index bedbaf0b9..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedCandidateGenerationParams.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object ConsumerEmbeddingBasedCandidateGenerationParams { - - object EnableTwHINParam - extends FSParam[Boolean]( - name = "consumer_embedding_based_candidate_generation_enable_twhin", - default = false - ) - - object EnableTwoTowerParam - extends FSParam[Boolean]( - name = "consumer_embedding_based_candidate_generation_enable_two_tower", - default = false - ) - - object EnableLogFavBasedSimClustersTripParam - extends FSParam[Boolean]( - name = "consumer_embedding_based_candidate_generation_enable_logfav_based_simclusters_trip", - default = false - ) - - object EnableFollowBasedSimClustersTripParam - extends FSParam[Boolean]( - name = "consumer_embedding_based_candidate_generation_enable_follow_based_simclusters_trip", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableTwHINParam, - EnableTwoTowerParam, - EnableFollowBasedSimClustersTripParam, - EnableLogFavBasedSimClustersTripParam - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableTwHINParam, - EnableTwoTowerParam, - EnableFollowBasedSimClustersTripParam, - EnableLogFavBasedSimClustersTripParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTripParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTripParams.docx new file mode 100644 index 000000000..07e52f815 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTripParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTripParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTripParams.scala deleted file mode 100644 index 4b43d42ab..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTripParams.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object ConsumerEmbeddingBasedTripParams { - object SourceIdParam - extends FSParam[String]( - name = "consumer_embedding_based_trip_source_id", - default = "EXPLR_TOPK_VID_48H_V3") - - object MaxNumCandidatesParam - extends FSBoundedParam[Int]( - name = "consumer_embedding_based_trip_max_num_candidates", - default = 80, - min = 0, - max = 200 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - SourceIdParam, - MaxNumCandidatesParam - ) - - lazy val config: BaseConfig = { - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - SourceIdParam - ) - - val intFSOverrides = - FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxNumCandidatesParam - ) - - BaseConfigBuilder() - .set(stringFSOverrides: _*) - .set(intFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwHINParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwHINParams.docx new file mode 100644 index 000000000..a75334680 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwHINParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwHINParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwHINParams.scala deleted file mode 100644 index bda14d5d4..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwHINParams.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil - -object ConsumerEmbeddingBasedTwHINParams { - object ModelIdParam - extends FSParam[String]( - name = "consumer_embedding_based_twhin_model_id", - default = ModelConfig.ConsumerBasedTwHINRegularUpdateAll20221024, - ) // Note: this default value does not match with ModelIds yet. This FS is a placeholder - - val AllParams: Seq[Param[_] with FSName] = Seq( - ModelIdParam - ) - - lazy val config: BaseConfig = { - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - ModelIdParam - ) - - BaseConfigBuilder() - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwoTowerParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwoTowerParams.docx new file mode 100644 index 000000000..a00c6f4da Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwoTowerParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwoTowerParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwoTowerParams.scala deleted file mode 100644 index 2a6474adc..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumerEmbeddingBasedTwoTowerParams.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.cr_mixer.model.ModelConfig.TwoTowerFavALL20220808 -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object ConsumerEmbeddingBasedTwoTowerParams { - object ModelIdParam - extends FSParam[String]( - name = "consumer_embedding_based_two_tower_model_id", - default = TwoTowerFavALL20220808, - ) // Note: this default value does not match with ModelIds yet. This FS is a placeholder - - val AllParams: Seq[Param[_] with FSName] = Seq( - ModelIdParam - ) - - lazy val config: BaseConfig = { - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - ModelIdParam - ) - - BaseConfigBuilder() - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserAdGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserAdGraphParams.docx new file mode 100644 index 000000000..e7018903f Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserAdGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserAdGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserAdGraphParams.scala deleted file mode 100644 index a730e0994..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserAdGraphParams.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object ConsumersBasedUserAdGraphParams { - - object EnableSourceParam - extends FSParam[Boolean]( - name = "consumers_based_user_ad_graph_enable_source", - default = false - ) - - // UTG-Lookalike - object MinCoOccurrenceParam - extends FSBoundedParam[Int]( - name = "consumers_based_user_ad_graph_min_co_occurrence", - default = 2, - min = 0, - max = 500 - ) - - object MinScoreParam - extends FSBoundedParam[Double]( - name = "consumers_based_user_ad_graph_min_score", - default = 0.0, - min = 0.0, - max = 10.0 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - MinCoOccurrenceParam, - MinScoreParam - ) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(MinCoOccurrenceParam) - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableSourceParam) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserTweetGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserTweetGraphParams.docx new file mode 100644 index 000000000..e6e03c334 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserTweetGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserTweetGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserTweetGraphParams.scala deleted file mode 100644 index 47c67887f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserTweetGraphParams.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -/** - * ConsumersBasedUserTweetGraph Params, there are multiple ways (e.g. FRS, RealGraphOon) to generate consumersSeedSet for ConsumersBasedUserTweetGraph - * for now we allow flexibility in tuning UTG params for different consumersSeedSet generation algo by giving the param name {consumerSeedSetAlgo}{ParamName} - */ - -object ConsumersBasedUserTweetGraphParams { - - object EnableSourceParam - extends FSParam[Boolean]( - name = "consumers_based_user_tweet_graph_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - ) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides() - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides() - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserVideoGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserVideoGraphParams.docx new file mode 100644 index 000000000..505ef9ef6 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserVideoGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserVideoGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserVideoGraphParams.scala deleted file mode 100644 index ab0133632..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ConsumersBasedUserVideoGraphParams.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -/** - * ConsumersBasedUserVideoGraph Params: there are multiple ways (e.g. FRS, RealGraphIn) to generate consumersSeedSet for ConsumersBasedUserTweetGraph - * for now we allow flexibility in tuning UVG params for different consumersSeedSet generation algo by giving the param name {consumerSeedSetAlgo}{ParamName} - */ - -object ConsumersBasedUserVideoGraphParams { - - object EnableSourceParam - extends FSParam[Boolean]( - name = "consumers_based_user_video_graph_enable_source", - default = false - ) - - // UTG-RealGraphIN - object RealGraphInMinCoOccurrenceParam - extends FSBoundedParam[Int]( - name = "consumers_based_user_video_graph_real_graph_in_min_co_occurrence", - default = 3, - min = 0, - max = 500 - ) - - object RealGraphInMinScoreParam - extends FSBoundedParam[Double]( - name = "consumers_based_user_video_graph_real_graph_in_min_score", - default = 2.0, - min = 0.0, - max = 10.0 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - RealGraphInMinCoOccurrenceParam, - RealGraphInMinScoreParam - ) - - lazy val config: BaseConfig = { - - val intOverrides = - FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(RealGraphInMinCoOccurrenceParam) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(RealGraphInMinScoreParam) - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CrMixerParamConfig.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CrMixerParamConfig.docx new file mode 100644 index 000000000..e579cd4bd Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CrMixerParamConfig.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CrMixerParamConfig.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CrMixerParamConfig.scala deleted file mode 100644 index ada50d965..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CrMixerParamConfig.scala +++ /dev/null @@ -1,122 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.CompositeConfig -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.Param - -object CrMixerParamConfig { - - lazy val config: CompositeConfig = new CompositeConfig( - configs = Seq( - AdsParams.config, - BlenderParams.config, - BypassInterleaveAndRankParams.config, - RankerParams.config, - ConsumerBasedWalsParams.config, - ConsumerEmbeddingBasedCandidateGenerationParams.config, - ConsumerEmbeddingBasedTripParams.config, - ConsumerEmbeddingBasedTwHINParams.config, - ConsumerEmbeddingBasedTwoTowerParams.config, - ConsumersBasedUserAdGraphParams.config, - ConsumersBasedUserTweetGraphParams.config, - ConsumersBasedUserVideoGraphParams.config, - CustomizedRetrievalBasedCandidateGenerationParams.config, - CustomizedRetrievalBasedOfflineInterestedInParams.config, - CustomizedRetrievalBasedFTROfflineInterestedInParams.config, - CustomizedRetrievalBasedTwhinParams.config, - EarlybirdFrsBasedCandidateGenerationParams.config, - FrsParams.config, - GlobalParams.config, - InterestedInParams.config, - ProducerBasedCandidateGenerationParams.config, - ProducerBasedUserAdGraphParams.config, - ProducerBasedUserTweetGraphParams.config, - RecentFollowsParams.config, - RecentNegativeSignalParams.config, - RecentNotificationsParams.config, - RecentOriginalTweetsParams.config, - RecentReplyTweetsParams.config, - RecentRetweetsParams.config, - RecentTweetFavoritesParams.config, - RelatedTweetGlobalParams.config, - RelatedVideoTweetGlobalParams.config, - RelatedTweetProducerBasedParams.config, - RelatedTweetTweetBasedParams.config, - RelatedVideoTweetTweetBasedParams.config, - RealGraphInParams.config, - RealGraphOonParams.config, - RepeatedProfileVisitsParams.config, - SimClustersANNParams.config, - TopicTweetParams.config, - TweetBasedCandidateGenerationParams.config, - TweetBasedUserAdGraphParams.config, - TweetBasedUserTweetGraphParams.config, - TweetBasedUserVideoGraphParams.config, - TweetSharesParams.config, - TweetBasedTwHINParams.config, - RealGraphOonParams.config, - GoodTweetClickParams.config, - GoodProfileClickParams.config, - UtegTweetGlobalParams.config, - VideoTweetFilterParams.config, - VideoViewTweetsParams.config, - UnifiedUSSSignalParams.config, - ), - simpleName = "CrMixerConfig" - ) - - val allParams: Seq[Param[_] with FSName] = { - AdsParams.AllParams ++ - BlenderParams.AllParams ++ - BypassInterleaveAndRankParams.AllParams ++ - RankerParams.AllParams ++ - ConsumerBasedWalsParams.AllParams ++ - ConsumerEmbeddingBasedCandidateGenerationParams.AllParams ++ - ConsumerEmbeddingBasedTripParams.AllParams ++ - ConsumerEmbeddingBasedTwHINParams.AllParams ++ - ConsumerEmbeddingBasedTwoTowerParams.AllParams ++ - ConsumersBasedUserAdGraphParams.AllParams ++ - ConsumersBasedUserTweetGraphParams.AllParams ++ - ConsumersBasedUserVideoGraphParams.AllParams ++ - CustomizedRetrievalBasedCandidateGenerationParams.AllParams ++ - CustomizedRetrievalBasedOfflineInterestedInParams.AllParams ++ - CustomizedRetrievalBasedFTROfflineInterestedInParams.AllParams ++ - CustomizedRetrievalBasedTwhinParams.AllParams ++ - EarlybirdFrsBasedCandidateGenerationParams.AllParams ++ - FrsParams.AllParams ++ - GlobalParams.AllParams ++ - InterestedInParams.AllParams ++ - ProducerBasedCandidateGenerationParams.AllParams ++ - ProducerBasedUserAdGraphParams.AllParams ++ - ProducerBasedUserTweetGraphParams.AllParams ++ - RecentFollowsParams.AllParams ++ - RecentNegativeSignalParams.AllParams ++ - RecentNotificationsParams.AllParams ++ - RecentOriginalTweetsParams.AllParams ++ - RecentReplyTweetsParams.AllParams ++ - RecentRetweetsParams.AllParams ++ - RecentTweetFavoritesParams.AllParams ++ - RelatedTweetGlobalParams.AllParams ++ - RelatedVideoTweetGlobalParams.AllParams ++ - RelatedTweetProducerBasedParams.AllParams ++ - RelatedTweetTweetBasedParams.AllParams ++ - RelatedVideoTweetTweetBasedParams.AllParams ++ - RepeatedProfileVisitsParams.AllParams ++ - SimClustersANNParams.AllParams ++ - TopicTweetParams.AllParams ++ - TweetBasedCandidateGenerationParams.AllParams ++ - TweetBasedUserAdGraphParams.AllParams ++ - TweetBasedUserTweetGraphParams.AllParams ++ - TweetBasedUserVideoGraphParams.AllParams ++ - TweetSharesParams.AllParams ++ - TweetBasedTwHINParams.AllParams ++ - RealGraphOonParams.AllParams ++ - RealGraphInParams.AllParams ++ - GoodTweetClickParams.AllParams ++ - GoodProfileClickParams.AllParams ++ - UtegTweetGlobalParams.AllParams ++ - VideoTweetFilterParams.AllParams ++ - VideoViewTweetsParams.AllParams ++ - UnifiedUSSSignalParams.AllParams - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedCandidateGenerationParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedCandidateGenerationParams.docx new file mode 100644 index 000000000..6fe522249 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedCandidateGenerationParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedCandidateGenerationParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedCandidateGenerationParams.scala deleted file mode 100644 index 966048b0f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedCandidateGenerationParams.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object CustomizedRetrievalBasedCandidateGenerationParams { - - // Offline SimClusters InterestedIn params - object EnableOfflineInterestedInParam - extends FSParam[Boolean]( - name = "customized_retrieval_based_candidate_generation_enable_offline_interestedin", - default = false - ) - - // Offline SimClusters FTR-based InterestedIn - object EnableOfflineFTRInterestedInParam - extends FSParam[Boolean]( - name = "customized_retrieval_based_candidate_generation_enable_ftr_offline_interestedin", - default = false - ) - - // TwHin Collab Filter Cluster params - object EnableTwhinCollabFilterClusterParam - extends FSParam[Boolean]( - name = "customized_retrieval_based_candidate_generation_enable_twhin_collab_filter_cluster", - default = false - ) - - // TwHin Multi Cluster params - object EnableTwhinMultiClusterParam - extends FSParam[Boolean]( - name = "customized_retrieval_based_candidate_generation_enable_twhin_multi_cluster", - default = false - ) - - object EnableRetweetBasedDiffusionParam - extends FSParam[Boolean]( - name = "customized_retrieval_based_candidate_generation_enable_retweet_based_diffusion", - default = false - ) - object CustomizedRetrievalBasedRetweetDiffusionSource - extends FSParam[String]( - name = - "customized_retrieval_based_candidate_generation_offline_retweet_based_diffusion_model_id", - default = ModelConfig.RetweetBasedDiffusion - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableOfflineInterestedInParam, - EnableOfflineFTRInterestedInParam, - EnableTwhinCollabFilterClusterParam, - EnableTwhinMultiClusterParam, - EnableRetweetBasedDiffusionParam, - CustomizedRetrievalBasedRetweetDiffusionSource - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableOfflineInterestedInParam, - EnableOfflineFTRInterestedInParam, - EnableTwhinCollabFilterClusterParam, - EnableTwhinMultiClusterParam, - EnableRetweetBasedDiffusionParam - ) - - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - CustomizedRetrievalBasedRetweetDiffusionSource - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedFTROfflineInterestedInParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedFTROfflineInterestedInParams.docx new file mode 100644 index 000000000..a4f269a3a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedFTROfflineInterestedInParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedFTROfflineInterestedInParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedFTROfflineInterestedInParams.scala deleted file mode 100644 index d6d1b0430..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedFTROfflineInterestedInParams.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.cr_mixer.param -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object CustomizedRetrievalBasedFTROfflineInterestedInParams { - object CustomizedRetrievalBasedFTROfflineInterestedInSource - extends FSParam[String]( - name = "customized_retrieval_based_ftr_offline_interestedin_model_id", - default = ModelConfig.OfflineFavDecayedSum - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - CustomizedRetrievalBasedFTROfflineInterestedInSource) - - lazy val config: BaseConfig = { - - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - CustomizedRetrievalBasedFTROfflineInterestedInSource - ) - - BaseConfigBuilder() - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedOfflineInterestedInParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedOfflineInterestedInParams.docx new file mode 100644 index 000000000..9a93459cb Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedOfflineInterestedInParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedOfflineInterestedInParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedOfflineInterestedInParams.scala deleted file mode 100644 index d5244e135..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedOfflineInterestedInParams.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object CustomizedRetrievalBasedOfflineInterestedInParams { - - // Model slots available for offline InterestedIn candidate generation - object CustomizedRetrievalBasedOfflineInterestedInSource - extends FSParam[String]( - name = "customized_retrieval_based_offline_interestedin_model_id", - default = ModelConfig.OfflineInterestedInFromKnownFor2020 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(CustomizedRetrievalBasedOfflineInterestedInSource) - - lazy val config: BaseConfig = { - - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - CustomizedRetrievalBasedOfflineInterestedInSource - ) - - BaseConfigBuilder() - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedTwhinParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedTwhinParams.docx new file mode 100644 index 000000000..d7e5eb6c9 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedTwhinParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedTwhinParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedTwhinParams.scala deleted file mode 100644 index 646cdb163..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/CustomizedRetrievalBasedTwhinParams.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object CustomizedRetrievalBasedTwhinParams { - - // Model slots available for TwhinCollab and MultiCluster - object CustomizedRetrievalBasedTwhinCollabFilterFollowSource - extends FSParam[String]( - name = "customized_retrieval_based_offline_twhin_collab_filter_follow_model_id", - default = ModelConfig.TwhinCollabFilterForFollow - ) - - object CustomizedRetrievalBasedTwhinCollabFilterEngagementSource - extends FSParam[String]( - name = "customized_retrieval_based_offline_twhin_collab_filter_engagement_model_id", - default = ModelConfig.TwhinCollabFilterForEngagement - ) - - object CustomizedRetrievalBasedTwhinMultiClusterFollowSource - extends FSParam[String]( - name = "customized_retrieval_based_offline_twhin_multi_cluster_follow_model_id", - default = ModelConfig.TwhinMultiClusterForFollow - ) - - object CustomizedRetrievalBasedTwhinMultiClusterEngagementSource - extends FSParam[String]( - name = "customized_retrieval_based_offline_twhin_multi_cluster_engagement_model_id", - default = ModelConfig.TwhinMultiClusterForEngagement - ) - - val AllParams: Seq[Param[_] with FSName] = - Seq( - CustomizedRetrievalBasedTwhinCollabFilterFollowSource, - CustomizedRetrievalBasedTwhinCollabFilterEngagementSource, - CustomizedRetrievalBasedTwhinMultiClusterFollowSource, - CustomizedRetrievalBasedTwhinMultiClusterEngagementSource, - ) - - lazy val config: BaseConfig = { - - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - CustomizedRetrievalBasedTwhinCollabFilterFollowSource, - CustomizedRetrievalBasedTwhinCollabFilterEngagementSource, - CustomizedRetrievalBasedTwhinMultiClusterFollowSource, - CustomizedRetrievalBasedTwhinMultiClusterEngagementSource, - ) - - BaseConfigBuilder() - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/EarlybirdFrsBasedCandidateGenerationParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/EarlybirdFrsBasedCandidateGenerationParams.docx new file mode 100644 index 000000000..480c80fea Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/EarlybirdFrsBasedCandidateGenerationParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/EarlybirdFrsBasedCandidateGenerationParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/EarlybirdFrsBasedCandidateGenerationParams.scala deleted file mode 100644 index 2a9ffb424..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/EarlybirdFrsBasedCandidateGenerationParams.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_ModelBased -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_RecencyBased -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_TensorflowBased -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object EarlybirdFrsBasedCandidateGenerationParams { - object CandidateGenerationEarlybirdSimilarityEngineType extends Enumeration { - protected case class SimilarityEngineType(rankingMode: EarlybirdSimilarityEngineType) - extends super.Val - import scala.language.implicitConversions - implicit def valueToEarlybirdRankingMode(x: Value): SimilarityEngineType = - x.asInstanceOf[SimilarityEngineType] - - val EarlybirdRankingMode_RecencyBased: SimilarityEngineType = SimilarityEngineType( - EarlybirdSimilarityEngineType_RecencyBased) - val EarlybirdRankingMode_ModelBased: SimilarityEngineType = SimilarityEngineType( - EarlybirdSimilarityEngineType_ModelBased) - val EarlybirdRankingMode_TensorflowBased: SimilarityEngineType = SimilarityEngineType( - EarlybirdSimilarityEngineType_TensorflowBased) - } - - object FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam - extends FSEnumParam[CandidateGenerationEarlybirdSimilarityEngineType.type]( - name = "frs_based_candidate_generation_earlybird_ranking_mode_id", - default = - CandidateGenerationEarlybirdSimilarityEngineType.EarlybirdRankingMode_RecencyBased, - enum = CandidateGenerationEarlybirdSimilarityEngineType - ) - - object FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser - extends FSBoundedParam[Int]( - name = "frs_based_candidate_generation_earlybird_max_tweets_per_user", - default = 100, - min = 0, - /** - * Note max should be equal to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser. - * Which is the size of the memcached result list. - */ - max = 100 - ) - - object FrsBasedCandidateGenerationEarlybirdMaxTweetAge - extends FSBoundedParam[Duration]( - name = "frs_based_candidate_generation_earlybird_max_tweet_age_hours", - default = 24.hours, - min = 12.hours, - /** - * Note max could be related to EarlybirdRecencyBasedCandidateStoreModule.DefaultMaxNumTweetPerUser. - * Which is the size of the memcached result list for recency based earlybird candidate source. - * E.g. if max = 720.hours, we may want to increase the DefaultMaxNumTweetPerUser. - */ - max = 96.hours - ) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromHours - } - - object FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies - extends FSParam[Boolean]( - name = "frs_based_candidate_generation_earlybird_filter_out_retweets_and_replies", - default = true - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam, - FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser, - FrsBasedCandidateGenerationEarlybirdMaxTweetAge, - FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies, - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies, - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides() - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - FrsBasedCandidateGenerationRecencyBasedEarlybirdMaxTweetsPerUser - ) - - val durationFSOverrides = - FeatureSwitchOverrideUtil.getDurationFSOverrides( - FrsBasedCandidateGenerationEarlybirdMaxTweetAge - ) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam, - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .set(intOverrides: _*) - .set(enumOverrides: _*) - .set(durationFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/FrsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/FrsParams.docx new file mode 100644 index 000000000..832b9ccf1 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/FrsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/FrsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/FrsParams.scala deleted file mode 100644 index 18bf1d474..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/FrsParams.scala +++ /dev/null @@ -1,131 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param -import com.twitter.follow_recommendations.thriftscala.DisplayLocation -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.logging.Logger -import com.twitter.finagle.stats.NullStatsReceiver - -object FrsParams { - object EnableSourceParam - extends FSParam[Boolean]( - name = "signal_frs_enable_source", - default = false - ) - - object EnableSourceGraphParam - extends FSParam[Boolean]( - name = "graph_frs_enable_source", - default = false - ) - - object MinScoreParam - extends FSBoundedParam[Double]( - name = "signal_frs_min_score", - default = 0.4, - min = 0.0, - max = 1.0 - ) - - object MaxConsumerSeedsNumParam - extends FSBoundedParam[Int]( - name = "graph_frs_max_user_seeds_num", - default = 200, - min = 0, - max = 1000 - ) - - /** - * These params below are only used for FrsTweetCandidateGenerator and shouldn't be used in other endpoints - * * FrsBasedCandidateGenerationMaxSeedsNumParam - * * FrsCandidateGenerationDisplayLocationParam - * * FrsCandidateGenerationDisplayLocation - * * FrsBasedCandidateGenerationMaxCandidatesNumParam - */ - object FrsBasedCandidateGenerationEnableVisibilityFilteringParam - extends FSParam[Boolean]( - name = "frs_based_candidate_generation_enable_vf", - default = true - ) - - object FrsBasedCandidateGenerationMaxSeedsNumParam - extends FSBoundedParam[Int]( - name = "frs_based_candidate_generation_max_seeds_num", - default = 100, - min = 0, - max = 800 - ) - - object FrsBasedCandidateGenerationDisplayLocation extends Enumeration { - protected case class FrsDisplayLocationValue(displayLocation: DisplayLocation) extends super.Val - import scala.language.implicitConversions - implicit def valueToDisplayLocationValue(x: Value): FrsDisplayLocationValue = - x.asInstanceOf[FrsDisplayLocationValue] - - val DisplayLocation_ContentRecommender: FrsDisplayLocationValue = FrsDisplayLocationValue( - DisplayLocation.ContentRecommender) - val DisplayLocation_Home: FrsDisplayLocationValue = FrsDisplayLocationValue( - DisplayLocation.HomeTimelineTweetRecs) - val DisplayLocation_Notifications: FrsDisplayLocationValue = FrsDisplayLocationValue( - DisplayLocation.TweetNotificationRecs) - } - - object FrsBasedCandidateGenerationDisplayLocationParam - extends FSEnumParam[FrsBasedCandidateGenerationDisplayLocation.type]( - name = "frs_based_candidate_generation_display_location_id", - default = FrsBasedCandidateGenerationDisplayLocation.DisplayLocation_Home, - enum = FrsBasedCandidateGenerationDisplayLocation - ) - - object FrsBasedCandidateGenerationMaxCandidatesNumParam - extends FSBoundedParam[Int]( - name = "frs_based_candidate_generation_max_candidates_num", - default = 100, - min = 0, - max = 2000 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - EnableSourceGraphParam, - MinScoreParam, - MaxConsumerSeedsNumParam, - FrsBasedCandidateGenerationMaxSeedsNumParam, - FrsBasedCandidateGenerationDisplayLocationParam, - FrsBasedCandidateGenerationMaxCandidatesNumParam, - FrsBasedCandidateGenerationEnableVisibilityFilteringParam - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - EnableSourceGraphParam, - FrsBasedCandidateGenerationEnableVisibilityFilteringParam - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxConsumerSeedsNumParam, - FrsBasedCandidateGenerationMaxSeedsNumParam, - FrsBasedCandidateGenerationMaxCandidatesNumParam) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - FrsBasedCandidateGenerationDisplayLocationParam, - ) - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .set(intOverrides: _*) - .set(enumOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GlobalParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GlobalParams.docx new file mode 100644 index 000000000..ccd0c35a9 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GlobalParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GlobalParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GlobalParams.scala deleted file mode 100644 index 77def9a2a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GlobalParams.scala +++ /dev/null @@ -1,106 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.simclusters_v2.common.ModelVersions -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -/** - * Instantiate Params that do not relate to a specific product. - * The params in this file correspond to config repo file - * [[https://sourcegraph.twitter.biz/config-git.twitter.biz/config/-/blob/features/cr-mixer/main/twistly_core.yml]] - */ -object GlobalParams { - - object MaxCandidatesPerRequestParam - extends FSBoundedParam[Int]( - name = "twistly_core_max_candidates_per_request", - default = 100, - min = 0, - max = 9000 - ) - - object ModelVersionParam - extends FSEnumParam[ModelVersions.Enum.type]( - name = "twistly_core_simclusters_model_version_id", - default = ModelVersions.Enum.Model20M145K2020, - enum = ModelVersions.Enum - ) - - object UnifiedMaxSourceKeyNum - extends FSBoundedParam[Int]( - name = "twistly_core_unified_max_sourcekey_num", - default = 15, - min = 0, - max = 100 - ) - - object MaxCandidateNumPerSourceKeyParam - extends FSBoundedParam[Int]( - name = "twistly_core_candidate_per_sourcekey_max_num", - default = 200, - min = 0, - max = 1000 - ) - - // 1 hours to 30 days - object MaxTweetAgeHoursParam - extends FSBoundedParam[Duration]( - name = "twistly_core_max_tweet_age_hours", - default = 720.hours, - min = 1.hours, - max = 720.hours - ) - with HasDurationConversion { - - override val durationConversion: DurationConversion = DurationConversion.FromHours - } - - val AllParams: Seq[Param[_] with FSName] = Seq( - MaxCandidatesPerRequestParam, - UnifiedMaxSourceKeyNum, - MaxCandidateNumPerSourceKeyParam, - ModelVersionParam, - MaxTweetAgeHoursParam - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides() - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxCandidatesPerRequestParam, - UnifiedMaxSourceKeyNum, - MaxCandidateNumPerSourceKeyParam - ) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - ModelVersionParam - ) - - val boundedDurationFSOverrides = - FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides(MaxTweetAgeHoursParam) - - val seqOverrides = FeatureSwitchOverrideUtil.getLongSeqFSOverrides() - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(intOverrides: _*) - .set(boundedDurationFSOverrides: _*) - .set(enumOverrides: _*) - .set(seqOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodProfileClickParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodProfileClickParams.docx new file mode 100644 index 000000000..5b19e2bbd Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodProfileClickParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodProfileClickParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodProfileClickParams.scala deleted file mode 100644 index 175dccfac..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodProfileClickParams.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param -import com.twitter.usersignalservice.thriftscala.SignalType - -object GoodProfileClickParams { - - object ClickMinDwellTimeParam extends Enumeration { - protected case class SignalTypeValue(signalType: SignalType) extends super.Val - import scala.language.implicitConversions - implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = - x.asInstanceOf[SignalTypeValue] - - val TotalDwellTime10s = SignalTypeValue(SignalType.GoodProfileClick) - val TotalDwellTime20s = SignalTypeValue(SignalType.GoodProfileClick20s) - val TotalDwellTime30s = SignalTypeValue(SignalType.GoodProfileClick30s) - - } - - object EnableSourceParam - extends FSParam[Boolean]( - name = "signal_good_profile_clicks_enable_source", - default = false - ) - - object ClickMinDwellTimeType - extends FSEnumParam[ClickMinDwellTimeParam.type]( - name = "signal_good_profile_clicks_min_dwelltime_type_id", - default = ClickMinDwellTimeParam.TotalDwellTime10s, - enum = ClickMinDwellTimeParam - ) - - val AllParams: Seq[Param[_] with FSName] = - Seq(EnableSourceParam, ClickMinDwellTimeType) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - ClickMinDwellTimeType - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(enumOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodTweetClickParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodTweetClickParams.docx new file mode 100644 index 000000000..696c43c18 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodTweetClickParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodTweetClickParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodTweetClickParams.scala deleted file mode 100644 index 949048821..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/GoodTweetClickParams.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param -import com.twitter.usersignalservice.thriftscala.SignalType - -object GoodTweetClickParams { - - object ClickMinDwellTimeParam extends Enumeration { - protected case class SignalTypeValue(signalType: SignalType) extends super.Val - import scala.language.implicitConversions - implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = - x.asInstanceOf[SignalTypeValue] - - val TotalDwellTime2s = SignalTypeValue(SignalType.GoodTweetClick) - val TotalDwellTime5s = SignalTypeValue(SignalType.GoodTweetClick5s) - val TotalDwellTime10s = SignalTypeValue(SignalType.GoodTweetClick10s) - val TotalDwellTime30s = SignalTypeValue(SignalType.GoodTweetClick30s) - - } - - object EnableSourceParam - extends FSParam[Boolean]( - name = "signal_good_tweet_clicks_enable_source", - default = false - ) - - object ClickMinDwellTimeType - extends FSEnumParam[ClickMinDwellTimeParam.type]( - name = "signal_good_tweet_clicks_min_dwelltime_type_id", - default = ClickMinDwellTimeParam.TotalDwellTime2s, - enum = ClickMinDwellTimeParam - ) - - object MaxSignalNumParam - extends FSBoundedParam[Int]( - name = "signal_good_tweet_clicks_max_signal_num", - default = 15, - min = 0, - max = 15 - ) - - val AllParams: Seq[Param[_] with FSName] = - Seq(EnableSourceParam, ClickMinDwellTimeType, MaxSignalNumParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - ClickMinDwellTimeType - ) - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxSignalNumParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(enumOverrides: _*) - .set(intOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/InterestedInParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/InterestedInParams.docx new file mode 100644 index 000000000..fc9600e88 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/InterestedInParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/InterestedInParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/InterestedInParams.scala deleted file mode 100644 index 503469ac3..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/InterestedInParams.scala +++ /dev/null @@ -1,213 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.simclusters_v2.thriftscala.{EmbeddingType => SimClustersEmbeddingType} -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object InterestedInParams { - - object SourceEmbedding extends Enumeration { - protected case class EmbeddingType(embeddingType: SimClustersEmbeddingType) extends super.Val - import scala.language.implicitConversions - implicit def valueToEmbeddingtype(x: Value): EmbeddingType = x.asInstanceOf[EmbeddingType] - - val UserInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.FilteredUserInterestedIn) - val UnfilteredUserInterestedIn: Value = EmbeddingType( - SimClustersEmbeddingType.UnfilteredUserInterestedIn) - val FromProducerEmbedding: Value = EmbeddingType( - SimClustersEmbeddingType.FilteredUserInterestedInFromPE) - val LogFavBasedUserInterestedInFromAPE: Value = EmbeddingType( - SimClustersEmbeddingType.LogFavBasedUserInterestedInFromAPE) - val FollowBasedUserInterestedInFromAPE: Value = EmbeddingType( - SimClustersEmbeddingType.FollowBasedUserInterestedInFromAPE) - val UserNextInterestedIn: Value = EmbeddingType(SimClustersEmbeddingType.UserNextInterestedIn) - // AddressBook based InterestedIn - val LogFavBasedUserInterestedAverageAddressBookFromIIAPE: Value = EmbeddingType( - SimClustersEmbeddingType.LogFavBasedUserInterestedAverageAddressBookFromIIAPE) - val LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( - SimClustersEmbeddingType.LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE) - val LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( - SimClustersEmbeddingType.LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE) - val LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( - SimClustersEmbeddingType.LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE) - val LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( - SimClustersEmbeddingType.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE) - val LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE: Value = EmbeddingType( - SimClustersEmbeddingType.LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE) - } - - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_source", - default = true - ) - - object InterestedInEmbeddingIdParam - extends FSEnumParam[SourceEmbedding.type]( - name = "twistly_interestedin_embedding_id", - default = SourceEmbedding.UnfilteredUserInterestedIn, - enum = SourceEmbedding - ) - - object MinScoreParam - extends FSBoundedParam[Double]( - name = "twistly_interestedin_min_score", - default = 0.072, - min = 0.0, - max = 1.0 - ) - - object EnableSourceSequentialModelParam - extends FSParam[Boolean]( - name = "twistly_interestedin_sequential_model_enable_source", - default = false - ) - - object NextInterestedInEmbeddingIdParam - extends FSEnumParam[SourceEmbedding.type]( - name = "twistly_interestedin_sequential_model_embedding_id", - default = SourceEmbedding.UserNextInterestedIn, - enum = SourceEmbedding - ) - - object MinScoreSequentialModelParam - extends FSBoundedParam[Double]( - name = "twistly_interestedin_sequential_model_min_score", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - object EnableSourceAddressBookParam - extends FSParam[Boolean]( - name = "twistly_interestedin_addressbook_enable_source", - default = false - ) - - object AddressBookInterestedInEmbeddingIdParam - extends FSEnumParam[SourceEmbedding.type]( - name = "twistly_interestedin_addressbook_embedding_id", - default = SourceEmbedding.LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE, - enum = SourceEmbedding - ) - - object MinScoreAddressBookParam - extends FSBoundedParam[Double]( - name = "twistly_interestedin_addressbook_min_score", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - // Prod SimClusters ANN param - // This is used to enable/disable querying of production SANN service. Useful when experimenting - // with replacements to it. - object EnableProdSimClustersANNParam - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_prod_simclusters_ann", - default = true - ) - - // Experimental SimClusters ANN params - object EnableExperimentalSimClustersANNParam - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_experimental_simclusters_ann", - default = false - ) - - // SimClusters ANN 1 cluster params - object EnableSimClustersANN1Param - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_simclusters_ann_1", - default = false - ) - - // SimClusters ANN 2 cluster params - object EnableSimClustersANN2Param - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_simclusters_ann_2", - default = false - ) - - // SimClusters ANN 3 cluster params - object EnableSimClustersANN3Param - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_simclusters_ann_3", - default = false - ) - - // SimClusters ANN 5 cluster params - object EnableSimClustersANN5Param - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_simclusters_ann_5", - default = false - ) - - // SimClusters ANN 4 cluster params - object EnableSimClustersANN4Param - extends FSParam[Boolean]( - name = "twistly_interestedin_enable_simclusters_ann_4", - default = false - ) - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - EnableSourceSequentialModelParam, - EnableSourceAddressBookParam, - EnableProdSimClustersANNParam, - EnableExperimentalSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - MinScoreParam, - MinScoreSequentialModelParam, - MinScoreAddressBookParam, - InterestedInEmbeddingIdParam, - NextInterestedInEmbeddingIdParam, - AddressBookInterestedInEmbeddingIdParam, - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - EnableSourceSequentialModelParam, - EnableSourceAddressBookParam, - EnableProdSimClustersANNParam, - EnableExperimentalSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - MinScoreParam, - MinScoreSequentialModelParam, - MinScoreAddressBookParam) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - InterestedInEmbeddingIdParam, - NextInterestedInEmbeddingIdParam, - AddressBookInterestedInEmbeddingIdParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .set(enumOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedCandidateGenerationParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedCandidateGenerationParams.docx new file mode 100644 index 000000000..9f3216c71 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedCandidateGenerationParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedCandidateGenerationParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedCandidateGenerationParams.scala deleted file mode 100644 index e9ae7feaa..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedCandidateGenerationParams.scala +++ /dev/null @@ -1,143 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object ProducerBasedCandidateGenerationParams { - // Source params. Not being used. It is always set to true in prod - object EnableSourceParam - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_source", - default = false - ) - - object UtgCombinationMethodParam - extends FSEnumParam[UnifiedSETweetCombinationMethod.type]( - name = "producer_based_candidate_generation_utg_combination_method_id", - default = UnifiedSETweetCombinationMethod.Frontload, - enum = UnifiedSETweetCombinationMethod - ) - - // UTG params - object EnableUTGParam - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_utg", - default = false - ) - - object EnableUAGParam - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_uag", - default = false - ) - - // SimClusters params - object EnableSimClustersANNParam - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_simclusters", - default = true - ) - - // Filter params - object SimClustersMinScoreParam - extends FSBoundedParam[Double]( - name = "producer_based_candidate_generation_filter_simclusters_min_score", - default = 0.7, - min = 0.0, - max = 1.0 - ) - - // Experimental SimClusters ANN params - object EnableExperimentalSimClustersANNParam - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_experimental_simclusters_ann", - default = false - ) - - // SimClusters ANN cluster 1 params - object EnableSimClustersANN1Param - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_simclusters_ann_1", - default = false - ) - - // SimClusters ANN cluster 2 params - object EnableSimClustersANN2Param - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_simclusters_ann_2", - default = false - ) - - // SimClusters ANN cluster 3 params - object EnableSimClustersANN3Param - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_simclusters_ann_3", - default = false - ) - - // SimClusters ANN cluster 5 params - object EnableSimClustersANN5Param - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_simclusters_ann_5", - default = false - ) - - object EnableSimClustersANN4Param - extends FSParam[Boolean]( - name = "producer_based_candidate_generation_enable_simclusters_ann_4", - default = false - ) - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - EnableUAGParam, - EnableUTGParam, - EnableSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam, - SimClustersMinScoreParam, - UtgCombinationMethodParam - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - EnableUAGParam, - EnableUTGParam, - EnableSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam - ) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - UtgCombinationMethodParam, - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(SimClustersMinScoreParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .set(enumOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserAdGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserAdGraphParams.docx new file mode 100644 index 000000000..daceb0000 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserAdGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserAdGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserAdGraphParams.scala deleted file mode 100644 index 197db074a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserAdGraphParams.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object ProducerBasedUserAdGraphParams { - - object MinCoOccurrenceParam - extends FSBoundedParam[Int]( - name = "producer_based_user_ad_graph_min_co_occurrence", - default = 2, - min = 0, - max = 500 - ) - - object MinScoreParam - extends FSBoundedParam[Double]( - name = "producer_based_user_ad_graph_min_score", - default = 3.0, - min = 0.0, - max = 10.0 - ) - - object MaxNumFollowersParam - extends FSBoundedParam[Int]( - name = "producer_based_user_ad_graph_max_num_followers", - default = 500, - min = 100, - max = 1000 - ) - - val AllParams: Seq[Param[_] with FSName] = - Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MinCoOccurrenceParam, - MaxNumFollowersParam, - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserTweetGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserTweetGraphParams.docx new file mode 100644 index 000000000..fdb50a3f1 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserTweetGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserTweetGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserTweetGraphParams.scala deleted file mode 100644 index 0747d0afd..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/ProducerBasedUserTweetGraphParams.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object ProducerBasedUserTweetGraphParams { - - object MinCoOccurrenceParam - extends FSBoundedParam[Int]( - name = "producer_based_user_tweet_graph_min_co_occurrence", - default = 4, - min = 0, - max = 500 - ) - - object MinScoreParam - extends FSBoundedParam[Double]( - name = "producer_based_user_tweet_graph_min_score", - default = 3.0, - min = 0.0, - max = 10.0 - ) - - object MaxNumFollowersParam - extends FSBoundedParam[Int]( - name = "producer_based_user_tweet_graph_max_num_followers", - default = 500, - min = 100, - max = 1000 - ) - - val AllParams: Seq[Param[_] with FSName] = - Seq(MinCoOccurrenceParam, MaxNumFollowersParam, MinScoreParam) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MinCoOccurrenceParam, - MaxNumFollowersParam, - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(MinScoreParam) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RankerParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RankerParams.docx new file mode 100644 index 000000000..78ea3b192 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RankerParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RankerParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RankerParams.scala deleted file mode 100644 index e7785ffb8..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RankerParams.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RankerParams { - - object MaxCandidatesToRank - extends FSBoundedParam[Int]( - name = "twistly_core_max_candidates_to_rank", - default = 2000, - min = 0, - max = 9999 - ) - - object EnableBlueVerifiedTopK - extends FSParam[Boolean]( - name = "twistly_core_blue_verified_top_k", - default = true - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - MaxCandidatesToRank, - EnableBlueVerifiedTopK - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableBlueVerifiedTopK) - - val boundedDurationFSOverrides = - FeatureSwitchOverrideUtil.getBoundedDurationFSOverrides() - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxCandidatesToRank - ) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - ) - val stringFSOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides() - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(boundedDurationFSOverrides: _*) - .set(intOverrides: _*) - .set(enumOverrides: _*) - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphInParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphInParams.docx new file mode 100644 index 000000000..99f3035a2 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphInParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphInParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphInParams.scala deleted file mode 100644 index 7614ca0eb..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphInParams.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi._ - -object RealGraphInParams { - object EnableSourceGraphParam - extends FSParam[Boolean]( - name = "graph_realgraphin_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceGraphParam, - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceGraphParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphOonParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphOonParams.docx new file mode 100644 index 000000000..3af3789ab Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphOonParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphOonParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphOonParams.scala deleted file mode 100644 index 8b303c55b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RealGraphOonParams.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RealGraphOonParams { - object EnableSourceParam - extends FSParam[Boolean]( - name = "signal_realgraphoon_enable_source", - default = false - ) - - object EnableSourceGraphParam - extends FSParam[Boolean]( - name = "graph_realgraphoon_enable_source", - default = false - ) - - object MaxConsumerSeedsNumParam - extends FSBoundedParam[Int]( - name = "graph_realgraphoon_max_user_seeds_num", - default = 200, - min = 0, - max = 1000 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - EnableSourceGraphParam, - MaxConsumerSeedsNumParam - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - EnableSourceGraphParam - ) - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(MaxConsumerSeedsNumParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(intOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentFollowsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentFollowsParams.docx new file mode 100644 index 000000000..2cd2266b3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentFollowsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentFollowsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentFollowsParams.scala deleted file mode 100644 index ecb75c82f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentFollowsParams.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RecentFollowsParams { - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_recentfollows_enable_source", - default = true - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNegativeSignalParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNegativeSignalParams.docx new file mode 100644 index 000000000..2b23eb7b8 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNegativeSignalParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNegativeSignalParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNegativeSignalParams.scala deleted file mode 100644 index 429d6daba..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNegativeSignalParams.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RecentNegativeSignalParams { - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_recentnegativesignals_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam - ) - - lazy val config: BaseConfig = { - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - ) - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides() - - BaseConfigBuilder() - .set(booleanOverrides: _*).set(doubleOverrides: _*).set(enumOverrides: _*).build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNotificationsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNotificationsParams.docx new file mode 100644 index 000000000..47a38fa09 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNotificationsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNotificationsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNotificationsParams.scala deleted file mode 100644 index 641118a05..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentNotificationsParams.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RecentNotificationsParams { - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_recentnotifications_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentOriginalTweetsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentOriginalTweetsParams.docx new file mode 100644 index 000000000..76c151ab3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentOriginalTweetsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentOriginalTweetsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentOriginalTweetsParams.scala deleted file mode 100644 index 5b485e61f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentOriginalTweetsParams.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RecentOriginalTweetsParams { - - // Source params - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_recentoriginaltweets_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableSourceParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentReplyTweetsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentReplyTweetsParams.docx new file mode 100644 index 000000000..f6c980854 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentReplyTweetsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentReplyTweetsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentReplyTweetsParams.scala deleted file mode 100644 index 7e6617c6d..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentReplyTweetsParams.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RecentReplyTweetsParams { - // Source params - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_recentreplytweets_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableSourceParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentRetweetsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentRetweetsParams.docx new file mode 100644 index 000000000..039cc8b58 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentRetweetsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentRetweetsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentRetweetsParams.scala deleted file mode 100644 index 93c1fe356..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentRetweetsParams.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RecentRetweetsParams { - - // Source params - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_recentretweets_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentTweetFavoritesParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentTweetFavoritesParams.docx new file mode 100644 index 000000000..05e83aca3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentTweetFavoritesParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentTweetFavoritesParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentTweetFavoritesParams.scala deleted file mode 100644 index 22d0d6a70..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RecentTweetFavoritesParams.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RecentTweetFavoritesParams { - // Source params - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_recenttweetfavorites_enable_source", - default = true - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetGlobalParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetGlobalParams.docx new file mode 100644 index 000000000..5ea37097b Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetGlobalParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetGlobalParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetGlobalParams.scala deleted file mode 100644 index f5e5ee21e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetGlobalParams.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RelatedTweetGlobalParams { - - object MaxCandidatesPerRequestParam - extends FSBoundedParam[Int]( - name = "related_tweet_core_max_candidates_per_request", - default = 100, - min = 0, - max = 500 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxCandidatesPerRequestParam - ) - - BaseConfigBuilder() - .set(intOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetProducerBasedParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetProducerBasedParams.docx new file mode 100644 index 000000000..6a504c96a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetProducerBasedParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetProducerBasedParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetProducerBasedParams.scala deleted file mode 100644 index 3851f6144..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetProducerBasedParams.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RelatedTweetProducerBasedParams { - - // UTG params - object EnableUTGParam - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_utg", - default = false - ) - - // SimClusters params - object EnableSimClustersANNParam - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_simclusters", - default = true - ) - - // Filter params - object SimClustersMinScoreParam - extends FSBoundedParam[Double]( - name = "related_tweet_producer_based_filter_simclusters_min_score", - default = 0.0, - min = 0.0, - max = 1.0 - ) - - // Experimental SimClusters ANN params - object EnableExperimentalSimClustersANNParam - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_experimental_simclusters_ann", - default = false - ) - - // SimClusters ANN cluster 1 params - object EnableSimClustersANN1Param - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_simclusters_ann_1", - default = false - ) - - // SimClusters ANN cluster 2 params - object EnableSimClustersANN2Param - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_simclusters_ann_2", - default = false - ) - - // SimClusters ANN cluster 3 params - object EnableSimClustersANN3Param - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_simclusters_ann_3", - default = false - ) - - // SimClusters ANN cluster 3 params - object EnableSimClustersANN5Param - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_simclusters_ann_5", - default = false - ) - - // SimClusters ANN cluster 4 params - object EnableSimClustersANN4Param - extends FSParam[Boolean]( - name = "related_tweet_producer_based_enable_simclusters_ann_4", - default = false - ) - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableUTGParam, - EnableSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam, - SimClustersMinScoreParam - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableUTGParam, - EnableSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam - ) - - val doubleOverrides = FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - SimClustersMinScoreParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetTweetBasedParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetTweetBasedParams.docx new file mode 100644 index 000000000..a1aa5964f Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetTweetBasedParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetTweetBasedParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetTweetBasedParams.scala deleted file mode 100644 index 10d01a5d1..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedTweetTweetBasedParams.scala +++ /dev/null @@ -1,141 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RelatedTweetTweetBasedParams { - - // UTG params - object EnableUTGParam - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_utg", - default = false - ) - - // UVG params - object EnableUVGParam - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_uvg", - default = false - ) - - // UAG params - object EnableUAGParam - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_uag", - default = false - ) - - // SimClusters params - object EnableSimClustersANNParam - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_simclusters", - default = true - ) - - // Experimental SimClusters ANN params - object EnableExperimentalSimClustersANNParam - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_experimental_simclusters_ann", - default = false - ) - - // SimClusters ANN cluster 1 params - object EnableSimClustersANN1Param - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_simclusters_ann_1", - default = false - ) - - // SimClusters ANN cluster 2 params - object EnableSimClustersANN2Param - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_simclusters_ann_2", - default = false - ) - - // SimClusters ANN cluster 3 params - object EnableSimClustersANN3Param - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_simclusters_ann_3", - default = false - ) - - // SimClusters ANN cluster 5 params - object EnableSimClustersANN5Param - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_simclusters_ann_5", - default = false - ) - - object EnableSimClustersANN4Param - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_simclusters_ann_4", - default = false - ) - // TwHIN params - object EnableTwHINParam - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_twhin", - default = false - ) - - // QIG params - object EnableQigSimilarTweetsParam - extends FSParam[Boolean]( - name = "related_tweet_tweet_based_enable_qig_similar_tweets", - default = false - ) - - // Filter params - object SimClustersMinScoreParam - extends FSBoundedParam[Double]( - name = "related_tweet_tweet_based_filter_simclusters_min_score", - default = 0.3, - min = 0.0, - max = 1.0 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableTwHINParam, - EnableQigSimilarTweetsParam, - EnableUTGParam, - EnableUVGParam, - EnableSimClustersANNParam, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam, - SimClustersMinScoreParam - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableTwHINParam, - EnableQigSimilarTweetsParam, - EnableUTGParam, - EnableUVGParam, - EnableSimClustersANNParam, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(SimClustersMinScoreParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetGlobalParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetGlobalParams.docx new file mode 100644 index 000000000..5e51eda2c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetGlobalParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetGlobalParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetGlobalParams.scala deleted file mode 100644 index eeed18e6c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetGlobalParams.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RelatedVideoTweetGlobalParams { - - object MaxCandidatesPerRequestParam - extends FSBoundedParam[Int]( - name = "related_video_tweet_core_max_candidates_per_request", - default = 100, - min = 0, - max = 500 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(MaxCandidatesPerRequestParam) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxCandidatesPerRequestParam - ) - - BaseConfigBuilder() - .set(intOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetTweetBasedParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetTweetBasedParams.docx new file mode 100644 index 000000000..2c3b8514c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetTweetBasedParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetTweetBasedParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetTweetBasedParams.scala deleted file mode 100644 index 3b40653bc..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RelatedVideoTweetTweetBasedParams.scala +++ /dev/null @@ -1,134 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RelatedVideoTweetTweetBasedParams { - - // UTG params - object EnableUTGParam - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_utg", - default = false - ) - - // SimClusters params - object EnableSimClustersANNParam - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_simclusters", - default = true - ) - - // Experimental SimClusters ANN params - object EnableExperimentalSimClustersANNParam - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_experimental_simclusters_ann", - default = false - ) - - // SimClusters ANN cluster 1 params - object EnableSimClustersANN1Param - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_simclusters_ann_1", - default = false - ) - - // SimClusters ANN cluster 2 params - object EnableSimClustersANN2Param - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_simclusters_ann_2", - default = false - ) - - // SimClusters ANN cluster 3 params - object EnableSimClustersANN3Param - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_simclusters_ann_3", - default = false - ) - - // SimClusters ANN cluster 5 params - object EnableSimClustersANN5Param - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_simclusters_ann_5", - default = false - ) - - // SimClusters ANN cluster 4 params - object EnableSimClustersANN4Param - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_simclusters_ann_4", - default = false - ) - // TwHIN params - object EnableTwHINParam - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_twhin", - default = false - ) - - // QIG params - object EnableQigSimilarTweetsParam - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_qig_similar_tweets", - default = false - ) - - // Filter params - object SimClustersMinScoreParam - extends FSBoundedParam[Double]( - name = "related_video_tweet_tweet_based_filter_simclusters_min_score", - default = 0.3, - min = 0.0, - max = 1.0 - ) - - object EnableUVGParam - extends FSParam[Boolean]( - name = "related_video_tweet_tweet_based_enable_uvg", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableTwHINParam, - EnableQigSimilarTweetsParam, - EnableUTGParam, - EnableUVGParam, - EnableSimClustersANNParam, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam, - SimClustersMinScoreParam - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableTwHINParam, - EnableQigSimilarTweetsParam, - EnableUTGParam, - EnableUVGParam, - EnableSimClustersANNParam, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(SimClustersMinScoreParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RepeatedProfileVisitsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RepeatedProfileVisitsParams.docx new file mode 100644 index 000000000..d9d298684 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RepeatedProfileVisitsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RepeatedProfileVisitsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RepeatedProfileVisitsParams.scala deleted file mode 100644 index 4cb205de9..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/RepeatedProfileVisitsParams.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.usersignalservice.thriftscala.SignalType -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object RepeatedProfileVisitsParams { - object ProfileMinVisitParam extends Enumeration { - protected case class SignalTypeValue(signalType: SignalType) extends super.Val - import scala.language.implicitConversions - implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = - x.asInstanceOf[SignalTypeValue] - - val TotalVisitsInPast180Days = SignalTypeValue(SignalType.RepeatedProfileVisit180dMinVisit6V1) - val TotalVisitsInPast90Days = SignalTypeValue(SignalType.RepeatedProfileVisit90dMinVisit6V1) - val TotalVisitsInPast14Days = SignalTypeValue(SignalType.RepeatedProfileVisit14dMinVisit2V1) - val TotalVisitsInPast180DaysNoNegative = SignalTypeValue( - SignalType.RepeatedProfileVisit180dMinVisit6V1NoNegative) - val TotalVisitsInPast90DaysNoNegative = SignalTypeValue( - SignalType.RepeatedProfileVisit90dMinVisit6V1NoNegative) - val TotalVisitsInPast14DaysNoNegative = SignalTypeValue( - SignalType.RepeatedProfileVisit14dMinVisit2V1NoNegative) - } - - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_repeatedprofilevisits_enable_source", - default = true - ) - - object MinScoreParam - extends FSBoundedParam[Double]( - name = "twistly_repeatedprofilevisits_min_score", - default = 0.5, - min = 0.0, - max = 1.0 - ) - - object ProfileMinVisitType - extends FSEnumParam[ProfileMinVisitParam.type]( - name = "twistly_repeatedprofilevisits_min_visit_type_id", - default = ProfileMinVisitParam.TotalVisitsInPast14Days, - enum = ProfileMinVisitParam - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam, ProfileMinVisitType) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam - ) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - ProfileMinVisitType - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(enumOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/SimClustersANNParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/SimClustersANNParams.docx new file mode 100644 index 000000000..0c41856f5 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/SimClustersANNParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/SimClustersANNParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/SimClustersANNParams.scala deleted file mode 100644 index b650d5123..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/SimClustersANNParams.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object SimClustersANNParams { - - // Different SimClusters ANN cluster has its own config id (model slot) - object SimClustersANNConfigId - extends FSParam[String]( - name = "similarity_simclusters_ann_simclusters_ann_config_id", - default = "Default" - ) - - object SimClustersANN1ConfigId - extends FSParam[String]( - name = "similarity_simclusters_ann_simclusters_ann_1_config_id", - default = "20220810" - ) - - object SimClustersANN2ConfigId - extends FSParam[String]( - name = "similarity_simclusters_ann_simclusters_ann_2_config_id", - default = "20220818" - ) - - object SimClustersANN3ConfigId - extends FSParam[String]( - name = "similarity_simclusters_ann_simclusters_ann_3_config_id", - default = "20220819" - ) - - object SimClustersANN5ConfigId - extends FSParam[String]( - name = "similarity_simclusters_ann_simclusters_ann_5_config_id", - default = "20221221" - ) - object SimClustersANN4ConfigId - extends FSParam[String]( - name = "similarity_simclusters_ann_simclusters_ann_4_config_id", - default = "20221220" - ) - object ExperimentalSimClustersANNConfigId - extends FSParam[String]( - name = "similarity_simclusters_ann_experimental_simclusters_ann_config_id", - default = "20220801" - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - SimClustersANNConfigId, - SimClustersANN1ConfigId, - SimClustersANN2ConfigId, - SimClustersANN3ConfigId, - SimClustersANN5ConfigId, - ExperimentalSimClustersANNConfigId - ) - - lazy val config: BaseConfig = { - val stringOverrides = FeatureSwitchOverrideUtil.getStringFSOverrides( - SimClustersANNConfigId, - SimClustersANN1ConfigId, - SimClustersANN2ConfigId, - SimClustersANN3ConfigId, - SimClustersANN5ConfigId, - ExperimentalSimClustersANNConfigId - ) - - BaseConfigBuilder() - .set(stringOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TopicTweetParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TopicTweetParams.docx new file mode 100644 index 000000000..12d478566 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TopicTweetParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TopicTweetParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TopicTweetParams.scala deleted file mode 100644 index 3ef683f52..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TopicTweetParams.scala +++ /dev/null @@ -1,115 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object TopicTweetParams { - object MaxTweetAge - extends FSBoundedParam[Duration]( - name = "topic_tweet_candidate_generation_max_tweet_age_hours", - default = 24.hours, - min = 12.hours, - max = 48.hours - ) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromHours - } - - object MaxTopicTweetCandidatesParam - extends FSBoundedParam[Int]( - name = "topic_tweet_max_candidates_num", - default = 200, - min = 0, - max = 1000 - ) - - object MaxSkitTfgCandidatesParam - extends FSBoundedParam[Int]( - name = "topic_tweet_skit_tfg_max_candidates_num", - default = 100, - min = 0, - max = 1000 - ) - - object MaxSkitHighPrecisionCandidatesParam - extends FSBoundedParam[Int]( - name = "topic_tweet_skit_high_precision_max_candidates_num", - default = 100, - min = 0, - max = 1000 - ) - - object MaxCertoCandidatesParam - extends FSBoundedParam[Int]( - name = "topic_tweet_certo_max_candidates_num", - default = 100, - min = 0, - max = 1000 - ) - - // The min prod score for Certo L2-normalized cosine candidates - object CertoScoreThresholdParam - extends FSBoundedParam[Double]( - name = "topic_tweet_certo_score_threshold", - default = 0.015, - min = 0, - max = 1 - ) - - object SemanticCoreVersionIdParam - extends FSParam[Long]( - name = "semantic_core_version_id", - default = 1380520918896713735L - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - CertoScoreThresholdParam, - MaxTopicTweetCandidatesParam, - MaxTweetAge, - MaxCertoCandidatesParam, - MaxSkitTfgCandidatesParam, - MaxSkitHighPrecisionCandidatesParam, - SemanticCoreVersionIdParam - ) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides() - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(CertoScoreThresholdParam) - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxCertoCandidatesParam, - MaxSkitTfgCandidatesParam, - MaxSkitHighPrecisionCandidatesParam, - MaxTopicTweetCandidatesParam - ) - - val longOverrides = FeatureSwitchOverrideUtil.getLongFSOverrides(SemanticCoreVersionIdParam) - - val durationFSOverrides = FeatureSwitchOverrideUtil.getDurationFSOverrides(MaxTweetAge) - - val enumOverrides = - FeatureSwitchOverrideUtil.getEnumFSOverrides(NullStatsReceiver, Logger(getClass)) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .set(intOverrides: _*) - .set(longOverrides: _*) - .set(enumOverrides: _*) - .set(durationFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedCandidateGenerationParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedCandidateGenerationParams.docx new file mode 100644 index 000000000..50017cb65 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedCandidateGenerationParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedCandidateGenerationParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedCandidateGenerationParams.scala deleted file mode 100644 index 7f94d2e41..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedCandidateGenerationParams.scala +++ /dev/null @@ -1,189 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object TweetBasedCandidateGenerationParams { - - // Source params. Not being used. It is always set to true in prod - object EnableSourceParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_source", - default = false - ) - - // UTG params - object EnableUTGParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_utg", - default = true - ) - - // SimClusters params - object EnableSimClustersANNParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_simclusters", - default = true - ) - - // Experimental SimClusters ANN params - object EnableExperimentalSimClustersANNParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_experimental_simclusters_ann", - default = false - ) - - // SimClusters ANN cluster 1 params - object EnableSimClustersANN1Param - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_simclusters_ann_1", - default = false - ) - - // SimClusters ANN cluster 2 params - object EnableSimClustersANN2Param - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_simclusters_ann_2", - default = false - ) - - // SimClusters ANN cluster 3 params - object EnableSimClustersANN3Param - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_simclusters_ann_3", - default = false - ) - - // SimClusters ANN cluster 3 params - object EnableSimClustersANN5Param - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_simclusters_ann_5", - default = false - ) - - // SimClusters ANN cluster 4 params - object EnableSimClustersANN4Param - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_simclusters_ann_4", - default = false - ) - // TwHIN params - object EnableTwHINParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_twhin", - default = false - ) - - // QIG params - object EnableQigSimilarTweetsParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_qig_similar_tweets", - default = false - ) - - object QigMaxNumSimilarTweetsParam - extends FSBoundedParam[Int]( - name = "tweet_based_candidate_generation_qig_max_num_similar_tweets", - default = 100, - min = 10, - max = 100 - ) - - // UVG params - object EnableUVGParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_uvg", - default = false - ) - - // UAG params - object EnableUAGParam - extends FSParam[Boolean]( - name = "tweet_based_candidate_generation_enable_uag", - default = false - ) - - // Filter params - object SimClustersMinScoreParam - extends FSBoundedParam[Double]( - name = "tweet_based_candidate_generation_filter_simclusters_min_score", - default = 0.5, - min = 0.0, - max = 1.0 - ) - - // for learning DDG that has a higher threshold for video based SANN - object SimClustersVideoBasedMinScoreParam - extends FSBoundedParam[Double]( - name = "tweet_based_candidate_generation_filter_simclusters_video_based_min_score", - default = 0.5, - min = 0.0, - max = 1.0 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableSourceParam, - EnableTwHINParam, - EnableQigSimilarTweetsParam, - EnableUTGParam, - EnableUVGParam, - EnableUAGParam, - EnableSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam, - SimClustersMinScoreParam, - SimClustersVideoBasedMinScoreParam, - QigMaxNumSimilarTweetsParam, - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - EnableTwHINParam, - EnableQigSimilarTweetsParam, - EnableUTGParam, - EnableUVGParam, - EnableUAGParam, - EnableSimClustersANNParam, - EnableSimClustersANN1Param, - EnableSimClustersANN2Param, - EnableSimClustersANN3Param, - EnableSimClustersANN5Param, - EnableSimClustersANN4Param, - EnableExperimentalSimClustersANNParam, - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - SimClustersMinScoreParam, - SimClustersVideoBasedMinScoreParam) - - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - ) - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - QigMaxNumSimilarTweetsParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(doubleOverrides: _*) - .set(enumOverrides: _*) - .set(intOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedTwHINParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedTwHINParams.docx new file mode 100644 index 000000000..a12883ab0 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedTwHINParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedTwHINParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedTwHINParams.scala deleted file mode 100644 index c4ecfc6fb..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedTwHINParams.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.cr_mixer.model.ModelConfig -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object TweetBasedTwHINParams { - object ModelIdParam - extends FSParam[String]( - name = "tweet_based_twhin_model_id", - default = ModelConfig.TweetBasedTwHINRegularUpdateAll20221024, - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(ModelIdParam) - - lazy val config: BaseConfig = { - val stringFSOverrides = - FeatureSwitchOverrideUtil.getStringFSOverrides( - ModelIdParam - ) - - BaseConfigBuilder() - .set(stringFSOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserAdGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserAdGraphParams.docx new file mode 100644 index 000000000..815030b61 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserAdGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserAdGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserAdGraphParams.scala deleted file mode 100644 index 9e994b16b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserAdGraphParams.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object TweetBasedUserAdGraphParams { - - object MinCoOccurrenceParam - extends FSBoundedParam[Int]( - name = "tweet_based_user_ad_graph_min_co_occurrence", - default = 1, - min = 0, - max = 500 - ) - - object ConsumersBasedMinScoreParam - extends FSBoundedParam[Double]( - name = "tweet_based_user_ad_graph_consumers_based_min_score", - default = 0.0, - min = 0.0, - max = 10.0 - ) - - object MaxConsumerSeedsNumParam - extends FSBoundedParam[Int]( - name = "tweet_based_user_ad_graph_max_user_seeds_num", - default = 100, - min = 0, - max = 300 - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - MinCoOccurrenceParam, - MaxConsumerSeedsNumParam, - ConsumersBasedMinScoreParam - ) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MinCoOccurrenceParam, - MaxConsumerSeedsNumParam - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(ConsumersBasedMinScoreParam) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(doubleOverrides: _*) - .build() - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserTweetGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserTweetGraphParams.docx new file mode 100644 index 000000000..96900b7c5 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserTweetGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserTweetGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserTweetGraphParams.scala deleted file mode 100644 index 8cc42f81f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserTweetGraphParams.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object TweetBasedUserTweetGraphParams { - - object MinCoOccurrenceParam - extends FSBoundedParam[Int]( - name = "tweet_based_user_tweet_graph_min_co_occurrence", - default = 3, - min = 0, - max = 500 - ) - - object TweetBasedMinScoreParam - extends FSBoundedParam[Double]( - name = "tweet_based_user_tweet_graph_tweet_based_min_score", - default = 0.5, - min = 0.0, - max = 10.0 - ) - - object ConsumersBasedMinScoreParam - extends FSBoundedParam[Double]( - name = "tweet_based_user_tweet_graph_consumers_based_min_score", - default = 4.0, - min = 0.0, - max = 10.0 - ) - object MaxConsumerSeedsNumParam - extends FSBoundedParam[Int]( - name = "tweet_based_user_tweet_graph_max_user_seeds_num", - default = 100, - min = 0, - max = 300 - ) - - object EnableCoverageExpansionOldTweetParam - extends FSParam[Boolean]( - name = "tweet_based_user_tweet_graph_enable_coverage_expansion_old_tweet", - default = false - ) - - object EnableCoverageExpansionAllTweetParam - extends FSParam[Boolean]( - name = "tweet_based_user_tweet_graph_enable_coverage_expansion_all_tweet", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableCoverageExpansionAllTweetParam, - EnableCoverageExpansionOldTweetParam, - MinCoOccurrenceParam, - MaxConsumerSeedsNumParam, - TweetBasedMinScoreParam, - ConsumersBasedMinScoreParam - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableCoverageExpansionAllTweetParam, - EnableCoverageExpansionOldTweetParam - ) - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MinCoOccurrenceParam, - MaxConsumerSeedsNumParam - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides( - TweetBasedMinScoreParam, - ConsumersBasedMinScoreParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(intOverrides: _*) - .set(doubleOverrides: _*) - .build() - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserVideoGraphParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserVideoGraphParams.docx new file mode 100644 index 000000000..0aa712646 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserVideoGraphParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserVideoGraphParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserVideoGraphParams.scala deleted file mode 100644 index 0de5d2df7..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetBasedUserVideoGraphParams.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object TweetBasedUserVideoGraphParams { - - object MinCoOccurrenceParam - extends FSBoundedParam[Int]( - name = "tweet_based_user_video_graph_min_co_occurrence", - default = 5, - min = 0, - max = 500 - ) - - object TweetBasedMinScoreParam - extends FSBoundedParam[Double]( - name = "tweet_based_user_video_graph_tweet_based_min_score", - default = 0.0, - min = 0.0, - max = 100.0 - ) - - object ConsumersBasedMinScoreParam - extends FSBoundedParam[Double]( - name = "tweet_based_user_video_graph_consumers_based_min_score", - default = 4.0, - min = 0.0, - max = 10.0 - ) - - object MaxConsumerSeedsNumParam - extends FSBoundedParam[Int]( - name = "tweet_based_user_video_graph_max_user_seeds_num", - default = 200, - min = 0, - max = 500 - ) - - object EnableCoverageExpansionOldTweetParam - extends FSParam[Boolean]( - name = "tweet_based_user_video_graph_enable_coverage_expansion_old_tweet", - default = false - ) - - object EnableCoverageExpansionAllTweetParam - extends FSParam[Boolean]( - name = "tweet_based_user_video_graph_enable_coverage_expansion_all_tweet", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - MinCoOccurrenceParam, - MaxConsumerSeedsNumParam, - TweetBasedMinScoreParam, - EnableCoverageExpansionOldTweetParam, - EnableCoverageExpansionAllTweetParam - ) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MinCoOccurrenceParam, - MaxConsumerSeedsNumParam - ) - - val doubleOverrides = - FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(TweetBasedMinScoreParam) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(doubleOverrides: _*) - .build() - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetSharesParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetSharesParams.docx new file mode 100644 index 000000000..70ab045ff Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetSharesParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetSharesParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetSharesParams.scala deleted file mode 100644 index 1602441b0..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/TweetSharesParams.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param - -object TweetSharesParams { - object EnableSourceParam - extends FSParam[Boolean]( - name = "twistly_tweetshares_enable_source", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq(EnableSourceParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedSETweetCombinationMethod.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedSETweetCombinationMethod.docx new file mode 100644 index 000000000..04d7fb1bf Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedSETweetCombinationMethod.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedSETweetCombinationMethod.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedSETweetCombinationMethod.scala deleted file mode 100644 index f5b92f138..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedSETweetCombinationMethod.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.cr_mixer.param - -import scala.language.implicitConversions - -object UnifiedSETweetCombinationMethod extends Enumeration { - - protected case class CombinationType(s: String) extends super.Val - - implicit def valueToCombinationType(x: Value): CombinationType = x.asInstanceOf[CombinationType] - - val Default: Value = CombinationType("") - val Interleave: Value = CombinationType("Interleave") - val Frontload: Value = CombinationType("Frontload") - val Backfill: Value = CombinationType("Backfill") -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedUSSSignalParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedUSSSignalParams.docx new file mode 100644 index 000000000..ef2909d50 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedUSSSignalParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedUSSSignalParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedUSSSignalParams.scala deleted file mode 100644 index 071cabc0c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UnifiedUSSSignalParams.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.cr_mixer.param -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param -import com.twitter.usersignalservice.thriftscala.SignalType -import scala.language.implicitConversions - -object UnifiedUSSSignalParams { - - object TweetAggregationTypeParam extends Enumeration { - protected case class SignalTypeValue(signalType: SignalType) extends super.Val - - implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = - x.asInstanceOf[SignalTypeValue] - - val UniformAggregation = SignalTypeValue(SignalType.TweetBasedUnifiedUniformSignal) - val EngagementAggregation = SignalTypeValue( - SignalType.TweetBasedUnifiedEngagementWeightedSignal) - } - - object ProducerAggregationTypeParam extends Enumeration { - protected case class SignalTypeValue(signalType: SignalType) extends super.Val - - import scala.language.implicitConversions - - implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = - x.asInstanceOf[SignalTypeValue] - - val UniformAggregation = SignalTypeValue(SignalType.ProducerBasedUnifiedUniformSignal) - val EngagementAggregation = SignalTypeValue( - SignalType.ProducerBasedUnifiedEngagementWeightedSignal) - - } - - object ReplaceIndividualUSSSourcesParam - extends FSParam[Boolean]( - name = "twistly_agg_replace_enable_source", - default = false - ) - - object EnableTweetAggSourceParam - extends FSParam[Boolean]( - name = "twistly_agg_tweet_agg_enable_source", - default = false - ) - - object TweetAggTypeParam - extends FSEnumParam[TweetAggregationTypeParam.type]( - name = "twistly_agg_tweet_agg_type_id", - default = TweetAggregationTypeParam.EngagementAggregation, - enum = TweetAggregationTypeParam - ) - - object UnifiedTweetSourceNumberParam - extends FSBoundedParam[Int]( - name = "twistly_agg_tweet_agg_source_number", - default = 0, - min = 0, - max = 100, - ) - - object EnableProducerAggSourceParam - extends FSParam[Boolean]( - name = "twistly_agg_producer_agg_enable_source", - default = false - ) - - object ProducerAggTypeParam - extends FSEnumParam[ProducerAggregationTypeParam.type]( - name = "twistly_agg_producer_agg_type_id", - default = ProducerAggregationTypeParam.EngagementAggregation, - enum = ProducerAggregationTypeParam - ) - - object UnifiedProducerSourceNumberParam - extends FSBoundedParam[Int]( - name = "twistly_agg_producer_agg_source_number", - default = 0, - min = 0, - max = 100, - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableTweetAggSourceParam, - EnableProducerAggSourceParam, - TweetAggTypeParam, - ProducerAggTypeParam, - UnifiedTweetSourceNumberParam, - UnifiedProducerSourceNumberParam, - ReplaceIndividualUSSSourcesParam - ) - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableTweetAggSourceParam, - EnableProducerAggSourceParam, - ReplaceIndividualUSSSourcesParam, - ) - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - UnifiedProducerSourceNumberParam, - UnifiedTweetSourceNumberParam) - val enumOverrides = FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - TweetAggTypeParam, - ProducerAggTypeParam - ) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(intOverrides: _*) - .set(enumOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UtegTweetGlobalParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UtegTweetGlobalParams.docx new file mode 100644 index 000000000..39715bfe4 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UtegTweetGlobalParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UtegTweetGlobalParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UtegTweetGlobalParams.scala deleted file mode 100644 index 29f5a7818..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/UtegTweetGlobalParams.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.Param -import com.twitter.util.Duration - -object UtegTweetGlobalParams { - - object MaxUtegCandidatesToRequestParam - extends FSBoundedParam[Int]( - name = "max_uteg_candidates_to_request", - default = 800, - min = 10, - max = 200 - ) - - object CandidateRefreshSinceTimeOffsetHoursParam - extends FSBoundedParam[Duration]( - name = "candidate_refresh_since_time_offset_hours", - default = 48.hours, - min = 1.hours, - max = 96.hours - ) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromHours - } - - object EnableTLRHealthFilterParam - extends FSParam[Boolean]( - name = "enable_uteg_tlr_health_filter", - default = true - ) - - object EnableRepliesToNonFollowedUsersFilterParam - extends FSParam[Boolean]( - name = "enable_uteg_replies_to_non_followed_users_filter", - default = false - ) - - object EnableRetweetFilterParam - extends FSParam[Boolean]( - name = "enable_uteg_retweet_filter", - default = true - ) - - object EnableInNetworkFilterParam - extends FSParam[Boolean]( - name = "enable_uteg_in_network_filter", - default = true - ) - - val AllParams: Seq[Param[_] with FSName] = - Seq( - MaxUtegCandidatesToRequestParam, - CandidateRefreshSinceTimeOffsetHoursParam, - EnableTLRHealthFilterParam, - EnableRepliesToNonFollowedUsersFilterParam, - EnableRetweetFilterParam, - EnableInNetworkFilterParam - ) - - lazy val config: BaseConfig = { - - val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides( - MaxUtegCandidatesToRequestParam - ) - - val durationFSOverrides = - FeatureSwitchOverrideUtil.getDurationFSOverrides( - CandidateRefreshSinceTimeOffsetHoursParam - ) - - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableTLRHealthFilterParam, - EnableRepliesToNonFollowedUsersFilterParam, - EnableRetweetFilterParam, - EnableInNetworkFilterParam - ) - - BaseConfigBuilder() - .set(intOverrides: _*) - .set(durationFSOverrides: _*) - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoTweetFilterParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoTweetFilterParams.docx new file mode 100644 index 000000000..84071a596 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoTweetFilterParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoTweetFilterParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoTweetFilterParams.scala deleted file mode 100644 index 3a93d0a1a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoTweetFilterParams.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Param - -object VideoTweetFilterParams { - - object EnableVideoTweetFilterParam - extends FSParam[Boolean]( - name = "video_tweet_filter_enable_filter", - default = false - ) - - val AllParams: Seq[Param[_] with FSName] = Seq( - EnableVideoTweetFilterParam - ) - - lazy val config: BaseConfig = { - - val booleanOverrides = - FeatureSwitchOverrideUtil.getBooleanFSOverrides(EnableVideoTweetFilterParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .build() - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoViewTweetsParams.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoViewTweetsParams.docx new file mode 100644 index 000000000..ce4f555fb Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoViewTweetsParams.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoViewTweetsParams.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoViewTweetsParams.scala deleted file mode 100644 index 44f508d89..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/VideoViewTweetsParams.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.cr_mixer.param - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.Logger -import com.twitter.timelines.configapi.BaseConfig -import com.twitter.timelines.configapi.BaseConfigBuilder -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSName -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil -import com.twitter.timelines.configapi.Param -import com.twitter.usersignalservice.thriftscala.SignalType - -object VideoViewTweetsParams { - object EnableSourceParam - extends FSParam[Boolean]( - name = "signal_videoviewtweets_enable_source", - default = false - ) - - object EnableSourceImpressionParam - extends FSParam[Boolean]( - name = "signal_videoviewtweets_enableimpression_source", - default = false - ) - - object VideoViewTweetType extends Enumeration { - protected case class SignalTypeValue(signalType: SignalType) extends super.Val - import scala.language.implicitConversions - implicit def valueToSignalTypeValue(x: Value): SignalTypeValue = - x.asInstanceOf[SignalTypeValue] - - val VideoTweetQualityView: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dQualityV1) - val VideoTweetPlayback50: SignalTypeValue = SignalTypeValue(SignalType.VideoView90dPlayback50V1) - } - - object VideoViewTweetTypeParam - extends FSEnumParam[VideoViewTweetType.type]( - name = "signal_videoviewtweets_videoviewtype_id", - default = VideoViewTweetType.VideoTweetQualityView, - enum = VideoViewTweetType - ) - - val AllParams: Seq[Param[_] with FSName] = - Seq(EnableSourceParam, EnableSourceImpressionParam, VideoViewTweetTypeParam) - - lazy val config: BaseConfig = { - val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides( - EnableSourceParam, - EnableSourceImpressionParam, - ) - val enumOverrides = - FeatureSwitchOverrideUtil.getEnumFSOverrides( - NullStatsReceiver, - Logger(getClass), - VideoViewTweetTypeParam) - - BaseConfigBuilder() - .set(booleanOverrides: _*) - .set(enumOverrides: _*) - .build() - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/BUILD deleted file mode 100644 index 730986d64..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "decider/src/main/scala", - "finagle/finagle-base-http/src/main", - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "servo/decider", - "src/scala/com/twitter/simclusters_v2/common", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/BUILD.docx new file mode 100644 index 000000000..b7c5b8501 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/CrMixerDecider.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/CrMixerDecider.docx new file mode 100644 index 000000000..06b72b786 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/CrMixerDecider.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/CrMixerDecider.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/CrMixerDecider.scala deleted file mode 100644 index 8c909ca05..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/CrMixerDecider.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.cr_mixer.param.decider - -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.decider.Recipient -import com.twitter.decider.SimpleRecipient -import com.twitter.simclusters_v2.common.DeciderGateBuilderWithIdHashing -import javax.inject.Inject - -case class CrMixerDecider @Inject() (decider: Decider) { - - def isAvailable(feature: String, recipient: Option[Recipient]): Boolean = { - decider.isAvailable(feature, recipient) - } - - lazy val deciderGateBuilder = new DeciderGateBuilderWithIdHashing(decider) - - /** - * When useRandomRecipient is set to false, the decider is either completely on or off. - * When useRandomRecipient is set to true, the decider is on for the specified % of traffic. - */ - def isAvailable(feature: String, useRandomRecipient: Boolean = true): Boolean = { - if (useRandomRecipient) isAvailable(feature, Some(RandomRecipient)) - else isAvailable(feature, None) - } - - /*** - * Decide whether the decider is available for a specific id using SimpleRecipient(id). - */ - def isAvailableForId( - id: Long, - deciderConstants: String - ): Boolean = { - // Note: SimpleRecipient does expose a `val isUser = true` field which is not correct if the Id is not a user Id. - // However this field does not appear to be used anywhere in source. - decider.isAvailable(deciderConstants, Some(SimpleRecipient(id))) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.docx new file mode 100644 index 000000000..5b1618bf4 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.scala deleted file mode 100644 index 518ea53db..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.scala +++ /dev/null @@ -1,67 +0,0 @@ -package com.twitter.cr_mixer.param.decider - -import com.twitter.servo.decider.DeciderKeyEnum - -object DeciderConstants { - val enableHealthSignalsScoreDeciderKey = "enable_tweet_health_score" - val enableUTGRealTimeTweetEngagementScoreDeciderKey = "enable_utg_realtime_tweet_engagement_score" - val enableUserAgathaScoreDeciderKey = "enable_user_agatha_score" - val enableUserTweetEntityGraphTrafficDeciderKey = "enable_user_tweet_entity_graph_traffic" - val enableUserTweetGraphTrafficDeciderKey = "enable_user_tweet_graph_traffic" - val enableUserVideoGraphTrafficDeciderKey = "enable_user_video_graph_traffic" - val enableUserAdGraphTrafficDeciderKey = "enable_user_ad_graph_traffic" - val enableSimClustersANN2DarkTrafficDeciderKey = "enable_simclusters_ann_2_dark_traffic" - val enableQigSimilarTweetsTrafficDeciderKey = "enable_qig_similar_tweets_traffic" - val enableFRSTrafficDeciderKey = "enable_frs_traffic" - val upperFunnelPerStepScribeRate = "upper_funnel_per_step_scribe_rate" - val kafkaMessageScribeSampleRate = "kafka_message_scribe_sample_rate" - val enableRealGraphMhStoreDeciderKey = "enable_real_graph_mh_store" - val topLevelApiDdgMetricsScribeRate = "top_level_api_ddg_metrics_scribe_rate" - val adsRecommendationsPerExperimentScribeRate = "ads_recommendations_per_experiment_scribe_rate" - val enableScribeForBlueVerifiedTweetCandidates = - "enable_scribe_for_blue_verified_tweet_candidates" - - val enableUserStateStoreDeciderKey = "enable_user_state_store" - val enableUserMediaRepresentationStoreDeciderKey = - "enable_user_media_representation_store" - val enableMagicRecsRealTimeAggregatesStoreDeciderKey = - "enable_magic_recs_real_time_aggregates_store" - - val enableEarlybirdTrafficDeciderKey = "enable_earlybird_traffic" - - val enableTopicTweetTrafficDeciderKey = "enable_topic_tweet_traffic" - - val getTweetRecommendationsCacheRate = "get_tweet_recommendations_cache_rate" -} - -object DeciderKey extends DeciderKeyEnum { - - val enableHealthSignalsScoreDeciderKey: Value = Value( - DeciderConstants.enableHealthSignalsScoreDeciderKey - ) - - val enableUtgRealTimeTweetEngagementScoreDeciderKey: Value = Value( - DeciderConstants.enableUTGRealTimeTweetEngagementScoreDeciderKey - ) - val enableUserAgathaScoreDeciderKey: Value = Value( - DeciderConstants.enableUserAgathaScoreDeciderKey - ) - val enableUserMediaRepresentationStoreDeciderKey: Value = Value( - DeciderConstants.enableUserMediaRepresentationStoreDeciderKey - ) - - val enableMagicRecsRealTimeAggregatesStore: Value = Value( - DeciderConstants.enableMagicRecsRealTimeAggregatesStoreDeciderKey - ) - - val enableUserStateStoreDeciderKey: Value = Value( - DeciderConstants.enableUserStateStoreDeciderKey - ) - - val enableRealGraphMhStoreDeciderKey: Value = Value( - DeciderConstants.enableRealGraphMhStoreDeciderKey - ) - - val enableEarlybirdTrafficDeciderKey: Value = Value( - DeciderConstants.enableEarlybirdTrafficDeciderKey) -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/EndpointLoadShedder.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/EndpointLoadShedder.docx new file mode 100644 index 000000000..7c9197749 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/EndpointLoadShedder.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/EndpointLoadShedder.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/EndpointLoadShedder.scala deleted file mode 100644 index a53e629a9..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/EndpointLoadShedder.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.cr_mixer.param.decider - -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Future -import javax.inject.Inject -import scala.util.control.NoStackTrace - -/* - Provides deciders-controlled load shedding for a given Product from a given endpoint. - The format of the decider keys is: - - enable_loadshedding__ - E.g.: - enable_loadshedding_getTweetRecommendations_Notifications - - Deciders are fractional, so a value of 50.00 will drop 50% of responses. If a decider key is not - defined for a particular endpoint/product combination, those requests will always be - served. - - We should therefore aim to define keys for the endpoints/product we care most about in decider.yml, - so that we can control them during incidents. - */ -case class EndpointLoadShedder @Inject() ( - decider: Decider, - statsReceiver: StatsReceiver) { - import EndpointLoadShedder._ - - // Fall back to False for any undefined key - private val deciderWithFalseFallback: Decider = decider.orElse(Decider.False) - private val keyPrefix = "enable_loadshedding" - private val scopedStats = statsReceiver.scope("EndpointLoadShedder") - - def apply[T](endpointName: String, product: String)(serve: => Future[T]): Future[T] = { - /* - Checks if either per-product or top-level load shedding is enabled - If both are enabled at different percentages, load shedding will not be perfectly calculable due - to salting of hash (i.e. 25% load shed for Product x + 25% load shed for overall does not - result in 50% load shed for x) - */ - val keyTyped = s"${keyPrefix}_${endpointName}_$product" - val keyTopLevel = s"${keyPrefix}_${endpointName}" - - if (deciderWithFalseFallback.isAvailable(keyTopLevel, recipient = Some(RandomRecipient))) { - scopedStats.counter(keyTopLevel).incr - Future.exception(LoadSheddingException) - } else if (deciderWithFalseFallback.isAvailable(keyTyped, recipient = Some(RandomRecipient))) { - scopedStats.counter(keyTyped).incr - Future.exception(LoadSheddingException) - } else serve - } -} - -object EndpointLoadShedder { - object LoadSheddingException extends Exception with NoStackTrace -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/BUILD deleted file mode 100644 index 139ecd4c7..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "content-recommender/thrift/src/main/thrift:content-recommender-common-scala", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "decider/src/main/scala", - "frigate/frigate-common:base", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util:stats_util", - "hydra/common/libraries/src/main/scala/com/twitter/hydra/common/model_config", - "hydra/partition/thrift/src/main/thrift:thrift-scala", - "hydra/root/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/BUILD.docx new file mode 100644 index 000000000..fda46de2b Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/DefaultRanker.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/DefaultRanker.docx new file mode 100644 index 000000000..48667964a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/DefaultRanker.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/DefaultRanker.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/DefaultRanker.scala deleted file mode 100644 index 2ae91642b..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/DefaultRanker.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.cr_mixer.ranker - -import com.twitter.cr_mixer.model.BlendedCandidate -import com.twitter.cr_mixer.model.RankedCandidate -import com.twitter.util.Future -import javax.inject.Singleton - -/** - * Keep the same order as the input. - */ -@Singleton -class DefaultRanker() { - def rank( - candidates: Seq[BlendedCandidate], - ): Future[Seq[RankedCandidate]] = { - val candidateSize = candidates.size - val rankedCandidates = candidates.zipWithIndex.map { - case (candidate, index) => - candidate.toRankedCandidate((candidateSize - index).toDouble) - } - Future.value(rankedCandidates) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/SwitchRanker.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/SwitchRanker.docx new file mode 100644 index 000000000..6306c2626 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/SwitchRanker.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/SwitchRanker.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/SwitchRanker.scala deleted file mode 100644 index da44f664e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/ranker/SwitchRanker.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.cr_mixer.ranker - -import com.twitter.cr_mixer.model.BlendedCandidate -import com.twitter.cr_mixer.model.CrCandidateGeneratorQuery -import com.twitter.cr_mixer.model.RankedCandidate -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Future -import com.twitter.util.JavaTimer -import com.twitter.util.Time -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Singleton - -/** - * CR-Mixer internal ranker - */ -@Singleton -class SwitchRanker @Inject() ( - defaultRanker: DefaultRanker, - globalStats: StatsReceiver) { - private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName) - implicit val timer: Timer = new JavaTimer(true) - - def rank( - query: CrCandidateGeneratorQuery, - candidates: Seq[BlendedCandidate], - ): Future[Seq[RankedCandidate]] = { - defaultRanker.rank(candidates) - } - -} - -object SwitchRanker { - - /** Prefers candidates generated from sources with the latest timestamps. - * The newer the source signal, the higher a candidate ranks. - * This ordering biases against consumer-based candidates because their timestamp defaults to 0 - */ - val TimestampOrder: Ordering[RankedCandidate] = - math.Ordering - .by[RankedCandidate, Time]( - _.reasonChosen.sourceInfoOpt - .flatMap(_.sourceEventTime) - .getOrElse(Time.fromMilliseconds(0L))) - .reverse -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/BUILD deleted file mode 100644 index 8e6ae5049..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "decider/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle/finagle-core/src/main", - "frigate/frigate-common:base", - "frigate/frigate-common:util", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "scrooge/scrooge-serializer/src/main/scala", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "util-internal/scribe/src/main/scala/com/twitter/logging", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/BUILD.docx new file mode 100644 index 000000000..0a1233247 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/ScribeCategory.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/ScribeCategory.docx new file mode 100644 index 000000000..e817f25e4 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/ScribeCategory.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/ScribeCategory.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/ScribeCategory.scala deleted file mode 100644 index b86c9174f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/scribe/ScribeCategory.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.cr_mixer.scribe - -/** - * Categories define scribe categories used in cr-mixer service. - */ -object ScribeCategories { - lazy val AllCategories = - List(AbDecider, TopLevelApiDdgMetrics, TweetsRecs) - - /** - * AbDecider represents scribe logs for experiments - */ - lazy val AbDecider: ScribeCategory = ScribeCategory( - "abdecider_scribe", - "client_event" - ) - - /** - * Top-Level Client event scribe logs, to record changes in system metrics (e.g. latency, - * candidates returned, empty rate ) per experiment bucket, and store them in DDG metric group - */ - lazy val TopLevelApiDdgMetrics: ScribeCategory = ScribeCategory( - "top_level_api_ddg_metrics_scribe", - "client_event" - ) - - lazy val TweetsRecs: ScribeCategory = ScribeCategory( - "get_tweets_recommendations_scribe", - "cr_mixer_get_tweets_recommendations" - ) - - lazy val VITTweetsRecs: ScribeCategory = ScribeCategory( - "get_vit_tweets_recommendations_scribe", - "cr_mixer_get_vit_tweets_recommendations" - ) - - lazy val RelatedTweets: ScribeCategory = ScribeCategory( - "get_related_tweets_scribe", - "cr_mixer_get_related_tweets" - ) - - lazy val UtegTweets: ScribeCategory = ScribeCategory( - "get_uteg_tweets_scribe", - "cr_mixer_get_uteg_tweets" - ) - - lazy val AdsRecommendations: ScribeCategory = ScribeCategory( - "get_ads_recommendations_scribe", - "cr_mixer_get_ads_recommendations" - ) -} - -/** - * Category represents each scribe log data. - * - * @param loggerFactoryNode loggerFactory node name in cr-mixer associated with this scribe category - * @param scribeCategory scribe category name (globally unique at Twitter) - */ -case class ScribeCategory( - loggerFactoryNode: String, - scribeCategory: String) { - def getProdLoggerFactoryNode: String = loggerFactoryNode - def getStagingLoggerFactoryNode: String = "staging_" + loggerFactoryNode -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/BUILD.bazel b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/BUILD.bazel deleted file mode 100644 index 8fa46c772..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - "stitch/stitch-core", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/BUILD.docx new file mode 100644 index 000000000..ad91f85fe Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/CrMixerAlertNotificationConfig.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/CrMixerAlertNotificationConfig.docx new file mode 100644 index 000000000..e31ef578c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/CrMixerAlertNotificationConfig.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/CrMixerAlertNotificationConfig.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/CrMixerAlertNotificationConfig.scala deleted file mode 100644 index df0572ef1..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/service/CrMixerAlertNotificationConfig.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.cr_mixer.service - -import com.twitter.product_mixer.core.functional_component.common.alert.Destination -import com.twitter.product_mixer.core.functional_component.common.alert.NotificationGroup - -/** - * Notifications (email, pagerduty, etc) can be specific per-alert but it is common for multiple - * products to share notification configuration. - * - * Our configuration uses only email notifications because SampleMixer is a demonstration service - * with neither internal nor customer-facing users. You will likely want to use a PagerDuty - * destination instead. For example: - * {{{ - * critical = Destination(pagerDutyKey = Some("your-pagerduty-key")) - * }}} - * - * - * For more information about how to get a PagerDuty key, see: - * https://docbird.twitter.biz/mon/how-to-guides.html?highlight=notificationgroup#set-up-email-pagerduty-and-slack-notifications - */ -object CrMixerAlertNotificationConfig { - val DefaultNotificationGroup: NotificationGroup = NotificationGroup( - warn = Destination(emails = Seq("no-reply@twitter.com")), - critical = Destination(emails = Seq("no-reply@twitter.com")) - ) -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/BUILD deleted file mode 100644 index c7ae7c752..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/BUILD +++ /dev/null @@ -1,74 +0,0 @@ -scala_library( - sources = [ - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/com/twitter/storehaus:memcache", - "3rdparty/jvm/io/grpc:grpc-api", - "3rdparty/jvm/io/grpc:grpc-auth", - "3rdparty/jvm/io/grpc:grpc-core", - "3rdparty/jvm/io/grpc:grpc-netty", - "3rdparty/jvm/io/grpc:grpc-protobuf", - "3rdparty/jvm/io/grpc:grpc-stub", - "3rdparty/jvm/io/opil:tensorflow-serving-client", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "ann/src/main/scala/com/twitter/ann/hnsw", - "ann/src/main/thrift/com/twitter/ann/common:ann-common-scala", - "configapi/configapi-core", - "content-recommender/thrift/src/main/thrift:thrift-scala", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/exception", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/util", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "decider/src/main/scala", - "finagle-internal/finagle-grpc/src/main/scala", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "frigate/frigate-common:base", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util:stats_util", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "mediaservices/commons/src/main/scala:futuretracker", - "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", - "qig-ranker/thrift/src/main/thrift:thrift-scala", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/simclustersann/multicluster", - "simclusters-ann/thrift/src/main/thrift:thrift-scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/search/common/schema/base", - "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/scala/com/twitter/cortex/ml/embeddings/common:Helpers", - "src/scala/com/twitter/ml/featurestore/lib", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/ml/api:embedding-scala", - "src/thrift/com/twitter/recos:recos-common-scala", - "src/thrift/com/twitter/recos/user_ad_graph:user_ad_graph-scala", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/recos/user_tweet_graph:user_tweet_graph-scala", - "src/thrift/com/twitter/recos/user_video_graph:user_video_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:ranking-scala", - "src/thrift/com/twitter/search/query_interaction_graph/service:qig-service-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/topic_recos:topic_recos-thrift-scala", - "src/thrift/com/twitter/trends/trip_v1:trip-tweets-thrift-scala", - "src/thrift/com/twitter/twistly:twistly-scala", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/BUILD.docx new file mode 100644 index 000000000..7c9c89a1b Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/CertoTopicTweetSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/CertoTopicTweetSimilarityEngine.docx new file mode 100644 index 000000000..8df474030 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/CertoTopicTweetSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/CertoTopicTweetSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/CertoTopicTweetSimilarityEngine.scala deleted file mode 100644 index a57085d0f..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/CertoTopicTweetSimilarityEngine.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TopicTweetWithScore -import com.twitter.cr_mixer.param.TopicTweetParams -import com.twitter.cr_mixer.similarity_engine.CertoTopicTweetSimilarityEngine._ -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.topic_recos.thriftscala._ -import com.twitter.util.Future - -@Singleton -case class CertoTopicTweetSimilarityEngine @Inject() ( - @Named(ModuleNames.CertoStratoStoreName) certoStratoStore: ReadableStore[ - TopicId, - Seq[TweetWithScores] - ], - statsReceiver: StatsReceiver) - extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] { - - private val name: String = this.getClass.getSimpleName - private val stats = statsReceiver.scope(name) - - override def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] = { - StatsUtil.trackOptionItemsStats(stats) { - topTweetsByFollowerL2NormalizedScore.get(query).map { - _.map { topicTopTweets => - topicTopTweets.map { topicTweet => - TopicTweetWithScore( - tweetId = topicTweet.tweetId, - score = topicTweet.scores.followerL2NormalizedCosineSimilarity8HrHalfLife, - similarityEngineType = SimilarityEngineType.CertoTopicTweet - ) - } - } - } - } - } - - private val topTweetsByFollowerL2NormalizedScore: ReadableStore[EngineQuery[Query], Seq[ - TweetWithScores - ]] = { - ReadableStore.fromFnFuture { query: EngineQuery[Query] => - StatsUtil.trackOptionItemsStats(stats) { - for { - topKTweetsWithScores <- certoStratoStore.get(query.storeQuery.topicId) - } yield { - topKTweetsWithScores.map( - _.filter( - _.scores.followerL2NormalizedCosineSimilarity8HrHalfLife >= query.storeQuery.certoScoreTheshold) - .take(query.storeQuery.maxCandidates)) - } - } - } - } -} - -object CertoTopicTweetSimilarityEngine { - - // Query is used as a cache key. Do not add any user level information in this. - case class Query( - topicId: TopicId, - maxCandidates: Int, - certoScoreTheshold: Double) - - def fromParams( - topicId: TopicId, - isVideoOnly: Boolean, - params: configapi.Params, - ): EngineQuery[Query] = { - - val maxCandidates = if (isVideoOnly) { - params(TopicTweetParams.MaxCertoCandidatesParam) * 2 - } else { - params(TopicTweetParams.MaxCertoCandidatesParam) - } - - EngineQuery( - Query( - topicId = topicId, - maxCandidates = maxCandidates, - certoScoreTheshold = params(TopicTweetParams.CertoScoreThresholdParam) - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.docx new file mode 100644 index 000000000..e7f70625e Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.scala deleted file mode 100644 index 599704fa7..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.scala +++ /dev/null @@ -1,246 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.SourceInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.ConsumerBasedWalsParams -import com.twitter.cr_mixer.similarity_engine.ConsumerBasedWalsSimilarityEngine.Query -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.cr_mixer.thriftscala.SourceType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Future -import io.grpc.ManagedChannel -import tensorflow.serving.Predict.PredictRequest -import tensorflow.serving.Predict.PredictResponse -import tensorflow.serving.PredictionServiceGrpc -import org.tensorflow.example.Feature -import org.tensorflow.example.Int64List -import org.tensorflow.example.FloatList -import org.tensorflow.example.Features -import org.tensorflow.example.Example -import tensorflow.serving.Model -import org.tensorflow.framework.TensorProto -import org.tensorflow.framework.DataType -import org.tensorflow.framework.TensorShapeProto -import com.twitter.finagle.grpc.FutureConverters -import java.util.ArrayList -import java.lang -import com.twitter.util.Return -import com.twitter.util.Throw -import java.util.concurrent.ConcurrentHashMap -import scala.jdk.CollectionConverters._ - -// Stats object maintain a set of stats that are specific to the Wals Engine. -case class WalsStats(scope: String, scopedStats: StatsReceiver) { - - val requestStat = scopedStats.scope(scope) - val inputSignalSize = requestStat.stat("input_signal_size") - - val latency = requestStat.stat("latency_ms") - val latencyOnError = requestStat.stat("error_latency_ms") - val latencyOnSuccess = requestStat.stat("success_latency_ms") - - val requests = requestStat.counter("requests") - val success = requestStat.counter("success") - val failures = requestStat.scope("failures") - - def onFailure(t: Throwable, startTimeMs: Long) { - val duration = System.currentTimeMillis() - startTimeMs - latency.add(duration) - latencyOnError.add(duration) - failures.counter(t.getClass.getName).incr() - } - - def onSuccess(startTimeMs: Long) { - val duration = System.currentTimeMillis() - startTimeMs - latency.add(duration) - latencyOnSuccess.add(duration) - success.incr() - } -} - -// StatsMap maintains a mapping from Model's input signature to a stats receiver -// The Wals model suports multiple input signature which can run different graphs internally and -// can have a different performance profile. -// Invoking StatsReceiver.stat() on each request can create a new stat object and can be expensive -// in performance critical paths. -object WalsStatsMap { - val mapping = new ConcurrentHashMap[String, WalsStats]() - - def get(scope: String, scopedStats: StatsReceiver): WalsStats = { - mapping.computeIfAbsent(scope, (scope) => WalsStats(scope, scopedStats)) - } -} - -case class ConsumerBasedWalsSimilarityEngine( - homeNaviGRPCClient: ManagedChannel, - adsFavedNaviGRPCClient: ManagedChannel, - adsMonetizableNaviGRPCClient: ManagedChannel, - statsReceiver: StatsReceiver) - extends ReadableStore[ - Query, - Seq[TweetWithScore] - ] { - - override def get( - query: ConsumerBasedWalsSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - val startTimeMs = System.currentTimeMillis() - val stats = - WalsStatsMap.get( - query.wilyNsName + "/" + query.modelSignatureName, - statsReceiver.scope("NaviPredictionService") - ) - stats.requests.incr() - stats.inputSignalSize.add(query.sourceIds.size) - try { - // avoid inference calls is source signals are empty - if (query.sourceIds.isEmpty) { - Future.value(Some(Seq.empty)) - } else { - val grpcClient = query.wilyNsName match { - case "navi-wals-recommended-tweets-home-client" => homeNaviGRPCClient - case "navi-wals-ads-faved-tweets" => adsFavedNaviGRPCClient - case "navi-wals-ads-monetizable-tweets" => adsFavedNaviGRPCClient - // default to homeNaviGRPCClient - case _ => homeNaviGRPCClient - } - val stub = PredictionServiceGrpc.newFutureStub(grpcClient) - val inferRequest = getModelInput(query) - - FutureConverters - .RichListenableFuture(stub.predict(inferRequest)).toTwitter - .transform { - case Return(resp) => - stats.onSuccess(startTimeMs) - Future.value(Some(getModelOutput(query, resp))) - case Throw(e) => - stats.onFailure(e, startTimeMs) - Future.exception(e) - } - } - } catch { - case e: Throwable => Future.exception(e) - } - } - - def getFeaturesForRecommendations(query: ConsumerBasedWalsSimilarityEngine.Query): Example = { - val tweetIds = new ArrayList[lang.Long]() - val tweetFaveWeight = new ArrayList[lang.Float]() - - query.sourceIds.foreach { sourceInfo => - val weight = sourceInfo.sourceType match { - case SourceType.TweetFavorite | SourceType.Retweet => 1.0f - // currently no-op - as we do not get negative signals - case SourceType.TweetDontLike | SourceType.TweetReport | SourceType.AccountMute | - SourceType.AccountBlock => - 0.0f - case _ => 0.0f - } - sourceInfo.internalId match { - case InternalId.TweetId(tweetId) => - tweetIds.add(tweetId) - tweetFaveWeight.add(weight) - case _ => - throw new IllegalArgumentException( - s"Invalid InternalID - does not contain TweetId for Source Signal: ${sourceInfo}") - } - } - - val tweetIdsFeature = - Feature - .newBuilder().setInt64List( - Int64List - .newBuilder().addAllValue(tweetIds).build() - ).build() - - val tweetWeightsFeature = Feature - .newBuilder().setFloatList( - FloatList.newBuilder().addAllValue(tweetFaveWeight).build()).build() - - val features = Features - .newBuilder() - .putFeature("tweet_ids", tweetIdsFeature) - .putFeature("tweet_weights", tweetWeightsFeature) - .build() - Example.newBuilder().setFeatures(features).build() - } - - def getModelInput(query: ConsumerBasedWalsSimilarityEngine.Query): PredictRequest = { - val tfExample = getFeaturesForRecommendations(query) - - val inferenceRequest = PredictRequest - .newBuilder() - .setModelSpec( - Model.ModelSpec - .newBuilder() - .setName(query.modelName) - .setSignatureName(query.modelSignatureName)) - .putInputs( - query.modelInputName, - TensorProto - .newBuilder() - .setDtype(DataType.DT_STRING) - .setTensorShape(TensorShapeProto - .newBuilder() - .addDim(TensorShapeProto.Dim.newBuilder().setSize(1))) - .addStringVal(tfExample.toByteString) - .build() - ).build() - inferenceRequest - } - - def getModelOutput(query: Query, response: PredictResponse): Seq[TweetWithScore] = { - val outputName = query.modelOutputName - if (response.containsOutputs(outputName)) { - val tweetList = response.getOutputsMap - .get(outputName) - .getInt64ValList.asScala - tweetList.zip(tweetList.size to 1 by -1).map { (tweetWithScore) => - TweetWithScore(tweetWithScore._1, tweetWithScore._2.toLong) - } - } else { - Seq.empty - } - } -} - -object ConsumerBasedWalsSimilarityEngine { - case class Query( - sourceIds: Seq[SourceInfo], - modelName: String, - modelInputName: String, - modelOutputName: String, - modelSignatureName: String, - wilyNsName: String, - ) - - def fromParams( - sourceIds: Seq[SourceInfo], - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query( - sourceIds, - params(ConsumerBasedWalsParams.ModelNameParam), - params(ConsumerBasedWalsParams.ModelInputNameParam), - params(ConsumerBasedWalsParams.ModelOutputNameParam), - params(ConsumerBasedWalsParams.ModelSignatureNameParam), - params(ConsumerBasedWalsParams.WilyNsNameParam), - ), - params - ) - } - - def toSimilarityEngineInfo( - score: Double - ): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.ConsumerBasedWalsANN, - modelId = None, - score = Some(score)) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngine.docx new file mode 100644 index 000000000..1f5b66f77 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngine.scala deleted file mode 100644 index 82a074208..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTripSimilarityEngine.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.TripTweetWithScore -import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedTripParams -import com.twitter.cr_mixer.util.InterleaveUtil -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.common.ClusterId -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.timelines.configapi.Params -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.Cluster -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.ClusterDomain -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripTweet -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import com.twitter.util.Future - -case class TripEngineQuery( - modelId: String, - sourceId: InternalId, - tripSourceId: String, - maxResult: Int, - params: Params) - -case class ConsumerEmbeddingBasedTripSimilarityEngine( - embeddingStoreLookUpMap: Map[String, ReadableStore[UserId, SimClustersEmbedding]], - tripCandidateSource: ReadableStore[TripDomain, Seq[TripTweet]], - statsReceiver: StatsReceiver, -) extends ReadableStore[TripEngineQuery, Seq[TripTweetWithScore]] { - import ConsumerEmbeddingBasedTripSimilarityEngine._ - - private val scopedStats = statsReceiver.scope(name) - private def fetchTopClusters(query: TripEngineQuery): Future[Option[Seq[ClusterId]]] = { - query.sourceId match { - case InternalId.UserId(userId) => - val embeddingStore = embeddingStoreLookUpMap.getOrElse( - query.modelId, - throw new IllegalArgumentException( - s"${this.getClass.getSimpleName}: " + - s"ModelId ${query.modelId} does not exist for embeddingStore" - ) - ) - embeddingStore.get(userId).map(_.map(_.topClusterIds(MaxClusters))) - case _ => - Future.None - } - } - private def fetchCandidates( - topClusters: Seq[ClusterId], - tripSourceId: String - ): Future[Seq[Seq[TripTweetWithScore]]] = { - Future - .collect { - topClusters.map { clusterId => - tripCandidateSource - .get( - TripDomain( - sourceId = tripSourceId, - clusterDomain = Some( - ClusterDomain(simCluster = Some(Cluster(clusterIntId = Some(clusterId))))))).map { - _.map { - _.collect { - case TripTweet(tweetId, score) => - TripTweetWithScore(tweetId, score) - } - }.getOrElse(Seq.empty).take(MaxNumResultsPerCluster) - } - } - } - } - - override def get(engineQuery: TripEngineQuery): Future[Option[Seq[TripTweetWithScore]]] = { - val fetchTopClustersStat = scopedStats.scope(engineQuery.modelId).scope("fetchTopClusters") - val fetchCandidatesStat = scopedStats.scope(engineQuery.modelId).scope("fetchCandidates") - - for { - topClustersOpt <- StatsUtil.trackOptionStats(fetchTopClustersStat) { - fetchTopClusters(engineQuery) - } - candidates <- StatsUtil.trackItemsStats(fetchCandidatesStat) { - topClustersOpt match { - case Some(topClusters) => fetchCandidates(topClusters, engineQuery.tripSourceId) - case None => Future.Nil - } - } - } yield { - val interleavedTweets = InterleaveUtil.interleave(candidates) - val dedupCandidates = interleavedTweets - .groupBy(_.tweetId).flatMap { - case (_, tweetWithScoreSeq) => tweetWithScoreSeq.sortBy(-_.score).take(1) - }.toSeq.take(engineQuery.maxResult) - Some(dedupCandidates) - } - } -} - -object ConsumerEmbeddingBasedTripSimilarityEngine { - private val MaxClusters: Int = 8 - private val MaxNumResultsPerCluster: Int = 25 - private val name: String = this.getClass.getSimpleName - - def fromParams( - modelId: String, - sourceId: InternalId, - params: configapi.Params - ): TripEngineQuery = { - TripEngineQuery( - modelId = modelId, - sourceId = sourceId, - tripSourceId = params(ConsumerEmbeddingBasedTripParams.SourceIdParam), - maxResult = params(ConsumerEmbeddingBasedTripParams.MaxNumCandidatesParam), - params = params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngine.docx new file mode 100644 index 000000000..59200da7f Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngine.scala deleted file mode 100644 index ed722f3eb..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwHINSimilarityEngine.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedTwHINParams -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.timelines.configapi - -object ConsumerEmbeddingBasedTwHINSimilarityEngine { - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): HnswANNEngineQuery = { - HnswANNEngineQuery( - sourceId = sourceId, - modelId = params(ConsumerEmbeddingBasedTwHINParams.ModelIdParam), - params = params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngine.docx new file mode 100644 index 000000000..fa5a72efa Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngine.scala deleted file mode 100644 index c63d517d6..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerEmbeddingBasedTwoTowerSimilarityEngine.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.param.ConsumerEmbeddingBasedTwoTowerParams -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.timelines.configapi - -object ConsumerEmbeddingBasedTwoTowerSimilarityEngine { - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): HnswANNEngineQuery = { - HnswANNEngineQuery( - sourceId = sourceId, - modelId = params(ConsumerEmbeddingBasedTwoTowerParams.ModelIdParam), - params = params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngine.docx new file mode 100644 index 000000000..9330ad811 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngine.scala deleted file mode 100644 index 585edc584..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserAdGraphSimilarityEngine.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.ConsumersBasedUserAdGraphParams -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest -import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Future -import javax.inject.Singleton - -/** - * This store uses the graph based input (a list of userIds) - * to query consumersBasedUserAdGraph and get their top engaged ad tweets - */ -@Singleton -case class ConsumersBasedUserAdGraphSimilarityEngine( - consumersBasedUserAdGraphStore: ReadableStore[ - ConsumersBasedRelatedAdRequest, - RelatedAdResponse - ], - statsReceiver: StatsReceiver) - extends ReadableStore[ - ConsumersBasedUserAdGraphSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - override def get( - query: ConsumersBasedUserAdGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - val consumersBasedRelatedAdRequest = - ConsumersBasedRelatedAdRequest( - query.seedWithScores.keySet.toSeq, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - minScore = Some(query.minScore), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours) - ) - consumersBasedUserAdGraphStore - .get(consumersBasedRelatedAdRequest) - .map { relatedAdResponseOpt => - relatedAdResponseOpt.map { relatedAdResponse => - relatedAdResponse.adTweets.map { tweet => - TweetWithScore(tweet.adTweetId, tweet.score) - } - } - } - } -} - -object ConsumersBasedUserAdGraphSimilarityEngine { - - case class Query( - seedWithScores: Map[UserId, Double], - maxResults: Int, - minCooccurrence: Int, - minScore: Double, - maxTweetAgeInHours: Int) - - def toSimilarityEngineInfo( - score: Double - ): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.ConsumersBasedUserAdGraph, - modelId = None, - score = Some(score)) - } - - def fromParams( - seedWithScores: Map[UserId, Double], - params: configapi.Params, - ): EngineQuery[Query] = { - - EngineQuery( - Query( - seedWithScores = seedWithScores, - maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), - minCooccurrence = params(ConsumersBasedUserAdGraphParams.MinCoOccurrenceParam), - minScore = params(ConsumersBasedUserAdGraphParams.MinScoreParam), - maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngine.docx new file mode 100644 index 000000000..b4e4328d9 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngine.scala deleted file mode 100644 index 633f5ee6d..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumersBasedUserVideoGraphSimilarityEngine.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.ConsumersBasedUserVideoGraphParams -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest -import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Future -import javax.inject.Singleton - -/** - * This store uses the graph based input (a list of userIds) - * to query consumersBasedUserVideoGraph and get their top engaged tweets - */ -@Singleton -case class ConsumersBasedUserVideoGraphSimilarityEngine( - consumersBasedUserVideoGraphStore: ReadableStore[ - ConsumersBasedRelatedTweetRequest, - RelatedTweetResponse - ], - statsReceiver: StatsReceiver) - extends ReadableStore[ - ConsumersBasedUserVideoGraphSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - override def get( - query: ConsumersBasedUserVideoGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - val consumersBasedRelatedTweetRequest = - ConsumersBasedRelatedTweetRequest( - query.seedWithScores.keySet.toSeq, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - minScore = Some(query.minScore), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours) - ) - consumersBasedUserVideoGraphStore - .get(consumersBasedRelatedTweetRequest) - .map { relatedTweetResponseOpt => - relatedTweetResponseOpt.map { relatedTweetResponse => - relatedTweetResponse.tweets.map { tweet => - TweetWithScore(tweet.tweetId, tweet.score) - } - } - } - } -} - -object ConsumersBasedUserVideoGraphSimilarityEngine { - - case class Query( - seedWithScores: Map[UserId, Double], - maxResults: Int, - minCooccurrence: Int, - minScore: Double, - maxTweetAgeInHours: Int) - - def toSimilarityEngineInfo( - score: Double - ): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.ConsumersBasedUserVideoGraph, - modelId = None, - score = Some(score)) - } - - def fromParamsForRealGraphIn( - seedWithScores: Map[UserId, Double], - params: configapi.Params, - ): EngineQuery[Query] = { - - EngineQuery( - Query( - seedWithScores = seedWithScores, - maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), - minCooccurrence = - params(ConsumersBasedUserVideoGraphParams.RealGraphInMinCoOccurrenceParam), - minScore = params(ConsumersBasedUserVideoGraphParams.RealGraphInMinScoreParam), - maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/DiffusionBasedSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/DiffusionBasedSimilarityEngine.docx new file mode 100644 index 000000000..cf884c5aa Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/DiffusionBasedSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/DiffusionBasedSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/DiffusionBasedSimilarityEngine.scala deleted file mode 100644 index a1bc0e248..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/DiffusionBasedSimilarityEngine.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.simclusters_v2.thriftscala.TweetsWithScore -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Future -import javax.inject.Singleton - -@Singleton -case class DiffusionBasedSimilarityEngine( - retweetBasedDiffusionRecsMhStore: ReadableStore[Long, TweetsWithScore], - statsReceiver: StatsReceiver) - extends ReadableStore[ - DiffusionBasedSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - override def get( - query: DiffusionBasedSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - - query.sourceId match { - case InternalId.UserId(userId) => - retweetBasedDiffusionRecsMhStore.get(userId).map { - _.map { tweetsWithScore => - { - tweetsWithScore.tweets - .map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) - } - } - } - case _ => - Future.None - } - } -} - -object DiffusionBasedSimilarityEngine { - - val defaultScore: Double = 0.0 - - case class Query( - sourceId: InternalId, - ) - - def toSimilarityEngineInfo( - query: LookupEngineQuery[Query], - score: Double - ): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.DiffusionBasedTweet, - modelId = Some(query.lookupKey), - score = Some(score)) - } - - def fromParams( - sourceId: InternalId, - modelId: String, - params: configapi.Params, - ): LookupEngineQuery[Query] = { - LookupEngineQuery( - Query(sourceId = sourceId), - modelId, - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdModelBasedSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdModelBasedSimilarityEngine.docx new file mode 100644 index 000000000..f5179553b Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdModelBasedSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdModelBasedSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdModelBasedSimilarityEngine.scala deleted file mode 100644 index da82b4eb1..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdModelBasedSimilarityEngine.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery -import com.twitter.cr_mixer.similarity_engine.EarlybirdSimilarityEngineBase._ -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.EarlybirdClientId -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.FacetsToFetch -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.MetadataOptions -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.search.common.ranking.thriftscala.ThriftRankingParams -import com.twitter.search.common.ranking.thriftscala.ThriftScoringFunctionType -import com.twitter.search.earlybird.thriftscala.EarlybirdRequest -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.search.earlybird.thriftscala.ThriftSearchQuery -import com.twitter.search.earlybird.thriftscala.ThriftSearchRankingMode -import com.twitter.search.earlybird.thriftscala.ThriftSearchRelevanceOptions -import com.twitter.simclusters_v2.common.UserId -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class EarlybirdModelBasedSimilarityEngine @Inject() ( - earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, - timeoutConfig: TimeoutConfig, - stats: StatsReceiver) - extends EarlybirdSimilarityEngineBase[EarlybirdModelBasedSearchQuery] { - import EarlybirdModelBasedSimilarityEngine._ - override val statsReceiver: StatsReceiver = stats.scope(this.getClass.getSimpleName) - override def getEarlybirdRequest( - query: EarlybirdModelBasedSearchQuery - ): Option[EarlybirdRequest] = - if (query.seedUserIds.nonEmpty) - Some( - EarlybirdRequest( - searchQuery = getThriftSearchQuery(query), - clientId = Some(EarlybirdClientId), - timeoutMs = timeoutConfig.earlybirdServerTimeout.inMilliseconds.intValue(), - clientRequestID = Some(s"${Trace.id.traceId}"), - )) - else None -} - -object EarlybirdModelBasedSimilarityEngine { - case class EarlybirdModelBasedSearchQuery( - seedUserIds: Seq[UserId], - maxNumTweets: Int, - oldestTweetTimestampInSec: Option[UserId], - frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]]) - extends EarlybirdSearchQuery - - /** - * Used by Push Service - */ - val RealGraphScoringModel = "frigate_unified_engagement_rg" - val MaxHitsToProcess = 1000 - val MaxConsecutiveSameUser = 1 - - private def getModelBasedRankingParams( - authorSpecificScoreAdjustments: Map[Long, Double] - ): ThriftRankingParams = ThriftRankingParams( - `type` = Some(ThriftScoringFunctionType.ModelBased), - selectedModels = Some(Map(RealGraphScoringModel -> 1.0)), - applyBoosts = false, - authorSpecificScoreAdjustments = Some(authorSpecificScoreAdjustments) - ) - - private def getRelevanceOptions( - authorSpecificScoreAdjustments: Map[Long, Double], - ): ThriftSearchRelevanceOptions = { - ThriftSearchRelevanceOptions( - maxConsecutiveSameUser = Some(MaxConsecutiveSameUser), - rankingParams = Some(getModelBasedRankingParams(authorSpecificScoreAdjustments)), - maxHitsToProcess = Some(MaxHitsToProcess), - orderByRelevance = true - ) - } - - private def getThriftSearchQuery(query: EarlybirdModelBasedSearchQuery): ThriftSearchQuery = - ThriftSearchQuery( - serializedQuery = Some(f"(* [since_time ${query.oldestTweetTimestampInSec.getOrElse(0)}])"), - fromUserIDFilter64 = Some(query.seedUserIds), - numResults = query.maxNumTweets, - maxHitsToProcess = MaxHitsToProcess, - rankingMode = ThriftSearchRankingMode.Relevance, - relevanceOptions = - Some(getRelevanceOptions(query.frsUserToScoresForScoreAdjustment.getOrElse(Map.empty))), - facetFieldNames = Some(FacetsToFetch), - resultMetadataOptions = Some(MetadataOptions), - searcherId = None - ) -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdRecencyBasedSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdRecencyBasedSimilarityEngine.docx new file mode 100644 index 000000000..e2fe967f3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdRecencyBasedSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdRecencyBasedSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdRecencyBasedSimilarityEngine.scala deleted file mode 100644 index 988d666a4..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdRecencyBasedSimilarityEngine.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TweetWithAuthor -import com.twitter.cr_mixer.similarity_engine.EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -case class EarlybirdRecencyBasedSimilarityEngine @Inject() ( - @Named(ModuleNames.EarlybirdRecencyBasedWithoutRetweetsRepliesTweetsCache) - earlybirdRecencyBasedWithoutRetweetsRepliesTweetsCacheStore: ReadableStore[ - UserId, - Seq[TweetId] - ], - @Named(ModuleNames.EarlybirdRecencyBasedWithRetweetsRepliesTweetsCache) - earlybirdRecencyBasedWithRetweetsRepliesTweetsCacheStore: ReadableStore[ - UserId, - Seq[TweetId] - ], - timeoutConfig: TimeoutConfig, - stats: StatsReceiver) - extends ReadableStore[EarlybirdRecencyBasedSearchQuery, Seq[TweetWithAuthor]] { - import EarlybirdRecencyBasedSimilarityEngine._ - val statsReceiver: StatsReceiver = stats.scope(this.getClass.getSimpleName) - - override def get( - query: EarlybirdRecencyBasedSearchQuery - ): Future[Option[Seq[TweetWithAuthor]]] = { - Future - .collect { - if (query.filterOutRetweetsAndReplies) { - query.seedUserIds.map { seedUserId => - StatsUtil.trackOptionItemsStats(statsReceiver.scope("WithoutRetweetsAndReplies")) { - earlybirdRecencyBasedWithoutRetweetsRepliesTweetsCacheStore - .get(seedUserId).map(_.map(_.map(tweetId => - TweetWithAuthor(tweetId = tweetId, authorId = seedUserId)))) - } - } - } else { - query.seedUserIds.map { seedUserId => - StatsUtil.trackOptionItemsStats(statsReceiver.scope("WithRetweetsAndReplies")) { - earlybirdRecencyBasedWithRetweetsRepliesTweetsCacheStore - .get(seedUserId) - .map(_.map(_.map(tweetId => - TweetWithAuthor(tweetId = tweetId, authorId = seedUserId)))) - } - } - } - } - .map { tweetWithAuthorList => - val earliestTweetId = SnowflakeId.firstIdFor(Time.now - query.maxTweetAge) - tweetWithAuthorList - .flatMap(_.getOrElse(Seq.empty)) - .filter(tweetWithAuthor => - tweetWithAuthor.tweetId >= earliestTweetId // tweet age filter - && !query.excludedTweetIds - .contains(tweetWithAuthor.tweetId)) // excluded tweet filter - .sortBy(tweetWithAuthor => - -SnowflakeId.unixTimeMillisFromId(tweetWithAuthor.tweetId)) // sort by recency - .take(query.maxNumTweets) // take most recent N tweets - } - .map(result => Some(result)) - } - -} - -object EarlybirdRecencyBasedSimilarityEngine { - case class EarlybirdRecencyBasedSearchQuery( - seedUserIds: Seq[UserId], - maxNumTweets: Int, - excludedTweetIds: Set[TweetId], - maxTweetAge: Duration, - filterOutRetweetsAndReplies: Boolean) - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngine.docx new file mode 100644 index 000000000..74783af3a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngine.scala deleted file mode 100644 index be23134eb..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngine.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.TweetWithAuthor -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class EarlybirdSimilarityEngine[ - Query, - EarlybirdSimilarityEngineStore <: ReadableStore[Query, Seq[TweetWithAuthor]] -]( - implementingStore: EarlybirdSimilarityEngineStore, - override val identifier: SimilarityEngineType, - globalStats: StatsReceiver, - engineConfig: SimilarityEngineConfig, -) extends SimilarityEngine[EngineQuery[Query], TweetWithAuthor] { - private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) - - def getScopedStats: StatsReceiver = scopedStats - - def getCandidates(query: EngineQuery[Query]): Future[Option[Seq[TweetWithAuthor]]] = { - SimilarityEngine.getFromFn( - implementingStore.get, - query.storeQuery, - engineConfig, - query.params, - scopedStats - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineBase.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineBase.docx new file mode 100644 index 000000000..caedc3685 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineBase.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineBase.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineBase.scala deleted file mode 100644 index ab4eb408e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineBase.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.TweetWithAuthor -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.earlybird.thriftscala.EarlybirdRequest -import com.twitter.search.earlybird.thriftscala.EarlybirdResponseCode -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.simclusters_v2.common.UserId -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -/** - * This trait is a base trait for Earlybird similarity engines. All Earlybird similarity - * engines extend from it and override the construction method for EarlybirdRequest - */ -trait EarlybirdSimilarityEngineBase[EarlybirdSearchQuery] - extends ReadableStore[EarlybirdSearchQuery, Seq[TweetWithAuthor]] { - def earlybirdSearchClient: EarlybirdService.MethodPerEndpoint - - def statsReceiver: StatsReceiver - - def getEarlybirdRequest(query: EarlybirdSearchQuery): Option[EarlybirdRequest] - - override def get(query: EarlybirdSearchQuery): Future[Option[Seq[TweetWithAuthor]]] = { - getEarlybirdRequest(query) - .map { earlybirdRequest => - earlybirdSearchClient - .search(earlybirdRequest).map { response => - response.responseCode match { - case EarlybirdResponseCode.Success => - val earlybirdSearchResult = - response.searchResults - .map( - _.results - .map(searchResult => - TweetWithAuthor( - searchResult.id, - // fromUserId should be there since MetadataOptions.getFromUserId = true - searchResult.metadata.map(_.fromUserId).getOrElse(0))).toSeq) - statsReceiver.scope("result").stat("size").add(earlybirdSearchResult.size) - earlybirdSearchResult - case e => - statsReceiver.scope("failures").counter(e.getClass.getSimpleName).incr() - Some(Seq.empty) - } - } - }.getOrElse(Future.None) - } -} - -object EarlybirdSimilarityEngineBase { - trait EarlybirdSearchQuery { - def seedUserIds: Seq[UserId] - def maxNumTweets: Int - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineRouter.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineRouter.docx new file mode 100644 index 000000000..14c42b60d Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineRouter.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineRouter.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineRouter.scala deleted file mode 100644 index 3237f13f8..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdSimilarityEngineRouter.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_ModelBased -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_RecencyBased -import com.twitter.cr_mixer.model.EarlybirdSimilarityEngineType_TensorflowBased -import com.twitter.cr_mixer.model.TweetWithAuthor -import com.twitter.cr_mixer.param.EarlybirdFrsBasedCandidateGenerationParams -import com.twitter.cr_mixer.param.EarlybirdFrsBasedCandidateGenerationParams.FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam -import com.twitter.cr_mixer.param.FrsParams.FrsBasedCandidateGenerationMaxCandidatesNumParam -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class EarlybirdSimilarityEngineRouter @Inject() ( - earlybirdRecencyBasedSimilarityEngine: EarlybirdSimilarityEngine[ - EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery, - EarlybirdRecencyBasedSimilarityEngine - ], - earlybirdModelBasedSimilarityEngine: EarlybirdSimilarityEngine[ - EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery, - EarlybirdModelBasedSimilarityEngine - ], - earlybirdTensorflowBasedSimilarityEngine: EarlybirdSimilarityEngine[ - EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery, - EarlybirdTensorflowBasedSimilarityEngine - ], - timeoutConfig: TimeoutConfig, - statsReceiver: StatsReceiver) - extends ReadableStore[EarlybirdSimilarityEngineRouter.Query, Seq[TweetWithAuthor]] { - import EarlybirdSimilarityEngineRouter._ - - override def get( - k: EarlybirdSimilarityEngineRouter.Query - ): Future[Option[Seq[TweetWithAuthor]]] = { - k.rankingMode match { - case EarlybirdSimilarityEngineType_RecencyBased => - earlybirdRecencyBasedSimilarityEngine.getCandidates(recencyBasedQueryFromParams(k)) - case EarlybirdSimilarityEngineType_ModelBased => - earlybirdModelBasedSimilarityEngine.getCandidates(modelBasedQueryFromParams(k)) - case EarlybirdSimilarityEngineType_TensorflowBased => - earlybirdTensorflowBasedSimilarityEngine.getCandidates(tensorflowBasedQueryFromParams(k)) - } - } -} - -object EarlybirdSimilarityEngineRouter { - case class Query( - searcherUserId: Option[UserId], - seedUserIds: Seq[UserId], - maxNumTweets: Int, - excludedTweetIds: Set[TweetId], - rankingMode: EarlybirdSimilarityEngineType, - frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]], - maxTweetAge: Duration, - filterOutRetweetsAndReplies: Boolean, - params: configapi.Params) - - def queryFromParams( - searcherUserId: Option[UserId], - seedUserIds: Seq[UserId], - excludedTweetIds: Set[TweetId], - frsUserToScoresForScoreAdjustment: Option[Map[UserId, Double]], - params: configapi.Params - ): Query = - Query( - searcherUserId, - seedUserIds, - maxNumTweets = params(FrsBasedCandidateGenerationMaxCandidatesNumParam), - excludedTweetIds, - rankingMode = - params(FrsBasedCandidateGenerationEarlybirdSimilarityEngineTypeParam).rankingMode, - frsUserToScoresForScoreAdjustment, - maxTweetAge = params( - EarlybirdFrsBasedCandidateGenerationParams.FrsBasedCandidateGenerationEarlybirdMaxTweetAge), - filterOutRetweetsAndReplies = params( - EarlybirdFrsBasedCandidateGenerationParams.FrsBasedCandidateGenerationEarlybirdFilterOutRetweetsAndReplies), - params - ) - - private def recencyBasedQueryFromParams( - query: Query - ): EngineQuery[EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery] = - EngineQuery( - EarlybirdRecencyBasedSimilarityEngine.EarlybirdRecencyBasedSearchQuery( - seedUserIds = query.seedUserIds, - maxNumTweets = query.maxNumTweets, - excludedTweetIds = query.excludedTweetIds, - maxTweetAge = query.maxTweetAge, - filterOutRetweetsAndReplies = query.filterOutRetweetsAndReplies - ), - query.params - ) - - private def tensorflowBasedQueryFromParams( - query: Query, - ): EngineQuery[EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery] = - EngineQuery( - EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery( - searcherUserId = query.searcherUserId, - seedUserIds = query.seedUserIds, - maxNumTweets = query.maxNumTweets, - // hard code the params below for now. Will move to FS after shipping the ddg - beforeTweetIdExclusive = None, - afterTweetIdExclusive = - Some(SnowflakeId.firstIdFor((Time.now - query.maxTweetAge).inMilliseconds)), - filterOutRetweetsAndReplies = query.filterOutRetweetsAndReplies, - useTensorflowRanking = true, - excludedTweetIds = query.excludedTweetIds, - maxNumHitsPerShard = 1000 - ), - query.params - ) - private def modelBasedQueryFromParams( - query: Query, - ): EngineQuery[EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery] = - EngineQuery( - EarlybirdModelBasedSimilarityEngine.EarlybirdModelBasedSearchQuery( - seedUserIds = query.seedUserIds, - maxNumTweets = query.maxNumTweets, - oldestTweetTimestampInSec = Some(query.maxTweetAge.ago.inSeconds), - frsUserToScoresForScoreAdjustment = query.frsUserToScoresForScoreAdjustment - ), - query.params - ) -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.docx new file mode 100644 index 000000000..0826d480a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.scala deleted file mode 100644 index 8df6ec711..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/EarlybirdTensorflowBasedSimilarityEngine.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.search.earlybird.thriftscala.EarlybirdRequest -import com.twitter.search.earlybird.thriftscala.EarlybirdService -import com.twitter.search.earlybird.thriftscala.ThriftSearchQuery -import com.twitter.util.Time -import com.twitter.search.common.query.thriftjava.thriftscala.CollectorParams -import com.twitter.search.common.ranking.thriftscala.ThriftRankingParams -import com.twitter.search.common.ranking.thriftscala.ThriftScoringFunctionType -import com.twitter.search.earlybird.thriftscala.ThriftSearchRelevanceOptions -import javax.inject.Inject -import javax.inject.Singleton -import EarlybirdSimilarityEngineBase._ -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.similarity_engine.EarlybirdTensorflowBasedSimilarityEngine.EarlybirdTensorflowBasedSearchQuery -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.EarlybirdClientId -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.FacetsToFetch -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetCollectorTerminationParams -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetEarlybirdQuery -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.MetadataOptions -import com.twitter.cr_mixer.util.EarlybirdSearchUtil.GetNamedDisjunctions -import com.twitter.search.earlybird.thriftscala.ThriftSearchRankingMode -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.common.UserId -import com.twitter.util.Duration - -@Singleton -case class EarlybirdTensorflowBasedSimilarityEngine @Inject() ( - earlybirdSearchClient: EarlybirdService.MethodPerEndpoint, - timeoutConfig: TimeoutConfig, - stats: StatsReceiver) - extends EarlybirdSimilarityEngineBase[EarlybirdTensorflowBasedSearchQuery] { - import EarlybirdTensorflowBasedSimilarityEngine._ - override val statsReceiver: StatsReceiver = stats.scope(this.getClass.getSimpleName) - override def getEarlybirdRequest( - query: EarlybirdTensorflowBasedSearchQuery - ): Option[EarlybirdRequest] = { - if (query.seedUserIds.nonEmpty) - Some( - EarlybirdRequest( - searchQuery = getThriftSearchQuery(query, timeoutConfig.earlybirdServerTimeout), - clientHost = None, - clientRequestID = None, - clientId = Some(EarlybirdClientId), - clientRequestTimeMs = Some(Time.now.inMilliseconds), - cachingParams = None, - timeoutMs = timeoutConfig.earlybirdServerTimeout.inMilliseconds.intValue(), - facetRequest = None, - termStatisticsRequest = None, - debugMode = 0, - debugOptions = None, - searchSegmentId = None, - returnStatusType = None, - successfulResponseThreshold = None, - querySource = None, - getOlderResults = Some(false), - followedUserIds = Some(query.seedUserIds), - adjustedProtectedRequestParams = None, - adjustedFullArchiveRequestParams = None, - getProtectedTweetsOnly = Some(false), - retokenizeSerializedQuery = None, - skipVeryRecentTweets = true, - experimentClusterToUse = None - )) - else None - } -} - -object EarlybirdTensorflowBasedSimilarityEngine { - case class EarlybirdTensorflowBasedSearchQuery( - searcherUserId: Option[UserId], - seedUserIds: Seq[UserId], - maxNumTweets: Int, - beforeTweetIdExclusive: Option[TweetId], - afterTweetIdExclusive: Option[TweetId], - filterOutRetweetsAndReplies: Boolean, - useTensorflowRanking: Boolean, - excludedTweetIds: Set[TweetId], - maxNumHitsPerShard: Int) - extends EarlybirdSearchQuery - - private def getThriftSearchQuery( - query: EarlybirdTensorflowBasedSearchQuery, - processingTimeout: Duration - ): ThriftSearchQuery = - ThriftSearchQuery( - serializedQuery = GetEarlybirdQuery( - query.beforeTweetIdExclusive, - query.afterTweetIdExclusive, - query.excludedTweetIds, - query.filterOutRetweetsAndReplies).map(_.serialize), - fromUserIDFilter64 = Some(query.seedUserIds), - numResults = query.maxNumTweets, - // Whether to collect conversation IDs. Remove it for now. - // collectConversationId = Gate.True(), // true for Home - rankingMode = ThriftSearchRankingMode.Relevance, - relevanceOptions = Some(getRelevanceOptions), - collectorParams = Some( - CollectorParams( - // numResultsToReturn defines how many results each EB shard will return to search root - numResultsToReturn = 1000, - // terminationParams.maxHitsToProcess is used for early terminating per shard results fetching. - terminationParams = - GetCollectorTerminationParams(query.maxNumHitsPerShard, processingTimeout) - )), - facetFieldNames = Some(FacetsToFetch), - resultMetadataOptions = Some(MetadataOptions), - searcherId = query.searcherUserId, - searchStatusIds = None, - namedDisjunctionMap = GetNamedDisjunctions(query.excludedTweetIds) - ) - - // The specific values of recap relevance/reranking options correspond to - // experiment: enable_recap_reranking_2988,timeline_internal_disable_recap_filter - // bucket : enable_rerank,disable_filter - private def getRelevanceOptions: ThriftSearchRelevanceOptions = { - ThriftSearchRelevanceOptions( - proximityScoring = true, - maxConsecutiveSameUser = Some(2), - rankingParams = Some(getTensorflowBasedRankingParams), - maxHitsToProcess = Some(500), - maxUserBlendCount = Some(3), - proximityPhraseWeight = 9.0, - returnAllResults = Some(true) - ) - } - - private def getTensorflowBasedRankingParams: ThriftRankingParams = { - ThriftRankingParams( - `type` = Some(ThriftScoringFunctionType.TensorflowBased), - selectedTensorflowModel = Some("timelines_rectweet_replica"), - minScore = -1.0e100, - applyBoosts = false, - authorSpecificScoreAdjustments = None - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/FilterUtil.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/FilterUtil.docx new file mode 100644 index 000000000..b4510c056 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/FilterUtil.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/FilterUtil.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/FilterUtil.scala deleted file mode 100644 index 4cd94d2bb..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/FilterUtil.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SourceInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.util.Duration -import com.twitter.util.Time - -object FilterUtil { - - /** Returns a list of tweets that are generated less than `maxTweetAgeHours` hours ago */ - def tweetAgeFilter( - candidates: Seq[TweetWithScore], - maxTweetAgeHours: Duration - ): Seq[TweetWithScore] = { - // Tweet IDs are approximately chronological (see http://go/snowflake), - // so we are building the earliest tweet id once - // The per-candidate logic here then be candidate.tweetId > earliestPermittedTweetId, which is far cheaper. - // See @cyao's phab on CrMixer generic age filter for reference https://phabricator.twitter.biz/D903188 - val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAgeHours) - candidates.filter { candidate => candidate.tweetId >= earliestTweetId } - } - - /** Returns a list of tweet sources that are generated less than `maxTweetAgeHours` hours ago */ - def tweetSourceAgeFilter( - candidates: Seq[SourceInfo], - maxTweetSignalAgeHoursParam: Duration - ): Seq[SourceInfo] = { - // Tweet IDs are approximately chronological (see http://go/snowflake), - // so we are building the earliest tweet id once - // This filter applies to source signals. Some candidate source calls can be avoided if source signals - // can be filtered. - val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetSignalAgeHoursParam) - candidates.filter { candidate => - candidate.internalId match { - case InternalId.TweetId(tweetId) => tweetId >= earliestTweetId - case _ => false - } - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/HnswANNSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/HnswANNSimilarityEngine.docx new file mode 100644 index 000000000..00159ff23 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/HnswANNSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/HnswANNSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/HnswANNSimilarityEngine.scala deleted file mode 100644 index 4a1422ce9..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/HnswANNSimilarityEngine.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.ann.common.thriftscala.AnnQueryService -import com.twitter.ann.common.thriftscala.Distance -import com.twitter.ann.common.thriftscala.NearestNeighborQuery -import com.twitter.ann.hnsw.HnswCommon -import com.twitter.ann.hnsw.HnswParams -import com.twitter.bijection.Injection -import com.twitter.cortex.ml.embeddings.common.TweetKind -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.MemCacheConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec -import com.twitter.ml.api.thriftscala.{Embedding => ThriftEmbedding} -import com.twitter.ml.featurestore.lib -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Params -import com.twitter.util.Future - -case class HnswANNEngineQuery( - modelId: String, - sourceId: InternalId, - params: Params, -) { - val cacheKey: String = s"${modelId}_${sourceId.toString}" -} - -/** - * This Engine looks for tweets whose similarity is close to a Source Dense Embedding. - * Only support Long based embedding lookup. UserId or TweetId. - * - * It provides HNSW specific implementations - * - * @param memCacheConfigOpt If specified, it will wrap the underlying store with a MemCache layer - * You should only enable this for cacheable queries, e.x. TweetIds. - * consumer based UserIds are generally not possible to cache. - */ -class HnswANNSimilarityEngine( - embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]], - annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint], - globalStats: StatsReceiver, - override val identifier: SimilarityEngineType, - engineConfig: SimilarityEngineConfig, - memCacheConfigOpt: Option[MemCacheConfig[HnswANNEngineQuery]] = None) - extends SimilarityEngine[HnswANNEngineQuery, TweetWithScore] { - - private val MaxNumResults: Int = 200 - private val ef: Int = 800 - private val TweetIdByteInjection: Injection[lib.TweetId, Array[Byte]] = TweetKind.byteInjection - - private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) - - def getScopedStats: StatsReceiver = scopedStats - - private def fetchEmbedding( - query: HnswANNEngineQuery, - ): Future[Option[ThriftEmbedding]] = { - val embeddingStore = embeddingStoreLookUpMap.getOrElse( - query.modelId, - throw new IllegalArgumentException( - s"${this.getClass.getSimpleName} ${identifier.toString}: " + - s"ModelId ${query.modelId} does not exist for embeddingStore" - ) - ) - - embeddingStore.get(query.sourceId) - } - - private def fetchCandidates( - query: HnswANNEngineQuery, - embedding: ThriftEmbedding - ): Future[Seq[TweetWithScore]] = { - val annService = annServiceLookUpMap.getOrElse( - query.modelId, - throw new IllegalArgumentException( - s"${this.getClass.getSimpleName} ${identifier.toString}: " + - s"ModelId ${query.modelId} does not exist for annStore" - ) - ) - - val hnswParams = HnswCommon.RuntimeParamsInjection.apply(HnswParams(ef)) - - val annQuery = - NearestNeighborQuery(embedding, withDistance = true, hnswParams, MaxNumResults) - - annService - .query(annQuery) - .map( - _.nearestNeighbors - .map { nearestNeighbor => - val candidateId = TweetIdByteInjection - .invert(ArrayByteBufferCodec.decode(nearestNeighbor.id)) - .toOption - .map(_.tweetId) - (candidateId, nearestNeighbor.distance) - }.collect { - case (Some(candidateId), Some(distance)) => - TweetWithScore(candidateId, toScore(distance)) - }) - } - - // Convert Distance to a score such that higher scores mean more similar. - def toScore(distance: Distance): Double = { - distance match { - case Distance.EditDistance(editDistance) => - // (-Infinite, 0.0] - 0.0 - editDistance.distance - case Distance.L2Distance(l2Distance) => - // (-Infinite, 0.0] - 0.0 - l2Distance.distance - case Distance.CosineDistance(cosineDistance) => - // [0.0 - 1.0] - 1.0 - cosineDistance.distance - case Distance.InnerProductDistance(innerProductDistance) => - // (-Infinite, Infinite) - 1.0 - innerProductDistance.distance - case Distance.UnknownUnionField(_) => - throw new IllegalStateException( - s"${this.getClass.getSimpleName} does not recognize $distance.toString" - ) - } - } - - private[similarity_engine] def getEmbeddingAndCandidates( - query: HnswANNEngineQuery - ): Future[Option[Seq[TweetWithScore]]] = { - - val fetchEmbeddingStat = scopedStats.scope(query.modelId).scope("fetchEmbedding") - val fetchCandidatesStat = scopedStats.scope(query.modelId).scope("fetchCandidates") - - for { - embeddingOpt <- StatsUtil.trackOptionStats(fetchEmbeddingStat) { fetchEmbedding(query) } - candidates <- StatsUtil.trackItemsStats(fetchCandidatesStat) { - - embeddingOpt match { - case Some(embedding) => fetchCandidates(query, embedding) - case None => Future.Nil - } - } - } yield { - Some(candidates) - } - } - - // Add memcache wrapper, if specified - private val store = { - val uncachedStore = ReadableStore.fromFnFuture(getEmbeddingAndCandidates) - - memCacheConfigOpt match { - case Some(config) => - SimilarityEngine.addMemCache( - underlyingStore = uncachedStore, - memCacheConfig = config, - statsReceiver = scopedStats - ) - case _ => uncachedStore - } - } - - def toSimilarityEngineInfo( - query: HnswANNEngineQuery, - score: Double - ): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = this.identifier, - modelId = Some(query.modelId), - score = Some(score)) - } - - override def getCandidates( - engineQuery: HnswANNEngineQuery - ): Future[Option[Seq[TweetWithScore]]] = { - val versionedStats = globalStats.scope(engineQuery.modelId) - SimilarityEngine.getFromFn( - store.get, - engineQuery, - engineConfig, - engineQuery.params, - versionedStats - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/LookupSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/LookupSimilarityEngine.docx new file mode 100644 index 000000000..93fbe8b9c Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/LookupSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/LookupSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/LookupSimilarityEngine.scala deleted file mode 100644 index c4e469899..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/LookupSimilarityEngine.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.MemCacheConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Params -import com.twitter.util.Future - -case class LookupEngineQuery[Query]( - storeQuery: Query, // the actual Query type of the underlying store - lookupKey: String, - params: Params, -) - -/** - * This Engine provides a map interface for looking up different model implementations. - * It provides modelId level monitoring for free. - * - * Example use cases include OfflineSimClusters lookup - * - * - * @param versionedStoreMap A mapping from a modelId to a corresponding implementation - * @param memCacheConfigOpt If specified, it will wrap the underlying store with a MemCache layer - * You should only enable this for cacheable queries, e.x. TweetIds. - * consumer based UserIds are generally not possible to cache. - */ -class LookupSimilarityEngine[Query, Candidate <: Serializable]( - versionedStoreMap: Map[String, ReadableStore[Query, Seq[Candidate]]], // key = modelId - override val identifier: SimilarityEngineType, - globalStats: StatsReceiver, - engineConfig: SimilarityEngineConfig, - memCacheConfigOpt: Option[MemCacheConfig[Query]] = None) - extends SimilarityEngine[LookupEngineQuery[Query], Candidate] { - - private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) - - private val underlyingLookupMap = { - memCacheConfigOpt match { - case Some(config) => - versionedStoreMap.map { - case (modelId, store) => - ( - modelId, - SimilarityEngine.addMemCache( - underlyingStore = store, - memCacheConfig = config, - keyPrefix = Some(modelId), - statsReceiver = scopedStats - ) - ) - } - case _ => versionedStoreMap - } - } - - override def getCandidates( - engineQuery: LookupEngineQuery[Query] - ): Future[Option[Seq[Candidate]]] = { - val versionedStore = - underlyingLookupMap - .getOrElse( - engineQuery.lookupKey, - throw new IllegalArgumentException( - s"${this.getClass.getSimpleName} ${identifier.toString}: ModelId ${engineQuery.lookupKey} does not exist" - ) - ) - - SimilarityEngine.getFromFn( - fn = versionedStore.get, - storeQuery = engineQuery.storeQuery, - engineConfig = engineConfig, - params = engineQuery.params, - scopedStats = scopedStats.scope(engineQuery.lookupKey) - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ModelBasedANNStore.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ModelBasedANNStore.docx new file mode 100644 index 000000000..be599f573 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ModelBasedANNStore.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ModelBasedANNStore.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ModelBasedANNStore.scala deleted file mode 100644 index 064bb8b1a..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ModelBasedANNStore.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.ann.common.thriftscala.AnnQueryService -import com.twitter.ann.common.thriftscala.Distance -import com.twitter.ann.common.thriftscala.NearestNeighborQuery -import com.twitter.ann.common.thriftscala.NearestNeighborResult -import com.twitter.ann.hnsw.HnswCommon -import com.twitter.ann.hnsw.HnswParams -import com.twitter.bijection.Injection -import com.twitter.conversions.DurationOps._ -import com.twitter.cortex.ml.embeddings.common.TweetKind -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.mediaservices.commons.codec.ArrayByteBufferCodec -import com.twitter.ml.api.thriftscala.{Embedding => ThriftEmbedding} -import com.twitter.ml.featurestore.lib -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Duration -import com.twitter.util.Future -import javax.inject.Singleton - -/** - * This store looks for tweets whose similarity is close to a Source Dense Embedding. - * Only support Long based embedding lookup. UserId or TweetId - */ -@Singleton -class ModelBasedANNStore( - embeddingStoreLookUpMap: Map[String, ReadableStore[InternalId, ThriftEmbedding]], - annServiceLookUpMap: Map[String, AnnQueryService.MethodPerEndpoint], - globalStats: StatsReceiver) - extends ReadableStore[ - ModelBasedANNStore.Query, - Seq[TweetWithScore] - ] { - - import ModelBasedANNStore._ - - private val stats = globalStats.scope(this.getClass.getSimpleName) - private val fetchEmbeddingStat = stats.scope("fetchEmbedding") - private val fetchCandidatesStat = stats.scope("fetchCandidates") - - override def get(query: Query): Future[Option[Seq[TweetWithScore]]] = { - for { - maybeEmbedding <- StatsUtil.trackOptionStats(fetchEmbeddingStat.scope(query.modelId)) { - fetchEmbedding(query) - } - maybeCandidates <- StatsUtil.trackOptionStats(fetchCandidatesStat.scope(query.modelId)) { - maybeEmbedding match { - case Some(embedding) => - fetchCandidates(query, embedding) - case None => - Future.None - } - } - } yield { - maybeCandidates.map( - _.nearestNeighbors - .map { nearestNeighbor => - val candidateId = TweetIdByteInjection - .invert(ArrayByteBufferCodec.decode(nearestNeighbor.id)) - .toOption - .map(_.tweetId) - (candidateId, nearestNeighbor.distance) - }.collect { - case (Some(candidateId), Some(distance)) => - TweetWithScore(candidateId, toScore(distance)) - }) - } - } - - private def fetchEmbedding(query: Query): Future[Option[ThriftEmbedding]] = { - embeddingStoreLookUpMap.get(query.modelId) match { - case Some(embeddingStore) => - embeddingStore.get(query.sourceId) - case _ => - Future.None - } - } - - private def fetchCandidates( - query: Query, - embedding: ThriftEmbedding - ): Future[Option[NearestNeighborResult]] = { - val hnswParams = HnswCommon.RuntimeParamsInjection.apply(HnswParams(query.ef)) - - annServiceLookUpMap.get(query.modelId) match { - case Some(annService) => - val annQuery = - NearestNeighborQuery(embedding, withDistance = true, hnswParams, MaxNumResults) - annService.query(annQuery).map(v => Some(v)) - case _ => - Future.None - } - } -} - -object ModelBasedANNStore { - - val MaxNumResults: Int = 200 - val MaxTweetCandidateAge: Duration = 1.day - - val TweetIdByteInjection: Injection[lib.TweetId, Array[Byte]] = TweetKind.byteInjection - - // For more information about HNSW algorithm: https://docbird.twitter.biz/ann/hnsw.html - case class Query( - sourceId: InternalId, - modelId: String, - similarityEngineType: SimilarityEngineType, - ef: Int = 800) - - def toScore(distance: Distance): Double = { - distance match { - case Distance.L2Distance(l2Distance) => - // (-Infinite, 0.0] - 0.0 - l2Distance.distance - case Distance.CosineDistance(cosineDistance) => - // [0.0 - 1.0] - 1.0 - cosineDistance.distance - case Distance.InnerProductDistance(innerProductDistance) => - // (-Infinite, Infinite) - 1.0 - innerProductDistance.distance - case _ => - 0.0 - } - } - def toSimilarityEngineInfo(query: Query, score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = query.similarityEngineType, - modelId = Some(query.modelId), - score = Some(score)) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUnifiedSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUnifiedSimilarityEngine.docx new file mode 100644 index 000000000..5f6017678 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUnifiedSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUnifiedSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUnifiedSimilarityEngine.scala deleted file mode 100644 index f782ae037..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUnifiedSimilarityEngine.scala +++ /dev/null @@ -1,641 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.CandidateGenerationInfo -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.SourceInfo -import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.param.ProducerBasedCandidateGenerationParams -import com.twitter.cr_mixer.param.UnifiedSETweetCombinationMethod -import com.twitter.cr_mixer.param.RelatedTweetProducerBasedParams -import com.twitter.cr_mixer.param.SimClustersANNParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.cr_mixer.thriftscala.SourceType -import com.twitter.cr_mixer.util.InterleaveUtil -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.common.ModelVersions -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Duration -import com.twitter.util.Future -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.mutable.ArrayBuffer - -/** - * This store looks for similar tweets from UserTweetGraph for a Source ProducerId - * For a query producerId,User Tweet Graph (UTG), - * lets us find out which tweets the query producer's followers co-engaged - */ -@Singleton -case class ProducerBasedUnifiedSimilarityEngine( - @Named(ModuleNames.ProducerBasedUserTweetGraphSimilarityEngine) - producerBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ - ProducerBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ], - simClustersANNSimilarityEngine: StandardSimilarityEngine[ - SimClustersANNSimilarityEngine.Query, - TweetWithScore - ], - statsReceiver: StatsReceiver) - extends ReadableStore[ProducerBasedUnifiedSimilarityEngine.Query, Seq[ - TweetWithCandidateGenerationInfo - ]] { - - import ProducerBasedUnifiedSimilarityEngine._ - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - - override def get( - query: Query - ): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] = { - query.sourceInfo.internalId match { - case _: InternalId.UserId => - StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { - val sannCandidatesFut = if (query.enableSimClustersANN) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANNQuery) - } else Future.None - - val sann1CandidatesFut = - if (query.enableSimClustersANN1) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN1Query) - } else Future.None - - val sann2CandidatesFut = - if (query.enableSimClustersANN2) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN2Query) - } else Future.None - - val sann3CandidatesFut = - if (query.enableSimClustersANN3) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN3Query) - } else Future.None - - val sann4CandidatesFut = - if (query.enableSimClustersANN4) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN4Query) - } else Future.None - - val sann5CandidatesFut = - if (query.enableSimClustersANN5) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN5Query) - } else Future.None - - val experimentalSANNCandidatesFut = - if (query.enableExperimentalSimClustersANN) { - simClustersANNSimilarityEngine.getCandidates(query.experimentalSimClustersANNQuery) - } else Future.None - - val utgCandidatesFut = if (query.enableUtg) { - producerBasedUserTweetGraphSimilarityEngine.getCandidates(query.utgQuery) - } else Future.None - - Future - .join( - sannCandidatesFut, - sann1CandidatesFut, - sann2CandidatesFut, - sann3CandidatesFut, - sann4CandidatesFut, - sann5CandidatesFut, - experimentalSANNCandidatesFut, - utgCandidatesFut - ).map { - case ( - simClustersAnnCandidates, - simClustersAnn1Candidates, - simClustersAnn2Candidates, - simClustersAnn3Candidates, - simClustersAnn4Candidates, - simClustersAnn5Candidates, - experimentalSANNCandidates, - userTweetGraphCandidates) => - val filteredSANNTweets = simClustersCandidateMinScoreFilter( - simClustersAnnCandidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANNQuery.storeQuery.simClustersANNConfigId) - - val filteredExperimentalSANNTweets = simClustersCandidateMinScoreFilter( - experimentalSANNCandidates.toSeq.flatten, - query.simClustersMinScore, - query.experimentalSimClustersANNQuery.storeQuery.simClustersANNConfigId) - - val filteredSANN1Tweets = simClustersCandidateMinScoreFilter( - simClustersAnn1Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN1Query.storeQuery.simClustersANNConfigId) - - val filteredSANN2Tweets = simClustersCandidateMinScoreFilter( - simClustersAnn2Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN2Query.storeQuery.simClustersANNConfigId) - - val filteredSANN3Tweets = simClustersCandidateMinScoreFilter( - simClustersAnn3Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN3Query.storeQuery.simClustersANNConfigId) - - val filteredSANN4Tweets = simClustersCandidateMinScoreFilter( - simClustersAnn4Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN4Query.storeQuery.simClustersANNConfigId) - - val filteredSANN5Tweets = simClustersCandidateMinScoreFilter( - simClustersAnn5Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN5Query.storeQuery.simClustersANNConfigId) - - val filteredUTGTweets = - userTweetGraphFilter(userTweetGraphCandidates.toSeq.flatten) - - val sannTweetsWithCGInfo = filteredSANNTweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANNQuery, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sann1TweetsWithCGInfo = filteredSANN1Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN1Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sann2TweetsWithCGInfo = filteredSANN2Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN2Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val sann3TweetsWithCGInfo = filteredSANN3Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN3Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val sann4TweetsWithCGInfo = filteredSANN4Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN4Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val sann5TweetsWithCGInfo = filteredSANN5Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN5Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val experimentalSANNTweetsWithCGInfo = filteredExperimentalSANNTweets.map { - tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo( - query.experimentalSimClustersANNQuery, - tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val utgTweetsWithCGInfo = filteredUTGTweets.map { tweetWithScore => - val similarityEngineInfo = - ProducerBasedUserTweetGraphSimilarityEngine - .toSimilarityEngineInfo(tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val candidateSourcesToBeInterleaved = - ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]]( - sannTweetsWithCGInfo, - sann1TweetsWithCGInfo, - sann2TweetsWithCGInfo, - sann3TweetsWithCGInfo, - sann4TweetsWithCGInfo, - sann5TweetsWithCGInfo, - experimentalSANNTweetsWithCGInfo, - ) - - if (query.utgCombinationMethod == UnifiedSETweetCombinationMethod.Interleave) { - candidateSourcesToBeInterleaved += utgTweetsWithCGInfo - } - - val interleavedCandidates = - InterleaveUtil.interleave(candidateSourcesToBeInterleaved) - - val candidateSourcesToBeOrdered = - ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]](interleavedCandidates) - - if (query.utgCombinationMethod == UnifiedSETweetCombinationMethod.Frontload) - candidateSourcesToBeOrdered.prepend(utgTweetsWithCGInfo) - - val candidatesFromGivenOrderCombination = - SimilaritySourceOrderingUtil.keepGivenOrder(candidateSourcesToBeOrdered) - - val unifiedCandidatesWithUnifiedCGInfo = candidatesFromGivenOrderCombination.map { - candidate => - /*** - * when a candidate was made by interleave/keepGivenOrder, - * then we apply getProducerBasedUnifiedCGInfo() to override with the unified CGInfo - * - * in contributingSE list for interleave. We only have the chosen SE available. - * This is hard to add for interleave, and we plan to add it later after abstraction improvement. - */ - TweetWithCandidateGenerationInfo( - tweetId = candidate.tweetId, - candidateGenerationInfo = getProducerBasedUnifiedCGInfo( - candidate.candidateGenerationInfo.sourceInfoOpt, - candidate.getSimilarityScore, - candidate.candidateGenerationInfo.contributingSimilarityEngines - ) // getSimilarityScore comes from either unifiedScore or single score - ) - } - stats.stat("unified_candidate_size").add(unifiedCandidatesWithUnifiedCGInfo.size) - val truncatedCandidates = - unifiedCandidatesWithUnifiedCGInfo.take(query.maxCandidateNumPerSourceKey) - stats.stat("truncatedCandidates_size").add(truncatedCandidates.size) - - Some(truncatedCandidates) - - } - } - - case _ => - stats.counter("sourceId_is_not_userId_cnt").incr() - Future.None - } - } - - private def simClustersCandidateMinScoreFilter( - simClustersAnnCandidates: Seq[TweetWithScore], - simClustersMinScore: Double, - simClustersANNConfigId: String - ): Seq[TweetWithScore] = { - val filteredCandidates = simClustersAnnCandidates - .filter { candidate => - candidate.score > simClustersMinScore - } - - stats.stat(simClustersANNConfigId, "simClustersAnnCandidates_size").add(filteredCandidates.size) - stats.counter(simClustersANNConfigId, "simClustersAnnRequests").incr() - if (filteredCandidates.isEmpty) - stats.counter(simClustersANNConfigId, "emptyFilteredSimClustersAnnCandidates").incr() - - filteredCandidates.map { candidate => - TweetWithScore(candidate.tweetId, candidate.score) - } - } - - /** A no-op filter as UTG filter already happened at UTG service side */ - private def userTweetGraphFilter( - userTweetGraphCandidates: Seq[TweetWithScore] - ): Seq[TweetWithScore] = { - val filteredCandidates = userTweetGraphCandidates - - stats.stat("userTweetGraphCandidates_size").add(userTweetGraphCandidates.size) - if (filteredCandidates.isEmpty) stats.counter("emptyFilteredUserTweetGraphCandidates").incr() - - filteredCandidates.map { candidate => - TweetWithScore(candidate.tweetId, candidate.score) - } - } - -} -object ProducerBasedUnifiedSimilarityEngine { - - /*** - * Every candidate will have the CG Info with ProducerBasedUnifiedSimilarityEngine - * as they are generated by a composite of Similarity Engines. - * Additionally, we store the contributing SEs (eg., SANN, UTG). - */ - private def getProducerBasedUnifiedCGInfo( - sourceInfoOpt: Option[SourceInfo], - unifiedScore: Double, - contributingSimilarityEngines: Seq[SimilarityEngineInfo] - ): CandidateGenerationInfo = { - CandidateGenerationInfo( - sourceInfoOpt, - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.ProducerBasedUnifiedSimilarityEngine, - modelId = None, // We do not assign modelId for a unified similarity engine - score = Some(unifiedScore) - ), - contributingSimilarityEngines - ) - } - - case class Query( - sourceInfo: SourceInfo, - maxCandidateNumPerSourceKey: Int, - maxTweetAgeHours: Duration, - // SimClusters - enableSimClustersANN: Boolean, - simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableExperimentalSimClustersANN: Boolean, - experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN1: Boolean, - simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN2: Boolean, - simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN4: Boolean, - simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN3: Boolean, - simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN5: Boolean, - simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - simClustersMinScore: Double, - // UTG - enableUtg: Boolean, - utgCombinationMethod: UnifiedSETweetCombinationMethod.Value, - utgQuery: EngineQuery[ProducerBasedUserTweetGraphSimilarityEngine.Query]) - - def fromParams( - sourceInfo: SourceInfo, - params: configapi.Params, - ): EngineQuery[Query] = { - val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) - val maxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - // SimClusters - val enableSimClustersANN = params( - ProducerBasedCandidateGenerationParams.EnableSimClustersANNParam) - val simClustersModelVersion = - ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) - val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) - // SimClusters - Experimental SANN Similarity Engine - val enableExperimentalSimClustersANN = params( - ProducerBasedCandidateGenerationParams.EnableExperimentalSimClustersANNParam) - val experimentalSimClustersANNConfigId = params( - SimClustersANNParams.ExperimentalSimClustersANNConfigId) - // SimClusters - SANN cluster 1 Similarity Engine - val enableSimClustersANN1 = params( - ProducerBasedCandidateGenerationParams.EnableSimClustersANN1Param) - val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) - // SimClusters - SANN cluster 2 Similarity Engine - val enableSimClustersANN2 = params( - ProducerBasedCandidateGenerationParams.EnableSimClustersANN2Param) - val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) - // SimClusters - SANN cluster 3 Similarity Engine - val enableSimClustersANN3 = params( - ProducerBasedCandidateGenerationParams.EnableSimClustersANN3Param) - val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) - // SimClusters - SANN cluster 5 Similarity Engine - val enableSimClustersANN5 = params( - ProducerBasedCandidateGenerationParams.EnableSimClustersANN5Param) - val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) - val enableSimClustersANN4 = params( - ProducerBasedCandidateGenerationParams.EnableSimClustersANN4Param) - val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) - - val simClustersMinScore = params( - ProducerBasedCandidateGenerationParams.SimClustersMinScoreParam) - - // SimClusters ANN Query - val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANNConfigId, - params - ) - val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - experimentalSimClustersANNConfigId, - params - ) - val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN1ConfigId, - params - ) - val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN2ConfigId, - params - ) - val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN3ConfigId, - params - ) - val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN5ConfigId, - params - ) - val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN4ConfigId, - params - ) - // UTG - val enableUtg = params(ProducerBasedCandidateGenerationParams.EnableUTGParam) - val utgCombinationMethod = params( - ProducerBasedCandidateGenerationParams.UtgCombinationMethodParam) - - EngineQuery( - Query( - sourceInfo = sourceInfo, - maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, - maxTweetAgeHours = maxTweetAgeHours, - enableSimClustersANN = enableSimClustersANN, - simClustersANNQuery = simClustersANNQuery, - enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, - experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, - enableSimClustersANN1 = enableSimClustersANN1, - simClustersANN1Query = simClustersANN1Query, - enableSimClustersANN2 = enableSimClustersANN2, - simClustersANN2Query = simClustersANN2Query, - enableSimClustersANN3 = enableSimClustersANN3, - simClustersANN3Query = simClustersANN3Query, - enableSimClustersANN5 = enableSimClustersANN5, - simClustersANN5Query = simClustersANN5Query, - enableSimClustersANN4 = enableSimClustersANN4, - simClustersANN4Query = simClustersANN4Query, - simClustersMinScore = simClustersMinScore, - enableUtg = enableUtg, - utgCombinationMethod = utgCombinationMethod, - utgQuery = ProducerBasedUserTweetGraphSimilarityEngine - .fromParams(sourceInfo.internalId, params) - ), - params - ) - } - - def fromParamsForRelatedTweet( - internalId: InternalId, - params: configapi.Params - ): EngineQuery[Query] = { - val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) - val maxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - // SimClusters - val enableSimClustersANN = params(RelatedTweetProducerBasedParams.EnableSimClustersANNParam) - val simClustersModelVersion = - ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) - val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) - val simClustersMinScore = - params(RelatedTweetProducerBasedParams.SimClustersMinScoreParam) - // SimClusters - Experimental SANN Similarity Engine - val enableExperimentalSimClustersANN = params( - RelatedTweetProducerBasedParams.EnableExperimentalSimClustersANNParam) - val experimentalSimClustersANNConfigId = params( - SimClustersANNParams.ExperimentalSimClustersANNConfigId) - // SimClusters - SANN cluster 1 Similarity Engine - val enableSimClustersANN1 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN1Param) - val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) - // SimClusters - SANN cluster 2 Similarity Engine - val enableSimClustersANN2 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN2Param) - val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) - // SimClusters - SANN cluster 3 Similarity Engine - val enableSimClustersANN3 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN3Param) - val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) - // SimClusters - SANN cluster 5 Similarity Engine - val enableSimClustersANN5 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN5Param) - val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) - - val enableSimClustersANN4 = params(RelatedTweetProducerBasedParams.EnableSimClustersANN4Param) - val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) - // Build SANN Query - val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANNConfigId, - params - ) - val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - experimentalSimClustersANNConfigId, - params - ) - val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN1ConfigId, - params - ) - val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN2ConfigId, - params - ) - val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN3ConfigId, - params - ) - val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN5ConfigId, - params - ) - val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.FavBasedProducer, - simClustersModelVersion, - simClustersANN4ConfigId, - params - ) - // UTG - val enableUtg = params(RelatedTweetProducerBasedParams.EnableUTGParam) - val utgCombinationMethod = params( - ProducerBasedCandidateGenerationParams.UtgCombinationMethodParam) - - // SourceType.RequestUserId is a placeholder. - val sourceInfo = SourceInfo(SourceType.RequestUserId, internalId, None) - - EngineQuery( - Query( - sourceInfo = sourceInfo, - maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, - maxTweetAgeHours = maxTweetAgeHours, - enableSimClustersANN = enableSimClustersANN, - simClustersANNQuery = simClustersANNQuery, - enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, - experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, - enableSimClustersANN1 = enableSimClustersANN1, - simClustersANN1Query = simClustersANN1Query, - enableSimClustersANN2 = enableSimClustersANN2, - simClustersANN2Query = simClustersANN2Query, - enableSimClustersANN3 = enableSimClustersANN3, - simClustersANN3Query = simClustersANN3Query, - enableSimClustersANN5 = enableSimClustersANN5, - simClustersANN5Query = simClustersANN5Query, - enableSimClustersANN4 = enableSimClustersANN4, - simClustersANN4Query = simClustersANN4Query, - simClustersMinScore = simClustersMinScore, - enableUtg = enableUtg, - utgQuery = ProducerBasedUserTweetGraphSimilarityEngine.fromParams(internalId, params), - utgCombinationMethod = utgCombinationMethod - ), - params - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserAdGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserAdGraphSimilarityEngine.docx new file mode 100644 index 000000000..cf044f745 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserAdGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserAdGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserAdGraphSimilarityEngine.scala deleted file mode 100644 index c9ebc91e7..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserAdGraphSimilarityEngine.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.param.ProducerBasedUserAdGraphParams -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.user_ad_graph.thriftscala.ProducerBasedRelatedAdRequest -import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Singleton -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.timelines.configapi - -/** - * This store looks for similar tweets from UserAdGraph for a Source ProducerId - * For a query producerId,User Tweet Graph (UAG), - * lets us find out which ad tweets the query producer's followers co-engaged - */ -@Singleton -case class ProducerBasedUserAdGraphSimilarityEngine( - userAdGraphService: UserAdGraph.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends ReadableStore[ProducerBasedUserAdGraphSimilarityEngine.Query, Seq[ - TweetWithScore - ]] { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - - override def get( - query: ProducerBasedUserAdGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - query.sourceId match { - case InternalId.UserId(producerId) => - StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { - val relatedAdRequest = - ProducerBasedRelatedAdRequest( - producerId, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - minScore = Some(query.minScore), - maxNumFollowers = Some(query.maxNumFollowers), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours), - ) - - userAdGraphService.producerBasedRelatedAds(relatedAdRequest).map { relatedAdResponse => - val candidates = - relatedAdResponse.adTweets.map(tweet => TweetWithScore(tweet.adTweetId, tweet.score)) - Some(candidates) - } - } - case _ => - Future.value(None) - } - } -} - -object ProducerBasedUserAdGraphSimilarityEngine { - - def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.ProducerBasedUserAdGraph, - modelId = None, - score = Some(score)) - } - - case class Query( - sourceId: InternalId, - maxResults: Int, - minCooccurrence: Int, // require at least {minCooccurrence} lhs user engaged with returned tweet - minScore: Double, - maxNumFollowers: Int, // max number of lhs users - maxTweetAgeInHours: Int) - - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query( - sourceId = sourceId, - maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), - minCooccurrence = params(ProducerBasedUserAdGraphParams.MinCoOccurrenceParam), - maxNumFollowers = params(ProducerBasedUserAdGraphParams.MaxNumFollowersParam), - maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, - minScore = params(ProducerBasedUserAdGraphParams.MinScoreParam) - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngine.docx new file mode 100644 index 000000000..888265aa3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngine.scala deleted file mode 100644 index 6e7ca95bd..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ProducerBasedUserTweetGraphSimilarityEngine.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.ProducerBasedUserTweetGraphParams -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.user_tweet_graph.thriftscala.ProducerBasedRelatedTweetRequest -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Singleton -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.timelines.configapi -import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph - -/** - * This store looks for similar tweets from UserTweetGraph for a Source ProducerId - * For a query producerId,User Tweet Graph (UTG), - * lets us find out which tweets the query producer's followers co-engaged - */ -@Singleton -case class ProducerBasedUserTweetGraphSimilarityEngine( - userTweetGraphService: UserTweetGraph.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends ReadableStore[ProducerBasedUserTweetGraphSimilarityEngine.Query, Seq[ - TweetWithScore - ]] { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - - override def get( - query: ProducerBasedUserTweetGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - query.sourceId match { - case InternalId.UserId(producerId) => - StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { - val relatedTweetRequest = - ProducerBasedRelatedTweetRequest( - producerId, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - minScore = Some(query.minScore), - maxNumFollowers = Some(query.maxNumFollowers), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours), - ) - - userTweetGraphService.producerBasedRelatedTweets(relatedTweetRequest).map { - relatedTweetResponse => - val candidates = - relatedTweetResponse.tweets.map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) - Some(candidates) - } - } - case _ => - Future.value(None) - } - } -} - -object ProducerBasedUserTweetGraphSimilarityEngine { - - def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.ProducerBasedUserTweetGraph, - modelId = None, - score = Some(score)) - } - - case class Query( - sourceId: InternalId, - maxResults: Int, - minCooccurrence: Int, // require at least {minCooccurrence} lhs user engaged with returned tweet - minScore: Double, - maxNumFollowers: Int, // max number of lhs users - maxTweetAgeInHours: Int) - - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query( - sourceId = sourceId, - maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), - minCooccurrence = params(ProducerBasedUserTweetGraphParams.MinCoOccurrenceParam), - maxNumFollowers = params(ProducerBasedUserTweetGraphParams.MaxNumFollowersParam), - maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, - minScore = params(ProducerBasedUserTweetGraphParams.MinScoreParam) - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimClustersANNSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimClustersANNSimilarityEngine.docx new file mode 100644 index 000000000..dcebfd54a Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimClustersANNSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimClustersANNSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimClustersANNSimilarityEngine.scala deleted file mode 100644 index 228627c87..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimClustersANNSimilarityEngine.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.config.SimClustersANNConfig -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclustersann.thriftscala.SimClustersANNService -import com.twitter.simclustersann.thriftscala.{Query => SimClustersANNQuery} -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Future -import javax.inject.Singleton -import com.twitter.cr_mixer.exception.InvalidSANNConfigException -import com.twitter.relevance_platform.simclustersann.multicluster.ServiceNameMapper - -@Singleton -case class SimClustersANNSimilarityEngine( - simClustersANNServiceNameToClientMapper: Map[String, SimClustersANNService.MethodPerEndpoint], - statsReceiver: StatsReceiver) - extends ReadableStore[ - SimClustersANNSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - private val name: String = this.getClass.getSimpleName - private val stats = statsReceiver.scope(name) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - - private def getSimClustersANNService( - query: SimClustersANNQuery - ): Option[SimClustersANNService.MethodPerEndpoint] = { - ServiceNameMapper - .getServiceName( - query.sourceEmbeddingId.modelVersion, - query.config.candidateEmbeddingType).flatMap(serviceName => - simClustersANNServiceNameToClientMapper.get(serviceName)) - } - - override def get( - query: SimClustersANNSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { - - getSimClustersANNService(query.simClustersANNQuery) match { - case Some(simClustersANNService) => - simClustersANNService.getTweetCandidates(query.simClustersANNQuery).map { - simClustersANNTweetCandidates => - val tweetWithScores = simClustersANNTweetCandidates.map { candidate => - TweetWithScore(candidate.tweetId, candidate.score) - } - Some(tweetWithScores) - } - case None => - throw InvalidSANNConfigException( - "No SANN Cluster configured to serve this query, check CandidateEmbeddingType and ModelVersion") - } - } - } -} - -object SimClustersANNSimilarityEngine { - case class Query( - simClustersANNQuery: SimClustersANNQuery, - simClustersANNConfigId: String) - - def toSimilarityEngineInfo( - query: EngineQuery[Query], - score: Double - ): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.SimClustersANN, - modelId = Some( - s"SimClustersANN_${query.storeQuery.simClustersANNQuery.sourceEmbeddingId.embeddingType.toString}_" + - s"${query.storeQuery.simClustersANNQuery.sourceEmbeddingId.modelVersion.toString}_" + - s"${query.storeQuery.simClustersANNConfigId}"), - score = Some(score) - ) - } - - def fromParams( - internalId: InternalId, - embeddingType: EmbeddingType, - modelVersion: ModelVersion, - simClustersANNConfigId: String, - params: configapi.Params, - ): EngineQuery[Query] = { - - // SimClusters EmbeddingId and ANNConfig - val simClustersEmbeddingId = - SimClustersEmbeddingId(embeddingType, modelVersion, internalId) - val simClustersANNConfig = - SimClustersANNConfig - .getConfig(embeddingType.toString, modelVersion.toString, simClustersANNConfigId) - - EngineQuery( - Query( - SimClustersANNQuery( - sourceEmbeddingId = simClustersEmbeddingId, - config = simClustersANNConfig.toSANNConfigThrift - ), - simClustersANNConfigId - ), - params - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilarityEngine.docx new file mode 100644 index 000000000..57ec29565 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilarityEngine.scala deleted file mode 100644 index 6bc332f75..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilarityEngine.scala +++ /dev/null @@ -1,169 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.param.decider.CrMixerDecider -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.GlobalRequestTimeoutException -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.mux.ServerApplicationError -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.injection.SeqObjectInjection -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.Params -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.TimeoutException -import com.twitter.util.logging.Logging -import org.apache.thrift.TApplicationException - -/** - * A SimilarityEngine is a wrapper which, given a [[Query]], returns a list of [[Candidate]] - * The main purposes of a SimilarityEngine is to provide a consistent interface for candidate - * generation logic, and provides default functions, including: - * - Identification - * - Observability - * - Timeout settings - * - Exception Handling - * - Gating by Deciders & FeatureSwitch settings - * - (coming soon): Dark traffic - * - * Note: - * A SimilarityEngine by itself is NOT meant to be cacheable. - * Caching should be implemented in the underlying ReadableStore that provides the [[Candidate]]s - * - * Please keep extension of this class local this directory only - * - */ -trait SimilarityEngine[Query, Candidate] { - - /** - * Uniquely identifies a similarity engine. - * Avoid using the same engine type for more than one engine, it will cause stats to double count - */ - private[similarity_engine] def identifier: SimilarityEngineType - - def getCandidates(query: Query): Future[Option[Seq[Candidate]]] - -} - -object SimilarityEngine extends Logging { - case class SimilarityEngineConfig( - timeout: Duration, - gatingConfig: GatingConfig) - - /** - * Controls for whether or not this Engine is enabled. - * In our previous design, we were expecting a Sim Engine will only take one set of Params, - * and that’s why we decided to have GatingConfig and the EnableFeatureSwitch in the trait. - * However, we now have two candidate generation pipelines: Tweet Rec, Related Tweets - * and they are now having their own set of Params, but EnableFeatureSwitch can only put in 1 fixed value. - * We need some further refactor work to make it more flexible. - * - * @param deciderConfig Gate the Engine by a decider. If specified, - * @param enableFeatureSwitch. DO NOT USE IT FOR NOW. It needs some refactorting. Please set it to None (SD-20268) - */ - case class GatingConfig( - deciderConfig: Option[DeciderConfig], - enableFeatureSwitch: Option[ - FSParam[Boolean] - ]) // Do NOT use the enableFeatureSwitch. It needs some refactoring. - - case class DeciderConfig( - decider: CrMixerDecider, - deciderString: String) - - case class MemCacheConfig[K]( - cacheClient: Client, - ttl: Duration, - asyncUpdate: Boolean = false, - keyToString: K => String) - - private[similarity_engine] def isEnabled( - params: Params, - gatingConfig: GatingConfig - ): Boolean = { - val enabledByDecider = - gatingConfig.deciderConfig.forall { config => - config.decider.isAvailable(config.deciderString) - } - - val enabledByFS = gatingConfig.enableFeatureSwitch.forall(params.apply) - - enabledByDecider && enabledByFS - } - - // Default key hasher for memcache keys - val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - - /** - * Add a MemCache wrapper to a ReadableStore with a preset key and value injection functions - * Note: The [[Query]] object needs to be cacheable, - * i.e. it cannot be a runtime objects or complex objects, for example, configapi.Params - * - * @param underlyingStore un-cached store implementation - * @param keyPrefix a prefix differentiates 2 stores if they share the same key space. - * e.x. 2 implementations of ReadableStore[UserId, Seq[Candidiate] ] - * can use prefix "store_v1", "store_v2" - * @return A ReadableStore with a MemCache wrapper - */ - private[similarity_engine] def addMemCache[Query, Candidate <: Serializable]( - underlyingStore: ReadableStore[Query, Seq[Candidate]], - memCacheConfig: MemCacheConfig[Query], - keyPrefix: Option[String] = None, - statsReceiver: StatsReceiver - ): ReadableStore[Query, Seq[Candidate]] = { - val prefix = keyPrefix.getOrElse("") - - ObservedMemcachedReadableStore.fromCacheClient[Query, Seq[Candidate]]( - backingStore = underlyingStore, - cacheClient = memCacheConfig.cacheClient, - ttl = memCacheConfig.ttl, - asyncUpdate = memCacheConfig.asyncUpdate, - )( - valueInjection = LZ4Injection.compose(SeqObjectInjection[Candidate]()), - keyToString = { k: Query => s"CRMixer:$prefix${memCacheConfig.keyToString(k)}" }, - statsReceiver = statsReceiver - ) - } - - private val timer = com.twitter.finagle.util.DefaultTimer - - /** - * Applies runtime configs, like stats, timeouts, exception handling, onto fn - */ - private[similarity_engine] def getFromFn[Query, Candidate]( - fn: Query => Future[Option[Seq[Candidate]]], - storeQuery: Query, - engineConfig: SimilarityEngineConfig, - params: Params, - scopedStats: StatsReceiver - ): Future[Option[Seq[Candidate]]] = { - if (isEnabled(params, engineConfig.gatingConfig)) { - scopedStats.counter("gate_enabled").incr() - - StatsUtil - .trackOptionItemsStats(scopedStats) { - fn.apply(storeQuery).raiseWithin(engineConfig.timeout)(timer) - } - .rescue { - case _: TimeoutException | _: GlobalRequestTimeoutException | _: TApplicationException | - _: ClientDiscardedRequestException | - _: ServerApplicationError // TApplicationException inside - => - debug("Failed to fetch. request aborted or timed out") - Future.None - case e => - error("Failed to fetch. request aborted or timed out", e) - Future.None - } - } else { - scopedStats.counter("gate_disabled").incr() - Future.None - } - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilaritySourceOrderingUtil.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilaritySourceOrderingUtil.docx new file mode 100644 index 000000000..018bf6ee3 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilaritySourceOrderingUtil.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilaritySourceOrderingUtil.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilaritySourceOrderingUtil.scala deleted file mode 100644 index b3da2b631..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SimilaritySourceOrderingUtil.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo -import com.twitter.simclusters_v2.common.TweetId -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer - -object SimilaritySourceOrderingUtil { - /** - * This function flatten and dedup input candidates according to the order in the input Seq - * [[candidate10, candidate11], [candidate20, candidate21]] => [candidate10, candidate11, candidate20, candidate21] - */ - def keepGivenOrder( - candidates: Seq[Seq[TweetWithCandidateGenerationInfo]], - ): Seq[TweetWithCandidateGenerationInfo] = { - - val seen = mutable.Set[TweetId]() - val combinedCandidates = candidates.flatten - val result = ArrayBuffer[TweetWithCandidateGenerationInfo]() - - combinedCandidates.foreach { candidate => - val candidateTweetId = candidate.tweetId - val seenCandidate = seen.contains(candidateTweetId) // de-dup - if (!seenCandidate) { - result += candidate - seen.add(candidate.tweetId) - } - } - //convert result to immutable seq - result.toList - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitHighPrecisionTopicTweetSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitHighPrecisionTopicTweetSimilarityEngine.docx new file mode 100644 index 000000000..f7fcdb592 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitHighPrecisionTopicTweetSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitHighPrecisionTopicTweetSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitHighPrecisionTopicTweetSimilarityEngine.scala deleted file mode 100644 index 37701e79e..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitHighPrecisionTopicTweetSimilarityEngine.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.contentrecommender.thriftscala.AlgorithmType -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TopicTweetWithScore -import com.twitter.cr_mixer.param.TopicTweetParams -import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine._ -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.topic_recos.thriftscala.TopicTweet -import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey -import com.twitter.util.Future - -@Singleton -case class SkitHighPrecisionTopicTweetSimilarityEngine @Inject() ( - @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ - TopicTweetPartitionFlatKey, - Seq[TopicTweet] - ], - statsReceiver: StatsReceiver) - extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] { - - private val name: String = this.getClass.getSimpleName - private val stats = statsReceiver.scope(name) - - override def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] = { - StatsUtil.trackOptionItemsStats(stats) { - fetch(query).map { tweets => - val topTweets = - tweets - .sortBy(-_.favCount) - .take(query.storeQuery.maxCandidates) - .map { tweet => - TopicTweetWithScore( - tweetId = tweet.tweetId, - score = tweet.favCount, - similarityEngineType = SimilarityEngineType.SkitHighPrecisionTopicTweet - ) - } - Some(topTweets) - } - } - } - - private def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]] = { - val latestTweetTimeInHour = System.currentTimeMillis() / 1000 / 60 / 60 - - val earliestTweetTimeInHour = latestTweetTimeInHour - - math.min(MaxTweetAgeInHours, query.storeQuery.maxTweetAge.inHours) - val timedKeys = for (timePartition <- earliestTweetTimeInHour to latestTweetTimeInHour) yield { - - TopicTweetPartitionFlatKey( - entityId = query.storeQuery.topicId.entityId, - timePartition = timePartition, - algorithmType = Some(AlgorithmType.SemanticCoreTweet), - tweetEmbeddingType = Some(EmbeddingType.LogFavBasedTweet), - language = query.storeQuery.topicId.language.getOrElse("").toLowerCase, - country = None, // Disable country. It is not used. - semanticCoreAnnotationVersionId = Some(query.storeQuery.semanticCoreVersionId) - ) - } - - getTweetsForKeys( - timedKeys, - query.storeQuery.topicId - ) - } - - /** - * Given a set of keys, multiget the underlying Strato store, combine and flatten the results. - */ - private def getTweetsForKeys( - keys: Seq[TopicTweetPartitionFlatKey], - sourceTopic: TopicId - ): Future[Seq[SkitTopicTweet]] = { - Future - .collect { skitStratoStore.multiGet(keys.toSet).values.toSeq } - .map { combinedResults => - val topTweets = combinedResults.flatten.flatten - topTweets.map { tweet => - SkitTopicTweet( - tweetId = tweet.tweetId, - favCount = tweet.scores.favCount.getOrElse(0L), - cosineSimilarityScore = tweet.scores.cosineSimilarity.getOrElse(0.0), - sourceTopic = sourceTopic - ) - } - } - } -} - -object SkitHighPrecisionTopicTweetSimilarityEngine { - - def fromParams( - topicId: TopicId, - isVideoOnly: Boolean, - params: configapi.Params, - ): EngineQuery[Query] = { - val maxCandidates = if (isVideoOnly) { - params(TopicTweetParams.MaxSkitHighPrecisionCandidatesParam) * 2 - } else { - params(TopicTweetParams.MaxSkitHighPrecisionCandidatesParam) - } - - EngineQuery( - Query( - topicId = topicId, - maxCandidates = maxCandidates, - maxTweetAge = params(TopicTweetParams.MaxTweetAge), - semanticCoreVersionId = params(TopicTweetParams.SemanticCoreVersionIdParam) - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitTopicTweetSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitTopicTweetSimilarityEngine.docx new file mode 100644 index 000000000..f69bfdad8 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitTopicTweetSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitTopicTweetSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitTopicTweetSimilarityEngine.scala deleted file mode 100644 index 44bb4b319..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/SkitTopicTweetSimilarityEngine.scala +++ /dev/null @@ -1,143 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.google.inject.Inject -import com.google.inject.Singleton -import com.google.inject.name.Named -import com.twitter.contentrecommender.thriftscala.AlgorithmType -import com.twitter.conversions.DurationOps._ -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.TopicTweetWithScore -import com.twitter.cr_mixer.param.TopicTweetParams -import com.twitter.cr_mixer.similarity_engine.SkitTopicTweetSimilarityEngine._ -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.topic_recos.thriftscala.TopicTweet -import com.twitter.topic_recos.thriftscala.TopicTweetPartitionFlatKey -import com.twitter.util.Duration -import com.twitter.util.Future - -@Singleton -case class SkitTopicTweetSimilarityEngine @Inject() ( - @Named(ModuleNames.SkitStratoStoreName) skitStratoStore: ReadableStore[ - TopicTweetPartitionFlatKey, - Seq[TopicTweet] - ], - statsReceiver: StatsReceiver) - extends ReadableStore[EngineQuery[Query], Seq[TopicTweetWithScore]] { - - private val name: String = this.getClass.getSimpleName - private val stats = statsReceiver.scope(name) - - override def get(query: EngineQuery[Query]): Future[Option[Seq[TopicTweetWithScore]]] = { - StatsUtil.trackOptionItemsStats(stats) { - fetch(query).map { tweets => - val topTweets = - tweets - .sortBy(-_.cosineSimilarityScore) - .take(query.storeQuery.maxCandidates) - .map { tweet => - TopicTweetWithScore( - tweetId = tweet.tweetId, - score = tweet.cosineSimilarityScore, - similarityEngineType = SimilarityEngineType.SkitTfgTopicTweet - ) - } - Some(topTweets) - } - } - } - - private def fetch(query: EngineQuery[Query]): Future[Seq[SkitTopicTweet]] = { - val latestTweetTimeInHour = System.currentTimeMillis() / 1000 / 60 / 60 - - val earliestTweetTimeInHour = latestTweetTimeInHour - - math.min(MaxTweetAgeInHours, query.storeQuery.maxTweetAge.inHours) - val timedKeys = for (timePartition <- earliestTweetTimeInHour to latestTweetTimeInHour) yield { - - TopicTweetPartitionFlatKey( - entityId = query.storeQuery.topicId.entityId, - timePartition = timePartition, - algorithmType = Some(AlgorithmType.TfgTweet), - tweetEmbeddingType = Some(EmbeddingType.LogFavBasedTweet), - language = query.storeQuery.topicId.language.getOrElse("").toLowerCase, - country = None, // Disable country. It is not used. - semanticCoreAnnotationVersionId = Some(query.storeQuery.semanticCoreVersionId), - simclustersModelVersion = Some(ModelVersion.Model20m145k2020) - ) - } - - getTweetsForKeys( - timedKeys, - query.storeQuery.topicId - ) - } - - /** - * Given a set of keys, multiget the underlying Strato store, combine and flatten the results. - */ - private def getTweetsForKeys( - keys: Seq[TopicTweetPartitionFlatKey], - sourceTopic: TopicId - ): Future[Seq[SkitTopicTweet]] = { - Future - .collect { skitStratoStore.multiGet(keys.toSet).values.toSeq } - .map { combinedResults => - val topTweets = combinedResults.flatten.flatten - topTweets.map { tweet => - SkitTopicTweet( - tweetId = tweet.tweetId, - favCount = tweet.scores.favCount.getOrElse(0L), - cosineSimilarityScore = tweet.scores.cosineSimilarity.getOrElse(0.0), - sourceTopic = sourceTopic - ) - } - } - } -} - -object SkitTopicTweetSimilarityEngine { - - val MaxTweetAgeInHours: Int = 7.days.inHours // Simple guard to prevent overloading - - // Query is used as a cache key. Do not add any user level information in this. - case class Query( - topicId: TopicId, - maxCandidates: Int, - maxTweetAge: Duration, - semanticCoreVersionId: Long) - - case class SkitTopicTweet( - sourceTopic: TopicId, - tweetId: TweetId, - favCount: Long, - cosineSimilarityScore: Double) - - def fromParams( - topicId: TopicId, - isVideoOnly: Boolean, - params: configapi.Params, - ): EngineQuery[Query] = { - val maxCandidates = if (isVideoOnly) { - params(TopicTweetParams.MaxSkitTfgCandidatesParam) * 2 - } else { - params(TopicTweetParams.MaxSkitTfgCandidatesParam) - } - - EngineQuery( - Query( - topicId = topicId, - maxCandidates = maxCandidates, - maxTweetAge = params(TopicTweetParams.MaxTweetAge), - semanticCoreVersionId = params(TopicTweetParams.SemanticCoreVersionIdParam) - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/StandardSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/StandardSimilarityEngine.docx new file mode 100644 index 000000000..0a454846e Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/StandardSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/StandardSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/StandardSimilarityEngine.scala deleted file mode 100644 index ae71c3736..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/StandardSimilarityEngine.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.MemCacheConfig -import com.twitter.cr_mixer.similarity_engine.SimilarityEngine.SimilarityEngineConfig -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Params -import com.twitter.util.Future - -/** - * @tparam Query ReadableStore's input type. - */ -case class EngineQuery[Query]( - storeQuery: Query, - params: Params, -) - -/** - * A straight forward SimilarityEngine implementation that wraps a ReadableStore - * - * @param implementingStore Provides the candidate retrieval's implementations - * @param memCacheConfig If specified, it will wrap the underlying store with a MemCache layer - * You should only enable this for cacheable queries, e.x. TweetIds. - * consumer based UserIds are generally not possible to cache. - * @tparam Query ReadableStore's input type - * @tparam Candidate ReadableStore's return type is Seq[[[Candidate]]] - */ -class StandardSimilarityEngine[Query, Candidate <: Serializable]( - implementingStore: ReadableStore[Query, Seq[Candidate]], - override val identifier: SimilarityEngineType, - globalStats: StatsReceiver, - engineConfig: SimilarityEngineConfig, - memCacheConfig: Option[MemCacheConfig[Query]] = None) - extends SimilarityEngine[EngineQuery[Query], Candidate] { - - private val scopedStats = globalStats.scope("similarityEngine", identifier.toString) - - def getScopedStats: StatsReceiver = scopedStats - - // Add memcache wrapper, if specified - private val store = { - memCacheConfig match { - case Some(config) => - SimilarityEngine.addMemCache( - underlyingStore = implementingStore, - memCacheConfig = config, - statsReceiver = scopedStats - ) - case _ => implementingStore - } - } - - override def getCandidates( - engineQuery: EngineQuery[Query] - ): Future[Option[Seq[Candidate]]] = { - SimilarityEngine.getFromFn( - store.get, - engineQuery.storeQuery, - engineConfig, - engineQuery.params, - scopedStats - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedQigSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedQigSimilarityEngine.docx new file mode 100644 index 000000000..dbf43e025 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedQigSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedQigSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedQigSimilarityEngine.scala deleted file mode 100644 index 317f09727..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedQigSimilarityEngine.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats -import com.twitter.product_mixer.core.thriftscala.ClientContext -import com.twitter.qig_ranker.thriftscala.Product -import com.twitter.qig_ranker.thriftscala.ProductContext -import com.twitter.qig_ranker.thriftscala.QigRanker -import com.twitter.qig_ranker.thriftscala.QigRankerProductResponse -import com.twitter.qig_ranker.thriftscala.QigRankerRequest -import com.twitter.qig_ranker.thriftscala.QigRankerResponse -import com.twitter.qig_ranker.thriftscala.TwistlySimilarTweetsProductContext -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Future -import javax.inject.Singleton - -/** - * This store looks for similar tweets from QueryInteractionGraph (QIG) for a source tweet id. - * For a given query tweet, QIG returns us the similar tweets that have an overlap of engagements - * (with the query tweet) on different search queries - */ -@Singleton -case class TweetBasedQigSimilarityEngine( - qigRanker: QigRanker.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends ReadableStore[ - TweetBasedQigSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - - override def get( - query: TweetBasedQigSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - query.sourceId match { - case InternalId.TweetId(tweetId) => - val qigSimilarTweetsRequest = getQigSimilarTweetsRequest(tweetId) - - Stats.trackOption(fetchCandidatesStat) { - qigRanker - .getSimilarCandidates(qigSimilarTweetsRequest) - .map { qigSimilarTweetsResponse => - getCandidatesFromQigResponse(qigSimilarTweetsResponse) - } - } - case _ => - Future.value(None) - } - } - - private def getQigSimilarTweetsRequest( - tweetId: Long - ): QigRankerRequest = { - // Note: QigRanker needs a non-empty userId to be passed to return results. - // We are passing in a dummy userId until we fix this on QigRanker side - val clientContext = ClientContext(userId = Some(0L)) - val productContext = ProductContext.TwistlySimilarTweetsProductContext( - TwistlySimilarTweetsProductContext(tweetId = tweetId)) - - QigRankerRequest( - clientContext = clientContext, - product = Product.TwistlySimilarTweets, - productContext = Some(productContext), - ) - } - - private def getCandidatesFromQigResponse( - qigSimilarTweetsResponse: QigRankerResponse - ): Option[Seq[TweetWithScore]] = { - qigSimilarTweetsResponse.productResponse match { - case QigRankerProductResponse - .TwistlySimilarTweetCandidatesResponse(response) => - val tweetsWithScore = response.similarTweets - .map { similarTweetResult => - TweetWithScore( - similarTweetResult.tweetResult.tweetId, - similarTweetResult.tweetResult.score.getOrElse(0L)) - } - Some(tweetsWithScore) - - case _ => None - } - } -} - -object TweetBasedQigSimilarityEngine { - - def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.Qig, - modelId = None, - score = Some(score)) - } - - case class Query(sourceId: InternalId) - - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query(sourceId = sourceId), - params - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUnifiedSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUnifiedSimilarityEngine.docx new file mode 100644 index 000000000..7488b8168 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUnifiedSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUnifiedSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUnifiedSimilarityEngine.scala deleted file mode 100644 index 6b84e2f67..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUnifiedSimilarityEngine.scala +++ /dev/null @@ -1,962 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.CandidateGenerationInfo -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.SourceInfo -import com.twitter.cr_mixer.model.TweetWithCandidateGenerationInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.param.RelatedTweetTweetBasedParams -import com.twitter.cr_mixer.param.RelatedVideoTweetTweetBasedParams -import com.twitter.cr_mixer.param.SimClustersANNParams -import com.twitter.cr_mixer.param.TweetBasedCandidateGenerationParams -import com.twitter.cr_mixer.param.TweetBasedTwHINParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.cr_mixer.thriftscala.SourceType -import com.twitter.cr_mixer.util.InterleaveUtil -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.simclusters_v2.common.ModelVersions -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.mutable.ArrayBuffer - -/** - * This store fetches similar tweets from multiple tweet based candidate sources - * and combines them using different methods obtained from query params - */ -@Singleton -case class TweetBasedUnifiedSimilarityEngine( - @Named(ModuleNames.TweetBasedUserTweetGraphSimilarityEngine) - tweetBasedUserTweetGraphSimilarityEngine: StandardSimilarityEngine[ - TweetBasedUserTweetGraphSimilarityEngine.Query, - TweetWithScore - ], - @Named(ModuleNames.TweetBasedUserVideoGraphSimilarityEngine) - tweetBasedUserVideoGraphSimilarityEngine: StandardSimilarityEngine[ - TweetBasedUserVideoGraphSimilarityEngine.Query, - TweetWithScore - ], - simClustersANNSimilarityEngine: StandardSimilarityEngine[ - SimClustersANNSimilarityEngine.Query, - TweetWithScore - ], - @Named(ModuleNames.TweetBasedQigSimilarityEngine) - tweetBasedQigSimilarTweetsSimilarityEngine: StandardSimilarityEngine[ - TweetBasedQigSimilarityEngine.Query, - TweetWithScore - ], - @Named(ModuleNames.TweetBasedTwHINANNSimilarityEngine) - tweetBasedTwHINANNSimilarityEngine: HnswANNSimilarityEngine, - statsReceiver: StatsReceiver) - extends ReadableStore[ - TweetBasedUnifiedSimilarityEngine.Query, - Seq[TweetWithCandidateGenerationInfo] - ] { - - import TweetBasedUnifiedSimilarityEngine._ - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - - override def get( - query: Query - ): Future[Option[Seq[TweetWithCandidateGenerationInfo]]] = { - - query.sourceInfo.internalId match { - case _: InternalId.TweetId => - StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { - val twhinQuery = - HnswANNEngineQuery( - sourceId = query.sourceInfo.internalId, - modelId = query.twhinModelId, - params = query.params) - val utgCandidatesFut = - if (query.enableUtg) - tweetBasedUserTweetGraphSimilarityEngine.getCandidates(query.utgQuery) - else Future.None - - val uvgCandidatesFut = - if (query.enableUvg) - tweetBasedUserVideoGraphSimilarityEngine.getCandidates(query.uvgQuery) - else Future.None - - val sannCandidatesFut = if (query.enableSimClustersANN) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANNQuery) - } else Future.None - - val sann1CandidatesFut = - if (query.enableSimClustersANN1) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN1Query) - } else Future.None - - val sann2CandidatesFut = - if (query.enableSimClustersANN2) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN2Query) - } else Future.None - - val sann3CandidatesFut = - if (query.enableSimClustersANN3) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN3Query) - } else Future.None - - val sann5CandidatesFut = - if (query.enableSimClustersANN5) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN5Query) - } else Future.None - - val sann4CandidatesFut = - if (query.enableSimClustersANN4) { - simClustersANNSimilarityEngine.getCandidates(query.simClustersANN4Query) - } else Future.None - - val experimentalSANNCandidatesFut = - if (query.enableExperimentalSimClustersANN) { - simClustersANNSimilarityEngine.getCandidates(query.experimentalSimClustersANNQuery) - } else Future.None - - val qigCandidatesFut = - if (query.enableQig) - tweetBasedQigSimilarTweetsSimilarityEngine.getCandidates(query.qigQuery) - else Future.None - - val twHINCandidateFut = if (query.enableTwHIN) { - tweetBasedTwHINANNSimilarityEngine.getCandidates(twhinQuery) - } else Future.None - - Future - .join( - utgCandidatesFut, - sannCandidatesFut, - sann1CandidatesFut, - sann2CandidatesFut, - sann3CandidatesFut, - sann5CandidatesFut, - sann4CandidatesFut, - experimentalSANNCandidatesFut, - qigCandidatesFut, - twHINCandidateFut, - uvgCandidatesFut - ).map { - case ( - userTweetGraphCandidates, - simClustersANNCandidates, - simClustersANN1Candidates, - simClustersANN2Candidates, - simClustersANN3Candidates, - simClustersANN5Candidates, - simClustersANN4Candidates, - experimentalSANNCandidates, - qigSimilarTweetsCandidates, - twhinCandidates, - userVideoGraphCandidates) => - val filteredUTGTweets = - userTweetGraphFilter(userTweetGraphCandidates.toSeq.flatten) - val filteredUVGTweets = - userVideoGraphFilter(userVideoGraphCandidates.toSeq.flatten) - val filteredSANNTweets = simClustersCandidateMinScoreFilter( - simClustersANNCandidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANNQuery.storeQuery.simClustersANNConfigId) - - val filteredSANN1Tweets = simClustersCandidateMinScoreFilter( - simClustersANN1Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN1Query.storeQuery.simClustersANNConfigId) - - val filteredSANN2Tweets = simClustersCandidateMinScoreFilter( - simClustersANN2Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN2Query.storeQuery.simClustersANNConfigId) - - val filteredSANN3Tweets = simClustersCandidateMinScoreFilter( - simClustersANN3Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN3Query.storeQuery.simClustersANNConfigId) - - val filteredSANN4Tweets = simClustersCandidateMinScoreFilter( - simClustersANN4Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN4Query.storeQuery.simClustersANNConfigId) - - val filteredSANN5Tweets = simClustersCandidateMinScoreFilter( - simClustersANN5Candidates.toSeq.flatten, - query.simClustersMinScore, - query.simClustersANN5Query.storeQuery.simClustersANNConfigId) - - val filteredExperimentalSANNTweets = simClustersCandidateMinScoreFilter( - experimentalSANNCandidates.toSeq.flatten, - query.simClustersVideoBasedMinScore, - query.experimentalSimClustersANNQuery.storeQuery.simClustersANNConfigId) - - val filteredQigTweets = qigSimilarTweetsFilter( - qigSimilarTweetsCandidates.toSeq.flatten, - query.qigMaxTweetAgeHours, - query.qigMaxNumSimilarTweets - ) - - val filteredTwHINTweets = twhinFilter( - twhinCandidates.toSeq.flatten.sortBy(-_.score), - query.twhinMaxTweetAgeHours, - tweetBasedTwHINANNSimilarityEngine.getScopedStats - ) - val utgTweetsWithCGInfo = filteredUTGTweets.map { tweetWithScore => - val similarityEngineInfo = TweetBasedUserTweetGraphSimilarityEngine - .toSimilarityEngineInfo(tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val uvgTweetsWithCGInfo = filteredUVGTweets.map { tweetWithScore => - val similarityEngineInfo = TweetBasedUserVideoGraphSimilarityEngine - .toSimilarityEngineInfo(tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sannTweetsWithCGInfo = filteredSANNTweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANNQuery, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sann1TweetsWithCGInfo = filteredSANN1Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN1Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sann2TweetsWithCGInfo = filteredSANN2Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN2Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sann3TweetsWithCGInfo = filteredSANN3Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN3Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sann4TweetsWithCGInfo = filteredSANN4Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN4Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val sann5TweetsWithCGInfo = filteredSANN5Tweets.map { tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo(query.simClustersANN5Query, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val experimentalSANNTweetsWithCGInfo = filteredExperimentalSANNTweets.map { - tweetWithScore => - val similarityEngineInfo = SimClustersANNSimilarityEngine - .toSimilarityEngineInfo( - query.experimentalSimClustersANNQuery, - tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - val qigTweetsWithCGInfo = filteredQigTweets.map { tweetWithScore => - val similarityEngineInfo = TweetBasedQigSimilarityEngine - .toSimilarityEngineInfo(tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val twHINTweetsWithCGInfo = filteredTwHINTweets.map { tweetWithScore => - val similarityEngineInfo = tweetBasedTwHINANNSimilarityEngine - .toSimilarityEngineInfo(twhinQuery, tweetWithScore.score) - TweetWithCandidateGenerationInfo( - tweetWithScore.tweetId, - CandidateGenerationInfo( - Some(query.sourceInfo), - similarityEngineInfo, - Seq(similarityEngineInfo) - )) - } - - val candidateSourcesToBeInterleaved = - ArrayBuffer[Seq[TweetWithCandidateGenerationInfo]]( - sannTweetsWithCGInfo, - experimentalSANNTweetsWithCGInfo, - sann1TweetsWithCGInfo, - sann2TweetsWithCGInfo, - sann3TweetsWithCGInfo, - sann5TweetsWithCGInfo, - sann4TweetsWithCGInfo, - qigTweetsWithCGInfo, - uvgTweetsWithCGInfo, - utgTweetsWithCGInfo, - twHINTweetsWithCGInfo - ) - - val interleavedCandidates = - InterleaveUtil.interleave(candidateSourcesToBeInterleaved) - - val unifiedCandidatesWithUnifiedCGInfo = - interleavedCandidates.map { candidate => - /*** - * when a candidate was made by interleave/keepGivenOrder, - * then we apply getTweetBasedUnifiedCGInfo() to override with the unified CGInfo - * - * we'll not have ALL SEs that generated the tweet - * in contributingSE list for interleave. We only have the chosen SE available. - */ - TweetWithCandidateGenerationInfo( - tweetId = candidate.tweetId, - candidateGenerationInfo = getTweetBasedUnifiedCGInfo( - candidate.candidateGenerationInfo.sourceInfoOpt, - candidate.getSimilarityScore, - candidate.candidateGenerationInfo.contributingSimilarityEngines - ) // getSimilarityScore comes from either unifiedScore or single score - ) - } - stats - .stat("unified_candidate_size").add(unifiedCandidatesWithUnifiedCGInfo.size) - - val truncatedCandidates = - unifiedCandidatesWithUnifiedCGInfo.take(query.maxCandidateNumPerSourceKey) - stats.stat("truncatedCandidates_size").add(truncatedCandidates.size) - - Some(truncatedCandidates) - } - } - - case _ => - stats.counter("sourceId_is_not_tweetId_cnt").incr() - Future.None - } - } - - private def simClustersCandidateMinScoreFilter( - simClustersAnnCandidates: Seq[TweetWithScore], - simClustersMinScore: Double, - simClustersANNConfigId: String - ): Seq[TweetWithScore] = { - val filteredCandidates = simClustersAnnCandidates - .filter { candidate => - candidate.score > simClustersMinScore - } - - stats.stat(simClustersANNConfigId, "simClustersAnnCandidates_size").add(filteredCandidates.size) - stats.counter(simClustersANNConfigId, "simClustersAnnRequests").incr() - if (filteredCandidates.isEmpty) - stats.counter(simClustersANNConfigId, "emptyFilteredSimClustersAnnCandidates").incr() - - filteredCandidates.map { candidate => - TweetWithScore(candidate.tweetId, candidate.score) - } - } - - /** Returns a list of tweets that are generated less than `maxTweetAgeHours` hours ago */ - private def tweetAgeFilter( - candidates: Seq[TweetWithScore], - maxTweetAgeHours: Duration - ): Seq[TweetWithScore] = { - // Tweet IDs are approximately chronological (see http://go/snowflake), - // so we are building the earliest tweet id once - // The per-candidate logic here then be candidate.tweetId > earliestPermittedTweetId, which is far cheaper. - val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAgeHours) - candidates.filter { candidate => candidate.tweetId >= earliestTweetId } - } - - private def twhinFilter( - twhinCandidates: Seq[TweetWithScore], - twhinMaxTweetAgeHours: Duration, - simEngineStats: StatsReceiver - ): Seq[TweetWithScore] = { - simEngineStats.stat("twhinCandidates_size").add(twhinCandidates.size) - val candidates = twhinCandidates.map { candidate => - TweetWithScore(candidate.tweetId, candidate.score) - } - - val filteredCandidates = tweetAgeFilter(candidates, twhinMaxTweetAgeHours) - simEngineStats.stat("filteredTwhinCandidates_size").add(filteredCandidates.size) - if (filteredCandidates.isEmpty) simEngineStats.counter("emptyFilteredTwhinCandidates").incr() - - filteredCandidates - } - - /** A no-op filter as UTG filtering already happens on UTG service side */ - private def userTweetGraphFilter( - userTweetGraphCandidates: Seq[TweetWithScore] - ): Seq[TweetWithScore] = { - val filteredCandidates = userTweetGraphCandidates - - stats.stat("userTweetGraphCandidates_size").add(userTweetGraphCandidates.size) - if (filteredCandidates.isEmpty) stats.counter("emptyFilteredUserTweetGraphCandidates").incr() - - filteredCandidates.map { candidate => - TweetWithScore(candidate.tweetId, candidate.score) - } - } - - /** A no-op filter as UVG filtering already happens on UVG service side */ - private def userVideoGraphFilter( - userVideoGraphCandidates: Seq[TweetWithScore] - ): Seq[TweetWithScore] = { - val filteredCandidates = userVideoGraphCandidates - - stats.stat("userVideoGraphCandidates_size").add(userVideoGraphCandidates.size) - if (filteredCandidates.isEmpty) stats.counter("emptyFilteredUserVideoGraphCandidates").incr() - - filteredCandidates.map { candidate => - TweetWithScore(candidate.tweetId, candidate.score) - } - } - private def qigSimilarTweetsFilter( - qigSimilarTweetsCandidates: Seq[TweetWithScore], - qigMaxTweetAgeHours: Duration, - qigMaxNumSimilarTweets: Int - ): Seq[TweetWithScore] = { - val ageFilteredCandidates = tweetAgeFilter(qigSimilarTweetsCandidates, qigMaxTweetAgeHours) - stats.stat("ageFilteredQigSimilarTweetsCandidates_size").add(ageFilteredCandidates.size) - - val filteredCandidates = ageFilteredCandidates.take(qigMaxNumSimilarTweets) - if (filteredCandidates.isEmpty) stats.counter("emptyFilteredQigSimilarTweetsCandidates").incr() - - filteredCandidates - } - - /*** - * Every candidate will have the CG Info with TweetBasedUnifiedSimilarityEngine - * as they are generated by a composite of Similarity Engines. - * Additionally, we store the contributing SEs (eg., SANN, UTG). - */ - private def getTweetBasedUnifiedCGInfo( - sourceInfoOpt: Option[SourceInfo], - unifiedScore: Double, - contributingSimilarityEngines: Seq[SimilarityEngineInfo] - ): CandidateGenerationInfo = { - CandidateGenerationInfo( - sourceInfoOpt, - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.TweetBasedUnifiedSimilarityEngine, - modelId = None, // We do not assign modelId for a unified similarity engine - score = Some(unifiedScore) - ), - contributingSimilarityEngines - ) - } -} - -object TweetBasedUnifiedSimilarityEngine { - - case class Query( - sourceInfo: SourceInfo, - maxCandidateNumPerSourceKey: Int, - enableSimClustersANN: Boolean, - simClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableExperimentalSimClustersANN: Boolean, - experimentalSimClustersANNQuery: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN1: Boolean, - simClustersANN1Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN2: Boolean, - simClustersANN2Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN3: Boolean, - simClustersANN3Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN5: Boolean, - simClustersANN5Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - enableSimClustersANN4: Boolean, - simClustersANN4Query: EngineQuery[SimClustersANNSimilarityEngine.Query], - simClustersMinScore: Double, - simClustersVideoBasedMinScore: Double, - twhinModelId: String, - enableTwHIN: Boolean, - twhinMaxTweetAgeHours: Duration, - qigMaxTweetAgeHours: Duration, - qigMaxNumSimilarTweets: Int, - enableUtg: Boolean, - utgQuery: EngineQuery[TweetBasedUserTweetGraphSimilarityEngine.Query], - enableUvg: Boolean, - uvgQuery: EngineQuery[TweetBasedUserVideoGraphSimilarityEngine.Query], - enableQig: Boolean, - qigQuery: EngineQuery[TweetBasedQigSimilarityEngine.Query], - params: configapi.Params) - - def fromParams( - sourceInfo: SourceInfo, - params: configapi.Params, - ): EngineQuery[Query] = { - // SimClusters - val enableSimClustersANN = - params(TweetBasedCandidateGenerationParams.EnableSimClustersANNParam) - - val simClustersModelVersion = - ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) - val simClustersMinScore = params(TweetBasedCandidateGenerationParams.SimClustersMinScoreParam) - val simClustersVideoBasedMinScore = params( - TweetBasedCandidateGenerationParams.SimClustersVideoBasedMinScoreParam) - val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) - // SimClusters - Experimental SANN Similarity Engine (Video based SE) - val enableExperimentalSimClustersANN = - params(TweetBasedCandidateGenerationParams.EnableExperimentalSimClustersANNParam) - - val experimentalSimClustersANNConfigId = params( - SimClustersANNParams.ExperimentalSimClustersANNConfigId) - // SimClusters - SANN cluster 1 Similarity Engine - val enableSimClustersANN1 = - params(TweetBasedCandidateGenerationParams.EnableSimClustersANN1Param) - - val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) - // SimClusters - SANN cluster 2 Similarity Engine - val enableSimClustersANN2 = - params(TweetBasedCandidateGenerationParams.EnableSimClustersANN2Param) - val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) - // SimClusters - SANN cluster 3 Similarity Engine - val enableSimClustersANN3 = - params(TweetBasedCandidateGenerationParams.EnableSimClustersANN3Param) - val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) - // SimClusters - SANN cluster 5 Similarity Engine - val enableSimClustersANN5 = - params(TweetBasedCandidateGenerationParams.EnableSimClustersANN5Param) - val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) - // SimClusters - SANN cluster 4 Similarity Engine - val enableSimClustersANN4 = - params(TweetBasedCandidateGenerationParams.EnableSimClustersANN4Param) - val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) - // SimClusters ANN Queries for different SANN clusters - val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANNConfigId, - params - ) - val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - experimentalSimClustersANNConfigId, - params - ) - val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN1ConfigId, - params - ) - val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN2ConfigId, - params - ) - val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN3ConfigId, - params - ) - val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN5ConfigId, - params - ) - val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( - sourceInfo.internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN4ConfigId, - params - ) - // TweetBasedCandidateGeneration - val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) - // TwHIN - val twhinModelId = params(TweetBasedTwHINParams.ModelIdParam) - val enableTwHIN = - params(TweetBasedCandidateGenerationParams.EnableTwHINParam) - - val twhinMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - - // QIG - val enableQig = - params(TweetBasedCandidateGenerationParams.EnableQigSimilarTweetsParam) - val qigMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - val qigMaxNumSimilarTweets = params( - TweetBasedCandidateGenerationParams.QigMaxNumSimilarTweetsParam) - - // UTG - val enableUtg = - params(TweetBasedCandidateGenerationParams.EnableUTGParam) - // UVG - val enableUvg = - params(TweetBasedCandidateGenerationParams.EnableUVGParam) - EngineQuery( - Query( - sourceInfo = sourceInfo, - maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, - enableSimClustersANN = enableSimClustersANN, - simClustersANNQuery = simClustersANNQuery, - enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, - experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, - enableSimClustersANN1 = enableSimClustersANN1, - simClustersANN1Query = simClustersANN1Query, - enableSimClustersANN2 = enableSimClustersANN2, - simClustersANN2Query = simClustersANN2Query, - enableSimClustersANN3 = enableSimClustersANN3, - simClustersANN3Query = simClustersANN3Query, - enableSimClustersANN5 = enableSimClustersANN5, - simClustersANN5Query = simClustersANN5Query, - enableSimClustersANN4 = enableSimClustersANN4, - simClustersANN4Query = simClustersANN4Query, - simClustersMinScore = simClustersMinScore, - simClustersVideoBasedMinScore = simClustersVideoBasedMinScore, - twhinModelId = twhinModelId, - enableTwHIN = enableTwHIN, - twhinMaxTweetAgeHours = twhinMaxTweetAgeHours, - qigMaxTweetAgeHours = qigMaxTweetAgeHours, - qigMaxNumSimilarTweets = qigMaxNumSimilarTweets, - enableUtg = enableUtg, - utgQuery = TweetBasedUserTweetGraphSimilarityEngine - .fromParams(sourceInfo.internalId, params), - enableQig = enableQig, - qigQuery = TweetBasedQigSimilarityEngine.fromParams(sourceInfo.internalId, params), - enableUvg = enableUvg, - uvgQuery = - TweetBasedUserVideoGraphSimilarityEngine.fromParams(sourceInfo.internalId, params), - params = params - ), - params - ) - } - - def fromParamsForRelatedTweet( - internalId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - // SimClusters - val enableSimClustersANN = params(RelatedTweetTweetBasedParams.EnableSimClustersANNParam) - val simClustersModelVersion = - ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) - val simClustersMinScore = params(RelatedTweetTweetBasedParams.SimClustersMinScoreParam) - val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) - val enableExperimentalSimClustersANN = - params(RelatedTweetTweetBasedParams.EnableExperimentalSimClustersANNParam) - val experimentalSimClustersANNConfigId = params( - SimClustersANNParams.ExperimentalSimClustersANNConfigId) - // SimClusters - SANN cluster 1 Similarity Engine - val enableSimClustersANN1 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN1Param) - val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) - // SimClusters - SANN cluster 2 Similarity Engine - val enableSimClustersANN2 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN2Param) - val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) - // SimClusters - SANN cluster 3 Similarity Engine - val enableSimClustersANN3 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN3Param) - val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) - // SimClusters - SANN cluster 5 Similarity Engine - val enableSimClustersANN5 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN5Param) - val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) - // SimClusters - SANN cluster 4 Similarity Engine - val enableSimClustersANN4 = params(RelatedTweetTweetBasedParams.EnableSimClustersANN4Param) - val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) - // SimClusters ANN Queries for different SANN clusters - val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANNConfigId, - params - ) - val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - experimentalSimClustersANNConfigId, - params - ) - val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN1ConfigId, - params - ) - val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN2ConfigId, - params - ) - val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN3ConfigId, - params - ) - val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN5ConfigId, - params - ) - val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN4ConfigId, - params - ) - // TweetBasedCandidateGeneration - val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) - // TwHIN - val twhinModelId = params(TweetBasedTwHINParams.ModelIdParam) - val enableTwHIN = params(RelatedTweetTweetBasedParams.EnableTwHINParam) - val twhinMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - // QIG - val enableQig = params(RelatedTweetTweetBasedParams.EnableQigSimilarTweetsParam) - val qigMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - val qigMaxNumSimilarTweets = params( - TweetBasedCandidateGenerationParams.QigMaxNumSimilarTweetsParam) - // UTG - val enableUtg = params(RelatedTweetTweetBasedParams.EnableUTGParam) - // UVG - val enableUvg = params(RelatedTweetTweetBasedParams.EnableUVGParam) - // SourceType.RequestTweetId is a placeholder. - val sourceInfo = SourceInfo(SourceType.RequestTweetId, internalId, None) - - EngineQuery( - Query( - sourceInfo = sourceInfo, - maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, - enableSimClustersANN = enableSimClustersANN, - simClustersMinScore = simClustersMinScore, - simClustersVideoBasedMinScore = simClustersMinScore, - simClustersANNQuery = simClustersANNQuery, - enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, - experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, - enableSimClustersANN1 = enableSimClustersANN1, - simClustersANN1Query = simClustersANN1Query, - enableSimClustersANN2 = enableSimClustersANN2, - simClustersANN2Query = simClustersANN2Query, - enableSimClustersANN3 = enableSimClustersANN3, - simClustersANN3Query = simClustersANN3Query, - enableSimClustersANN5 = enableSimClustersANN5, - simClustersANN5Query = simClustersANN5Query, - enableSimClustersANN4 = enableSimClustersANN4, - simClustersANN4Query = simClustersANN4Query, - twhinModelId = twhinModelId, - enableTwHIN = enableTwHIN, - twhinMaxTweetAgeHours = twhinMaxTweetAgeHours, - qigMaxTweetAgeHours = qigMaxTweetAgeHours, - qigMaxNumSimilarTweets = qigMaxNumSimilarTweets, - enableUtg = enableUtg, - utgQuery = TweetBasedUserTweetGraphSimilarityEngine - .fromParams(sourceInfo.internalId, params), - enableQig = enableQig, - qigQuery = TweetBasedQigSimilarityEngine.fromParams(sourceInfo.internalId, params), - enableUvg = enableUvg, - uvgQuery = - TweetBasedUserVideoGraphSimilarityEngine.fromParams(sourceInfo.internalId, params), - params = params, - ), - params - ) - } - def fromParamsForRelatedVideoTweet( - internalId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - // SimClusters - val enableSimClustersANN = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANNParam) - val simClustersModelVersion = - ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam)) - val simClustersMinScore = params(RelatedVideoTweetTweetBasedParams.SimClustersMinScoreParam) - val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId) - val enableExperimentalSimClustersANN = params( - RelatedVideoTweetTweetBasedParams.EnableExperimentalSimClustersANNParam) - val experimentalSimClustersANNConfigId = params( - SimClustersANNParams.ExperimentalSimClustersANNConfigId) - // SimClusters - SANN cluster 1 Similarity Engine - val enableSimClustersANN1 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN1Param) - val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId) - // SimClusters - SANN cluster 2 Similarity Engine - val enableSimClustersANN2 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN2Param) - val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId) - // SimClusters - SANN cluster 3 Similarity Engine - val enableSimClustersANN3 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN3Param) - val simClustersANN3ConfigId = params(SimClustersANNParams.SimClustersANN3ConfigId) - // SimClusters - SANN cluster 5 Similarity Engine - val enableSimClustersANN5 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN5Param) - val simClustersANN5ConfigId = params(SimClustersANNParams.SimClustersANN5ConfigId) - - // SimClusters - SANN cluster 4 Similarity Engine - val enableSimClustersANN4 = params(RelatedVideoTweetTweetBasedParams.EnableSimClustersANN4Param) - val simClustersANN4ConfigId = params(SimClustersANNParams.SimClustersANN4ConfigId) - // SimClusters ANN Queries for different SANN clusters - val simClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANNConfigId, - params - ) - val experimentalSimClustersANNQuery = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - experimentalSimClustersANNConfigId, - params - ) - val simClustersANN1Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN1ConfigId, - params - ) - val simClustersANN2Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN2ConfigId, - params - ) - val simClustersANN3Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN3ConfigId, - params - ) - val simClustersANN5Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN5ConfigId, - params - ) - - val simClustersANN4Query = SimClustersANNSimilarityEngine.fromParams( - internalId, - EmbeddingType.LogFavLongestL2EmbeddingTweet, - simClustersModelVersion, - simClustersANN4ConfigId, - params - ) - // TweetBasedCandidateGeneration - val maxCandidateNumPerSourceKey = params(GlobalParams.MaxCandidateNumPerSourceKeyParam) - // TwHIN - val twhinModelId = params(TweetBasedTwHINParams.ModelIdParam) - val enableTwHIN = params(RelatedVideoTweetTweetBasedParams.EnableTwHINParam) - val twhinMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - // QIG - val enableQig = params(RelatedVideoTweetTweetBasedParams.EnableQigSimilarTweetsParam) - val qigMaxTweetAgeHours = params(GlobalParams.MaxTweetAgeHoursParam) - val qigMaxNumSimilarTweets = params( - TweetBasedCandidateGenerationParams.QigMaxNumSimilarTweetsParam) - // UTG - val enableUtg = params(RelatedVideoTweetTweetBasedParams.EnableUTGParam) - - // SourceType.RequestTweetId is a placeholder. - val sourceInfo = SourceInfo(SourceType.RequestTweetId, internalId, None) - - val enableUvg = params(RelatedVideoTweetTweetBasedParams.EnableUVGParam) - EngineQuery( - Query( - sourceInfo = sourceInfo, - maxCandidateNumPerSourceKey = maxCandidateNumPerSourceKey, - enableSimClustersANN = enableSimClustersANN, - simClustersMinScore = simClustersMinScore, - simClustersVideoBasedMinScore = simClustersMinScore, - simClustersANNQuery = simClustersANNQuery, - enableExperimentalSimClustersANN = enableExperimentalSimClustersANN, - experimentalSimClustersANNQuery = experimentalSimClustersANNQuery, - enableSimClustersANN1 = enableSimClustersANN1, - simClustersANN1Query = simClustersANN1Query, - enableSimClustersANN2 = enableSimClustersANN2, - simClustersANN2Query = simClustersANN2Query, - enableSimClustersANN3 = enableSimClustersANN3, - simClustersANN3Query = simClustersANN3Query, - enableSimClustersANN5 = enableSimClustersANN5, - simClustersANN5Query = simClustersANN5Query, - enableSimClustersANN4 = enableSimClustersANN4, - simClustersANN4Query = simClustersANN4Query, - twhinModelId = twhinModelId, - enableTwHIN = enableTwHIN, - twhinMaxTweetAgeHours = twhinMaxTweetAgeHours, - qigMaxTweetAgeHours = qigMaxTweetAgeHours, - qigMaxNumSimilarTweets = qigMaxNumSimilarTweets, - enableUtg = enableUtg, - utgQuery = TweetBasedUserTweetGraphSimilarityEngine - .fromParams(sourceInfo.internalId, params), - enableUvg = enableUvg, - uvgQuery = - TweetBasedUserVideoGraphSimilarityEngine.fromParams(sourceInfo.internalId, params), - enableQig = enableQig, - qigQuery = TweetBasedQigSimilarityEngine.fromParams(sourceInfo.internalId, params), - params = params - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserAdGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserAdGraphSimilarityEngine.docx new file mode 100644 index 000000000..b70b8d4ae Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserAdGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserAdGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserAdGraphSimilarityEngine.scala deleted file mode 100644 index 365bead35..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserAdGraphSimilarityEngine.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.param.TweetBasedUserAdGraphParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.recos.user_ad_graph.thriftscala.ConsumersBasedRelatedAdRequest -import com.twitter.recos.user_ad_graph.thriftscala.RelatedAdResponse -import com.twitter.recos.user_ad_graph.thriftscala.UserAdGraph -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers -import com.twitter.util.Future -import javax.inject.Singleton - -/** - * This store looks for similar tweets from UserAdGraph for a Source TweetId - * For a query tweet,User Ad Graph (UAG) - * lets us find out which other tweets share a lot of the same engagers with the query tweet - */ -@Singleton -case class TweetBasedUserAdGraphSimilarityEngine( - userAdGraphService: UserAdGraph.MethodPerEndpoint, - tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], - statsReceiver: StatsReceiver) - extends ReadableStore[ - TweetBasedUserAdGraphSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - import TweetBasedUserAdGraphSimilarityEngine._ - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCoverageExpansionCandidatesStat = stats.scope("fetchCoverageExpansionCandidates") - override def get( - query: TweetBasedUserAdGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - query.sourceId match { - case InternalId.TweetId(tweetId) => getCandidates(tweetId, query) - case _ => - Future.value(None) - } - } - - // We first fetch tweet's recent engaged users as consumeSeedSet from MH store, - // then query consumersBasedUTG using the consumerSeedSet - private def getCandidates( - tweetId: TweetId, - query: TweetBasedUserAdGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - StatsUtil - .trackOptionItemsStats(fetchCoverageExpansionCandidatesStat) { - tweetEngagedUsersStore - .get(tweetId).flatMap { - _.map { tweetRecentEngagedUsers => - val consumerSeedSet = - tweetRecentEngagedUsers.recentEngagedUsers - .map { _.userId }.take(query.maxConsumerSeedsNum) - val consumersBasedRelatedAdRequest = - ConsumersBasedRelatedAdRequest( - consumerSeedSet = consumerSeedSet, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - excludeTweetIds = Some(Seq(tweetId)), - minScore = Some(query.consumersBasedMinScore), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours) - ) - toTweetWithScore(userAdGraphService - .consumersBasedRelatedAds(consumersBasedRelatedAdRequest).map { Some(_) }) - }.getOrElse(Future.value(None)) - } - } - } - -} - -object TweetBasedUserAdGraphSimilarityEngine { - - def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.TweetBasedUserAdGraph, - modelId = None, - score = Some(score)) - } - private def toTweetWithScore( - relatedAdResponseFut: Future[Option[RelatedAdResponse]] - ): Future[Option[Seq[TweetWithScore]]] = { - relatedAdResponseFut.map { relatedAdResponseOpt => - relatedAdResponseOpt.map { relatedAdResponse => - val candidates = - relatedAdResponse.adTweets.map(tweet => TweetWithScore(tweet.adTweetId, tweet.score)) - - candidates - } - } - } - - case class Query( - sourceId: InternalId, - maxResults: Int, - minCooccurrence: Int, - consumersBasedMinScore: Double, - maxTweetAgeInHours: Int, - maxConsumerSeedsNum: Int, - ) - - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query( - sourceId = sourceId, - maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), - minCooccurrence = params(TweetBasedUserAdGraphParams.MinCoOccurrenceParam), - consumersBasedMinScore = params(TweetBasedUserAdGraphParams.ConsumersBasedMinScoreParam), - maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, - maxConsumerSeedsNum = params(TweetBasedUserAdGraphParams.MaxConsumerSeedsNumParam), - ), - params - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserTweetGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserTweetGraphSimilarityEngine.docx new file mode 100644 index 000000000..a5357a316 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserTweetGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserTweetGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserTweetGraphSimilarityEngine.scala deleted file mode 100644 index 316f980a7..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserTweetGraphSimilarityEngine.scala +++ /dev/null @@ -1,184 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.param.TweetBasedUserTweetGraphParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.recos.user_tweet_graph.thriftscala.RelatedTweetResponse -import com.twitter.recos.user_tweet_graph.thriftscala.TweetBasedRelatedTweetRequest -import com.twitter.recos.user_tweet_graph.thriftscala.ConsumersBasedRelatedTweetRequest -import com.twitter.recos.user_tweet_graph.thriftscala.UserTweetGraph -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers -import com.twitter.util.Future -import javax.inject.Singleton -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.timelines.configapi -import com.twitter.util.Duration -import com.twitter.util.Time -import scala.concurrent.duration.HOURS - -/** - * This store looks for similar tweets from UserTweetGraph for a Source TweetId - * For a query tweet,User Tweet Graph (UTG), - * lets us find out which other tweets share a lot of the same engagers with the query tweet - * one-pager: go/UTG - */ -@Singleton -case class TweetBasedUserTweetGraphSimilarityEngine( - userTweetGraphService: UserTweetGraph.MethodPerEndpoint, - tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], - statsReceiver: StatsReceiver) - extends ReadableStore[ - TweetBasedUserTweetGraphSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - import TweetBasedUserTweetGraphSimilarityEngine._ - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - private val fetchCoverageExpansionCandidatesStat = stats.scope("fetchCoverageExpansionCandidates") - - override def get( - query: TweetBasedUserTweetGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - query.sourceId match { - case InternalId.TweetId(tweetId) if query.enableCoverageExpansionAllTweet => - getCoverageExpansionCandidates(tweetId, query) - - case InternalId.TweetId(tweetId) if query.enableCoverageExpansionOldTweet => // For Home - if (isOldTweet(tweetId)) getCoverageExpansionCandidates(tweetId, query) - else getCandidates(tweetId, query) - - case InternalId.TweetId(tweetId) => getCandidates(tweetId, query) - case _ => - Future.value(None) - } - } - - // This is the main candidate source - private def getCandidates( - tweetId: TweetId, - query: TweetBasedUserTweetGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { - val tweetBasedRelatedTweetRequest = { - TweetBasedRelatedTweetRequest( - tweetId, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - excludeTweetIds = Some(Seq(tweetId)), - minScore = Some(query.tweetBasedMinScore), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours) - ) - } - toTweetWithScore( - userTweetGraphService.tweetBasedRelatedTweets(tweetBasedRelatedTweetRequest).map { - Some(_) - }) - } - } - - // function for DDGs, for coverage expansion algo, we first fetch tweet's recent engaged users as consumeSeedSet from MH store, - // and query consumersBasedUTG using the consumeSeedSet - private def getCoverageExpansionCandidates( - tweetId: TweetId, - query: TweetBasedUserTweetGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - StatsUtil - .trackOptionItemsStats(fetchCoverageExpansionCandidatesStat) { - tweetEngagedUsersStore - .get(tweetId).flatMap { - _.map { tweetRecentEngagedUsers => - val consumerSeedSet = - tweetRecentEngagedUsers.recentEngagedUsers - .map { _.userId }.take(query.maxConsumerSeedsNum) - val consumersBasedRelatedTweetRequest = - ConsumersBasedRelatedTweetRequest( - consumerSeedSet = consumerSeedSet, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - excludeTweetIds = Some(Seq(tweetId)), - minScore = Some(query.consumersBasedMinScore), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours) - ) - - toTweetWithScore(userTweetGraphService - .consumersBasedRelatedTweets(consumersBasedRelatedTweetRequest).map { Some(_) }) - }.getOrElse(Future.value(None)) - } - } - } - -} - -object TweetBasedUserTweetGraphSimilarityEngine { - - def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.TweetBasedUserTweetGraph, - modelId = None, - score = Some(score)) - } - - private val oldTweetCap: Duration = Duration(48, HOURS) - - private def toTweetWithScore( - relatedTweetResponseFut: Future[Option[RelatedTweetResponse]] - ): Future[Option[Seq[TweetWithScore]]] = { - relatedTweetResponseFut.map { relatedTweetResponseOpt => - relatedTweetResponseOpt.map { relatedTweetResponse => - val candidates = - relatedTweetResponse.tweets.map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) - candidates - } - } - } - - private def isOldTweet(tweetId: TweetId): Boolean = { - SnowflakeId - .timeFromIdOpt(tweetId).forall { tweetTime => tweetTime < Time.now - oldTweetCap } - // If there's no snowflake timestamp, we have no idea when this tweet happened. - } - - case class Query( - sourceId: InternalId, - maxResults: Int, - minCooccurrence: Int, - tweetBasedMinScore: Double, - consumersBasedMinScore: Double, - maxTweetAgeInHours: Int, - maxConsumerSeedsNum: Int, - enableCoverageExpansionOldTweet: Boolean, - enableCoverageExpansionAllTweet: Boolean, - ) - - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query( - sourceId = sourceId, - maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), - minCooccurrence = params(TweetBasedUserTweetGraphParams.MinCoOccurrenceParam), - tweetBasedMinScore = params(TweetBasedUserTweetGraphParams.TweetBasedMinScoreParam), - consumersBasedMinScore = params(TweetBasedUserTweetGraphParams.ConsumersBasedMinScoreParam), - maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, - maxConsumerSeedsNum = params(TweetBasedUserTweetGraphParams.MaxConsumerSeedsNumParam), - enableCoverageExpansionOldTweet = - params(TweetBasedUserTweetGraphParams.EnableCoverageExpansionOldTweetParam), - enableCoverageExpansionAllTweet = - params(TweetBasedUserTweetGraphParams.EnableCoverageExpansionAllTweetParam), - ), - params - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserVideoGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserVideoGraphSimilarityEngine.docx new file mode 100644 index 000000000..c4d9d9b39 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserVideoGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserVideoGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserVideoGraphSimilarityEngine.scala deleted file mode 100644 index 6190cd4fb..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TweetBasedUserVideoGraphSimilarityEngine.scala +++ /dev/null @@ -1,184 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.param.GlobalParams -import com.twitter.cr_mixer.param.TweetBasedUserVideoGraphParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.StatsUtil -import com.twitter.recos.user_video_graph.thriftscala.RelatedTweetResponse -import com.twitter.recos.user_video_graph.thriftscala.ConsumersBasedRelatedTweetRequest -import com.twitter.recos.user_video_graph.thriftscala.TweetBasedRelatedTweetRequest -import com.twitter.recos.user_video_graph.thriftscala.UserVideoGraph -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.timelines.configapi -import com.twitter.twistly.thriftscala.TweetRecentEngagedUsers -import com.twitter.util.Duration -import javax.inject.Singleton -import com.twitter.util.Future -import com.twitter.util.Time -import scala.concurrent.duration.HOURS - -/** - * This store looks for similar tweets from UserVideoGraph for a Source TweetId - * For a query tweet,User Video Graph (UVG), - * lets us find out which other video tweets share a lot of the same engagers with the query tweet - */ -@Singleton -case class TweetBasedUserVideoGraphSimilarityEngine( - userVideoGraphService: UserVideoGraph.MethodPerEndpoint, - tweetEngagedUsersStore: ReadableStore[TweetId, TweetRecentEngagedUsers], - statsReceiver: StatsReceiver) - extends ReadableStore[ - TweetBasedUserVideoGraphSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - import TweetBasedUserVideoGraphSimilarityEngine._ - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val fetchCandidatesStat = stats.scope("fetchCandidates") - private val fetchCoverageExpansionCandidatesStat = stats.scope("fetchCoverageExpansionCandidates") - - override def get( - query: TweetBasedUserVideoGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - - query.sourceId match { - case InternalId.TweetId(tweetId) if query.enableCoverageExpansionAllTweet => - getCoverageExpansionCandidates(tweetId, query) - - case InternalId.TweetId(tweetId) if query.enableCoverageExpansionOldTweet => // For Home - if (isOldTweet(tweetId)) getCoverageExpansionCandidates(tweetId, query) - else getCandidates(tweetId, query) - - case InternalId.TweetId(tweetId) => getCandidates(tweetId, query) - case _ => - Future.value(None) - } - } - - private def getCandidates( - tweetId: TweetId, - query: TweetBasedUserVideoGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - StatsUtil.trackOptionItemsStats(fetchCandidatesStat) { - val tweetBasedRelatedTweetRequest = { - TweetBasedRelatedTweetRequest( - tweetId, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - excludeTweetIds = Some(Seq(tweetId)), - minScore = Some(query.tweetBasedMinScore), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours) - ) - } - toTweetWithScore( - userVideoGraphService.tweetBasedRelatedTweets(tweetBasedRelatedTweetRequest).map { - Some(_) - }) - } - } - - private def getCoverageExpansionCandidates( - tweetId: TweetId, - query: TweetBasedUserVideoGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - StatsUtil - .trackOptionItemsStats(fetchCoverageExpansionCandidatesStat) { - tweetEngagedUsersStore - .get(tweetId).flatMap { - _.map { tweetRecentEngagedUsers => - val consumerSeedSet = - tweetRecentEngagedUsers.recentEngagedUsers - .map { - _.userId - }.take(query.maxConsumerSeedsNum) - val consumersBasedRelatedTweetRequest = - ConsumersBasedRelatedTweetRequest( - consumerSeedSet = consumerSeedSet, - maxResults = Some(query.maxResults), - minCooccurrence = Some(query.minCooccurrence), - excludeTweetIds = Some(Seq(tweetId)), - minScore = Some(query.consumersBasedMinScore), - maxTweetAgeInHours = Some(query.maxTweetAgeInHours) - ) - - toTweetWithScore(userVideoGraphService - .consumersBasedRelatedTweets(consumersBasedRelatedTweetRequest).map { - Some(_) - }) - }.getOrElse(Future.value(None)) - } - } - } - -} - -object TweetBasedUserVideoGraphSimilarityEngine { - - private val oldTweetCap: Duration = Duration(24, HOURS) - - def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.TweetBasedUserVideoGraph, - modelId = None, - score = Some(score)) - } - - private def toTweetWithScore( - relatedTweetResponseFut: Future[Option[RelatedTweetResponse]] - ): Future[Option[Seq[TweetWithScore]]] = { - relatedTweetResponseFut.map { relatedTweetResponseOpt => - relatedTweetResponseOpt.map { relatedTweetResponse => - val candidates = - relatedTweetResponse.tweets.map(tweet => TweetWithScore(tweet.tweetId, tweet.score)) - candidates - } - } - } - - private def isOldTweet(tweetId: TweetId): Boolean = { - SnowflakeId - .timeFromIdOpt(tweetId).forall { tweetTime => tweetTime < Time.now - oldTweetCap } - // If there's no snowflake timestamp, we have no idea when this tweet happened. - } - - case class Query( - sourceId: InternalId, - maxResults: Int, - minCooccurrence: Int, - tweetBasedMinScore: Double, - consumersBasedMinScore: Double, - maxTweetAgeInHours: Int, - maxConsumerSeedsNum: Int, - enableCoverageExpansionOldTweet: Boolean, - enableCoverageExpansionAllTweet: Boolean) - - def fromParams( - sourceId: InternalId, - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query( - sourceId = sourceId, - maxResults = params(GlobalParams.MaxCandidateNumPerSourceKeyParam), - minCooccurrence = params(TweetBasedUserVideoGraphParams.MinCoOccurrenceParam), - tweetBasedMinScore = params(TweetBasedUserVideoGraphParams.TweetBasedMinScoreParam), - consumersBasedMinScore = params(TweetBasedUserVideoGraphParams.ConsumersBasedMinScoreParam), - maxTweetAgeInHours = params(GlobalParams.MaxTweetAgeHoursParam).inHours, - maxConsumerSeedsNum = params(TweetBasedUserVideoGraphParams.MaxConsumerSeedsNumParam), - enableCoverageExpansionOldTweet = - params(TweetBasedUserVideoGraphParams.EnableCoverageExpansionOldTweetParam), - enableCoverageExpansionAllTweet = - params(TweetBasedUserVideoGraphParams.EnableCoverageExpansionAllTweetParam) - ), - params - ) - } - -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TwhinCollabFilterSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TwhinCollabFilterSimilarityEngine.docx new file mode 100644 index 000000000..5c59b7d78 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TwhinCollabFilterSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TwhinCollabFilterSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TwhinCollabFilterSimilarityEngine.scala deleted file mode 100644 index eccab6aa3..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/TwhinCollabFilterSimilarityEngine.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.cr_mixer.model.TweetWithScore -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Future -import javax.inject.Singleton - -@Singleton -case class TwhinCollabFilterSimilarityEngine( - twhinCandidatesStratoStore: ReadableStore[Long, Seq[TweetId]], - statsReceiver: StatsReceiver) - extends ReadableStore[ - TwhinCollabFilterSimilarityEngine.Query, - Seq[TweetWithScore] - ] { - - import TwhinCollabFilterSimilarityEngine._ - override def get( - query: TwhinCollabFilterSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScore]]] = { - - query.sourceId match { - case InternalId.UserId(userId) => - twhinCandidatesStratoStore.get(userId).map { - _.map { - _.map { tweetId => TweetWithScore(tweetId, defaultScore) } - } - } - case _ => - Future.None - } - } -} - -object TwhinCollabFilterSimilarityEngine { - - val defaultScore: Double = 1.0 - - case class TwhinCollabFilterView(clusterVersion: String) - - case class Query( - sourceId: InternalId, - ) - - def toSimilarityEngineInfo( - query: LookupEngineQuery[Query], - score: Double - ): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.TwhinCollabFilter, - modelId = Some(query.lookupKey), - score = Some(score)) - } - - def fromParams( - sourceId: InternalId, - modelId: String, - params: configapi.Params, - ): LookupEngineQuery[Query] = { - LookupEngineQuery( - Query(sourceId = sourceId), - modelId, - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/UserTweetEntityGraphSimilarityEngine.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/UserTweetEntityGraphSimilarityEngine.docx new file mode 100644 index 000000000..d03b216c0 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/UserTweetEntityGraphSimilarityEngine.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/UserTweetEntityGraphSimilarityEngine.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/UserTweetEntityGraphSimilarityEngine.scala deleted file mode 100644 index 9c61b3d1c..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/UserTweetEntityGraphSimilarityEngine.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.cr_mixer.similarity_engine - -import com.twitter.recos.recos_common.thriftscala.SocialProofType -import com.twitter.cr_mixer.model.SimilarityEngineInfo -import com.twitter.cr_mixer.model.TweetWithScoreAndSocialProof -import com.twitter.cr_mixer.param.UtegTweetGlobalParams -import com.twitter.cr_mixer.thriftscala.SimilarityEngineType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetEntityDisplayLocation -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph -import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendTweetEntityRequest -import com.twitter.recos.user_tweet_entity_graph.thriftscala.RecommendationType -import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityRecommendationUnion.TweetRec -import com.twitter.simclusters_v2.common.UserId -import com.twitter.simclusters_v2.common.TweetId -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.util.Duration -import com.twitter.util.Future -import javax.inject.Singleton - -@Singleton -case class UserTweetEntityGraphSimilarityEngine( - userTweetEntityGraph: UserTweetEntityGraph.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends ReadableStore[ - UserTweetEntityGraphSimilarityEngine.Query, - Seq[TweetWithScoreAndSocialProof] - ] { - - override def get( - query: UserTweetEntityGraphSimilarityEngine.Query - ): Future[Option[Seq[TweetWithScoreAndSocialProof]]] = { - val recommendTweetEntityRequest = - RecommendTweetEntityRequest( - requesterId = query.userId, - displayLocation = TweetEntityDisplayLocation.HomeTimeline, - recommendationTypes = Seq(RecommendationType.Tweet), - seedsWithWeights = query.seedsWithWeights, - maxResultsByType = Some(Map(RecommendationType.Tweet -> query.maxUtegCandidates)), - maxTweetAgeInMillis = Some(query.maxTweetAge.inMilliseconds), - excludedTweetIds = query.excludedTweetIds, - maxUserSocialProofSize = Some(UserTweetEntityGraphSimilarityEngine.MaxUserSocialProofSize), - maxTweetSocialProofSize = - Some(UserTweetEntityGraphSimilarityEngine.MaxTweetSocialProofSize), - minUserSocialProofSizes = Some(Map(RecommendationType.Tweet -> 1)), - tweetTypes = None, - socialProofTypes = query.socialProofTypes, - socialProofTypeUnions = None, - tweetAuthors = None, - maxEngagementAgeInMillis = None, - excludedTweetAuthors = None, - ) - - userTweetEntityGraph - .recommendTweets(recommendTweetEntityRequest) - .map { recommendTweetsResponse => - val candidates = recommendTweetsResponse.recommendations.flatMap { - case TweetRec(recommendation) => - Some( - TweetWithScoreAndSocialProof( - recommendation.tweetId, - recommendation.score, - recommendation.socialProofByType.toMap)) - case _ => None - } - Some(candidates) - } - } -} - -object UserTweetEntityGraphSimilarityEngine { - - private val MaxUserSocialProofSize = 10 - private val MaxTweetSocialProofSize = 10 - - def toSimilarityEngineInfo(score: Double): SimilarityEngineInfo = { - SimilarityEngineInfo( - similarityEngineType = SimilarityEngineType.Uteg, - modelId = None, - score = Some(score)) - } - - case class Query( - userId: UserId, - seedsWithWeights: Map[UserId, Double], - excludedTweetIds: Option[Seq[Long]] = None, - maxUtegCandidates: Int, - maxTweetAge: Duration, - socialProofTypes: Option[Seq[SocialProofType]]) - - def fromParams( - userId: UserId, - seedsWithWeights: Map[UserId, Double], - excludedTweetIds: Option[Seq[TweetId]] = None, - params: configapi.Params, - ): EngineQuery[Query] = { - EngineQuery( - Query( - userId = userId, - seedsWithWeights = seedsWithWeights, - excludedTweetIds = excludedTweetIds, - maxUtegCandidates = params(UtegTweetGlobalParams.MaxUtegCandidatesToRequestParam), - maxTweetAge = params(UtegTweetGlobalParams.CandidateRefreshSinceTimeOffsetHoursParam), - socialProofTypes = Some(Seq(SocialProofType.Favorite)) - ), - params - ) - } -} diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/BUILD b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/BUILD deleted file mode 100644 index 37b0a7585..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/BUILD +++ /dev/null @@ -1,32 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/src/jvm/com/twitter/storehaus:core", - "configapi/configapi-core", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/config", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param", - "cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider", - "cr-mixer/thrift/src/main/thrift:thrift-scala", - "decider/src/main/scala", - "finatra-internal/mtls-thriftmux/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "frigate/frigate-common:base", - "frigate/frigate-common:util", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/candidate", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/core_workflows/user_model:user_model-scala", - "src/thrift/com/twitter/hermit/stp:hermit-stp-scala", - "src/thrift/com/twitter/onboarding/relevance/coldstart_lookalike:coldstartlookalike-thrift-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", - "user-signal-service/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/BUILD.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/BUILD.docx new file mode 100644 index 000000000..112bf8be0 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/BUILD.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceGraphFetcher.docx b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceGraphFetcher.docx new file mode 100644 index 000000000..ad7d64b54 Binary files /dev/null and b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceGraphFetcher.docx differ diff --git a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceGraphFetcher.scala b/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceGraphFetcher.scala deleted file mode 100644 index 16162c67d..000000000 --- a/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/source_signal/FrsSourceGraphFetcher.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.cr_mixer.source_signal - -import com.twitter.cr_mixer.config.TimeoutConfig -import com.twitter.cr_mixer.model.GraphSourceInfo -import com.twitter.cr_mixer.model.ModuleNames -import com.twitter.cr_mixer.param.FrsParams -import com.twitter.cr_mixer.source_signal.FrsStore.FrsQueryResult -import com.twitter.cr_mixer.source_signal.SourceFetcher.FetcherQuery -import com.twitter.cr_mixer.thriftscala.SourceType -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -/*** - * This store fetches user recommendations from FRS (go/frs) for a given userId - */ -@Singleton -case class FrsSourceGraphFetcher @Inject() ( - @Named(ModuleNames.FrsStore) frsStore: ReadableStore[FrsStore.Query, Seq[FrsQueryResult]], - override val timeoutConfig: TimeoutConfig, - globalStats: StatsReceiver) - extends SourceGraphFetcher { - - override protected val stats: StatsReceiver = globalStats.scope(identifier) - override protected val graphSourceType: SourceType = SourceType.FollowRecommendation - - override def isEnabled(query: FetcherQuery): Boolean = { - query.params(FrsParams.EnableSourceGraphParam) - } - - override def fetchAndProcess( - query: FetcherQuery, - ): Future[Option[GraphSourceInfo]] = { - - val rawSignals = trackPerItemStats(query)( - frsStore - .get( - FrsStore - .Query(query.userId, query.params(FrsParams.MaxConsumerSeedsNumParam))).map { - _.map { - _.map { v => (v.userId, v.score) } - } - } - ) - rawSignals.map { - _.map { userWithScores => - convertGraphSourceInfo(userWithScores) - } - } - } -}