diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.docx new file mode 100644 index 000000000..71bd5dd37 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala deleted file mode 100644 index d83650de0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/ntab_caret_fatigue/RecTypeNtabCaretFatiguePredicate.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.FatiguePredicate -import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter -import com.twitter.frigate.pushservice.predicate.{ - TargetNtabCaretClickFatiguePredicate => CommonNtabCaretClickFatiguePredicate -} -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.util.Duration -import com.twitter.util.Future - -object RecTypeNtabCaretClickFatiguePredicate { - val defaultName = "RecTypeNtabCaretClickFatiguePredicateForCandidate" - - private def candidateFatiguePredicate( - genericTypeCategories: Seq[String], - crts: Set[CRT] - )( - implicit stats: StatsReceiver - ): NamedPredicate[ - PushCandidate - ] = { - val name = "f1TriggeredCRTBasedFatiguePredciate" - val scopedStats = stats.scope(s"predicate_$name") - Predicate - .fromAsync { candidate: PushCandidate => - if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { - if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) { - NtabCaretClickContFnFatiguePredicate - .ntabCaretClickContFnFatiguePredicates( - filterHistory = FatiguePredicate.recTypesOnlyFilter(crts), - filterCaretFeedbackHistory = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories), - filterInlineFeedbackHistory = - NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(crts) - ).apply(Seq(candidate)) - .map(_.headOption.getOrElse(false)) - } else Future.True - } else { - Future.True - } - }.withStats(scopedStats) - .withName(name) - } - - def apply( - genericTypeCategories: Seq[String], - crts: Set[CRT], - calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration, - useMostRecentDislikeTime: Boolean, - name: String = defaultName - )( - implicit globalStats: StatsReceiver - ): NamedPredicate[PushCandidate] = { - val scopedStats = globalStats.scope(name) - val commonNtabCaretClickFatiguePredicate = CommonNtabCaretClickFatiguePredicate( - filterCaretFeedbackHistory = - CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories), - filterHistory = FatiguePredicate.recTypesOnlyFilter(crts), - calculateFatiguePeriod = calculateFatiguePeriod, - useMostRecentDislikeTime = useMostRecentDislikeTime, - name = name - )(globalStats) - - Predicate - .fromAsync { candidate: PushCandidate => - if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) { - if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) { - commonNtabCaretClickFatiguePredicate - .apply(Seq(candidate.target)) - .map(_.headOption.getOrElse(false)) - } else Future.True - } else { - Future.True - } - }.andThen(candidateFatiguePredicate(genericTypeCategories, crts)) - .withStats(scopedStats) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.docx new file mode 100644 index 000000000..9dba85375 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala deleted file mode 100644 index 61c1f78cc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/package.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.frigate.pushservice - -import com.twitter.frigate.common.base.Candidate -import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.Predicate - -package object predicate { - implicit class CandidatesWithAuthorFollowPredicates( - predicate: Predicate[ - PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - ]) { - def applyOnlyToAuthorBeingFollowPredicates: Predicate[Candidate] = - predicate.optionalOn[Candidate]( - { - case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - if isInNetworkTweetType(candidate.commonRecType) => - Some(candidate) - case _ => - None - }, - missingResult = true - ) - } - - implicit class TweetCandidateWithTweetAuthor( - predicate: Predicate[ - PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - ]) { - def applyOnlyToBasicTweetPredicates: Predicate[Candidate] = - predicate.optionalOn[Candidate]( - { - case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap - if isInNetworkTweetType(candidate.commonRecType) => - Some(candidate) - case _ => - None - }, - missingResult = true - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.docx new file mode 100644 index 000000000..c87d06d75 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala deleted file mode 100644 index d7f38349b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/OpenOrNtabClickQualityPredicate.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.quality_model_predicate - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.util.Future - -object ExplicitOONCFilterPredicate extends QualityPredicateBase { - override lazy val name = "open_or_ntab_click_explicit_threshold" - - override lazy val thresholdExtractor = (t: Target) => - Future.value(t.params(PushFeatureSwitchParams.QualityPredicateExplicitThresholdParam)) - - override def scoreExtractor = (candidate: PushCandidate) => - candidate.mrWeightedOpenOrNtabClickRankingProbability -} - -object WeightedOpenOrNtabClickQualityPredicate extends QualityPredicateBase { - override lazy val name = "weighted_open_or_ntab_click_model" - - override lazy val thresholdExtractor = (t: Target) => { - Future.value(0.0) - } - - override def scoreExtractor = - (candidate: PushCandidate) => candidate.mrWeightedOpenOrNtabClickFilteringProbability -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.docx new file mode 100644 index 000000000..4384d5fd2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala deleted file mode 100644 index d22f8c68f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateCommon.scala +++ /dev/null @@ -1,165 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.quality_model_predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.target.TargetScoringDetails -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -object PDauCohort extends Enumeration { - type PDauCohort = Value - - val cohort1 = Value - val cohort2 = Value - val cohort3 = Value - val cohort4 = Value - val cohort5 = Value - val cohort6 = Value -} - -object PDauCohortUtil { - - case class DauThreshold( - threshold1: Double, - threshold2: Double, - threshold3: Double, - threshold4: Double, - threshold5: Double) - - val defaultDAUProb = 0.0 - - val dauProbThresholds = DauThreshold( - threshold1 = 0.05, - threshold2 = 0.14, - threshold3 = 0.33, - threshold4 = 0.7, - threshold5 = 0.959 - ) - - val finerThresholdMap = - Map( - PDauCohort.cohort2 -> List(0.05, 0.0539, 0.0563, 0.0600, 0.0681, 0.0733, 0.0800, 0.0849, - 0.0912, 0.0975, 0.1032, 0.1092, 0.1134, 0.1191, 0.1252, 0.1324, 0.14), - PDauCohort.cohort3 -> List(0.14, 0.1489, 0.1544, 0.1625, 0.1704, 0.1797, 0.1905, 0.2001, - 0.2120, 0.2248, 0.2363, 0.2500, 0.2650, 0.2801, 0.2958, 0.3119, 0.33), - PDauCohort.cohort4 -> List(0.33, 0.3484, 0.3686, 0.3893, 0.4126, 0.4350, 0.4603, 0.4856, - 0.5092, 0.5348, 0.5602, 0.5850, 0.6087, 0.6319, 0.6548, 0.6779, 0.7), - PDauCohort.cohort5 -> List(0.7, 0.7295, 0.7581, 0.7831, 0.8049, 0.8251, 0.8444, 0.8612, - 0.8786, 0.8936, 0.9043, 0.9175, 0.9290, 0.9383, 0.9498, 0.9587, 0.959) - ) - - def getBucket(targetUser: PushTypes.Target, doImpression: Boolean) = { - implicit val stats = targetUser.stats.scope("PDauCohortUtil") - if (doImpression) targetUser.getBucket _ else targetUser.getBucketWithoutImpression _ - } - - def threshold1(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold1 - - def threshold2(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold2 - - def threshold3(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold3 - - def threshold4(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold4 - - def threshold5(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold5 - - def thresholdForCohort(targetUser: PushTypes.Target, dauCohort: Int): Double = { - if (dauCohort == 0) 0.0 - else if (dauCohort == 1) threshold1(targetUser) - else if (dauCohort == 2) threshold2(targetUser) - else if (dauCohort == 3) threshold3(targetUser) - else if (dauCohort == 4) threshold4(targetUser) - else if (dauCohort == 5) threshold5(targetUser) - else 1.0 - } - - def getPDauCohort(dauProbability: Double, thresholds: DauThreshold): PDauCohort.Value = { - dauProbability match { - case dauProb if dauProb >= 0.0 && dauProb < thresholds.threshold1 => PDauCohort.cohort1 - case dauProb if dauProb >= thresholds.threshold1 && dauProb < thresholds.threshold2 => - PDauCohort.cohort2 - case dauProb if dauProb >= thresholds.threshold2 && dauProb < thresholds.threshold3 => - PDauCohort.cohort3 - case dauProb if dauProb >= thresholds.threshold3 && dauProb < thresholds.threshold4 => - PDauCohort.cohort4 - case dauProb if dauProb >= thresholds.threshold4 && dauProb < thresholds.threshold5 => - PDauCohort.cohort5 - case dauProb if dauProb >= thresholds.threshold5 && dauProb <= 1.0 => PDauCohort.cohort6 - } - } - - def getDauProb(target: TargetScoringDetails): Future[Double] = { - target.dauProbability.map { dauProb => - dauProb.map(_.probability).getOrElse(defaultDAUProb) - } - } - - def getPDauCohort(target: TargetScoringDetails): Future[PDauCohort.Value] = { - getDauProb(target).map { getPDauCohort(_, dauProbThresholds) } - } - - def getPDauCohortWithPDau(target: TargetScoringDetails): Future[(PDauCohort.Value, Double)] = { - getDauProb(target).map { prob => - (getPDauCohort(prob, dauProbThresholds), prob) - } - } - - def updateStats( - target: PushTypes.Target, - modelName: String, - predicateResult: Boolean - )( - implicit statsReceiver: StatsReceiver - ): Unit = { - val dauCohortOp = getPDauCohort(target) - dauCohortOp.map { dauCohort => - val cohortStats = statsReceiver.scope(modelName).scope(dauCohort.toString) - cohortStats.counter(s"filter_$predicateResult").incr() - } - if (target.isNewSignup) { - val newUserModelStats = statsReceiver.scope(modelName) - newUserModelStats.counter(s"new_user_filter_$predicateResult").incr() - } - } -} - -trait QualityPredicateBase { - def name: String - def thresholdExtractor: Target => Future[Double] - def scoreExtractor: PushCandidate => Future[Option[Double]] - def isPredicateEnabled: PushCandidate => Future[Boolean] = _ => Future.True - def comparator: (Double, Double) => Boolean = - (score: Double, threshold: Double) => score >= threshold - def updateCustomStats( - candidate: PushCandidate, - score: Double, - threshold: Double, - result: Boolean - )( - implicit statsReceiver: StatsReceiver - ): Unit = {} - - def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = { - Predicate - .fromAsync { candidate: PushCandidate => - isPredicateEnabled(candidate).flatMap { - case true => - scoreExtractor(candidate).flatMap { scoreOpt => - thresholdExtractor(candidate.target).map { threshold => - val score = scoreOpt.getOrElse(0.0) - val result = comparator(score, threshold) - PDauCohortUtil.updateStats(candidate.target, name, result) - updateCustomStats(candidate, score, threshold, result) - result - } - } - case _ => Future.True - } - } - .withStats(statsReceiver.scope(s"predicate_$name")) - .withName(name) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.docx new file mode 100644 index 000000000..87cfe0738 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala deleted file mode 100644 index 9ac360df0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/predicate/quality_model_predicate/QualityPredicateMap.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.predicate.quality_model_predicate - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.QualityPredicateEnum -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.hermit.predicate.NamedPredicate - -object QualityPredicateMap { - - def apply( - )( - implicit statsReceiver: StatsReceiver - ): Map[QualityPredicateEnum.Value, NamedPredicate[PushCandidate]] = { - Map( - QualityPredicateEnum.WeightedOpenOrNtabClick -> WeightedOpenOrNtabClickQualityPredicate(), - QualityPredicateEnum.ExplicitOpenOrNtabClickFilter -> ExplicitOONCFilterPredicate(), - QualityPredicateEnum.AlwaysTrue -> PredicatesForCandidate.alwaysTruePushCandidatePredicate, - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.docx new file mode 100644 index 000000000..ea220706a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala deleted file mode 100644 index de95c0695..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTBoostRanker.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType - -/** - * This Ranker re-ranks MR candidates, boosting input CRTs. - * Relative ranking between input CRTs and rest of the candidates doesn't change - * - * Ex: T: Tweet candidate, F: input CRT candidatess - * - * T3, F2, T1, T2, F1 => F2, F1, T3, T1, T2 - */ -case class CRTBoostRanker(statsReceiver: StatsReceiver) { - - private val recsToBoostStat = statsReceiver.stat("recs_to_boost") - private val otherRecsStat = statsReceiver.stat("other_recs") - - private def boostCrtToTop( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtToBoost: CommonRecommendationType - ): Seq[CandidateDetails[PushCandidate]] = { - val (upRankedCandidates, otherCandidates) = - inputCandidates.partition(_.candidate.commonRecType == crtToBoost) - recsToBoostStat.add(upRankedCandidates.size) - otherRecsStat.add(otherCandidates.size) - upRankedCandidates ++ otherCandidates - } - - final def boostCrtsToTop( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtsToBoost: Seq[CommonRecommendationType] - ): Seq[CandidateDetails[PushCandidate]] = { - crtsToBoost.headOption match { - case Some(crt) => - val upRankedCandidates = boostCrtToTop(inputCandidates, crt) - boostCrtsToTop(upRankedCandidates, crtsToBoost.tail) - case None => inputCandidates - } - } - - final def boostCrtsToTopStableOrder( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtsToBoost: Seq[CommonRecommendationType] - ): Seq[CandidateDetails[PushCandidate]] = { - val crtsToBoostSet = crtsToBoost.toSet - val (upRankedCandidates, otherCandidates) = inputCandidates.partition(candidateDetail => - crtsToBoostSet.contains(candidateDetail.candidate.commonRecType)) - - upRankedCandidates ++ otherCandidates - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.docx new file mode 100644 index 000000000..acd13e3d1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala deleted file mode 100644 index 4a8d74504..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/CRTDownRanker.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType - -/** - * This Ranker re-ranks MR candidates, down ranks input CRTs. - * Relative ranking between input CRTs and rest of the candidates doesn't change - * - * Ex: T: Tweet candidate, F: input CRT candidates - * - * T3, F2, T1, T2, F1 => T3, T1, T2, F2, F1 - */ -case class CRTDownRanker(statsReceiver: StatsReceiver) { - - private val recsToDownRankStat = statsReceiver.stat("recs_to_down_rank") - private val otherRecsStat = statsReceiver.stat("other_recs") - private val downRankerRequests = statsReceiver.counter("down_ranker_requests") - - private def downRank( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtToDownRank: CommonRecommendationType - ): Seq[CandidateDetails[PushCandidate]] = { - downRankerRequests.incr() - val (downRankedCandidates, otherCandidates) = - inputCandidates.partition(_.candidate.commonRecType == crtToDownRank) - recsToDownRankStat.add(downRankedCandidates.size) - otherRecsStat.add(otherCandidates.size) - otherCandidates ++ downRankedCandidates - } - - final def downRank( - inputCandidates: Seq[CandidateDetails[PushCandidate]], - crtsToDownRank: Seq[CommonRecommendationType] - ): Seq[CandidateDetails[PushCandidate]] = { - crtsToDownRank.headOption match { - case Some(crt) => - val downRankedCandidates = downRank(inputCandidates, crt) - downRank(downRankedCandidates, crtsToDownRank.tail) - case None => inputCandidates - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.docx new file mode 100644 index 000000000..b68fb73a5 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala deleted file mode 100644 index 5ab0c240a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/LoggedOutRanker.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class LoggedOutRanker(tweetyPieStore: ReadableStore[Long, TweetyPieResult], stats: StatsReceiver) { - private val statsReceiver = stats.scope(this.getClass.getSimpleName) - private val rankedCandidates = statsReceiver.counter("ranked_candidates_count") - - def rank( - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val tweetIds = candidates.map { cand => cand.candidate.asInstanceOf[TweetCandidate].tweetId } - val results = tweetyPieStore.multiGet(tweetIds.toSet).values.toSeq - val futureOfResults = Future.traverseSequentially(results)(r => r) - val tweetsFut = futureOfResults.map { tweetyPieResults => - tweetyPieResults.map(_.map(_.tweet)) - } - val sortedTweetsFuture = tweetsFut.map { tweets => - tweets - .map { tweet => - if (tweet.isDefined && tweet.get.counts.isDefined) { - tweet.get.id -> tweet.get.counts.get.favoriteCount.getOrElse(0L) - } else { - 0 -> 0L - } - }.sortBy(_._2)(Ordering[Long].reverse) - } - val finalCandidates = sortedTweetsFuture.map { sortedTweets => - sortedTweets - .map { tweet => - candidates.find(_.candidate.asInstanceOf[TweetCandidate].tweetId == tweet._1).orNull - }.filter { cand => cand != null } - } - finalCandidates.map { fc => - rankedCandidates.incr(fc.size) - } - finalCandidates - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.docx new file mode 100644 index 000000000..29bd9c60e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala deleted file mode 100644 index 372888a66..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/ModelBasedRanker.scala +++ /dev/null @@ -1,204 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.util.Future - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.params.PushConstants.OoncQualityCombinedScore - -object ModelBasedRanker { - - def rankBySpecifiedScore( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - scoreExtractor: PushCandidate => Future[Option[Double]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - val scoredCandidatesFutures = candidatesDetails.map { cand => - scoreExtractor(cand.candidate).map { scoreOp => (cand, scoreOp.getOrElse(0.0)) } - } - - Future.collect(scoredCandidatesFutures).map { scores => - val sorted = scores.sortBy { candidateDetails => -1 * candidateDetails._2 } - sorted.map(_._1) - } - } - - def populatePredictionScoreStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - scoreExtractor: PushCandidate => Future[Option[Double]], - predictionScoreStats: StatsReceiver - ): Unit = { - val scoreScaleFactorForStat = 10000 - val statName = "prediction_scores" - candidatesDetails.map { - case CandidateDetails(candidate, source) => - val crt = candidate.commonRecType - scoreExtractor(candidate).map { scoreOp => - val scaledScore = (scoreOp.getOrElse(0.0) * scoreScaleFactorForStat).toFloat - predictionScoreStats.scope("all_candidates").stat(statName).add(scaledScore) - predictionScoreStats.scope(crt.toString()).stat(statName).add(scaledScore) - } - } - } - - def populateMrWeightedOpenOrNtabClickScoreStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - predictionScoreStats: StatsReceiver - ): Unit = { - populatePredictionScoreStats( - candidatesDetails, - candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability, - predictionScoreStats - ) - } - - def populateMrQualityUprankingScoreStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - predictionScoreStats: StatsReceiver - ): Unit = { - populatePredictionScoreStats( - candidatesDetails, - candidate => candidate.mrQualityUprankingProbability, - predictionScoreStats - ) - } - - def rankByMrWeightedOpenOrNtabClickScore( - candidatesDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - rankBySpecifiedScore( - candidatesDetails, - candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability - ) - } - - def transformSigmoid( - score: Double, - weight: Double = 1.0, - bias: Double = 0.0 - ): Double = { - val base = -1.0 * (weight * score + bias) - val cappedBase = math.max(math.min(base, 100.0), -100.0) - 1.0 / (1.0 + math.exp(cappedBase)) - } - - def transformLinear( - score: Double, - bar: Double = 1.0 - ): Double = { - val positiveBar = math.abs(bar) - val cappedScore = math.max(math.min(score, positiveBar), -1.0 * positiveBar) - cappedScore / positiveBar - } - - def transformIdentity( - score: Double - ): Double = score - - def rankByQualityOoncCombinedScore( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - qualityScoreTransform: Double => Double, - qualityScoreBoost: Double = 1.0 - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - rankBySpecifiedScore( - candidatesDetails, - candidate => { - val ooncScoreFutOpt: Future[Option[Double]] = - candidate.mrWeightedOpenOrNtabClickRankingProbability - val qualityScoreFutOpt: Future[Option[Double]] = - candidate.mrQualityUprankingProbability - Future - .join( - ooncScoreFutOpt, - qualityScoreFutOpt - ).map { - case (Some(ooncScore), Some(qualityScore)) => - val transformedQualityScore = qualityScoreTransform(qualityScore) - val combinedScore = ooncScore * (1.0 + qualityScoreBoost * transformedQualityScore) - candidate - .cacheExternalScore(OoncQualityCombinedScore, Future.value(Some(combinedScore))) - Some(combinedScore) - case _ => None - } - } - ) - } - - def rerankByProducerQualityOoncCombinedScore( - candidateDetails: Seq[CandidateDetails[PushCandidate]] - )( - implicit stat: StatsReceiver - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val scopedStat = stat.scope("producer_quality_reranking") - val oonCandidates = candidateDetails.filter { - case CandidateDetails(pushCandidate: PushCandidate, _) => - tweetCandidateSelector(pushCandidate, MrQualityUprankingPartialTypeEnum.Oon) - } - - val rankedOonCandidatesFut = rankBySpecifiedScore( - oonCandidates, - candidate => { - val baseScoreFutureOpt: Future[Option[Double]] = { - val qualityCombinedScoreFutureOpt = - candidate.getExternalCachedScoreByName(OoncQualityCombinedScore) - val ooncScoreFutureOpt = candidate.mrWeightedOpenOrNtabClickRankingProbability - Future.join(qualityCombinedScoreFutureOpt, ooncScoreFutureOpt).map { - case (Some(qualityCombinedScore), _) => - scopedStat.counter("quality_combined_score").incr() - Some(qualityCombinedScore) - case (_, ooncScoreOpt) => - scopedStat.counter("oonc_score").incr() - ooncScoreOpt - } - } - baseScoreFutureOpt.map { - case Some(baseScore) => - val boostRatio = candidate.mrProducerQualityUprankingBoost.getOrElse(1.0) - if (boostRatio > 1.0) scopedStat.counter("author_uprank").incr() - else if (boostRatio < 1.0) scopedStat.counter("author_downrank").incr() - else scopedStat.counter("author_noboost").incr() - Some(baseScore * boostRatio) - case _ => - scopedStat.counter("empty_score").incr() - None - } - } - ) - - rankedOonCandidatesFut.map { rankedOonCandidates => - val sortedOonCandidateIterator = rankedOonCandidates.toIterator - candidateDetails.map { ooncRankedCandidate => - val isOon = tweetCandidateSelector( - ooncRankedCandidate.candidate, - MrQualityUprankingPartialTypeEnum.Oon) - - if (sortedOonCandidateIterator.hasNext && isOon) - sortedOonCandidateIterator.next() - else ooncRankedCandidate - } - } - } - - def tweetCandidateSelector( - pushCandidate: PushCandidate, - selectedCandidateType: MrQualityUprankingPartialTypeEnum.Value - ): Boolean = { - pushCandidate match { - case candidate: PushCandidate with TweetCandidate => - selectedCandidateType match { - case MrQualityUprankingPartialTypeEnum.Oon => - val crt = candidate.commonRecType - RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes - .contains(crt) - case _ => true - } - case _ => false - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.docx new file mode 100644 index 000000000..8e9a9ff72 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala deleted file mode 100644 index 26a3a4239..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/PushserviceRanker.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.Ranker -import com.twitter.util.Future - -trait PushserviceRanker[T, C] extends Ranker[T, C] { - - /** - * Initial Ranking of input candidates - */ - def initialRank(target: T, candidates: Seq[CandidateDetails[C]]): Future[Seq[CandidateDetails[C]]] - - /** - * Re-ranks input ranked candidates. Useful when a subset of candidates are ranked - * by a different logic, while preserving the initial ranking for the rest - */ - def reRank( - target: T, - rankedCandidates: Seq[CandidateDetails[C]] - ): Future[Seq[CandidateDetails[C]]] - - /** - * Final ranking that does Initial + Rerank - */ - override final def rank(target: T, candidates: Seq[CandidateDetails[C]]): ( - Future[Seq[CandidateDetails[C]]] - ) = { - initialRank(target, candidates).flatMap { rankedCandidates => reRank(target, rankedCandidates) } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.docx new file mode 100644 index 000000000..98c8e6188 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala deleted file mode 100644 index 3fdae08c3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHLightRanker.scala +++ /dev/null @@ -1,139 +0,0 @@ -package com.twitter.frigate.pushservice.rank -import com.twitter.contentrecommender.thriftscala.LightRankingCandidate -import com.twitter.contentrecommender.thriftscala.LightRankingFeatureHydrationContext -import com.twitter.contentrecommender.thriftscala.MagicRecsFeatureHydrationContext -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.RandomRanker -import com.twitter.frigate.common.base.Ranker -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.ml.featurestore.lib.UserId -import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker -import com.twitter.util.Future - -class RFPHLightRanker( - lightRanker: MagicRecsServeDataRecordLightRanker, - stats: StatsReceiver) - extends Ranker[Target, PushCandidate] { - - private val statsReceiver = stats.scope(this.getClass.getSimpleName) - - private val lightRankerCandidateCounter = statsReceiver.counter("light_ranker_candidate_count") - private val lightRankerRequestCounter = statsReceiver.counter("light_ranker_request_count") - private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking") - private val restrictLightRankingCounter: Counter = - lightRankingStats.counter("restrict_light_ranking") - private val selectedLightRankerScribedTargetCandidateCountStats: Stat = - lightRankingStats.stat("selected_light_ranker_scribed_target_candidate_count") - private val selectedLightRankerScribedCandidatesStats: Stat = - lightRankingStats.stat("selected_light_ranker_scribed_candidates") - private val lightRankingRandomBaselineStats: StatsReceiver = - statsReceiver.scope("light_ranking_random_baseline") - - override def rank( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val enableLightRanker = target.params(PushFeatureSwitchParams.EnableLightRankingParam) - val restrictLightRanker = target.params(PushParams.RestrictLightRankingParam) - val lightRankerSelectionThreshold = - target.params(PushFeatureSwitchParams.LightRankingNumberOfCandidatesParam) - val randomRanker = RandomRanker[Target, PushCandidate]()(lightRankingRandomBaselineStats) - - if (enableLightRanker && candidates.length > lightRankerSelectionThreshold && !target.scribeFeatureForRequestScribe) { - val (tweetCandidates, nonTweetCandidates) = - candidates.partition { - case CandidateDetails(pushCandidate: PushCandidate with TweetCandidate, source) => true - case _ => false - } - val lightRankerSelectedTweetCandidatesFut = { - if (restrictLightRanker) { - restrictLightRankingCounter.incr() - lightRankThenTake( - target, - tweetCandidates - .asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]], - PushConstants.RestrictLightRankingCandidatesThreshold - ) - } else if (target.params(PushFeatureSwitchParams.EnableRandomBaselineLightRankingParam)) { - randomRanker.rank(target, tweetCandidates).map { randomLightRankerCands => - randomLightRankerCands.take(lightRankerSelectionThreshold) - } - } else { - lightRankThenTake( - target, - tweetCandidates - .asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]], - lightRankerSelectionThreshold - ) - } - } - lightRankerSelectedTweetCandidatesFut.map { returnedTweetCandidates => - nonTweetCandidates ++ returnedTweetCandidates - } - } else if (target.scribeFeatureForRequestScribe) { - val downSampleRate: Double = - if (target.params(PushParams.DownSampleLightRankingScribeCandidatesParam)) - PushConstants.DownSampleLightRankingScribeCandidatesRate - else target.params(PushFeatureSwitchParams.LightRankingScribeCandidatesDownSamplingParam) - val selectedCandidateCounter: Int = math.ceil(candidates.size * downSampleRate).toInt - selectedLightRankerScribedTargetCandidateCountStats.add(selectedCandidateCounter.toFloat) - - randomRanker.rank(target, candidates).map { randomLightRankerCands => - val selectedCandidates = randomLightRankerCands.take(selectedCandidateCounter) - selectedLightRankerScribedCandidatesStats.add(selectedCandidates.size.toFloat) - selectedCandidates - } - } else Future.value(candidates) - } - - private def lightRankThenTake( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate with TweetCandidate]], - numOfCandidates: Int - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - lightRankerCandidateCounter.incr(candidates.length) - lightRankerRequestCounter.incr() - val lightRankerCandidates: Seq[LightRankingCandidate] = candidates.map { - case CandidateDetails(tweetCandidate, _) => - val tweetAuthor = tweetCandidate match { - case t: TweetCandidate with TweetAuthor => t.authorId - case _ => None - } - val hydrationContext: LightRankingFeatureHydrationContext = - LightRankingFeatureHydrationContext.MagicRecsHydrationContext( - MagicRecsFeatureHydrationContext( - tweetAuthor = tweetAuthor, - pushString = tweetCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString)) - ) - LightRankingCandidate( - tweetId = tweetCandidate.tweetId, - hydrationContext = Some(hydrationContext) - ) - } - val modelName = target.params(PushFeatureSwitchParams.LightRankingModelTypeParam) - val lightRankedCandidatesFut = { - lightRanker - .rank(UserId(target.targetId), lightRankerCandidates, modelName) - } - - lightRankedCandidatesFut.map { lightRankedCandidates => - val lrScoreMap = lightRankedCandidates.map { lrCand => - lrCand.tweetId -> lrCand.score - }.toMap - val candScoreMap: Seq[Option[Double]] = candidates.map { candidateDetails => - lrScoreMap.get(candidateDetails.candidate.tweetId) - } - sortCandidatesByScore(candidates, candScoreMap) - .take(numOfCandidates) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.docx new file mode 100644 index 000000000..6179d8487 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala deleted file mode 100644 index 83bdf3932..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/RFPHRanker.scala +++ /dev/null @@ -1,297 +0,0 @@ -package com.twitter.frigate.pushservice.rank -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.Ranker -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.ml.HealthFeatureGetter -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushMLModel -import com.twitter.frigate.pushservice.params.PushModelName -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.updateMediaCategoryStats -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.util.Future -import com.twitter.frigate.pushservice.params.MrQualityUprankingTransformTypeEnum -import com.twitter.storehaus.ReadableStore -import com.twitter.frigate.thriftscala.UserMediaRepresentation -import com.twitter.hss.api.thriftscala.UserHealthSignalResponse - -class RFPHRanker( - randomRanker: Ranker[Target, PushCandidate], - weightedOpenOrNtabClickModelScorer: PushMLModelScorer, - subscriptionCreatorRanker: SubscriptionCreatorRanker, - userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse], - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation], - stats: StatsReceiver) - extends PushserviceRanker[Target, PushCandidate] { - - private val statsReceiver = stats.scope(this.getClass.getSimpleName) - - private val boostCRTsRanker = CRTBoostRanker(statsReceiver.scope("boost_desired_crts")) - private val crtDownRanker = CRTDownRanker(statsReceiver.scope("down_rank_desired_crts")) - - private val crtsToDownRank = statsReceiver.stat("crts_to_downrank") - private val crtsToUprank = statsReceiver.stat("crts_to_uprank") - - private val randomRankingCounter = stats.counter("randomRanking") - private val mlRankingCounter = stats.counter("mlRanking") - private val disableAllRelevanceCounter = stats.counter("disableAllRelevance") - private val disableHeavyRankingCounter = stats.counter("disableHeavyRanking") - - private val heavyRankerCandidateCounter = stats.counter("heavy_ranker_candidate_count") - private val heavyRankerScoreStats = statsReceiver.scope("heavy_ranker_prediction_scores") - - private val producerUprankingCounter = statsReceiver.counter("producer_quality_upranking") - private val producerBoostedCounter = statsReceiver.counter("producer_boosted_candidates") - private val producerDownboostedCounter = statsReceiver.counter("producer_downboosted_candidates") - - override def initialRank( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - heavyRankerCandidateCounter.incr(candidates.size) - - updateMediaCategoryStats(candidates)(stats) - target.targetUserState - .flatMap { targetUserState => - val useRandomRanking = target.skipMlRanker || target.params( - PushParams.UseRandomRankingParam - ) - - if (useRandomRanking) { - randomRankingCounter.incr() - randomRanker.rank(target, candidates) - } else if (target.params(PushParams.DisableAllRelevanceParam)) { - disableAllRelevanceCounter.incr() - Future.value(candidates) - } else if (target.params(PushParams.DisableHeavyRankingParam) || target.params( - PushFeatureSwitchParams.DisableHeavyRankingModelFSParam)) { - disableHeavyRankingCounter.incr() - Future.value(candidates) - } else { - mlRankingCounter.incr() - - val scoredCandidatesFut = scoring(target, candidates) - - target.rankingModelParam.map { rankingModelParam => - val modelName = PushModelName( - PushMLModel.WeightedOpenOrNtabClickProbability, - target.params(rankingModelParam)).toString - ModelBasedRanker.populateMrWeightedOpenOrNtabClickScoreStats( - candidates, - heavyRankerScoreStats.scope(modelName) - ) - } - - if (target.params( - PushFeatureSwitchParams.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam)) { - val modelName = PushModelName( - PushMLModel.FilteringProbability, - target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam) - ).toString - ModelBasedRanker.populateMrQualityUprankingScoreStats( - candidates, - heavyRankerScoreStats.scope(modelName) - ) - } - - val ooncRankedCandidatesFut = - scoredCandidatesFut.flatMap(ModelBasedRanker.rankByMrWeightedOpenOrNtabClickScore) - - val qualityUprankedCandidatesFut = - if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) { - ooncRankedCandidatesFut.flatMap { ooncRankedCandidates => - val transformFunc: Double => Double = - target.params(PushFeatureSwitchParams.QualityUprankingTransformTypeParam) match { - case MrQualityUprankingTransformTypeEnum.Linear => - ModelBasedRanker.transformLinear( - _, - bar = target.params( - PushFeatureSwitchParams.QualityUprankingLinearBarForHeavyRankingParam)) - case MrQualityUprankingTransformTypeEnum.Sigmoid => - ModelBasedRanker.transformSigmoid( - _, - weight = target.params( - PushFeatureSwitchParams.QualityUprankingSigmoidWeightForHeavyRankingParam), - bias = target.params( - PushFeatureSwitchParams.QualityUprankingSigmoidBiasForHeavyRankingParam) - ) - case _ => ModelBasedRanker.transformIdentity - } - - ModelBasedRanker.rankByQualityOoncCombinedScore( - ooncRankedCandidates, - transformFunc, - target.params(PushFeatureSwitchParams.QualityUprankingBoostForHeavyRankingParam) - ) - } - } else ooncRankedCandidatesFut - - if (target.params( - PushFeatureSwitchParams.EnableProducersQualityBoostingForHeavyRankingParam)) { - producerUprankingCounter.incr() - qualityUprankedCandidatesFut.flatMap(cands => - ModelBasedRanker.rerankByProducerQualityOoncCombinedScore(cands)(statsReceiver)) - } else qualityUprankedCandidatesFut - } - } - } - - private def scoring( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - val ooncScoredCandidatesFut = target.rankingModelParam.map { rankingModelParam => - weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion( - target, - candidates, - rankingModelParam - ) - } - - val scoredCandidatesFut = { - if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) { - ooncScoredCandidatesFut.map { candidates => - weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion( - target = target, - candidatesDetails = candidates, - modelVersionParam = PushFeatureSwitchParams.QualityUprankingModelTypeParam, - overridePushMLModelOpt = Some(PushMLModel.FilteringProbability) - ) - } - } else ooncScoredCandidatesFut - } - - scoredCandidatesFut.foreach { candidates => - val oonCandidates = candidates.filter { - case CandidateDetails(pushCandidate: PushCandidate, _) => - ModelBasedRanker.tweetCandidateSelector( - pushCandidate, - MrQualityUprankingPartialTypeEnum.Oon) - } - setProducerQuality( - target, - oonCandidates, - userHealthSignalStore, - producerMediaRepresentationStore) - } - } - - private def setProducerQuality( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]], - userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse], - producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation] - ): Unit = { - lazy val boostRatio = - target.params(PushFeatureSwitchParams.QualityUprankingBoostForHighQualityProducersParam) - lazy val downboostRatio = - target.params(PushFeatureSwitchParams.QualityUprankingDownboostForLowQualityProducersParam) - candidates.foreach { - case CandidateDetails(pushCandidate, _) => - HealthFeatureGetter - .getFeatures(pushCandidate, producerMediaRepresentationStore, userHealthSignalStore).map { - featureMap => - val agathaNsfwScore = featureMap.numericFeatures.getOrElse("agathaNsfwScore", 0.5) - val textNsfwScore = featureMap.numericFeatures.getOrElse("textNsfwScore", 0.15) - val nudityRate = featureMap.numericFeatures.getOrElse("nudityRate", 0.0) - val activeFollowers = featureMap.numericFeatures.getOrElse("activeFollowers", 0.0) - val favorsRcvd28Days = featureMap.numericFeatures.getOrElse("favorsRcvd28Days", 0.0) - val tweets28Days = featureMap.numericFeatures.getOrElse("tweets28Days", 0.0) - val authorDislikeCount = featureMap.numericFeatures - .getOrElse("authorDislikeCount", 0.0) - val authorDislikeRate = featureMap.numericFeatures.getOrElse("authorDislikeRate", 0.0) - val authorReportRate = featureMap.numericFeatures.getOrElse("authorReportRate", 0.0) - val abuseStrikeTop2Percent = - featureMap.booleanFeatures.getOrElse("abuseStrikeTop2Percent", false) - val abuseStrikeTop1Percent = - featureMap.booleanFeatures.getOrElse("abuseStrikeTop1Percent", false) - val hasNsfwToken = featureMap.booleanFeatures.getOrElse("hasNsfwToken", false) - - if ((activeFollowers > 3000000) || - (activeFollowers > 1000000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent) || - (activeFollowers > 100000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent && - tweets28Days > 0 && favorsRcvd28Days / tweets28Days > 3000 && authorReportRate < 0.000001 && authorDislikeRate < 0.0005)) { - producerBoostedCounter.incr() - pushCandidate.setProducerQualityUprankingBoost(boostRatio) - } else if (activeFollowers < 5 || agathaNsfwScore > 0.9 || nudityRate > 0.03 || hasNsfwToken || abuseStrikeTop1Percent || - textNsfwScore > 0.4 || (authorDislikeRate > 0.005 && authorDislikeCount > 5) || - (tweets28Days > 56 && favorsRcvd28Days / tweets28Days < 100)) { - producerDownboostedCounter.incr() - pushCandidate.setProducerQualityUprankingBoost(downboostRatio) - } else pushCandidate.setProducerQualityUprankingBoost(1.0) - } - } - } - - private def rerankBySubscriptionCreatorRanker( - target: Target, - rankedCandidates: Future[Seq[CandidateDetails[PushCandidate]]], - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - if (target.params(PushFeatureSwitchParams.SoftRankCandidatesFromSubscriptionCreators)) { - val factor = target.params(PushFeatureSwitchParams.SoftRankFactorForSubscriptionCreators) - subscriptionCreatorRanker.boostByScoreFactor(rankedCandidates, factor) - } else - subscriptionCreatorRanker.boostSubscriptionCreator(rankedCandidates) - } - - override def reRank( - target: Target, - rankedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val numberOfF1Candidates = - rankedCandidates.count(candidateDetails => - RecTypes.isF1Type(candidateDetails.candidate.commonRecType)) - lazy val threshold = - target.params(PushFeatureSwitchParams.NumberOfF1CandidatesThresholdForOONBackfill) - lazy val enableOONBackfillBasedOnF1 = - target.params(PushFeatureSwitchParams.EnableOONBackfillBasedOnF1Candidates) - - val f1BoostedCandidates = - if (enableOONBackfillBasedOnF1 && numberOfF1Candidates > threshold) { - boostCRTsRanker.boostCrtsToTopStableOrder( - rankedCandidates, - RecTypes.f1FirstDegreeTypes.toSeq) - } else rankedCandidates - - val topTweetsByGeoDownRankedCandidates = - if (target.params(PushFeatureSwitchParams.BackfillRankTopTweetsByGeoCandidates)) { - crtDownRanker.downRank( - f1BoostedCandidates, - Seq(CommonRecommendationType.GeoPopTweet) - ) - } else f1BoostedCandidates - - val reRankedCandidatesWithBoostedCrts = { - val listOfCrtsToUpRank = target - .params(PushFeatureSwitchParams.ListOfCrtsToUpRank) - .flatMap(CommonRecommendationType.valueOf) - crtsToUprank.add(listOfCrtsToUpRank.size) - boostCRTsRanker.boostCrtsToTop(topTweetsByGeoDownRankedCandidates, listOfCrtsToUpRank) - } - - val reRankedCandidatesWithDownRankedCrts = { - val listOfCrtsToDownRank = target - .params(PushFeatureSwitchParams.ListOfCrtsToDownRank) - .flatMap(CommonRecommendationType.valueOf) - crtsToDownRank.add(listOfCrtsToDownRank.size) - crtDownRanker.downRank(reRankedCandidatesWithBoostedCrts, listOfCrtsToDownRank) - } - - val rerankBySubscriptionCreatorFut = { - if (target.params(PushFeatureSwitchParams.BoostCandidatesFromSubscriptionCreators)) { - rerankBySubscriptionCreatorRanker( - target, - Future.value(reRankedCandidatesWithDownRankedCrts)) - } else Future.value(reRankedCandidatesWithDownRankedCrts) - } - - rerankBySubscriptionCreatorFut - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.docx new file mode 100644 index 000000000..a15e152b4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala deleted file mode 100644 index 3a2bff9a5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/rank/SubscriptionCreatorRanker.scala +++ /dev/null @@ -1,110 +0,0 @@ -package com.twitter.frigate.pushservice.rank - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class SubscriptionCreatorRanker( - superFollowEligibilityUserStore: ReadableStore[Long, Boolean], - statsReceiver: StatsReceiver) { - - private val scopedStats = statsReceiver.scope("SubscriptionCreatorRanker") - private val boostStats = scopedStats.scope("boostSubscriptionCreator") - private val softUprankStats = scopedStats.scope("boostByScoreFactor") - private val boostTotalCandidates = boostStats.stat("total_input_candidates") - private val softRankTotalCandidates = softUprankStats.stat("total_input_candidates") - private val softRankNumCandidatesCreators = softUprankStats.counter("candidates_from_creators") - private val softRankNumCandidatesNonCreators = - softUprankStats.counter("candidates_not_from_creators") - private val boostNumCandidatesCreators = boostStats.counter("candidates_from_creators") - private val boostNumCandidatesNonCreators = - boostStats.counter("candidates_not_from_creators") - - def boostSubscriptionCreator( - inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - inputCandidatesFut.flatMap { inputCandidates => - boostTotalCandidates.add(inputCandidates.size) - val tweetAuthorIds = inputCandidates.flatMap { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId - case _ => None - }.toSet - - FutureOps - .mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds)) - .map { creatorAuthorMap => - val (upRankedCandidates, otherCandidates) = inputCandidates.partition { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId match { - case Some(authorId) => - creatorAuthorMap(authorId).getOrElse(false) - case _ => false - } - case _ => false - } - boostNumCandidatesCreators.incr(upRankedCandidates.size) - boostNumCandidatesNonCreators.incr(otherCandidates.size) - upRankedCandidates ++ otherCandidates - } - } - } - - def boostByScoreFactor( - inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]], - factor: Double = 1.0, - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - inputCandidatesFut.flatMap { inputCandidates => - softRankTotalCandidates.add(inputCandidates.size) - val tweetAuthorIds = inputCandidates.flatMap { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId - case _ => None - }.toSet - - FutureOps - .mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds)) - .flatMap { creatorAuthorMap => - val (upRankedCandidates, otherCandidates) = inputCandidates.partition { - case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) => - candidate.authorId match { - case Some(authorId) => - creatorAuthorMap(authorId).getOrElse(false) - case _ => false - } - case _ => false - } - softRankNumCandidatesCreators.incr(upRankedCandidates.size) - softRankNumCandidatesNonCreators.incr(otherCandidates.size) - - ModelBasedRanker.rankBySpecifiedScore( - inputCandidates, - candidate => { - val isFromCreator = candidate match { - case candidate: TweetCandidate with TweetAuthor => - candidate.authorId match { - case Some(authorId) => - creatorAuthorMap(authorId).getOrElse(false) - case _ => false - } - case _ => false - } - candidate.mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(score) => - if (isFromCreator) Some(score * factor) - else Some(score) - case _ => None - } - } - ) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.docx new file mode 100644 index 000000000..21e5d6ca2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala deleted file mode 100644 index f626d5b08..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/LoggedOutRefreshForPushHandler.scala +++ /dev/null @@ -1,259 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.CandidateSource -import com.twitter.frigate.common.base.FetchRankFlowWithHydratedCandidates -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.base.Stats.trackSeq -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator -import com.twitter.frigate.pushservice.predicate.LoggedOutPreRankingPredicates -import com.twitter.frigate.pushservice.predicate.LoggedOutTargetPredicates -import com.twitter.frigate.pushservice.rank.LoggedOutRanker -import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier -import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler -import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder -import com.twitter.frigate.pushservice.thriftscala.LoggedOutRequest -import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.SequentialPredicate -import com.twitter.util.Future - -class LoggedOutRefreshForPushHandler( - val loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder, - val loPushCandidateSourceGenerator: LoggedOutPushCandidateSourceGenerator, - candidateHydrator: PushCandidateHydrator, - val loRanker: LoggedOutRanker, - val loRfphNotifier: LoggedOutRefreshForPushNotifier, - loMrRequestScriberNode: String -)( - globalStats: StatsReceiver) - extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] { - - val log = MRLogger("LORefreshForPushHandler") - implicit val statsReceiver: StatsReceiver = - globalStats.scope("LORefreshForPushHandler") - private val loggedOutBuildStats = statsReceiver.scope("logged_out_build_target") - private val loggedOutProcessStats = statsReceiver.scope("logged_out_process") - private val loggedOutNotifyStats = statsReceiver.scope("logged_out_notify") - private val loCandidateHydrationStats: StatsReceiver = - statsReceiver.scope("logged_out_candidate_hydration") - val mrLORequestCandidateScribeStats = - statsReceiver.scope("mr_logged_out_request_scribe_candidates") - - val mrRequestScribeHandler = - new MrRequestScribeHandler(loMrRequestScriberNode, statsReceiver.scope("lo_mr_request_scribe")) - val loMrRequestTargetScribeStats = statsReceiver.scope("lo_mr_request_scribe_target") - - lazy val loCandSourceEligibleCounter: Counter = - loCandidateStats.counter("logged_out_cand_source_eligible") - lazy val loCandSourceNotEligibleCounter: Counter = - loCandidateStats.counter("logged_out_cand_source_not_eligible") - lazy val allCandidatesCounter: Counter = statsReceiver.counter("all_logged_out_candidates") - val allCandidatesFilteredPreRank = filterStats.counter("all_logged_out_candidates_filtered") - - override def targetPredicates(target: Target): List[Predicate[Target]] = List( - LoggedOutTargetPredicates.targetFatiguePredicate(), - LoggedOutTargetPredicates.loggedOutRecsHoldbackPredicate() - ) - - override def isTargetValid(target: Target): Future[Result] = { - val resultFut = - if (target.skipFilters) { - Future.value(OK) - } else { - predicateSeq(target).track(Seq(target)).map { resultArr => - trackTargetPredStats(resultArr(0)) - } - } - track(targetStats)(resultFut) - } - - override def rank( - target: Target, - candidateDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - loRanker.rank(candidateDetails) - } - - override def validCandidates( - target: Target, - candidates: Seq[PushCandidate] - ): Future[Seq[Result]] = { - Future.value(candidates.map { c => OK }) - } - - override def desiredCandidateCount(target: Target): Int = 1 - - private val loggedOutPreRankingPredicates = - LoggedOutPreRankingPredicates(filterStats.scope("logged_out_predicates")) - - private val loggedOutPreRankingPredicateChain = - new SequentialPredicate[PushCandidate](loggedOutPreRankingPredicates) - - override def filter( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - val predicateChain = loggedOutPreRankingPredicateChain - predicateChain - .track(candidates.map(_.candidate)) - .map { results => - val resultForPreRankingFiltering = - results - .zip(candidates) - .foldLeft( - ( - Seq.empty[CandidateDetails[PushCandidate]], - Seq.empty[CandidateResult[PushCandidate, Result]] - ) - ) { - case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => - result match { - case None => - (goodCandidates :+ candidateDetails, filteredCandidates) - - case Some(pred: NamedPredicate[_]) => - val r = Invalid(Some(pred.name)) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - case Some(_) => - val r = Invalid(Some("Filtered by un-named predicate")) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - } - } - resultForPreRankingFiltering match { - case (validCandidates, _) if validCandidates.isEmpty && candidates.nonEmpty => - allCandidatesFilteredPreRank.incr() - case _ => () - - } - resultForPreRankingFiltering - - } - - } - - override def candidateSources( - target: Target - ): Future[Seq[CandidateSource[Target, RawCandidate]]] = { - Future - .collect(loPushCandidateSourceGenerator.sources.map { cs => - cs.isCandidateSourceAvailable(target).map { isEligible => - if (isEligible) { - loCandSourceEligibleCounter.incr() - Some(cs) - } else { - loCandSourceNotEligibleCounter.incr() - None - } - } - }).map(_.flatten) - } - - override def process( - target: Target, - externalCandidates: Seq[RawCandidate] = Nil - ): Future[Response[PushCandidate, Result]] = { - isTargetValid(target).flatMap { - case OK => - for { - candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target)) - externalCandidateDetails = externalCandidates.map( - CandidateDetails(_, "logged_out_refresh_for_push_handler_external_candidates")) - allCandidates = candidatesFromSources ++ externalCandidateDetails - hydratedCandidatesWithCopy <- - trackSeq(loCandidateHydrationStats)(hydrateCandidates(allCandidates)) - (candidates, preRankingFilteredCandidates) <- - track(filterStats)(filter(target, hydratedCandidatesWithCopy)) - rankedCandidates <- trackSeq(rankingStats)(rank(target, candidates)) - allTakeCandidateResults <- track(takeStats)( - take(target, rankedCandidates, desiredCandidateCount(target)) - ) - _ <- track(mrLORequestCandidateScribeStats)( - mrRequestScribeHandler.scribeForCandidateFiltering( - target, - hydratedCandidatesWithCopy, - preRankingFilteredCandidates, - rankedCandidates, - rankedCandidates, - rankedCandidates, - allTakeCandidateResults - )) - - } yield { - val takeCandidateResults = allTakeCandidateResults.filterNot { candResult => - candResult.result == MoreThanDesiredCandidates - } - val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates - allCandidatesCounter.incr(allCandidateResults.size) - Response(OK, allCandidateResults) - } - - case result: Result => - for (_ <- track(loMrRequestTargetScribeStats)( - mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield { - Response(result, Nil) - } - } - } - - def buildTarget( - guestId: Long, - inputPushContext: Option[PushContext] - ): Future[Target] = - loPushTargetUserBuilder.buildTarget(guestId, inputPushContext) - - /** - * Hydrate candidate by querying downstream services - * - * @param candidates - candidates - * - * @return - hydrated candidates - */ - override def hydrateCandidates( - candidates: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates) - - override def batchForCandidatesCheck(target: Target): Int = 1 - - def refreshAndSend(request: LoggedOutRequest): Future[LoggedOutResponse] = { - for { - target <- track(loggedOutBuildStats)( - loPushTargetUserBuilder.buildTarget(request.guestId, request.context)) - response <- track(loggedOutProcessStats)(process(target, externalCandidates = Seq.empty)) - loggedOutRefreshResponse <- - track(loggedOutNotifyStats)(loRfphNotifier.checkResponseAndNotify(response)) - } yield { - loggedOutRefreshResponse - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.docx new file mode 100644 index 000000000..e3a7c665c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala deleted file mode 100644 index b8bf675bd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/PushCandidateHydrator.scala +++ /dev/null @@ -1,239 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.TrendTweetPushCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._ -import com.twitter.frigate.pushservice.util.MrUserStateUtil -import com.twitter.frigate.pushservice.util.RelationshipUtil -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class PushCandidateHydrator( - socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean], - safeUserStore: ReadableStore[Long, User], - apiListStore: ReadableStore[Long, ApiList], - candidateCopyCross: CandidateCopyExpansion -)( - implicit statsReceiver: StatsReceiver, - implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) { - - lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num") - lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates") - lazy val mrUserStateStat = statsReceiver.scope("mr_user_state") - - lazy val queryStep = statsReceiver.scope("query_step") - lazy val relationEdgeWithoutDuplicateInQueryStep = - queryStep.counter("number_of_relationEdge_without_duplicate_in_query_step") - lazy val relationEdgeWithoutDuplicateInQueryStepDistribution = - queryStep.stat("number_of_relationEdge_without_duplicate_in_query_step_distribution") - - case class Entities( - users: Set[Long] = Set.empty[Long], - relationshipEdges: Set[RelationEdge] = Set.empty[RelationEdge]) { - def merge(otherEntities: Entities): Entities = { - this.copy( - users = this.users ++ otherEntities.users, - relationshipEdges = - this.relationshipEdges ++ otherEntities.relationshipEdges - ) - } - } - - case class EntitiesMap( - userMap: Map[Long, User] = Map.empty[Long, User], - relationshipMap: Map[RelationEdge, Boolean] = Map.empty[RelationEdge, Boolean]) - - private def updateCandidateAndCrtStats( - candidate: RawCandidate, - candidateType: String, - numEntities: Int = 1 - ): Unit = { - statsReceiver - .scope(candidateType).scope(candidate.commonRecType.name).stat( - "totalEntitiesPerCandidateTypePerCrt").add(numEntities) - statsReceiver.scope(candidateType).stat("totalEntitiesPerCandidateType").add(numEntities) - } - - private def collectEntities( - candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]] - ): Entities = { - candidateDetailsSeq - .map { candidateDetails => - val pushCandidate = candidateDetails.candidate - - val userEntities = pushCandidate match { - case tweetWithSocialContext: RawCandidate with TweetWithSocialContextTraits => - val authorIdOpt = getAuthorIdFromTweetCandidate(tweetWithSocialContext) - val scUserIds = tweetWithSocialContext.socialContextUserIds.toSet - updateCandidateAndCrtStats(pushCandidate, "tweetWithSocialContext", scUserIds.size + 1) - Entities(users = scUserIds ++ authorIdOpt.toSet) - - case _ => Entities() - } - - val relationEntities = { - if (isInNetworkTweetType(pushCandidate.commonRecType)) { - Entities( - relationshipEdges = - RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(pushCandidate).toSet - ) - } else Entities() - } - - userEntities.merge(relationEntities) - } - .foldLeft(Entities()) { (e1, e2) => e1.merge(e2) } - - } - - /** - * This method calls Gizmoduck and Social Graph Service, keep the results in EntitiesMap - * and passed onto the update candidate phase in the hydration step - * - * @param entities contains all userIds and relationEdges for all candidates - * @return EntitiesMap contains userMap and relationshipMap - */ - private def queryEntities(entities: Entities): Future[EntitiesMap] = { - - relationEdgeWithoutDuplicateInQueryStep.incr(entities.relationshipEdges.size) - relationEdgeWithoutDuplicateInQueryStepDistribution.add(entities.relationshipEdges.size) - - val relationshipMapFuture = Future - .collect(socialGraphServiceProcessStore.multiGet(entities.relationshipEdges)) - .map { resultMap => - resultMap.collect { - case (relationshipEdge, Some(res)) => relationshipEdge -> res - case (relationshipEdge, None) => relationshipEdge -> false - } - } - - val userMapFuture = Future - .collect(safeUserStore.multiGet(entities.users)) - .map { userMap => - userMap.collect { - case (userId, Some(user)) => - userId -> user - } - } - - Future.join(userMapFuture, relationshipMapFuture).map { - case (uMap, rMap) => EntitiesMap(userMap = uMap, relationshipMap = rMap) - } - } - - /** - * @param candidateDetails: recommendation candidates for a user - * @return sequence of candidates tagged with push and ntab copy id - */ - private def expandCandidatesWithCopy( - candidateDetails: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = { - candidateCopyCross.expandCandidatesWithCopyId(candidateDetails) - } - - def updateCandidates( - candidateDetailsWithCopies: Seq[(CandidateDetails[RawCandidate], CopyIds)], - entitiesMaps: EntitiesMap - ): Seq[CandidateDetails[PushCandidate]] = { - candidateDetailsWithCopies.map { - case (candidateDetail, copyIds) => - val pushCandidate = candidateDetail.candidate - val userMap = entitiesMaps.userMap - val relationshipMap = entitiesMaps.relationshipMap - - val hydratedCandidate = pushCandidate match { - - case f1TweetCandidate: F1FirstDegree => - getHydratedCandidateForF1FirstDegreeTweet( - f1TweetCandidate, - userMap, - relationshipMap, - copyIds) - - case tweetRetweet: TweetRetweetCandidate => - getHydratedCandidateForTweetRetweet(tweetRetweet, userMap, copyIds) - - case tweetFavorite: TweetFavoriteCandidate => - getHydratedCandidateForTweetFavorite(tweetFavorite, userMap, copyIds) - - case tripTweetCandidate: OutOfNetworkTweetCandidate with TripCandidate => - getHydratedCandidateForTripTweetCandidate(tripTweetCandidate, userMap, copyIds) - - case outOfNetworkTweetCandidate: OutOfNetworkTweetCandidate with TopicCandidate => - getHydratedCandidateForOutOfNetworkTweetCandidate( - outOfNetworkTweetCandidate, - userMap, - copyIds) - - case topicProofTweetCandidate: TopicProofTweetCandidate => - getHydratedTopicProofTweetCandidate(topicProofTweetCandidate, userMap, copyIds) - - case subscribedSearchTweetCandidate: SubscribedSearchTweetCandidate => - getHydratedSubscribedSearchTweetCandidate( - subscribedSearchTweetCandidate, - userMap, - copyIds) - - case listRecommendation: ListPushCandidate => - getHydratedListCandidate(apiListStore, listRecommendation, copyIds) - - case discoverTwitterCandidate: DiscoverTwitterCandidate => - getHydratedCandidateForDiscoverTwitterCandidate(discoverTwitterCandidate, copyIds) - - case topTweetImpressionsCandidate: TopTweetImpressionsCandidate => - getHydratedCandidateForTopTweetImpressionsCandidate( - topTweetImpressionsCandidate, - copyIds) - - case trendTweetCandidate: TrendTweetCandidate => - new TrendTweetPushCandidate( - trendTweetCandidate, - trendTweetCandidate.authorId.flatMap(userMap.get), - copyIds) - - case unknownCandidate => - throw new IllegalArgumentException( - s"Incorrect candidate for hydration: ${unknownCandidate.commonRecType}") - } - - CandidateDetails( - hydratedCandidate, - source = candidateDetail.source - ) - } - } - - def apply( - candidateDetails: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val isLoggedOutRequest = - candidateDetails.headOption.exists(_.candidate.target.isLoggedOutUser) - if (!isLoggedOutRequest) { - candidateDetails.headOption.map { cd => - MrUserStateUtil.updateMrUserStateStats(cd.candidate.target)(mrUserStateStat) - } - } - - expandCandidatesWithCopy(candidateDetails).flatMap { candidateDetailsWithCopy => - candidateWithCopyNumStat.add(candidateDetailsWithCopy.size) - val entities = collectEntities(candidateDetailsWithCopy.map(_._1)) - queryEntities(entities).flatMap { entitiesMap => - val updatedCandidates = updateCandidates(candidateDetailsWithCopy, entitiesMap) - updatedCandidates.foreach { cand => - hydratedCandidateStat.counter(cand.candidate.commonRecType.name).incr() - } - Future.value(updatedCandidates) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.docx new file mode 100644 index 000000000..8ec736d44 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala deleted file mode 100644 index 6d1172cb9..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHFeatureHydrator.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.util.MrUserStateUtil -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.util.Future - -class RFPHFeatureHydrator( - featureHydrator: FeatureHydrator -)( - implicit globalStats: StatsReceiver) { - - implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - - //stat for feature hydration - private val featureHydrationEnabledCounter = statsReceiver.counter("featureHydrationEnabled") - private val mrUserStateStat = statsReceiver.scope("mr_user_state") - - private def hydrateFromRelevanceHydrator( - candidateDetails: Seq[CandidateDetails[PushCandidate]], - mrRequestContextForFeatureStore: MrRequestContextForFeatureStore - ): Future[Unit] = { - val pushCandidates = candidateDetails.map(_.candidate) - val candidatesAndContextsFut = Future.collect(pushCandidates.map { pc => - val contextFut = HydrationContextBuilder.build(pc) - contextFut.map { ctx => (pc, ctx) } - }) - candidatesAndContextsFut.flatMap { candidatesAndContexts => - val contexts = candidatesAndContexts.map(_._2) - val resultsFut = featureHydrator.hydrateCandidate(contexts, mrRequestContextForFeatureStore) - resultsFut.map { hydrationResult => - candidatesAndContexts.foreach { - case (pushCandidate, context) => - val resultFeatures = hydrationResult.getOrElse(context, FeatureMap()) - pushCandidate.mergeFeatures(resultFeatures) - } - } - } - } - - def candidateFeatureHydration( - candidateDetails: Seq[CandidateDetails[PushCandidate]], - mrRequestContextForFeatureStore: MrRequestContextForFeatureStore - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - candidateDetails.headOption match { - case Some(cand) => - val target = cand.candidate.target - MrUserStateUtil.updateMrUserStateStats(target)(mrUserStateStat) - if (target.params(PushParams.DisableAllRelevanceParam)) { - Future.value(candidateDetails) - } else { - featureHydrationEnabledCounter.incr() - for { - _ <- hydrateFromRelevanceHydrator(candidateDetails, mrRequestContextForFeatureStore) - } yield { - candidateDetails - } - } - case _ => Future.Nil - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.docx new file mode 100644 index 000000000..86c6699c2 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala deleted file mode 100644 index fe52428b3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHPrerankFilter.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.predicate.PreRankingPredicates -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.SequentialPredicate -import com.twitter.util._ - -class RFPHPrerankFilter( -)( - globalStats: StatsReceiver) { - def filter( - target: Target, - hydratedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - lazy val filterStats: StatsReceiver = globalStats.scope("RefreshForPushHandler/filter") - lazy val okFilterCounter: Counter = filterStats.counter("ok") - lazy val invalidFilterCounter: Counter = filterStats.counter("invalid") - lazy val invalidFilterStat: StatsReceiver = filterStats.scope("invalid") - lazy val invalidFilterReasonStat: StatsReceiver = invalidFilterStat.scope("reason") - val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered") - - lazy val preRankingPredicates = PreRankingPredicates( - filterStats.scope("predicates") - ) - - lazy val preRankingPredicateChain = - new SequentialPredicate[PushCandidate](preRankingPredicates) - - val predicateChain = if (target.pushContext.exists(_.predicatesToEnable.exists(_.nonEmpty))) { - val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil) - new SequentialPredicate[PushCandidate](preRankingPredicates.filter { pred => - predicatesToEnable.contains(pred.name) - }) - } else preRankingPredicateChain - - predicateChain - .track(hydratedCandidates.map(_.candidate)) - .map { results => - val resultForPreRankFiltering = results - .zip(hydratedCandidates) - .foldLeft( - ( - Seq.empty[CandidateDetails[PushCandidate]], - Seq.empty[CandidateResult[PushCandidate, Result]] - ) - ) { - case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => - result match { - case None => - okFilterCounter.incr() - (goodCandidates :+ candidateDetails, filteredCandidates) - - case Some(pred: NamedPredicate[_]) => - invalidFilterCounter.incr() - invalidFilterReasonStat.counter(pred.name).incr() - invalidFilterReasonStat - .scope(candidateDetails.candidate.commonRecType.toString).counter( - pred.name).incr() - - val r = Invalid(Some(pred.name)) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - case Some(_) => - invalidFilterCounter.incr() - invalidFilterReasonStat.counter("unknown").incr() - invalidFilterReasonStat - .scope(candidateDetails.candidate.commonRecType.toString).counter( - "unknown").incr() - - val r = Invalid(Some("Filtered by un-named predicate")) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - } - } - - resultForPreRankFiltering match { - case (validCandidates, _) if validCandidates.isEmpty && hydratedCandidates.nonEmpty => - allCandidatesFilteredPreRank.incr() - case _ => () - } - - resultForPreRankFiltering - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.docx new file mode 100644 index 000000000..615d01461 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala deleted file mode 100644 index 037479111..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHRestrictStep.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.target.TargetScoringDetails - -class RFPHRestrictStep()(implicit stats: StatsReceiver) { - - private val statsReceiver: StatsReceiver = stats.scope("RefreshForPushHandler") - private val restrictStepStats: StatsReceiver = statsReceiver.scope("restrict") - private val restrictStepNumCandidatesDroppedStat: Stat = - restrictStepStats.stat("candidates_dropped") - - /** - * Limit the number of candidates that enter the Take step - */ - def restrict( - target: TargetUser with TargetABDecider with TargetScoringDetails, - candidates: Seq[CandidateDetails[PushCandidate]] - ): (Seq[CandidateDetails[PushCandidate]], Seq[CandidateDetails[PushCandidate]]) = { - if (target.params(PushFeatureSwitchParams.EnableRestrictStep)) { - val restrictSizeParam = PushFeatureSwitchParams.RestrictStepSize - val (newCandidates, filteredCandidates) = candidates.splitAt(target.params(restrictSizeParam)) - val numDropped = candidates.length - newCandidates.length - restrictStepNumCandidatesDroppedStat.add(numDropped) - (newCandidates, filteredCandidates) - } else (candidates, Seq.empty) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.docx new file mode 100644 index 000000000..9579e5ad7 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala deleted file mode 100644 index c09b4348a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RFPHStatsRecorder.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType - -class RFPHStatsRecorder(implicit statsReceiver: StatsReceiver) { - - private val selectedCandidateScoreStats: StatsReceiver = - statsReceiver.scope("score_of_sent_candidate_times_10000") - - private val emptyScoreStats: StatsReceiver = - statsReceiver.scope("score_of_sent_candidate_empty") - - def trackPredictionScoreStats(candidate: PushCandidate): Unit = { - candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach { - case Some(s) => - selectedCandidateScoreStats - .stat("weighted_open_or_ntab_click_ranking") - .add((s * 10000).toFloat) - case None => - emptyScoreStats.counter("weighted_open_or_ntab_click_ranking").incr() - } - candidate.mrWeightedOpenOrNtabClickFilteringProbability.foreach { - case Some(s) => - selectedCandidateScoreStats - .stat("weighted_open_or_ntab_click_filtering") - .add((s * 10000).toFloat) - case None => - emptyScoreStats.counter("weighted_open_or_ntab_click_filtering").incr() - } - candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach { - case Some(s) => - selectedCandidateScoreStats - .scope(candidate.commonRecType.toString) - .stat("weighted_open_or_ntab_click_ranking") - .add((s * 10000).toFloat) - case None => - emptyScoreStats - .scope(candidate.commonRecType.toString) - .counter("weighted_open_or_ntab_click_ranking") - .incr() - } - } - - def refreshRequestExceptionStats( - exception: Throwable, - bStats: StatsReceiver - ): Unit = { - bStats.counter("failures").incr() - bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr() - } - - def loggedOutRequestExceptionStats( - exception: Throwable, - bStats: StatsReceiver - ): Unit = { - bStats.counter("logged_out_failures").incr() - bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr() - } - - def rankDistributionStats( - candidatesDetails: Seq[CandidateDetails[PushCandidate]], - numRecsPerTypeStat: (CommonRecommendationType => Stat) - ): Unit = { - candidatesDetails - .groupBy { c => - c.candidate.commonRecType - } - .mapValues { s => - s.size - } - .foreach { case (crt, numRecs) => numRecsPerTypeStat(crt).add(numRecs) } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.docx new file mode 100644 index 000000000..e8922469c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala deleted file mode 100644 index 17fb846cf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushHandler.scala +++ /dev/null @@ -1,292 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.base.Stats.trackSeq -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.adaptor._ -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.rank.RFPHLightRanker -import com.twitter.frigate.pushservice.rank.RFPHRanker -import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler -import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.frigate.pushservice.target.RFPHTargetPredicates -import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil -import com.twitter.frigate.pushservice.util.AdhocStatsUtil -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.frigate.pushservice.thriftscala.RefreshRequest -import com.twitter.frigate.pushservice.thriftscala.RefreshResponse -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.Predicate -import com.twitter.timelines.configapi.FeatureValue -import com.twitter.util._ - -case class ResultWithDebugInfo(result: Result, predicateResults: Seq[PredicateWithResult]) - -class RefreshForPushHandler( - val pushTargetUserBuilder: PushTargetUserBuilder, - val candSourceGenerator: PushCandidateSourceGenerator, - rfphRanker: RFPHRanker, - candidateHydrator: PushCandidateHydrator, - candidateValidator: RFPHCandidateValidator, - rfphTakeStepUtil: RFPHTakeStepUtil, - rfphRestrictStep: RFPHRestrictStep, - val rfphNotifier: RefreshForPushNotifier, - rfphStatsRecorder: RFPHStatsRecorder, - mrRequestScriberNode: String, - rfphFeatureHydrator: RFPHFeatureHydrator, - rfphPrerankFilter: RFPHPrerankFilter, - rfphLightRanker: RFPHLightRanker -)( - globalStats: StatsReceiver) - extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] { - - val log = MRLogger("RefreshForPushHandler") - - implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - private val maxCandidatesToBatchInTakeStat: Stat = - statsReceiver.stat("max_cands_to_batch_in_take") - - private val rfphRequestCounter = statsReceiver.counter("requests") - - private val buildTargetStats = statsReceiver.scope("build_target") - private val processStats = statsReceiver.scope("process") - private val notifyStats = statsReceiver.scope("notify") - - private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking") - private val reRankingStats: StatsReceiver = statsReceiver.scope("rerank") - private val featureHydrationLatency: StatsReceiver = - statsReceiver.scope("featureHydrationLatency") - private val candidateHydrationStats: StatsReceiver = statsReceiver.scope("candidate_hydration") - - lazy val candSourceEligibleCounter: Counter = - candidateStats.counter("cand_source_eligible") - lazy val candSourceNotEligibleCounter: Counter = - candidateStats.counter("cand_source_not_eligible") - - //pre-ranking stats - val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered") - - // total invalid candidates - val totalStats: StatsReceiver = statsReceiver.scope("total") - val totalInvalidCandidatesStat: Stat = totalStats.stat("candidates_invalid") - - val mrRequestScribeBuiltStats: Counter = statsReceiver.counter("mr_request_scribe_built") - - val mrRequestCandidateScribeStats = statsReceiver.scope("mr_request_scribe_candidates") - val mrRequestTargetScribeStats = statsReceiver.scope("mr_request_scribe_target") - - val mrRequestScribeHandler = - new MrRequestScribeHandler(mrRequestScriberNode, statsReceiver.scope("mr_request_scribe")) - - val adhocStatsUtil = new AdhocStatsUtil(statsReceiver.scope("adhoc_stats")) - - private def numRecsPerTypeStat(crt: CommonRecommendationType) = - fetchStats.scope(crt.toString).stat("dist") - - // static list of target predicates - private val targetPredicates = RFPHTargetPredicates(targetStats.scope("predicates")) - - def buildTarget( - userId: Long, - inputPushContext: Option[PushContext], - forcedFeatureValues: Option[Map[String, FeatureValue]] = None - ): Future[Target] = - pushTargetUserBuilder.buildTarget(userId, inputPushContext, forcedFeatureValues) - - override def targetPredicates(target: Target): List[Predicate[Target]] = targetPredicates - - override def isTargetValid(target: Target): Future[Result] = { - val resultFut = if (target.skipFilters) { - Future.value(trackTargetPredStats(None)) - } else { - predicateSeq(target).track(Seq(target)).map { resultArr => - trackTargetPredStats(resultArr(0)) - } - } - track(targetStats)(resultFut) - } - - override def candidateSources( - target: Target - ): Future[Seq[CandidateSource[Target, RawCandidate]]] = { - Future - .collect(candSourceGenerator.sources.map { cs => - cs.isCandidateSourceAvailable(target).map { isEligible => - if (isEligible) { - candSourceEligibleCounter.incr() - Some(cs) - } else { - candSourceNotEligibleCounter.incr() - None - } - } - }).map(_.flatten) - } - - override def updateCandidateCounter( - candidateResults: Seq[CandidateResult[PushCandidate, Result]] - ): Unit = { - candidateResults.foreach { - case candidateResult if candidateResult.result == OK => - okCandidateCounter.incr() - case candidateResult if candidateResult.result.isInstanceOf[Invalid] => - invalidCandidateCounter.incr() - case _ => - } - } - - override def hydrateCandidates( - candidates: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates) - - override def filter( - target: Target, - hydratedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = rfphPrerankFilter.filter(target, hydratedCandidates) - - def lightRankAndTake( - target: Target, - candidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - rfphLightRanker.rank(target, candidates) - } - - override def rank( - target: Target, - candidatesDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - val featureHydratedCandidatesFut = trackSeq(featureHydrationLatency)( - rfphFeatureHydrator - .candidateFeatureHydration(candidatesDetails, target.mrRequestContextForFeatureStore) - ) - featureHydratedCandidatesFut.flatMap { featureHydratedCandidates => - rfphStatsRecorder.rankDistributionStats(featureHydratedCandidates, numRecsPerTypeStat) - rfphRanker.initialRank(target, candidatesDetails) - } - } - - def reRank( - target: Target, - rankedCandidates: Seq[CandidateDetails[PushCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - rfphRanker.reRank(target, rankedCandidates) - } - - override def validCandidates( - target: Target, - candidates: Seq[PushCandidate] - ): Future[Seq[Result]] = { - Future.collect(candidates.map { candidate => - rfphTakeStepUtil.isCandidateValid(candidate, candidateValidator).map(res => res.result) - }) - } - - override def desiredCandidateCount(target: Target): Int = target.desiredCandidateCount - - override def batchForCandidatesCheck(target: Target): Int = { - val fsParam = PushFeatureSwitchParams.NumberOfMaxCandidatesToBatchInRFPHTakeStep - val maxToBatch = target.params(fsParam) - maxCandidatesToBatchInTakeStat.add(maxToBatch) - maxToBatch - } - - override def process( - target: Target, - externalCandidates: Seq[RawCandidate] = Nil - ): Future[Response[PushCandidate, Result]] = { - isTargetValid(target).flatMap { - case OK => - for { - candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target)) - externalCandidateDetails = externalCandidates.map( - CandidateDetails(_, "refresh_for_push_handler_external_candidate")) - allCandidates = candidatesFromSources ++ externalCandidateDetails - hydratedCandidatesWithCopy <- - trackSeq(candidateHydrationStats)(hydrateCandidates(allCandidates)) - _ = adhocStatsUtil.getCandidateSourceStats(hydratedCandidatesWithCopy) - (candidates, preRankingFilteredCandidates) <- - track(filterStats)(filter(target, hydratedCandidatesWithCopy)) - _ = adhocStatsUtil.getPreRankingFilterStats(preRankingFilteredCandidates) - lightRankerFilteredCandidates <- - trackSeq(lightRankingStats)(lightRankAndTake(target, candidates)) - _ = adhocStatsUtil.getLightRankingStats(lightRankerFilteredCandidates) - rankedCandidates <- trackSeq(rankingStats)(rank(target, lightRankerFilteredCandidates)) - _ = adhocStatsUtil.getRankingStats(rankedCandidates) - rerankedCandidates <- trackSeq(reRankingStats)(reRank(target, rankedCandidates)) - _ = adhocStatsUtil.getReRankingStats(rerankedCandidates) - (restrictedCandidates, restrictFilteredCandidates) = - rfphRestrictStep.restrict(target, rerankedCandidates) - allTakeCandidateResults <- track(takeStats)( - take(target, restrictedCandidates, desiredCandidateCount(target)) - ) - _ = adhocStatsUtil.getTakeCandidateResultStats(allTakeCandidateResults) - _ <- track(mrRequestCandidateScribeStats)( - mrRequestScribeHandler.scribeForCandidateFiltering( - target, - hydratedCandidatesWithCopy, - preRankingFilteredCandidates, - rankedCandidates, - rerankedCandidates, - restrictFilteredCandidates, - allTakeCandidateResults - )) - } yield { - - /** - * Take processes post restrict step candidates and returns both: - * 1. valid + invalid candidates - * 2. Candidates that are not processed (more than desired) + restricted candidates - * We need #2 only for importance sampling - */ - val takeCandidateResults = - allTakeCandidateResults.filterNot { candResult => - candResult.result == MoreThanDesiredCandidates - } - - val totalInvalidCandidates = { - preRankingFilteredCandidates.size + //pre-ranking filtered candidates - (rerankedCandidates.length - restrictedCandidates.length) + //candidates reject in restrict step - takeCandidateResults.count(_.result != OK) //candidates reject in take step - } - takeInvalidCandidateDist.add( - takeCandidateResults - .count(_.result != OK) - ) // take step invalid candidates - totalInvalidCandidatesStat.add(totalInvalidCandidates) - val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates - Response(OK, allCandidateResults) - } - - case result: Result => - for (_ <- track(mrRequestTargetScribeStats)( - mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield { - mrRequestScribeBuiltStats.incr() - Response(result, Nil) - } - } - } - - def refreshAndSend(request: RefreshRequest): Future[RefreshResponse] = { - rfphRequestCounter.incr() - for { - target <- track(buildTargetStats)( - pushTargetUserBuilder - .buildTarget(request.userId, request.context)) - response <- track(processStats)(process(target, externalCandidates = Seq.empty)) - refreshResponse <- track(notifyStats)(rfphNotifier.checkResponseAndNotify(response, target)) - } yield { - refreshResponse - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.docx new file mode 100644 index 000000000..155c68621 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala deleted file mode 100644 index ae68d46ea..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/RefreshForPushNotifier.scala +++ /dev/null @@ -1,128 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.config.CommonConstants -import com.twitter.frigate.common.util.PushServiceUtil.FilteredRefreshResponseFut -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.take.CandidateNotifier -import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.frigate.pushservice.thriftscala.RefreshResponse -import com.twitter.util.Future -import com.twitter.util.JavaTimer -import com.twitter.util.Timer - -class RefreshForPushNotifier( - rfphStatsRecorder: RFPHStatsRecorder, - candidateNotifier: CandidateNotifier -)( - globalStats: StatsReceiver) { - - private implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - - private val pushStats: StatsReceiver = statsReceiver.scope("push") - private val sendLatency: StatsReceiver = statsReceiver.scope("send_handler") - implicit private val timer: Timer = new JavaTimer(true) - - private def notify( - candidatesResult: CandidateResult[PushCandidate, Result], - target: Target, - receivers: Seq[StatsReceiver] - ): Future[RefreshResponse] = { - - val candidate = candidatesResult.candidate - - val predsResult = candidatesResult.result - - if (predsResult != OK) { - val invalidResult = predsResult - invalidResult match { - case Invalid(Some(reason)) => - Future.value(RefreshResponse(PushStatus.Filtered, Some(reason))) - case _ => - Future.value(RefreshResponse(PushStatus.Filtered, None)) - } - } else { - rfphStatsRecorder.trackPredictionScoreStats(candidate) - - val isQualityUprankingCandidate = candidate.mrQualityUprankingBoost.isDefined - val commonRecTypeStats = Seq( - statsReceiver.scope(candidate.commonRecType.toString), - globalStats.scope(candidate.commonRecType.toString) - ) - val qualityUprankingStats = Seq( - statsReceiver.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString), - globalStats.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString) - ) - - val receiversWithRecTypeStats = { - if (isQualityUprankingCandidate) { - receivers ++ commonRecTypeStats ++ qualityUprankingStats - } else { - receivers ++ commonRecTypeStats - } - } - track(sendLatency)(candidateNotifier.notify(candidate).map { res => - trackStatsForResponseToRequest( - candidate.commonRecType, - candidate.target, - res, - receiversWithRecTypeStats - )(globalStats) - RefreshResponse(res.status) - }) - } - } - - def checkResponseAndNotify( - response: Response[PushCandidate, Result], - targetUserContext: Target - ): Future[RefreshResponse] = { - val receivers = Seq(statsReceiver) - val refreshResponse = response match { - case Response(OK, processedCandidates) => - // valid rec candidates - val validCandidates = processedCandidates.filter(_.result == OK) - - // top rec candidate - validCandidates.headOption match { - case Some(candidatesResult) => - candidatesResult.result match { - case OK => - notify(candidatesResult, targetUserContext, receivers) - .onSuccess { nr => - pushStats.scope("result").counter(nr.status.name).incr() - } - case _ => - targetUserContext.isTeamMember.flatMap { isTeamMember => - FilteredRefreshResponseFut - } - } - case _ => - FilteredRefreshResponseFut - } - case Response(Invalid(reason), _) => - // invalid target with known reason - FilteredRefreshResponseFut.map(_.copy(targetFilteredBy = reason)) - case _ => - // invalid target - FilteredRefreshResponseFut - } - - val bStats = BroadcastStatsReceiver(receivers) - Stat - .timeFuture(bStats.stat("latency"))( - refreshResponse - .raiseWithin(CommonConstants.maxPushRequestDuration) - ) - .onFailure { exception => - rfphStatsRecorder.refreshRequestExceptionStats(exception, bStats) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.docx new file mode 100644 index 000000000..cfd9b3933 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala deleted file mode 100644 index 47426a386..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/BaseCopyFramework.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.MRNtabCopy -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.util.Future - -abstract class BaseCopyFramework(statsReceiver: StatsReceiver) { - - private val NoAvailableCopyStat = statsReceiver.scope("no_copy_for_crt") - private val NoAvailableNtabCopyStat = statsReceiver.scope("no_ntab_copy") - - /** - * Instantiate push copy filters - */ - protected final val copyFilters = new CopyFilters(statsReceiver.scope("filters")) - - /** - * - * The following method fetches all the push copies for a [[com.twitter.frigate.thriftscala.CommonRecommendationType]] - * associated with a candidate and then filters the eligible copies based on - * [[PushTypes.PushCandidate]] features. These filters are defined in - * [[CopyFilters]] - * - * @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate - * - * @return - set of eligible push copies for a given candidate - */ - protected[cross] final def getEligiblePushCopiesFromCandidate( - rawCandidate: RawCandidate - ): Future[Seq[MRPushCopy]] = { - val pushCopiesFromRectype = CandidateToCopy.getPushCopiesFromRectype(rawCandidate.commonRecType) - - if (pushCopiesFromRectype.isEmpty) { - NoAvailableCopyStat.counter(rawCandidate.commonRecType.name).incr() - throw new IllegalStateException(s"No Copy defined for CRT: " + rawCandidate.commonRecType) - } - pushCopiesFromRectype - .map(pushCopySet => copyFilters.execute(rawCandidate, pushCopySet.toSeq)) - .getOrElse(Future.value(Seq.empty)) - } - - /** - * - * This method essentially forms the base for cross-step for the MagicRecs Copy Framework. Given - * a recommendation type this returns a set of tuples wherein each tuple is a pair of push and - * ntab copy eligible for the said recommendation type - * - * @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate - * @return - Set of eligible [[MRPushCopy]], Option[[MRNtabCopy]] for a given recommendation type - */ - protected[cross] final def getEligiblePushAndNtabCopiesFromCandidate( - rawCandidate: RawCandidate - ): Future[Seq[(MRPushCopy, Option[MRNtabCopy])]] = { - - val eligiblePushCopies = getEligiblePushCopiesFromCandidate(rawCandidate) - - eligiblePushCopies.map { pushCopies => - val setBuilder = Set.newBuilder[(MRPushCopy, Option[MRNtabCopy])] - pushCopies.foreach { pushCopy => - val ntabCopies = CandidateToCopy.getNtabcopiesFromPushcopy(pushCopy) - val pushNtabCopyPairs = ntabCopies match { - case Some(ntabCopySet) => - if (ntabCopySet.isEmpty) { - NoAvailableNtabCopyStat.counter(s"copy_id: ${pushCopy.copyId}").incr() - Set(pushCopy -> None) - } // push copy only - else ntabCopySet.map(pushCopy -> Some(_)) - - case None => - Set.empty[(MRPushCopy, Option[MRNtabCopy])] // no push or ntab copy - } - setBuilder ++= pushNtabCopyPairs - } - setBuilder.result().toSeq - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.docx new file mode 100644 index 000000000..910cad6ab Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala deleted file mode 100644 index 9748c90ff..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyExpansion.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.util.MRNtabCopy -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.util.Future - -/** - * @param statsReceiver - stats receiver object - */ -class CandidateCopyExpansion(statsReceiver: StatsReceiver) - extends BaseCopyFramework(statsReceiver) { - - /** - * - * Given a [[CandidateDetails]] object representing a push recommendation candidate this method - * expands it to multiple candidates, each tagged with a push copy id and ntab copy id to - * represent the eligible copies for the given recommendation candidate - * - * @param candidateDetails - [[CandidateDetails]] objects containing a recommendation candidate - * - * @return - list of tuples of [[PushTypes.RawCandidate]] and [[CopyIds]] - */ - private final def crossCandidateDetailsWithCopyId( - candidateDetails: CandidateDetails[RawCandidate] - ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = { - val eligibleCopyPairs = getEligiblePushAndNtabCopiesFromCandidate(candidateDetails.candidate) - val copyPairs = eligibleCopyPairs.map(_.map { - case (pushCopy: MRPushCopy, ntabCopy: Option[MRNtabCopy]) => - CopyIds( - pushCopyId = Some(pushCopy.copyId), - ntabCopyId = ntabCopy.map(_.copyId) - ) - }) - - copyPairs.map(_.map((candidateDetails, _))) - } - - /** - * - * This method takes as input a list of [[CandidateDetails]] objects which contain the push - * recommendation candidates for a given target user. It expands each input candidate into - * multiple candidates, each tagged with a push copy id and ntab copy id to represent the eligible - * copies for the given recommendation candidate - * - * @param candidateDetailsSeq - list of fetched candidates for push recommendation - * @return - list of tuples of [[RawCandidate]] and [[CopyIds]] - */ - final def expandCandidatesWithCopyId( - candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = - Future.collect(candidateDetailsSeq.map(crossCandidateDetailsWithCopyId)).map(_.flatten) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.docx new file mode 100644 index 000000000..a4de38c4d Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala deleted file mode 100644 index 4eca41730..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateCopyPair.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate - -/** - * - * @param candidate: [[RawCandidate]] is a recommendation candidate - * @param pushCopy: [[MRPushCopy]] eligible for candidate - */ -case class CandidateCopyPair(candidate: RawCandidate, pushCopy: MRPushCopy) diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.docx new file mode 100644 index 000000000..a87761b86 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala deleted file mode 100644 index e7fbefe16..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CandidateToCopy.scala +++ /dev/null @@ -1,263 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.common.util._ -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ - -object CandidateToCopy { - - // Static map from a CommonRecommendationType to set of eligible push notification copies - private[cross] val rectypeToPushCopy: Map[CommonRecommendationType, Set[ - MRPushCopy - ]] = - Map[CommonRecommendationType, Set[MRPushCopy]]( - F1FirstdegreeTweet -> Set( - MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle - ), - F1FirstdegreePhoto -> Set( - MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle - ), - F1FirstdegreeVideo -> Set( - MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle - ), - TweetRetweet -> Set( - MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText - ), - TweetRetweetPhoto -> Set( - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText, - MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText - ), - TweetRetweetVideo -> Set( - MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText, - MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText, - MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText - ), - TweetFavorite -> Set( - MrPushCopyObjects.TweetLikeOneSocialContextWithText, - MrPushCopyObjects.TweetLikeTwoSocialContextWithText, - MrPushCopyObjects.TweetLikeMultipleSocialContextWithText - ), - TweetFavoritePhoto -> Set( - MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText, - MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText, - MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText - ), - TweetFavoriteVideo -> Set( - MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText, - MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText, - MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText - ), - UnreadBadgeCount -> Set(MrPushCopyObjects.UnreadBadgeCount), - InterestBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - InterestBasedPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - InterestBasedVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - UserFollow -> Set( - MrPushCopyObjects.UserFollowWithOneSocialContext, - MrPushCopyObjects.UserFollowWithTwoSocialContext, - MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext - ), - HermitUser -> Set( - MrPushCopyObjects.HermitUserWithOneSocialContext, - MrPushCopyObjects.HermitUserWithTwoSocialContext, - MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts - ), - TriangularLoopUser -> Set( - MrPushCopyObjects.TriangularLoopUserWithOneSocialContext, - MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts, - MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext - ), - ForwardAddressbookUserFollow -> Set(MrPushCopyObjects.ForwardAddressBookUserFollow), - NewsArticleNewsLanding -> Set(MrPushCopyObjects.NewsArticleNewsLandingCopy), - TopicProofTweet -> Set(MrPushCopyObjects.TopicProofTweet), - UserInterestinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - UserInterestinPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - UserInterestinVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - TwistlyTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - TwistlyPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - TwistlyVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - ElasticTimelineTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - ElasticTimelinePhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - ElasticTimelineVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - ExploreVideoTweet -> Set(MrPushCopyObjects.ExploreVideoTweet), - List -> Set(MrPushCopyObjects.ListRecommendation), - InterestBasedUserFollow -> Set(MrPushCopyObjects.UserFollowInterestBasedCopy), - PastEmailEngagementTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - PastEmailEngagementPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto), - PastEmailEngagementVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo), - ExplorePush -> Set(MrPushCopyObjects.ExplorePush), - ConnectTabPush -> Set(MrPushCopyObjects.ConnectTabPush), - ConnectTabWithUserPush -> Set(MrPushCopyObjects.ConnectTabWithUserPush), - AddressBookUploadPush -> Set(MrPushCopyObjects.AddressBookPush), - InterestPickerPush -> Set(MrPushCopyObjects.InterestPickerPush), - CompleteOnboardingPush -> Set(MrPushCopyObjects.CompleteOnboardingPush), - GeoPopTweet -> Set(MrPushCopyObjects.GeoPopPushCopy), - TagSpaceTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - FrsTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - TwhinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - MrModelingBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - DetopicTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - TweetImpressions -> Set(MrPushCopyObjects.TopTweetImpressions), - TrendTweet -> Set(MrPushCopyObjects.TrendTweet), - ReverseAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - ForwardAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - SpaceInNetwork -> Set(MrPushCopyObjects.SpaceHost), - SpaceOutOfNetwork -> Set(MrPushCopyObjects.SpaceHost), - SubscribedSearch -> Set(MrPushCopyObjects.SubscribedSearchTweet), - TripGeoTweet -> Set(MrPushCopyObjects.TripGeoTweetPushCopy), - CrowdSearchTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet), - Digest -> Set(MrPushCopyObjects.Digest), - TripHqTweet -> Set(MrPushCopyObjects.TripHqTweetPushCopy) - ) - - // Static map from a push copy to set of eligible ntab copies - private[cross] val pushcopyToNtabcopy: Map[MRPushCopy, Set[MRNtabCopy]] = - Map[MRPushCopy, Set[MRNtabCopy]]( - MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle -> Set( - MrNtabCopyObjects.FirstDegreeTweetRecent), - MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle -> Set( - MrNtabCopyObjects.FirstDegreeTweetRecent - ), - MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle -> Set( - MrNtabCopyObjects.FirstDegreeTweetRecent - ), - MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText -> Set( - MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText -> Set( - MrNtabCopyObjects.TweetRetweetVideoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText -> Set( - MrNtabCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetLikeOneSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetLikeTwoSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetLikeMultipleSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikePhotoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikePhotoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikePhotoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeVideoWithOneDisplaySocialContext - ), - MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeVideoWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText -> Set( - MrNtabCopyObjects.TweetLikeVideoWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.UnreadBadgeCount -> Set.empty[MRNtabCopy], - MrPushCopyObjects.RecommendedForYouTweet -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.RecommendedForYouPhoto -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.RecommendedForYouVideo -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.GeoPopPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.UserFollowWithOneSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext - ), - MrPushCopyObjects.UserFollowWithTwoSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.HermitUserWithOneSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext - ), - MrPushCopyObjects.HermitUserWithTwoSocialContext -> Set( - MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts - ), - MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts -> Set( - MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts - ), - MrPushCopyObjects.TriangularLoopUserWithOneSocialContext -> Set( - MrNtabCopyObjects.TriangularLoopUserWithOneSocialContext - ), - MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts -> Set( - MrNtabCopyObjects.TriangularLoopUserWithTwoSocialContexts - ), - MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext -> Set( - MrNtabCopyObjects.TriangularLoopUserOneDisplayAndKOtherSocialContext - ), - MrPushCopyObjects.NewsArticleNewsLandingCopy -> Set( - MrNtabCopyObjects.NewsArticleNewsLandingCopy - ), - MrPushCopyObjects.UserFollowInterestBasedCopy -> Set( - MrNtabCopyObjects.UserFollowInterestBasedCopy - ), - MrPushCopyObjects.ForwardAddressBookUserFollow -> Set( - MrNtabCopyObjects.ForwardAddressBookUserFollow), - MrPushCopyObjects.ConnectTabPush -> Set( - MrNtabCopyObjects.ConnectTabPush - ), - MrPushCopyObjects.ExplorePush -> Set.empty[MRNtabCopy], - MrPushCopyObjects.ConnectTabWithUserPush -> Set( - MrNtabCopyObjects.UserFollowInterestBasedCopy), - MrPushCopyObjects.AddressBookPush -> Set(MrNtabCopyObjects.AddressBook), - MrPushCopyObjects.InterestPickerPush -> Set(MrNtabCopyObjects.InterestPicker), - MrPushCopyObjects.CompleteOnboardingPush -> Set(MrNtabCopyObjects.CompleteOnboarding), - MrPushCopyObjects.TopicProofTweet -> Set(MrNtabCopyObjects.TopicProofTweet), - MrPushCopyObjects.TopTweetImpressions -> Set(MrNtabCopyObjects.TopTweetImpressions), - MrPushCopyObjects.TrendTweet -> Set(MrNtabCopyObjects.TrendTweet), - MrPushCopyObjects.SpaceHost -> Set(MrNtabCopyObjects.SpaceHost), - MrPushCopyObjects.SubscribedSearchTweet -> Set(MrNtabCopyObjects.SubscribedSearchTweet), - MrPushCopyObjects.TripGeoTweetPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy), - MrPushCopyObjects.Digest -> Set(MrNtabCopyObjects.Digest), - MrPushCopyObjects.TripHqTweetPushCopy -> Set(MrNtabCopyObjects.HighQualityTweet), - MrPushCopyObjects.ExploreVideoTweet -> Set(MrNtabCopyObjects.ExploreVideoTweet), - MrPushCopyObjects.ListRecommendation -> Set(MrNtabCopyObjects.ListRecommendation), - MrPushCopyObjects.MagicFanoutCreatorSubscription -> Set( - MrNtabCopyObjects.MagicFanoutCreatorSubscription), - MrPushCopyObjects.MagicFanoutNewCreator -> Set(MrNtabCopyObjects.MagicFanoutNewCreator) - ) - - /** - * - * @param crt - [[CommonRecommendationType]] used for a frigate push notification - * - * @return - Set of [[MRPushCopy]] objects representing push copies eligibile for a - * [[CommonRecommendationType]] - */ - def getPushCopiesFromRectype(crt: CommonRecommendationType): Option[Set[MRPushCopy]] = - rectypeToPushCopy.get(crt) - - /** - * - * @param pushcopy - [[MRPushCopy]] object representing a push notification copy - * @return - Set of [[MRNtabCopy]] objects that can be paired with a given [[MRPushCopy]] - */ - def getNtabcopiesFromPushcopy(pushcopy: MRPushCopy): Option[Set[MRNtabCopy]] = - pushcopyToNtabcopy.get(pushcopy) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.docx new file mode 100644 index 000000000..75c68fe63 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala deleted file mode 100644 index 0fe5f5cdd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyFilters.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.util.MRPushCopy -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -private[cross] class CopyFilters(statsReceiver: StatsReceiver) { - - private val copyPredicates = new CopyPredicates(statsReceiver.scope("copy_predicate")) - - def execute(rawCandidate: RawCandidate, pushCopies: Seq[MRPushCopy]): Future[Seq[MRPushCopy]] = { - val candidateCopyPairs: Seq[CandidateCopyPair] = - pushCopies.map(CandidateCopyPair(rawCandidate, _)) - - val compositePredicate: Predicate[CandidateCopyPair] = rawCandidate match { - case _: F1FirstDegree | _: OutOfNetworkTweetCandidate | _: EventCandidate | - _: TopicProofTweetCandidate | _: ListPushCandidate | _: HermitInterestBasedUserFollow | - _: UserFollowWithoutSocialContextCandidate | _: DiscoverTwitterCandidate | - _: TopTweetImpressionsCandidate | _: TrendTweetCandidate | - _: SubscribedSearchTweetCandidate | _: DigestCandidate => - copyPredicates.alwaysTruePredicate - - case _: SocialContextActions => copyPredicates.displaySocialContextPredicate - - case _ => copyPredicates.unrecognizedCandidatePredicate // block unrecognised candidates - } - - // apply predicate to all [[MRPushCopy]] objects - val filterResults: Future[Seq[Boolean]] = compositePredicate(candidateCopyPairs) - filterResults.map { results: Seq[Boolean] => - val seqBuilder = Seq.newBuilder[MRPushCopy] - results.zip(pushCopies).foreach { - case (result, pushCopy) => if (result) seqBuilder += pushCopy - } - seqBuilder.result() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.docx new file mode 100644 index 000000000..6ea7cae38 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala deleted file mode 100644 index 980af1554..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/refresh_handler/cross/CopyPredicates.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.frigate.pushservice.refresh_handler.cross - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.hermit.predicate.Predicate - -class CopyPredicates(statsReceiver: StatsReceiver) { - val alwaysTruePredicate = Predicate - .from { _: CandidateCopyPair => - true - }.withStats(statsReceiver.scope("always_true_copy_predicate")) - - val unrecognizedCandidatePredicate = alwaysTruePredicate.flip - .withStats(statsReceiver.scope("unrecognized_candidate")) - - val displaySocialContextPredicate = Predicate - .from { candidateCopyPair: CandidateCopyPair => - candidateCopyPair.candidate match { - case candidateWithScActions: RawCandidate with SocialContextActions => - val socialContextUserIds = candidateWithScActions.socialContextActions.map(_.userId) - val countSocialContext = socialContextUserIds.size - val pushCopy = candidateCopyPair.pushCopy - - countSocialContext match { - case 1 => pushCopy.hasOneDisplaySocialContext && !pushCopy.hasOtherSocialContext - case 2 => pushCopy.hasTwoDisplayContext && !pushCopy.hasOtherSocialContext - case c if c > 2 => - pushCopy.hasOneDisplaySocialContext && pushCopy.hasOtherSocialContext - case _ => false - } - - case _ => false - } - }.withStats(statsReceiver.scope("display_social_context_predicate")) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.docx new file mode 100644 index 000000000..6e2ed147c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala deleted file mode 100644 index 90095056a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/scriber/MrRequestScribeHandler.scala +++ /dev/null @@ -1,388 +0,0 @@ -package com.twitter.frigate.pushservice.scriber - -import com.twitter.bijection.Base64String -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.core_workflows.user_model.thriftscala.{UserState => ThriftUserState} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.tracing.Trace -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.scribe.thriftscala.CandidateFilteredOutStep -import com.twitter.frigate.scribe.thriftscala.CandidateRequestInfo -import com.twitter.frigate.scribe.thriftscala.MrRequestScribe -import com.twitter.frigate.scribe.thriftscala.TargetUserInfo -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.TweetNotification -import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} -import com.twitter.logging.Logger -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureType -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.nrel.heavyranker.PushPredictionHelper -import com.twitter.util.Future -import com.twitter.util.Time -import java.util.UUID -import scala.collection.mutable - -class MrRequestScribeHandler(mrRequestScriberNode: String, stats: StatsReceiver) { - - private val mrRequestScribeLogger = Logger(mrRequestScriberNode) - - private val mrRequestScribeTargetFilteringStats = - stats.counter("MrRequestScribeHandler_target_filtering") - private val mrRequestScribeCandidateFilteringStats = - stats.counter("MrRequestScribeHandler_candidate_filtering") - private val mrRequestScribeInvalidStats = - stats.counter("MrRequestScribeHandler_invalid_filtering") - private val mrRequestScribeUnsupportedFeatureTypeStats = - stats.counter("MrRequestScribeHandler_unsupported_feature_type") - private val mrRequestScribeNotIncludedFeatureStats = - stats.counter("MrRequestScribeHandler_not_included_features") - - private final val MrRequestScribeInjection: Injection[MrRequestScribe, String] = BinaryScalaCodec( - MrRequestScribe - ) andThen Injection.connect[Array[Byte], Base64String, String] - - /** - * - * @param target : Target user id - * @param result : Result for target filtering - * - * @return - */ - def scribeForTargetFiltering(target: Target, result: Result): Future[Option[MrRequestScribe]] = { - if (target.isLoggedOutUser || !enableTargetFilteringScribing(target)) { - Future.None - } else { - val predicate = result match { - case Invalid(reason) => reason - case _ => - mrRequestScribeInvalidStats.incr() - throw new IllegalStateException("Invalid reason for Target Filtering " + result) - } - buildScribeThrift(target, predicate, None).map { targetFilteredScribe => - writeAtTargetFilteringStep(target, targetFilteredScribe) - Some(targetFilteredScribe) - } - } - } - - /** - * - * @param target : Target user id - * @param hydratedCandidates : Candidates hydrated with details: impressionId, frigateNotification and source - * @param preRankingFilteredCandidates : Candidates result filtered out at preRanking filtering step - * @param rankedCandidates : Sorted candidates details ranked by ranking step - * @param rerankedCandidates : Sorted candidates details ranked by reranking step - * @param restrictFilteredCandidates : Candidates details filtered out at restrict step - * @param allTakeCandidateResults : Candidates results at take step, include the candidates we take and the candidates filtered out at take step [with different result] - * - * @return - */ - def scribeForCandidateFiltering( - target: Target, - hydratedCandidates: Seq[CandidateDetails[PushCandidate]], - preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]], - rankedCandidates: Seq[CandidateDetails[PushCandidate]], - rerankedCandidates: Seq[CandidateDetails[PushCandidate]], - restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]], - allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]] - ): Future[Seq[MrRequestScribe]] = { - if (target.isLoggedOutUser || target.isEmailUser) { - Future.Nil - } else if (enableCandidateFilteringScribing(target)) { - val hydrateFeature = - target.params(PushFeatureSwitchParams.EnableMrRequestScribingWithFeatureHydrating) || - target.scribeFeatureForRequestScribe - - val candidateRequestInfoSeq = generateCandidatesScribeInfo( - hydratedCandidates, - preRankingFilteredCandidates, - rankedCandidates, - rerankedCandidates, - restrictFilteredCandidates, - allTakeCandidateResults, - isFeatureHydratingEnabled = hydrateFeature - ) - val flattenStructure = - target.params(PushFeatureSwitchParams.EnableFlattenMrRequestScribing) || hydrateFeature - candidateRequestInfoSeq.flatMap { candidateRequestInfos => - if (flattenStructure) { - Future.collect { - candidateRequestInfos.map { candidateRequestInfo => - buildScribeThrift(target, None, Some(Seq(candidateRequestInfo))) - .map { mrRequestScribe => - writeAtCandidateFilteringStep(target, mrRequestScribe) - mrRequestScribe - } - } - } - } else { - buildScribeThrift(target, None, Some(candidateRequestInfos)) - .map { mrRequestScribe => - writeAtCandidateFilteringStep(target, mrRequestScribe) - Seq(mrRequestScribe) - } - } - } - } else Future.Nil - - } - - private def buildScribeThrift( - target: Target, - targetFilteredOutPredicate: Option[String], - candidatesRequestInfo: Option[Seq[CandidateRequestInfo]] - ): Future[MrRequestScribe] = { - Future - .join( - target.targetUserState, - generateTargetFeatureScribeInfo(target), - target.targetUser).map { - case (userStateOption, targetFeatureOption, gizmoduckUserOpt) => - val userState = userStateOption.map(userState => ThriftUserState(userState.id)) - val targetFeatures = - targetFeatureOption.map(ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord) - val traceId = Trace.id.traceId.toLong - - MrRequestScribe( - requestId = UUID.randomUUID.toString.replaceAll("-", ""), - scribedTimeMs = Time.now.inMilliseconds, - targetUserId = target.targetId, - targetUserInfo = Some( - TargetUserInfo( - userState, - features = targetFeatures, - userType = gizmoduckUserOpt.map(_.userType)) - ), - targetFilteredOutPredicate = targetFilteredOutPredicate, - candidates = candidatesRequestInfo, - traceId = Some(traceId) - ) - } - } - - private def generateTargetFeatureScribeInfo( - target: Target - ): Future[Option[DataRecord]] = { - val featureList = - target.params(PushFeatureSwitchParams.TargetLevelFeatureListForMrRequestScribing) - if (featureList.nonEmpty) { - PushPredictionHelper - .getDataRecordFromTargetFeatureMap( - target.targetId, - target.featureMap, - stats - ).map { dataRecord => - val richRecord = - new SRichDataRecord(dataRecord, PushQualityModelFeatureContext.featureContext) - - val selectedRecord = - SRichDataRecord(new DataRecord(), PushQualityModelFeatureContext.featureContext) - featureList.map { featureName => - val feature: Feature[_] = { - try { - PushQualityModelFeatureContext.featureContext.getFeature(featureName) - } catch { - case _: Exception => - mrRequestScribeNotIncludedFeatureStats.incr() - throw new IllegalStateException( - "Scribing features not included in FeatureContext: " + featureName) - } - } - - richRecord.getFeatureValueOpt(feature).foreach { featureVal => - feature.getFeatureType() match { - case FeatureType.BINARY => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[Boolean]], - featureVal.asInstanceOf[Boolean]) - case FeatureType.CONTINUOUS => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[Double]], - featureVal.asInstanceOf[Double]) - case FeatureType.STRING => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[String]], - featureVal.asInstanceOf[String]) - case FeatureType.DISCRETE => - selectedRecord.setFeatureValue( - feature.asInstanceOf[Feature[Long]], - featureVal.asInstanceOf[Long]) - case _ => - mrRequestScribeUnsupportedFeatureTypeStats.incr() - } - } - } - Some(selectedRecord.getRecord) - } - } else Future.None - } - - private def generateCandidatesScribeInfo( - hydratedCandidates: Seq[CandidateDetails[PushCandidate]], - preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]], - rankedCandidates: Seq[CandidateDetails[PushCandidate]], - rerankedCandidates: Seq[CandidateDetails[PushCandidate]], - restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]], - allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]], - isFeatureHydratingEnabled: Boolean - ): Future[Seq[CandidateRequestInfo]] = { - val candidatesMap = new mutable.HashMap[String, CandidateRequestInfo] - - hydratedCandidates.foreach { hydratedCandidate => - val frgNotif = hydratedCandidate.candidate.frigateNotification - val simplifiedTweetNotificationOpt = frgNotif.tweetNotification.map { tweetNotification => - TweetNotification( - tweetNotification.tweetId, - Seq.empty[TSocialContextAction], - tweetNotification.tweetAuthorId) - } - val simplifiedFrigateNotification = FrigateNotification( - frgNotif.commonRecommendationType, - frgNotif.notificationDisplayLocation, - tweetNotification = simplifiedTweetNotificationOpt - ) - candidatesMap(hydratedCandidate.candidate.impressionId) = CandidateRequestInfo( - candidateId = "", - candidateSource = hydratedCandidate.source.substring( - 0, - Math.min(6, hydratedCandidate.source.length) - ), - frigateNotification = Some(simplifiedFrigateNotification), - modelScore = None, - rankPosition = None, - rerankPosition = None, - features = None, - isSent = Some(false) - ) - } - - preRankingFilteredCandidates.foreach { preRankingFilteredCandidateResult => - candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) = - candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) - .copy( - candidateFilteredOutPredicate = preRankingFilteredCandidateResult.result match { - case Invalid(reason) => reason - case _ => { - mrRequestScribeInvalidStats.incr() - throw new IllegalStateException( - "Invalid reason for Candidate Filtering " + preRankingFilteredCandidateResult.result) - } - }, - candidateFilteredOutStep = Some(CandidateFilteredOutStep.PreRankFiltering) - ) - } - - for { - _ <- Future.collectToTry { - rankedCandidates.zipWithIndex.map { - case (rankedCandidateDetail, index) => - val modelScoresFut = { - val crt = rankedCandidateDetail.candidate.commonRecType - if (RecTypes.notEligibleForModelScoreTracking.contains(crt)) Future.None - else rankedCandidateDetail.candidate.modelScores.map(Some(_)) - } - - modelScoresFut.map { modelScores => - candidatesMap(rankedCandidateDetail.candidate.impressionId) = - candidatesMap(rankedCandidateDetail.candidate.impressionId).copy( - rankPosition = Some(index), - modelScore = modelScores - ) - } - } - } - - _ = rerankedCandidates.zipWithIndex.foreach { - case (rerankedCandidateDetail, index) => { - candidatesMap(rerankedCandidateDetail.candidate.impressionId) = - candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy( - rerankPosition = Some(index) - ) - } - } - - _ <- Future.collectToTry { - rerankedCandidates.map { rerankedCandidateDetail => - if (isFeatureHydratingEnabled) { - PushPredictionHelper - .getDataRecord( - rerankedCandidateDetail.candidate.target.targetHydrationContext, - rerankedCandidateDetail.candidate.target.featureMap, - rerankedCandidateDetail.candidate.candidateHydrationContext, - rerankedCandidateDetail.candidate.candidateFeatureMap(), - stats - ).map { features => - candidatesMap(rerankedCandidateDetail.candidate.impressionId) = - candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy( - features = Some( - ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(features)) - ) - } - } else Future.Unit - } - } - - _ = restrictFilteredCandidates.foreach { restrictFilteredCandidateDetatil => - candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) = - candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) - .copy(candidateFilteredOutStep = Some(CandidateFilteredOutStep.Restrict)) - } - - _ = allTakeCandidateResults.foreach { allTakeCandidateResult => - allTakeCandidateResult.result match { - case OK => - candidatesMap(allTakeCandidateResult.candidate.impressionId) = - candidatesMap(allTakeCandidateResult.candidate.impressionId).copy(isSent = Some(true)) - case Invalid(reason) => - candidatesMap(allTakeCandidateResult.candidate.impressionId) = - candidatesMap(allTakeCandidateResult.candidate.impressionId).copy( - candidateFilteredOutPredicate = reason, - candidateFilteredOutStep = Some(CandidateFilteredOutStep.PostRankFiltering)) - case _ => - mrRequestScribeInvalidStats.incr() - throw new IllegalStateException( - "Invalid reason for Candidate Filtering " + allTakeCandidateResult.result) - } - } - } yield candidatesMap.values.toSeq - } - - private def enableTargetFilteringScribing(target: Target): Boolean = { - target.params(PushParams.EnableMrRequestScribing) && target.params( - PushFeatureSwitchParams.EnableMrRequestScribingForTargetFiltering) - } - - private def enableCandidateFilteringScribing(target: Target): Boolean = { - target.params(PushParams.EnableMrRequestScribing) && target.params( - PushFeatureSwitchParams.EnableMrRequestScribingForCandidateFiltering) - } - - private def writeAtTargetFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = { - logToScribe(mrRequestScribe) - mrRequestScribeTargetFilteringStats.incr() - } - - private def writeAtCandidateFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = { - logToScribe(mrRequestScribe) - mrRequestScribeCandidateFilteringStats.incr() - } - - private def logToScribe(mrRequestScribe: MrRequestScribe): Unit = { - val logEntry: String = MrRequestScribeInjection(mrRequestScribe) - mrRequestScribeLogger.info(logEntry) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.docx new file mode 100644 index 000000000..c608431b0 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala deleted file mode 100644 index e235f76cb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandler.scala +++ /dev/null @@ -1,250 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateFilteringOnlyFlow -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.config.CommonConstants -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.util.InvalidRequestException -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableMagicFanoutNewsForYouNtabCopy -import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler -import com.twitter.frigate.pushservice.send_handler.generator.PushRequestToCandidate -import com.twitter.frigate.pushservice.take.SendHandlerNotifier -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator -import com.twitter.frigate.pushservice.target.PushTargetUserBuilder -import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest -import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil -import com.twitter.frigate.pushservice.thriftscala.PushRequest -import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.util._ - -/** - * A handler for sending PushRequests - */ -class SendHandler( - pushTargetUserBuilder: PushTargetUserBuilder, - preCandidateValidator: SendHandlerPreCandidateValidator, - postCandidateValidator: SendHandlerPostCandidateValidator, - sendHandlerNotifier: SendHandlerNotifier, - candidateHydrator: SendHandlerPushCandidateHydrator, - featureHydrator: FeatureHydrator, - sendHandlerPredicateUtil: SendHandlerPredicateUtil, - mrRequestScriberNode: String -)( - implicit val statsReceiver: StatsReceiver, - implicit val config: Config) - extends CandidateFilteringOnlyFlow[Target, RawCandidate, PushCandidate] { - - implicit private val timer: Timer = new JavaTimer(true) - val stats = statsReceiver.scope("SendHandler") - val log = MRLogger("SendHandler") - - private val buildTargetStats = stats.scope("build_target") - - private val candidateHydrationLatency: Stat = - stats.stat("candidateHydrationLatency") - - private val candidatePreValidatorLatency: Stat = - stats.stat("candidatePreValidatorLatency") - - private val candidatePostValidatorLatency: Stat = - stats.stat("candidatePostValidatorLatency") - - private val featureHydrationLatency: StatsReceiver = - stats.scope("featureHydrationLatency") - - private val mrRequestScribeHandler = - new MrRequestScribeHandler(mrRequestScriberNode, stats.scope("mr_request_scribe")) - - def apply(request: PushRequest): Future[PushResponse] = { - val receivers = Seq( - stats, - stats.scope(request.notification.commonRecommendationType.toString) - ) - val bStats = BroadcastStatsReceiver(receivers) - bStats.counter("requests").incr() - Stat - .timeFuture(bStats.stat("latency"))( - process(request).raiseWithin(CommonConstants.maxPushRequestDuration)) - .onSuccess { - case (pushResp, rawCandidate) => - trackStatsForResponseToRequest( - rawCandidate.commonRecType, - rawCandidate.target, - pushResp, - receivers)(statsReceiver) - if (!request.context.exists(_.darkWrite.contains(true))) { - config.requestScribe(PushRequestScribe(request, pushResp)) - } - } - .onFailure { ex => - bStats.counter("failures").incr() - bStats.scope("failures").counter(ex.getClass.getCanonicalName).incr() - } - .map { - case (pushResp, _) => pushResp - } - } - - private def process(request: PushRequest): Future[(PushResponse, RawCandidate)] = { - val recType = request.notification.commonRecommendationType - - track(buildTargetStats)( - pushTargetUserBuilder - .buildTarget( - request.userId, - request.context - ) - ).flatMap { targetUser => - val responseWithScribedInfo = request.context.exists { context => - context.responseWithScribedInfo.contains(true) - } - val newRequest = - if (request.notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent && - targetUser.params(EnableMagicFanoutNewsForYouNtabCopy)) { - val newNotification = request.notification.copy(ntabCopyId = - Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId)) - request.copy(notification = newNotification) - } else request - - if (RecTypes.isSendHandlerType(recType) || newRequest.context.exists( - _.allowCRT.contains(true))) { - - val rawCandidateFut = PushRequestToCandidate.generatePushCandidate( - newRequest.notification, - targetUser - ) - - rawCandidateFut.flatMap { rawCandidate => - val pushResponse = process(targetUser, Seq(rawCandidate)).flatMap { - sendHandlerNotifier.checkResponseAndNotify(_, responseWithScribedInfo) - } - - pushResponse.map { pushResponse => - (pushResponse, rawCandidate) - } - } - } else { - Future.exception(InvalidRequestException(s"${recType.name} not supported in SendHandler")) - } - } - } - - private def hydrateFeatures( - candidateDetails: Seq[CandidateDetails[PushCandidate]], - target: Target, - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - candidateDetails.headOption match { - case Some(candidateDetail) - if RecTypes.notEligibleForModelScoreTracking(candidateDetail.candidate.commonRecType) => - Future.value(candidateDetails) - - case Some(candidateDetail) => - val hydrationContextFut = HydrationContextBuilder.build(candidateDetail.candidate) - hydrationContextFut.flatMap { hc => - featureHydrator - .hydrateCandidate(Seq(hc), target.mrRequestContextForFeatureStore) - .map { hydrationResult => - val features = hydrationResult.getOrElse(hc, FeatureMap()) - candidateDetail.candidate.mergeFeatures(features) - candidateDetails - } - } - case _ => Future.Nil - } - } - - override def process( - target: Target, - externalCandidates: Seq[RawCandidate] - ): Future[Response[PushCandidate, Result]] = { - val candidate = externalCandidates.map(CandidateDetails(_, "realtime")) - - for { - hydratedCandidatesWithCopy <- hydrateCandidates(candidate) - - (candidates, preHydrationFilteredCandidates) <- track(filterStats)( - filter(target, hydratedCandidatesWithCopy) - ) - - featureHydratedCandidates <- - track(featureHydrationLatency)(hydrateFeatures(candidates, target)) - - allTakeCandidateResults <- track(takeStats)( - take(target, featureHydratedCandidates, desiredCandidateCount(target)) - ) - - _ <- mrRequestScribeHandler.scribeForCandidateFiltering( - target = target, - hydratedCandidates = hydratedCandidatesWithCopy, - preRankingFilteredCandidates = preHydrationFilteredCandidates, - rankedCandidates = featureHydratedCandidates, - rerankedCandidates = Seq.empty, - restrictFilteredCandidates = Seq.empty, // no restrict step - allTakeCandidateResults = allTakeCandidateResults - ) - } yield { - - /** - * We combine the results for all filtering steps and pass on in sequence to next step - * - * This is done to ensure the filtering reason for the candidate from multiple levels of - * filtering is carried all the way until [[PushResponse]] is built and returned from - * frigate-pushservice-send - */ - Response(OK, allTakeCandidateResults ++ preHydrationFilteredCandidates) - } - } - - override def hydrateCandidates( - candidates: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - Stat.timeFuture(candidateHydrationLatency)(candidateHydrator(candidates)) - } - - // Filter Step - pre-predicates and app specific predicates - override def filter( - target: Target, - hydratedCandidatesDetails: Seq[CandidateDetails[PushCandidate]] - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - Stat.timeFuture(candidatePreValidatorLatency)( - sendHandlerPredicateUtil.preValidationForCandidate( - hydratedCandidatesDetails, - preCandidateValidator - )) - } - - // Post Validation - Take step - override def validCandidates( - target: Target, - candidates: Seq[PushCandidate] - ): Future[Seq[Result]] = { - Stat.timeFuture(candidatePostValidatorLatency)(Future.collect(candidates.map { candidate => - sendHandlerPredicateUtil - .postValidationForCandidate(candidate, postCandidateValidator) - .map(res => res.result) - })) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.docx new file mode 100644 index 000000000..de9279050 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala deleted file mode 100644 index f8f102790..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/SendHandlerPushCandidateHydrator.scala +++ /dev/null @@ -1,184 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler - -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.common.util.MrNtabCopyObjects -import com.twitter.frigate.common.util.MrPushCopyObjects -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._ -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.UserId -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -case class SendHandlerPushCandidateHydrator( - lexServiceStore: ReadableStore[EventRequest, LiveEvent], - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - safeUserStore: ReadableStore[Long, User], - simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities], - audioSpaceStore: ReadableStore[String, AudioSpace], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore, - superFollowCreatorTweetCountStore: ReadableStore[UserId, Int] -)( - implicit statsReceiver: StatsReceiver, - implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) { - - lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num") - lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates") - - def updateCandidates( - candidateDetails: Seq[CandidateDetails[RawCandidate]], - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - - Future.collect { - candidateDetails.map { candidateDetail => - val pushCandidate = candidateDetail.candidate - - val copyIds = getCopyIdsByCRT(pushCandidate.commonRecType) - - val hydratedCandidateFut = pushCandidate match { - case magicFanoutNewsEventCandidate: MagicFanoutNewsEventCandidate => - getHydratedCandidateForMagicFanoutNewsEvent( - magicFanoutNewsEventCandidate, - copyIds, - lexServiceStore, - fanoutMetadataStore, - semanticCoreMegadataStore, - simClusterToEntityStore, - interestsLookupStore, - uttEntityHydrationStore - ) - - case scheduledSpaceSubscriberCandidate: ScheduledSpaceSubscriberCandidate => - getHydratedCandidateForScheduledSpaceSubscriber( - scheduledSpaceSubscriberCandidate, - safeUserStore, - copyIds, - audioSpaceStore - ) - case scheduledSpaceSpeakerCandidate: ScheduledSpaceSpeakerCandidate => - getHydratedCandidateForScheduledSpaceSpeaker( - scheduledSpaceSpeakerCandidate, - safeUserStore, - copyIds, - audioSpaceStore - ) - case magicFanoutSportsEventCandidate: MagicFanoutSportsEventCandidate with MagicFanoutSportsScoreInformation => - getHydratedCandidateForMagicFanoutSportsEvent( - magicFanoutSportsEventCandidate, - copyIds, - lexServiceStore, - fanoutMetadataStore, - semanticCoreMegadataStore, - interestsLookupStore, - uttEntityHydrationStore - ) - case magicFanoutProductLaunchCandidate: MagicFanoutProductLaunchCandidate => - getHydratedCandidateForMagicFanoutProductLaunch( - magicFanoutProductLaunchCandidate, - copyIds) - case creatorEventCandidate: MagicFanoutCreatorEventCandidate => - getHydratedCandidateForMagicFanoutCreatorEvent( - creatorEventCandidate, - safeUserStore, - copyIds, - superFollowCreatorTweetCountStore) - case _ => - throw new IllegalArgumentException("Incorrect candidate type when update candidates") - } - - hydratedCandidateFut.map { hydratedCandidate => - hydratedCandidateStat.counter(hydratedCandidate.commonRecType.name).incr() - CandidateDetails( - hydratedCandidate, - source = candidateDetail.source - ) - } - } - } - } - - private def getCopyIdsByCRT(crt: CommonRecommendationType): CopyIds = { - crt match { - case CommonRecommendationType.MagicFanoutNewsEvent => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewsPushCopy.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId), - aggregationId = None - ) - - case CommonRecommendationType.ScheduledSpaceSubscriber => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSubscriber.copyId), - ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSubscriber.copyId), - aggregationId = None - ) - case CommonRecommendationType.ScheduledSpaceSpeaker => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSpeaker.copyId), - ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSpeakerNow.copyId), - aggregationId = None - ) - case CommonRecommendationType.SpaceSpeaker => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.SpaceSpeaker.copyId), - ntabCopyId = Some(MrNtabCopyObjects.SpaceSpeaker.copyId), - aggregationId = None - ) - case CommonRecommendationType.SpaceHost => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.SpaceHost.copyId), - ntabCopyId = Some(MrNtabCopyObjects.SpaceHost.copyId), - aggregationId = None - ) - case CommonRecommendationType.MagicFanoutSportsEvent => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutSportsPushCopy.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutSportsCopy.copyId), - aggregationId = None - ) - case CommonRecommendationType.MagicFanoutProductLaunch => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutProductLaunch.copyId), - ntabCopyId = Some(MrNtabCopyObjects.ProductLaunch.copyId), - aggregationId = None - ) - case CommonRecommendationType.CreatorSubscriber => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutCreatorSubscription.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutCreatorSubscription.copyId), - aggregationId = None - ) - case CommonRecommendationType.NewCreator => - CopyIds( - pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewCreator.copyId), - ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewCreator.copyId), - aggregationId = None - ) - case _ => - throw new IllegalArgumentException("Incorrect candidate type when fetch copy ids") - } - } - - def apply( - candidateDetails: Seq[CandidateDetails[RawCandidate]] - ): Future[Seq[CandidateDetails[PushCandidate]]] = { - updateCandidates(candidateDetails) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.docx new file mode 100644 index 000000000..e82a60b65 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala deleted file mode 100644 index 45907fa8e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/CandidateGenerator.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -trait CandidateGenerator { - - /** - * Build RawCandidate from FrigateNotification - * @param target - * @param frigateNotification - * @return RawCandidate - */ - def getCandidate(target: Target, frigateNotification: FrigateNotification): Future[RawCandidate] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.docx new file mode 100644 index 000000000..8f72a3263 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala deleted file mode 100644 index 10a7acb89..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutCreatorEventCandidateGenerator.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate -import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object MagicFanoutCreatorEventCandidateGenerator extends CandidateGenerator { - override def getCandidate( - targetUser: PushTypes.Target, - notification: FrigateNotification - ): Future[PushTypes.RawCandidate] = { - - require( - notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber || notification.commonRecommendationType == CommonRecommendationType.NewCreator, - "MagicFanoutCreatorEvent: unexpected CRT " + notification.commonRecommendationType - ) - require( - notification.creatorSubscriptionNotification.isDefined, - "MagicFanoutCreatorEvent: creatorSubscriptionNotification is not defined") - require( - notification.creatorSubscriptionNotification.exists(_.magicFanoutPushId.isDefined), - "MagicFanoutCreatorEvent: magicFanoutPushId is not defined") - require( - notification.creatorSubscriptionNotification.exists(_.fanoutReasons.isDefined), - "MagicFanoutCreatorEvent: fanoutReasons is not defined") - require( - notification.creatorSubscriptionNotification.exists(_.creatorId.isDefined), - "MagicFanoutCreatorEvent: creatorId is not defined") - if (notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber) { - require( - notification.creatorSubscriptionNotification - .exists(_.subscriberId.isDefined), - "MagicFanoutCreatorEvent: subscriber id is not defined" - ) - } - - val creatorSubscriptionNotification = notification.creatorSubscriptionNotification.get - - val candidate = new RawCandidate with MagicFanoutCreatorEventCandidate { - - override val target: Target = targetUser - - override val pushId: Long = - creatorSubscriptionNotification.magicFanoutPushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - creatorSubscriptionNotification.fanoutReasons.get - - override val creatorFanoutType: CreatorFanoutType = - creatorSubscriptionNotification.creatorFanoutType - - override val commonRecType: CommonRecommendationType = - notification.commonRecommendationType - - override val frigateNotification: FrigateNotification = notification - - override val subscriberId: Option[Long] = creatorSubscriptionNotification.subscriberId - - override val creatorId: Long = creatorSubscriptionNotification.creatorId.get - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.docx new file mode 100644 index 000000000..686c1346c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala deleted file mode 100644 index 7b351c91a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutNewsEventCandidateGenerator.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails -import com.twitter.util.Future - -object MagicFanoutNewsEventCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: Target, - notification: FrigateNotification - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutNewsEvent]] - * AND pushId field should be set - **/ - require( - notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent, - "MagicFanoutNewsEvent: unexpected CRT " + notification.commonRecommendationType - ) - - require( - notification.magicFanoutEventNotification.exists(_.pushId.isDefined), - "MagicFanoutNewsEvent: pushId is not defined") - - val magicFanoutEventNotification = notification.magicFanoutEventNotification.get - - val candidate = new RawCandidate with MagicFanoutNewsEventCandidate { - - override val target: Target = targetUser - - override val eventId: Long = magicFanoutEventNotification.eventId - - override val pushId: Long = magicFanoutEventNotification.pushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty) - - override val momentId: Option[Long] = magicFanoutEventNotification.momentId - - override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage - - override val details: Option[MagicFanoutEventNotificationDetails] = - magicFanoutEventNotification.details - - override val frigateNotification: FrigateNotification = notification - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.docx new file mode 100644 index 000000000..0854cd9a5 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala deleted file mode 100644 index 6844b1b06..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutProductLaunchCandidateGenerator.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.ProductType -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object MagicFanoutProductLaunchCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: PushTypes.Target, - notification: FrigateNotification - ): Future[PushTypes.RawCandidate] = { - - require( - notification.commonRecommendationType == CommonRecommendationType.MagicFanoutProductLaunch, - "MagicFanoutProductLaunch: unexpected CRT " + notification.commonRecommendationType - ) - require( - notification.magicFanoutProductLaunchNotification.isDefined, - "MagicFanoutProductLaunch: magicFanoutProductLaunchNotification is not defined") - require( - notification.magicFanoutProductLaunchNotification.exists(_.magicFanoutPushId.isDefined), - "MagicFanoutProductLaunch: magicFanoutPushId is not defined") - require( - notification.magicFanoutProductLaunchNotification.exists(_.fanoutReasons.isDefined), - "MagicFanoutProductLaunch: fanoutReasons is not defined") - - val magicFanoutProductLaunchNotification = notification.magicFanoutProductLaunchNotification.get - - val candidate = new RawCandidate with MagicFanoutProductLaunchCandidate { - - override val target: Target = targetUser - - override val pushId: Long = - magicFanoutProductLaunchNotification.magicFanoutPushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - magicFanoutProductLaunchNotification.fanoutReasons.get - - override val productLaunchType: ProductType = - magicFanoutProductLaunchNotification.productLaunchType - - override val frigateNotification: FrigateNotification = notification - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.docx new file mode 100644 index 000000000..f392d6093 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala deleted file mode 100644 index cdd37833e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/MagicFanoutSportsEventCandidateGenerator.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate -import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate -import com.twitter.escherbird.common.thriftscala.Domains -import com.twitter.escherbird.common.thriftscala.QualifiedId -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.frigate.common.base.BaseGameScore -import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate -import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation -import com.twitter.frigate.common.base.TeamInfo -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.pushservice.exception.InvalidSportDomainException -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object MagicFanoutSportsEventCandidateGenerator { - - final def getCandidate( - targetUser: Target, - notification: FrigateNotification, - basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate], - baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate], - cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate], - soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate], - nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutSportsEvent]] - * AND pushId field should be set - * - * */ - require( - notification.commonRecommendationType == CommonRecommendationType.MagicFanoutSportsEvent, - "MagicFanoutSports: unexpected CRT " + notification.commonRecommendationType - ) - - require( - notification.magicFanoutEventNotification.exists(_.pushId.isDefined), - "MagicFanoutSportsEvent: pushId is not defined") - - val magicFanoutEventNotification = notification.magicFanoutEventNotification.get - val eventId = magicFanoutEventNotification.eventId - val _isScoreUpdate = magicFanoutEventNotification.isScoreUpdate.getOrElse(false) - - val gameScoresFut: Future[Option[BaseGameScore]] = { - if (_isScoreUpdate) { - semanticCoreMegadataStore - .get(SemanticEntityForQuery(PushConstants.SportsEventDomainId, eventId)) - .flatMap { - case Some(megadata) => - if (megadata.domains.contains(Domains.BasketballGame)) { - basketballGameScoreStore - .get(QualifiedId(Domains.BasketballGame.value, eventId)).map { - case Some(game) if game.status.isDefined => - val status = game.status.get - MagicFanoutSportsUtil.transformToGameScore(game.score, status) - case _ => None - } - } else if (megadata.domains.contains(Domains.BaseballGame)) { - baseballGameScoreStore - .get(QualifiedId(Domains.BaseballGame.value, eventId)).map { - case Some(game) if game.status.isDefined => - val status = game.status.get - MagicFanoutSportsUtil.transformToGameScore(game.runs, status) - case _ => None - } - } else if (megadata.domains.contains(Domains.NflFootballGame)) { - nflGameScoreStore - .get(QualifiedId(Domains.NflFootballGame.value, eventId)).map { - case Some(game) if game.status.isDefined => - val nflScore = MagicFanoutSportsUtil.transformNFLGameScore(game) - nflScore - case _ => None - } - } else if (megadata.domains.contains(Domains.SoccerMatch)) { - soccerMatchScoreStore - .get(QualifiedId(Domains.SoccerMatch.value, eventId)).map { - case Some(game) if game.status.isDefined => - val soccerScore = MagicFanoutSportsUtil.transformSoccerGameScore(game) - soccerScore - case _ => None - } - } else { - // The domains are not in our list of supported sports - throw new InvalidSportDomainException( - s"Domain for entity ${eventId} is not supported") - } - case _ => Future.None - } - } else Future.None - } - - val homeTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap { - case Some(gameScore) => - MagicFanoutSportsUtil.getTeamInfo(gameScore.home, semanticCoreMegadataStore) - case _ => Future.None - } - - val awayTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap { - case Some(gameScore) => - MagicFanoutSportsUtil.getTeamInfo(gameScore.away, semanticCoreMegadataStore) - case _ => Future.None - } - - val candidate = new RawCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation { - - override val target: Target = targetUser - - override val eventId: Long = magicFanoutEventNotification.eventId - - override val pushId: Long = magicFanoutEventNotification.pushId.get - - override val candidateMagicEventsReasons: Seq[MagicEventsReason] = - magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty) - - override val momentId: Option[Long] = magicFanoutEventNotification.momentId - - override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage - - override val details: Option[MagicFanoutEventNotificationDetails] = - magicFanoutEventNotification.details - - override val frigateNotification: FrigateNotification = notification - - override val homeTeamInfo: Future[Option[TeamInfo]] = homeTeamInfoFut - - override val awayTeamInfo: Future[Option[TeamInfo]] = awayTeamInfoFut - - override val gameScores: Future[Option[BaseGameScore]] = gameScoresFut - - override val isScoreUpdate: Boolean = _isScoreUpdate - } - - Future.value(candidate) - - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.docx new file mode 100644 index 000000000..914219858 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala deleted file mode 100644 index 8d7e81d3f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/PushRequestToCandidate.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.exception.UnsupportedCrtException -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.util.Future - -object PushRequestToCandidate { - final def generatePushCandidate( - frigateNotification: FrigateNotification, - target: Target - )( - implicit config: Config - ): Future[RawCandidate] = { - - val candidateGenerator: (Target, FrigateNotification) => Future[RawCandidate] = { - frigateNotification.commonRecommendationType match { - case CRT.MagicFanoutNewsEvent => MagicFanoutNewsEventCandidateGenerator.getCandidate - case CRT.ScheduledSpaceSubscriber => ScheduledSpaceSubscriberCandidateGenerator.getCandidate - case CRT.ScheduledSpaceSpeaker => ScheduledSpaceSpeakerCandidateGenerator.getCandidate - case CRT.MagicFanoutSportsEvent => - MagicFanoutSportsEventCandidateGenerator.getCandidate( - _, - _, - config.basketballGameScoreStore, - config.baseballGameScoreStore, - config.cricketMatchScoreStore, - config.soccerMatchScoreStore, - config.nflGameScoreStore, - config.semanticCoreMegadataStore - ) - case CRT.MagicFanoutProductLaunch => - MagicFanoutProductLaunchCandidateGenerator.getCandidate - case CRT.NewCreator => - MagicFanoutCreatorEventCandidateGenerator.getCandidate - case CRT.CreatorSubscriber => - MagicFanoutCreatorEventCandidateGenerator.getCandidate - case _ => - throw new UnsupportedCrtException( - "UnsupportedCrtException for SendHandler: " + frigateNotification.commonRecommendationType) - } - } - - candidateGenerator(target, frigateNotification) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.docx new file mode 100644 index 000000000..f8bc3bdda Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala deleted file mode 100644 index e7821db76..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSpeakerCandidateGenerator.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object ScheduledSpaceSpeakerCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: Target, - notification: FrigateNotification - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSpeaker]] - * - **/ - require( - notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSpeaker, - "ScheduledSpaceSpeaker: unexpected CRT " + notification.commonRecommendationType - ) - - val spaceNotification = notification.spaceNotification.getOrElse( - throw new IllegalStateException("ScheduledSpaceSpeaker notification object not defined")) - - require( - spaceNotification.hostUserId.isDefined, - "ScheduledSpaceSpeaker notification - hostUserId not defined" - ) - - val spaceHostId = spaceNotification.hostUserId - - require( - spaceNotification.scheduledStartTime.isDefined, - "ScheduledSpaceSpeaker notification - scheduledStartTime not defined" - ) - - val scheduledStartTime = spaceNotification.scheduledStartTime.get - - val candidate = new RawCandidate with ScheduledSpaceSpeakerCandidate { - override val target: Target = targetUser - override val frigateNotification: FrigateNotification = notification - override val spaceId: String = spaceNotification.broadcastId - override val hostId: Option[Long] = spaceHostId - override val startTime: Long = scheduledStartTime - override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers - override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.docx new file mode 100644 index 000000000..e6506bc77 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala deleted file mode 100644 index 484f17b2a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/send_handler/generator/ScheduledSpaceSubscriberCandidateGenerator.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.send_handler.generator - -import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Future - -object ScheduledSpaceSubscriberCandidateGenerator extends CandidateGenerator { - - override def getCandidate( - targetUser: Target, - notification: FrigateNotification - ): Future[RawCandidate] = { - - /** - * frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSubscriber]] - * - **/ - require( - notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSubscriber, - "ScheduledSpaceSubscriber: unexpected CRT " + notification.commonRecommendationType - ) - - val spaceNotification = notification.spaceNotification.getOrElse( - throw new IllegalStateException("ScheduledSpaceSubscriber notification object not defined")) - - require( - spaceNotification.hostUserId.isDefined, - "ScheduledSpaceSubscriber notification - hostUserId not defined" - ) - - val spaceHostId = spaceNotification.hostUserId - - require( - spaceNotification.scheduledStartTime.isDefined, - "ScheduledSpaceSubscriber notification - scheduledStartTime not defined" - ) - - val scheduledStartTime = spaceNotification.scheduledStartTime.get - - val candidate = new RawCandidate with ScheduledSpaceSubscriberCandidate { - override val target: Target = targetUser - override val frigateNotification: FrigateNotification = notification - override val spaceId: String = spaceNotification.broadcastId - override val hostId: Option[Long] = spaceHostId - override val startTime: Long = scheduledStartTime - override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers - override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners - } - - Future.value(candidate) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.docx new file mode 100644 index 000000000..6f5f52c14 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala deleted file mode 100644 index 1f4171030..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ContentMixerStore.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.content_mixer.thriftscala.ContentMixer -import com.twitter.content_mixer.thriftscala.ContentMixerRequest -import com.twitter.content_mixer.thriftscala.ContentMixerResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class ContentMixerStore(contentMixer: ContentMixer.MethodPerEndpoint) - extends ReadableStore[ContentMixerRequest, ContentMixerResponse] { - - override def get(request: ContentMixerRequest): Future[Option[ContentMixerResponse]] = { - contentMixer.getCandidates(request).map { response => - Some(response) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.docx new file mode 100644 index 000000000..0a53433c7 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala deleted file mode 100644 index b793ade7e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CopySelectionServiceStore.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.copyselectionservice.thriftscala._ -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class CopySelectionServiceStore(copySelectionServiceClient: CopySelectionService.FinagledClient) - extends ReadableStore[CopySelectionRequestV1, Copy] { - override def get(k: CopySelectionRequestV1): Future[Option[Copy]] = - copySelectionServiceClient.getSelectedCopy(CopySelectionRequest.V1(k)).map { - case CopySelectionResponse.V1(response) => - Some(response.selectedCopy) - case _ => throw CopyServiceException(CopyServiceErrorCode.VersionNotFound) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.docx new file mode 100644 index 000000000..eda440ab4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala deleted file mode 100644 index dba016e6c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/CrMixerTweetStore.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.cr_mixer.thriftscala.CrMixer -import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest -import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse -import com.twitter.cr_mixer.thriftscala.FrsTweetRequest -import com.twitter.cr_mixer.thriftscala.FrsTweetResponse -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.util.Future - -/** - * Store to get content recs from content recommender. - */ -case class CrMixerTweetStore( - crMixer: CrMixer.MethodPerEndpoint -)( - implicit statsReceiver: StatsReceiver = NullStatsReceiver) { - - private val requestsCounter = statsReceiver.counter("requests") - private val successCounter = statsReceiver.counter("success") - private val failuresCounter = statsReceiver.counter("failures") - private val nonEmptyCounter = statsReceiver.counter("non_empty") - private val emptyCounter = statsReceiver.counter("empty") - private val failuresScope = statsReceiver.scope("failures") - private val latencyStat = statsReceiver.stat("latency") - - private def updateStats[T](f: => Future[Option[T]]): Future[Option[T]] = { - requestsCounter.incr() - Stat - .timeFuture(latencyStat)(f) - .onSuccess { r => - if (r.isDefined) nonEmptyCounter.incr() else emptyCounter.incr() - successCounter.incr() - } - .onFailure { e => - { - failuresCounter.incr() - failuresScope.counter(e.getClass.getName).incr() - } - } - } - - def getTweetRecommendations( - request: CrMixerTweetRequest - ): Future[Option[CrMixerTweetResponse]] = { - updateStats(crMixer.getTweetRecommendations(request).map { response => - Some(response) - }) - } - - def getFRSTweetCandidates(request: FrsTweetRequest): Future[Option[FrsTweetResponse]] = { - updateStats(crMixer.getFrsBasedTweetRecommendations(request).map { response => - Some(response) - }) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.docx new file mode 100644 index 000000000..945d6005c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala deleted file mode 100644 index eeeb62d27..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/ExploreRankerStore.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.explore_ranker.thriftscala.ExploreRanker -import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse -import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -/** A Store for Video Tweet Recommendations from Explore - * - * @param exploreRankerService - */ -case class ExploreRankerStore(exploreRankerService: ExploreRanker.MethodPerEndpoint) - extends ReadableStore[ExploreRankerRequest, ExploreRankerResponse] { - - /** Method to get video recommendations - * - * @param request explore ranker request object - * @return - */ - override def get( - request: ExploreRankerRequest - ): Future[Option[ExploreRankerResponse]] = { - exploreRankerService.getRankedResults(request).map { response => - Some(response) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.docx new file mode 100644 index 000000000..491824ed1 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala deleted file mode 100644 index 0ab722cd4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/FollowRecommendationsStore.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService -import com.twitter.follow_recommendations.thriftscala.Recommendation -import com.twitter.follow_recommendations.thriftscala.RecommendationRequest -import com.twitter.follow_recommendations.thriftscala.RecommendationResponse -import com.twitter.follow_recommendations.thriftscala.UserRecommendation -import com.twitter.inject.Logging -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class FollowRecommendationsStore( - frsClient: FollowRecommendationsThriftService.MethodPerEndpoint, - statsReceiver: StatsReceiver) - extends ReadableStore[RecommendationRequest, RecommendationResponse] - with Logging { - - private val scopedStats = statsReceiver.scope(getClass.getSimpleName) - private val requests = scopedStats.counter("requests") - private val valid = scopedStats.counter("valid") - private val invalid = scopedStats.counter("invalid") - private val numTotalResults = scopedStats.stat("total_results") - private val numValidResults = scopedStats.stat("valid_results") - - override def get(request: RecommendationRequest): Future[Option[RecommendationResponse]] = { - requests.incr() - frsClient.getRecommendations(request).map { response => - numTotalResults.add(response.recommendations.size) - val validRecs = response.recommendations.filter { - case Recommendation.User(_: UserRecommendation) => - valid.incr() - true - case _ => - invalid.incr() - false - } - - numValidResults.add(validRecs.size) - Some( - RecommendationResponse( - recommendations = validRecs - )) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.docx new file mode 100644 index 000000000..198bef1bd Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala deleted file mode 100644 index 6c355c505..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/IbisStore.scala +++ /dev/null @@ -1,190 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.store -import com.twitter.frigate.common.store.Fail -import com.twitter.frigate.common.store.IbisRequestInfo -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.Sent -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.ibis2.service.thriftscala.Flags -import com.twitter.ibis2.service.thriftscala.FlowControl -import com.twitter.ibis2.service.thriftscala.Ibis2Request -import com.twitter.ibis2.service.thriftscala.Ibis2Response -import com.twitter.ibis2.service.thriftscala.Ibis2ResponseStatus -import com.twitter.ibis2.service.thriftscala.Ibis2Service -import com.twitter.ibis2.service.thriftscala.NotificationNotSentCode -import com.twitter.ibis2.service.thriftscala.TargetFanoutResult.NotSentReason -import com.twitter.util.Future - -trait Ibis2Store extends store.Ibis2Store { - def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] -} - -case class PushIbis2Store( - ibisClient: Ibis2Service.MethodPerEndpoint -)( - implicit val statsReceiver: StatsReceiver = NullStatsReceiver) - extends Ibis2Store { - private val log = MRLogger(this.getClass.getSimpleName) - private val stats = statsReceiver.scope("ibis_v2_store") - private val statsByCrt = stats.scope("byCrt") - private val requestsByCrt = statsByCrt.scope("requests") - private val failuresByCrt = statsByCrt.scope("failures") - private val successByCrt = statsByCrt.scope("success") - - private val statsByIbisModel = stats.scope("byIbisModel") - private val requestsByIbisModel = statsByIbisModel.scope("requests") - private val failuresByIbisModel = statsByIbisModel.scope("failures") - private val successByIbisModel = statsByIbisModel.scope("success") - - private[this] def ibisSend( - ibis2Request: Ibis2Request, - commonRecommendationType: CommonRecommendationType - ): Future[IbisResponse] = { - val ibisModel = ibis2Request.modelName - - val bStats = if (ibis2Request.flags.getOrElse(Flags()).darkWrite.contains(true)) { - BroadcastStatsReceiver( - Seq( - stats, - stats.scope("dark_write") - ) - ) - } else BroadcastStatsReceiver(Seq(stats)) - - bStats.counter("requests").incr() - requestsByCrt.counter(commonRecommendationType.name).incr() - requestsByIbisModel.counter(ibisModel).incr() - - retry(ibisClient, ibis2Request, 3, bStats) - .map { response => - bStats.counter(response.status.status.name).incr() - successByCrt.counter(response.status.status.name, commonRecommendationType.name).incr() - successByIbisModel.counter(response.status.status.name, ibisModel).incr() - response.status.status match { - case Ibis2ResponseStatus.SuccessWithDeliveries | - Ibis2ResponseStatus.SuccessNoDeliveries => - IbisResponse(Sent, Some(response)) - case _ => - IbisResponse(Fail, Some(response)) - } - } - .onFailure { ex => - bStats.counter("failures").incr() - val exceptionName = ex.getClass.getCanonicalName - bStats.scope("failures").counter(exceptionName).incr() - failuresByCrt.counter(exceptionName, commonRecommendationType.name).incr() - failuresByIbisModel.counter(exceptionName, ibisModel).incr() - } - } - - private def getNotifNotSentReason( - ibis2Response: Ibis2Response - ): Option[NotificationNotSentCode] = { - ibis2Response.status.fanoutResults match { - case Some(fanoutResult) => - fanoutResult.pushResult.flatMap { pushResult => - pushResult.results.headOption match { - case Some(NotSentReason(notSentInfo)) => Some(notSentInfo.notSentCode) - case _ => None - } - } - case _ => None - } - } - - def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = { - val requestWithIID = if (ibis2Request.flowControl.exists(_.externalIid.isDefined)) { - ibis2Request - } else { - ibis2Request.copy( - flowControl = Some( - ibis2Request.flowControl - .getOrElse(FlowControl()) - .copy(externalIid = Some(candidate.impressionId)) - ) - ) - } - - val commonRecommendationType = candidate.frigateNotification.commonRecommendationType - - ibisSend(requestWithIID, commonRecommendationType) - .onSuccess { response => - response.ibis2Response.foreach { ibis2Response => - getNotifNotSentReason(ibis2Response).foreach { notifNotSentCode => - stats.scope(ibis2Response.status.status.name).counter(s"$notifNotSentCode").incr() - } - if (ibis2Response.status.status != Ibis2ResponseStatus.SuccessWithDeliveries) { - log.warning( - s"Request dropped on ibis for ${ibis2Request.recipientSelector.recipientId}: $ibis2Response") - } - } - } - .onFailure { ex => - log.warning( - s"Ibis Request failure: ${ex.getClass.getCanonicalName} \n For IbisRequest: $ibis2Request") - log.error(ex, ex.getMessage) - } - } - - // retry request when Ibis2ResponseStatus is PreFanoutError - def retry( - ibisClient: Ibis2Service.MethodPerEndpoint, - request: Ibis2Request, - retryCount: Int, - bStats: StatsReceiver - ): Future[Ibis2Response] = { - ibisClient.sendNotification(request).flatMap { response => - response.status.status match { - case Ibis2ResponseStatus.PreFanoutError if retryCount > 0 => - bStats.scope("requests").counter("retry").incr() - bStats.counter(response.status.status.name).incr() - retry(ibisClient, request, retryCount - 1, bStats) - case _ => - Future.value(response) - } - } - } - - override def send( - ibis2Request: Ibis2Request, - requestInfo: IbisRequestInfo - ): Future[IbisResponse] = { - ibisSend(ibis2Request, requestInfo.commonRecommendationType) - } -} - -case class StagingIbis2Store(remoteIbis2Store: PushIbis2Store) extends Ibis2Store { - - final def addDarkWriteFlagIbis2Request( - isTeamMember: Boolean, - ibis2Request: Ibis2Request - ): Ibis2Request = { - val flags = - ibis2Request.flags.getOrElse(Flags()) - val darkWrite: Boolean = !isTeamMember || flags.darkWrite.getOrElse(false) - ibis2Request.copy(flags = Some(flags.copy(darkWrite = Some(darkWrite)))) - } - - override def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = { - candidate.target.isTeamMember.flatMap { isTeamMember => - val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request) - remoteIbis2Store.send(ibis2Req, candidate) - } - } - - override def send( - ibis2Request: Ibis2Request, - requestInfo: IbisRequestInfo - ): Future[IbisResponse] = { - requestInfo.isTeamMember.flatMap { isTeamMember => - val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request) - remoteIbis2Store.send(ibis2Req, requestInfo) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.docx new file mode 100644 index 000000000..51d5bc1a8 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala deleted file mode 100644 index 80fc0ea7e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/InterestDiscoveryStore.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService -import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest -import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class InterestDiscoveryStore( - client: InterestsDiscoveryService.MethodPerEndpoint) - extends ReadableStore[RecommendedListsRequest, RecommendedListsResponse] { - - override def get(request: RecommendedListsRequest): Future[Option[RecommendedListsResponse]] = { - client.getListRecos(request).map(Some(_)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.docx new file mode 100644 index 000000000..3a5dc3038 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala deleted file mode 100644 index 73fc28837..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LabeledPushRecsDecideredStore.scala +++ /dev/null @@ -1,156 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.candidate.TargetDecider -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.history.HistoryStoreKeyContext -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.data_pipeline.thriftscala._ -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.hermit.store.labeled_push_recs.LabeledPushRecsJoinedWithNotificationHistoryStore -import com.twitter.logging.Logger -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future -import com.twitter.util.Time - -case class LabeledPushRecsVerifyingStoreKey( - historyStoreKey: HistoryStoreKeyContext, - useHydratedDataset: Boolean, - verifyHydratedDatasetResults: Boolean) { - def userId: Long = historyStoreKey.targetUserId -} - -case class LabeledPushRecsVerifyingStoreResponse( - userHistory: UserHistoryValue, - unequalNotificationsUnhydratedToHydrated: Option[ - Map[(Time, FrigateNotification), FrigateNotification] - ], - missingFromHydrated: Option[Map[Time, FrigateNotification]]) - -case class LabeledPushRecsVerifyingStore( - labeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue], - historyStore: PushServiceHistoryStore -)( - implicit stats: StatsReceiver) - extends ReadableStore[LabeledPushRecsVerifyingStoreKey, LabeledPushRecsVerifyingStoreResponse] { - - private def getByJoiningWithRealHistory( - key: HistoryStoreKeyContext - ): Future[Option[UserHistoryValue]] = { - val historyFut = historyStore.get(key, Some(365.days)) - val toJoinWithRealHistoryFut = labeledPushRecsStore.get(UserHistoryKey.UserId(key.targetUserId)) - Future.join(historyFut, toJoinWithRealHistoryFut).map { - case (_, None) => None - case (History(realtimeHistoryMap), Some(uhValue)) => - Some( - LabeledPushRecsJoinedWithNotificationHistoryStore - .joinLabeledPushRecsSentWithNotificationHistory(uhValue, realtimeHistoryMap, stats) - ) - } - } - - private def processUserHistoryValue(uhValue: UserHistoryValue): Map[Time, FrigateNotification] = { - uhValue.events - .getOrElse(Nil) - .collect { - case Event( - EventType.LabeledPushRecSend, - Some(tsMillis), - Some(EventUnion.LabeledPushRecSendEvent(lprs: LabeledPushRecSendEvent)) - ) if lprs.pushRecSendEvent.frigateNotification.isDefined => - Time.fromMilliseconds(tsMillis) -> lprs.pushRecSendEvent.frigateNotification.get - } - .toMap - } - - override def get( - key: LabeledPushRecsVerifyingStoreKey - ): Future[Option[LabeledPushRecsVerifyingStoreResponse]] = { - val uhKey = UserHistoryKey.UserId(key.userId) - if (!key.useHydratedDataset) { - getByJoiningWithRealHistory(key.historyStoreKey).map { uhValueOpt => - uhValueOpt.map { uhValue => LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) } - } - } else { - labeledPushRecsStore.get(uhKey).flatMap { hydratedValueOpt: Option[UserHistoryValue] => - if (!key.verifyHydratedDatasetResults) { - Future.value(hydratedValueOpt.map { uhValue => - LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) - }) - } else { - getByJoiningWithRealHistory(key.historyStoreKey).map { - joinedWithRealHistoryOpt: Option[UserHistoryValue] => - val joinedWithRealHistoryMap = - joinedWithRealHistoryOpt.map(processUserHistoryValue).getOrElse(Map.empty) - val hydratedMap = hydratedValueOpt.map(processUserHistoryValue).getOrElse(Map.empty) - val unequal = joinedWithRealHistoryMap.flatMap { - case (time, frigateNotif) => - hydratedMap.get(time).collect { - case n if n != frigateNotif => ((time, frigateNotif), n) - } - } - val missing = joinedWithRealHistoryMap.filter { - case (time, frigateNotif) => !hydratedMap.contains(time) - } - hydratedValueOpt.map { hydratedValue => - LabeledPushRecsVerifyingStoreResponse(hydratedValue, Some(unequal), Some(missing)) - } - } - } - } - } - } -} - -case class LabeledPushRecsStoreKey(target: TargetDecider, historyStoreKey: HistoryStoreKeyContext) { - def userId: Long = historyStoreKey.targetUserId -} - -case class LabeledPushRecsDecideredStore( - verifyingStore: ReadableStore[ - LabeledPushRecsVerifyingStoreKey, - LabeledPushRecsVerifyingStoreResponse - ], - useHydratedLabeledSendsDatasetDeciderKey: String, - verifyHydratedLabeledSendsForHistoryDeciderKey: String -)( - implicit globalStats: StatsReceiver) - extends ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] { - private val log = Logger() - private val stats = globalStats.scope("LabeledPushRecsDecideredStore") - private val numComparisons = stats.counter("num_comparisons") - private val numMissingStat = stats.stat("num_missing") - private val numUnequalStat = stats.stat("num_unequal") - - override def get(key: LabeledPushRecsStoreKey): Future[Option[UserHistoryValue]] = { - val useHydrated = key.target.isDeciderEnabled( - useHydratedLabeledSendsDatasetDeciderKey, - stats, - useRandomRecipient = true - ) - - val verifyHydrated = if (useHydrated) { - key.target.isDeciderEnabled( - verifyHydratedLabeledSendsForHistoryDeciderKey, - stats, - useRandomRecipient = true - ) - } else false - - val newKey = LabeledPushRecsVerifyingStoreKey(key.historyStoreKey, useHydrated, verifyHydrated) - verifyingStore.get(newKey).map { - case None => None - case Some(LabeledPushRecsVerifyingStoreResponse(uhValue, unequalOpt, missingOpt)) => - (unequalOpt, missingOpt) match { - case (Some(unequal), Some(missing)) => - numComparisons.incr() - numMissingStat.add(missing.size) - numUnequalStat.add(unequal.size) - case _ => //no-op - } - Some(uhValue) - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.docx new file mode 100644 index 000000000..b13730beb Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala deleted file mode 100644 index b11cdc0dd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/LexServiceStore.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.livevideo.common.ids.EventId -import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient -import com.twitter.livevideo.timeline.domain.v2.Event -import com.twitter.livevideo.timeline.domain.v2.LookupContext -import com.twitter.stitch.storehaus.ReadableStoreOfStitch -import com.twitter.stitch.NotFound -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore - -case class EventRequest(eventId: Long, lookupContext: LookupContext = LookupContext.default) - -object LexServiceStore { - def apply( - liveVideoTimelineClient: LiveVideoTimelineClient - ): ReadableStore[EventRequest, Event] = { - ReadableStoreOfStitch { eventRequest => - liveVideoTimelineClient.getEvent( - EventId(eventRequest.eventId), - eventRequest.lookupContext) rescue { - case NotFound => Stitch.NotFound - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.docx new file mode 100644 index 000000000..56055ce85 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala deleted file mode 100644 index 9e9bc37b7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/NTabHistoryStore.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.hermit.store.common.ReadableWritableStore -import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint -import com.twitter.storage.client.manhattan.kv.impl.Component -import com.twitter.storage.client.manhattan.kv.impl.DescriptorP1L1 -import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.util.Future - -case class NTabHistoryStore(mhEndpoint: ManhattanKVEndpoint, dataset: String) - extends ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] { - - private val keyDesc: DescriptorP1L1.EmptyKey[Long, String] = - KeyDescriptor(Component(LongInjection), Component(StringInjection)) - - private val genericNotifKeyValDesc: ValueDescriptor.EmptyValue[GenericNotificationOverrideKey] = - ValueDescriptor[GenericNotificationOverrideKey]( - BinaryCompactScalaInjection(GenericNotificationOverrideKey) - ) - - override def get(key: (Long, String)): Future[Option[GenericNotificationOverrideKey]] = { - val (userId, impressionId) = key - val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId) - - Stitch - .run(mhEndpoint.get(mhKey, genericNotifKeyValDesc)) - .map { optionMhValue => - optionMhValue.map(_.contents) - } - } - - override def put(keyValue: ((Long, String), GenericNotificationOverrideKey)): Future[Unit] = { - val ((userId, impressionId), genericNotifOverrideKey) = keyValue - val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId) - val mhVal = genericNotifKeyValDesc.withValue(genericNotifOverrideKey) - Stitch.run(mhEndpoint.insert(mhKey, mhVal)) - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.docx new file mode 100644 index 000000000..d528f3635 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala deleted file mode 100644 index 33b119c79..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OCFPromptHistoryStore.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment -import com.twitter.stitch.Stitch -import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection -import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection -import com.twitter.storage.client.manhattan.kv.impl.Component -import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor -import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor -import com.twitter.storage.client.manhattan.kv.ManhattanKVClient -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder -import com.twitter.storage.client.manhattan.kv.NoMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Omega -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time - -case class OCFHistoryStoreKey(userId: Long, fatigueDuration: Duration, fatigueGroup: String) - -class OCFPromptHistoryStore( - manhattanAppId: String, - dataset: String, - mtlsParams: ManhattanKVClientMtlsParams = NoMtlsParams -)( - implicit stats: StatsReceiver) - extends ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] { - - import ManhattanInjections._ - - private val client = ManhattanKVClient( - appId = manhattanAppId, - dest = Omega.wilyName, - mtlsParams = mtlsParams, - label = "ocf_history_store" - ) - private val endpoint = ManhattanKVEndpointBuilder(client, defaultMaxTimeout = 5.seconds) - .statsReceiver(stats.scope("ocf_history_store")) - .build() - - private val limitResultsTo = 1 - - private val datasetKey = keyDesc.withDataset(dataset) - - override def get(storeKey: OCFHistoryStoreKey): Future[Option[FatigueFlowEnrollment]] = { - val userId = storeKey.userId - val fatigueGroup = storeKey.fatigueGroup - val fatigueLength = storeKey.fatigueDuration.inMilliseconds - val currentTime = Time.now.inMilliseconds - val fullKey = datasetKey - .withPkey(userId) - .from(fatigueGroup) - .to(fatigueGroup, fatigueLength - currentTime) - - Stitch - .run(endpoint.slice(fullKey, valDesc, limit = Some(limitResultsTo))) - .map { results => - if (results.nonEmpty) { - val (_, mhValue) = results.head - Some(mhValue.contents) - } else None - } - } -} - -object ManhattanInjections { - val keyDesc = KeyDescriptor(Component(LongInjection), Component(StringInjection, LongInjection)) - val valDesc = ValueDescriptor(BinaryScalaInjection(FatigueFlowEnrollment)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.docx new file mode 100644 index 000000000..5413777c6 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala deleted file mode 100644 index d7ecfa7e4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OnlineUserHistoryStore.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.conversions.DurationOps._ -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.store.RealTimeClientEventStore -import com.twitter.frigate.data_pipeline.common.HistoryJoin -import com.twitter.frigate.data_pipeline.thriftscala.Event -import com.twitter.frigate.data_pipeline.thriftscala.EventUnion -import com.twitter.frigate.data_pipeline.thriftscala.PushRecSendEvent -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time - -case class OnlineUserHistoryKey( - userId: Long, - offlineUserHistory: Option[UserHistoryValue], - history: Option[History]) - -case class OnlineUserHistoryStore( - realTimeClientEventStore: RealTimeClientEventStore, - duration: Duration = 3.days) - extends ReadableStore[OnlineUserHistoryKey, UserHistoryValue] { - - override def get(key: OnlineUserHistoryKey): Future[Option[UserHistoryValue]] = { - val now = Time.now - - val pushRecSends = key.history - .getOrElse(History(Nil.toMap)) - .sortedPushDmHistory - .filter(_._1 > now - (duration + 1.day)) - .map { - case (time, frigateNotification) => - val pushRecSendEvent = PushRecSendEvent( - frigateNotification = Some(frigateNotification), - impressionId = frigateNotification.impressionId - ) - pushRecSendEvent -> time - } - - realTimeClientEventStore - .get(key.userId, now - duration, now) - .map { attributedEventHistory => - val attributedClientEvents = attributedEventHistory.sortedHistory.flatMap { - case (time, event) => - event.eventUnion match { - case Some(eventUnion: EventUnion.AttributedPushRecClientEvent) => - Some((eventUnion.attributedPushRecClientEvent, event.eventType, time)) - case _ => None - } - } - - val realtimeLabeledSends: Seq[Event] = HistoryJoin.getLabeledPushRecSends( - pushRecSends, - attributedClientEvents, - Seq(), - Seq(), - Seq(), - now - ) - - key.offlineUserHistory.map { offlineUserHistory => - val combinedEvents = offlineUserHistory.events.map { offlineEvents => - (offlineEvents ++ realtimeLabeledSends) - .map { event => - event.timestampMillis -> event - } - .toMap - .values - .toSeq - .sortBy { event => - -1 * event.timestampMillis.getOrElse(0L) - } - } - - offlineUserHistory.copy(events = combinedEvents) - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.docx new file mode 100644 index 000000000..0022edbc7 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala deleted file mode 100644 index 85d3f5afa..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/OpenAppUserStore.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.strato.generated.client.rux.open_app.UsersInOpenAppDdgOnUserClientColumn - -object OpenAppUserStore { - def apply(stratoClient: Client): ReadableStore[Long, Boolean] = { - val fetcher = new UsersInOpenAppDdgOnUserClientColumn(stratoClient).fetcher - StratoFetchableStore.withUnitView(fetcher).mapValues(_ => true) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.docx new file mode 100644 index 000000000..29272cca6 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala deleted file mode 100644 index 4af473656..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SocialGraphServiceProcessStore.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.frigate.pushservice.params.PushQPSLimitConstants.SocialGraphServiceBatchSize -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -case class SocialGraphServiceProcessStore(edgeStore: ReadableStore[RelationEdge, Boolean]) - extends ReadableStore[RelationEdge, Boolean] { - override def multiGet[T <: RelationEdge]( - relationEdges: Set[T] - ): Map[T, Future[Option[Boolean]]] = { - val splitSet = relationEdges.grouped(SocialGraphServiceBatchSize).toSet - splitSet - .map { relationship => - edgeStore.multiGet(relationship) - }.foldLeft(Map.empty[T, Future[Option[Boolean]]]) { (map1, map2) => - map1 ++ map2 - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.docx new file mode 100644 index 000000000..e4f4321f4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala deleted file mode 100644 index b2de4cf26..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/SoftUserFollowingStore.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.gizmoduck.thriftscala.UserType -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.Client -import com.twitter.strato.client.UserId -import com.twitter.strato.config.FlockCursors.BySource.Begin -import com.twitter.strato.config.FlockCursors.Continue -import com.twitter.strato.config.FlockCursors.End -import com.twitter.strato.config.FlockPage -import com.twitter.strato.generated.client.socialgraph.service.soft_users.softUserFollows.EdgeBySourceClientColumn -import com.twitter.util.Future - -object SoftUserFollowingStore { - type ViewerFollowingCursor = EdgeBySourceClientColumn.Cursor - val MaxPagesToFetch = 2 - val PageLimit = 50 -} - -class SoftUserFollowingStore(stratoClient: Client) extends ReadableStore[User, Seq[Long]] { - import SoftUserFollowingStore._ - private val softUserFollowingEdgesPaginator = new EdgeBySourceClientColumn(stratoClient).paginator - - private def accumulateIds(cursor: ViewerFollowingCursor, pagesToFetch: Int): Stitch[Seq[Long]] = - softUserFollowingEdgesPaginator.paginate(cursor).flatMap { - case FlockPage(data, next, _) => - next match { - case cont: Continue if pagesToFetch > 1 => - Stitch - .join( - Stitch.value(data.map(_.to).map(_.value)), - accumulateIds(cont, pagesToFetch - 1)) - .map { - case (a, b) => a ++ b - } - - case _: End | _: Continue => - // end pagination if last page has been fetched or [[MaxPagesToFetch]] have been fetched - Stitch.value(data.map(_.to).map(_.value)) - } - } - - private def softFollowingFromStrato( - sourceId: Long, - pageLimit: Int, - pagesToFetch: Int - ): Stitch[Seq[Long]] = { - val begin = Begin[UserId, UserId](UserId(sourceId), pageLimit) - accumulateIds(begin, pagesToFetch) - } - - override def get(user: User): Future[Option[Seq[Long]]] = { - user.userType match { - case UserType.Soft => - Stitch.run(softFollowingFromStrato(user.id, PageLimit, MaxPagesToFetch)).map(Option(_)) - case _ => Future.None - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.docx new file mode 100644 index 000000000..9e7704dd3 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala deleted file mode 100644 index 6acf1d136..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetImpressionsStore.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.util.Future - -/** - * Store to get inbound Tweet impressions count for a specific Tweet id. - */ -class TweetImpressionsStore(stratoClient: StratoClient) extends ReadableStore[Long, String] { - - private val column = "rux/impression.Tweet" - private val store = StratoFetchableStore.withUnitView[Long, String](stratoClient, column) - - def getCounts(tweetId: Long): Future[Option[Long]] = { - store.get(tweetId).map(_.map(_.toLong)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.docx new file mode 100644 index 000000000..698315945 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala deleted file mode 100644 index 618d8da32..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/TweetTranslationStore.scala +++ /dev/null @@ -1,211 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.context.TwitterContext -import com.twitter.context.thriftscala.Viewer -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.TwitterContextPermit -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.kujaku.domain.thriftscala.CacheUsageType -import com.twitter.kujaku.domain.thriftscala.MachineTranslation -import com.twitter.kujaku.domain.thriftscala.MachineTranslationResponse -import com.twitter.kujaku.domain.thriftscala.TranslationSource -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.generated.client.translation.service.IsTweetTranslatableClientColumn -import com.twitter.strato.generated.client.translation.service.platform.MachineTranslateTweetClientColumn -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.util.Future -import com.twitter.util.logging.Logging - -object TweetTranslationStore { - case class Key( - target: Target, - tweetId: Long, - tweet: Option[Tweet], - crt: CommonRecommendationType) - - case class Value( - translatedTweetText: String, - localizedSourceLanguage: String) - - val allowedCRTs = Set[CommonRecommendationType]( - CommonRecommendationType.TwistlyTweet - ) -} - -case class TweetTranslationStore( - translateTweetStore: ReadableStore[ - MachineTranslateTweetClientColumn.Key, - MachineTranslationResponse - ], - isTweetTranslatableStore: ReadableStore[IsTweetTranslatableClientColumn.Key, Boolean], - statsReceiver: StatsReceiver) - extends ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value] - with Logging { - - private val stats = statsReceiver.scope("tweetTranslationStore") - private val isTranslatableCounter = stats.counter("tweetIsTranslatable") - private val notTranslatableCounter = stats.counter("tweetIsNotTranslatable") - private val protectedUserCounter = stats.counter("protectedUser") - private val notProtectedUserCounter = stats.counter("notProtectedUser") - private val validLanguageCounter = stats.counter("validTweetLanguage") - private val invalidLanguageCounter = stats.counter("invalidTweetLanguage") - private val validCrtCounter = stats.counter("validCrt") - private val invalidCrtCounter = stats.counter("invalidCrt") - private val paramEnabledCounter = stats.counter("paramEnabled") - private val paramDisabledCounter = stats.counter("paramDisabled") - - private val twitterContext = TwitterContext(TwitterContextPermit) - - override def get(k: TweetTranslationStore.Key): Future[Option[TweetTranslationStore.Value]] = { - k.target.inferredUserDeviceLanguage.flatMap { - case Some(deviceLanguage) => - setTwitterContext(k.target, deviceLanguage) { - translateTweet( - target = k.target, - tweetId = k.tweetId, - tweet = k.tweet, - crt = k.crt, - deviceLanguage = deviceLanguage).map { responseOpt => - responseOpt.flatMap { response => - response.translatorLocalizedSourceLanguage - .map { localizedSourceLanguage => - TweetTranslationStore.Value( - translatedTweetText = response.translation, - localizedSourceLanguage = localizedSourceLanguage - ) - }.filter { _ => - response.translationSource == TranslationSource.Google - } - } - } - } - case None => Future.None - } - - } - - // Don't sent protected tweets to external API for translation - private def checkProtectedUser(target: Target): Future[Boolean] = { - target.targetUser.map(_.flatMap(_.safety).forall(_.isProtected)).onSuccess { - case true => protectedUserCounter.incr() - case false => notProtectedUserCounter.incr() - } - } - - private def isTweetTranslatable( - target: Target, - tweetId: Long, - tweet: Option[Tweet], - crt: CommonRecommendationType, - deviceLanguage: String - ): Future[Boolean] = { - val tweetLangOpt = tweet.flatMap(_.language) - val isValidLanguage = tweetLangOpt.exists { tweetLang => - tweetLang.confidence > 0.5 && - tweetLang.language != deviceLanguage - } - - if (isValidLanguage) { - validLanguageCounter.incr() - } else { - invalidLanguageCounter.incr() - } - - val isValidCrt = TweetTranslationStore.allowedCRTs.contains(crt) - if (isValidCrt) { - validCrtCounter.incr() - } else { - invalidCrtCounter.incr() - } - - if (isValidCrt && isValidLanguage && target.params(PushParams.EnableIsTweetTranslatableCheck)) { - checkProtectedUser(target).flatMap { - case false => - val isTweetTranslatableKey = IsTweetTranslatableClientColumn.Key( - tweetId = tweetId, - destinationLanguage = Some(deviceLanguage), - translationSource = Some(TranslationSource.Google.name), - excludePreferredLanguages = Some(true) - ) - isTweetTranslatableStore - .get(isTweetTranslatableKey).map { resultOpt => - resultOpt.getOrElse(false) - }.onSuccess { - case true => isTranslatableCounter.incr() - case false => notTranslatableCounter.incr() - } - case true => - Future.False - } - } else { - Future.False - } - } - - private def translateTweet( - tweetId: Long, - deviceLanguage: String - ): Future[Option[MachineTranslation]] = { - val translateKey = MachineTranslateTweetClientColumn.Key( - tweetId = tweetId, - destinationLanguage = deviceLanguage, - translationSource = TranslationSource.Google, - translatableEntityTypes = Seq(), - onlyCached = false, - cacheUsageType = CacheUsageType.Default - ) - translateTweetStore.get(translateKey).map { - _.collect { - case MachineTranslationResponse.Result(result) => result - } - } - } - - private def translateTweet( - target: Target, - tweetId: Long, - tweet: Option[Tweet], - crt: CommonRecommendationType, - deviceLanguage: String - ): Future[Option[MachineTranslation]] = { - isTweetTranslatable(target, tweetId, tweet, crt, deviceLanguage).flatMap { - case true => - val isEnabledByParam = target.params(PushFeatureSwitchParams.EnableTweetTranslation) - if (isEnabledByParam) { - paramEnabledCounter.incr() - translateTweet(tweetId, deviceLanguage) - } else { - paramDisabledCounter.incr() - Future.None - } - case false => - Future.None - } - } - - private def setTwitterContext[Rep]( - target: Target, - deviceLanguage: String - )( - f: => Future[Rep] - ): Future[Rep] = { - twitterContext() match { - case Some(viewer) if viewer.userId.nonEmpty && viewer.authenticatedUserId.nonEmpty => - // If the context is already setup with a user ID just use it - f - case _ => - // If not, create a new context containing the viewer user id - twitterContext.let( - Viewer( - userId = Some(target.targetId), - requestLanguageCode = Some(deviceLanguage), - authenticatedUserId = Some(target.targetId) - )) { - f - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.docx new file mode 100644 index 000000000..4997d8749 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala deleted file mode 100644 index bd96bf690..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/store/UttEntityHydrationStore.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.frigate.pushservice.store - -import com.twitter.escherbird.util.uttclient.CachedUttClientV2 -import com.twitter.escherbird.util.uttclient.InvalidUttEntityException -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.stitch.Stitch -import com.twitter.topiclisting.TopicListingViewerContext -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.topiclisting.utt.LocalizedEntityFactory -import com.twitter.util.Future - -/** - * - * @param viewerContext: [[TopicListingViewerContext]] for filtering topic - * @param semanticCoreEntityIds: list of semantic core entities to hydrate - */ -case class UttEntityHydrationQuery( - viewerContext: TopicListingViewerContext, - semanticCoreEntityIds: Seq[Long]) - -/** - * - * @param cachedUttClientV2 - * @param statsReceiver - */ -class UttEntityHydrationStore( - cachedUttClientV2: CachedUttClientV2, - statsReceiver: StatsReceiver, - log: Logger) { - - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val uttEntityNotFound = stats.counter("invalid_utt_entity") - private val deviceLanguageMismatch = stats.counter("language_mismatch") - - /** - * SemanticCore recommends setting language and country code to None to fetch all localized topic - * names and apply filtering for locales on our end - * - * We use [[LocalizedEntityFactory]] from [[Topiclisting]] library to filter out topic name based - * on user locale - * - * Some(LocalizedEntity) - LocalizedUttEntity found - * None - LocalizedUttEntity not found - */ - def getLocalizedTopicEntities( - query: UttEntityHydrationQuery - ): Future[Seq[Option[LocalizedEntity]]] = Stitch.run { - Stitch.collect { - query.semanticCoreEntityIds.map { semanticCoreEntityId => - val uttEntity = cachedUttClientV2.cachedGetUttEntity( - language = None, - country = None, - version = None, - entityId = semanticCoreEntityId) - - uttEntity - .map { uttEntityMetadata => - val localizedEntity = LocalizedEntityFactory.getLocalizedEntity( - uttEntityMetadata, - query.viewerContext, - enableInternationalTopics = true, - enableTopicDescription = true) - // update counter - localizedEntity.foreach { entity => - if (!entity.nameMatchesDeviceLanguage) deviceLanguageMismatch.incr() - } - - localizedEntity - }.handle { - case e: InvalidUttEntityException => - log.error(e.getMessage) - uttEntityNotFound.incr() - None - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.docx new file mode 100644 index 000000000..5d2a45533 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala deleted file mode 100644 index 1eb8cbc04..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/CandidateNotifier.scala +++ /dev/null @@ -1,160 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.store.Fail -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.InvalidConfiguration -import com.twitter.frigate.common.store.NoRequest -import com.twitter.frigate.common.store.Sent -import com.twitter.frigate.common.util.CasLock -import com.twitter.frigate.common.util.PushServiceUtil.InvalidConfigResponse -import com.twitter.frigate.common.util.PushServiceUtil.NtabWriteOnlyResponse -import com.twitter.frigate.common.util.PushServiceUtil.SendFailedResponse -import com.twitter.frigate.common.util.PushServiceUtil.SentResponse -import com.twitter.frigate.pushservice.predicate.CasLockPredicate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.history._ -import com.twitter.frigate.pushservice.util.CopyUtil -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.frigate.pushservice.util.OverrideNotificationUtil -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future - -class CandidateNotifier( - notificationSender: NotificationSender, - casLock: CasLock, - historyWriter: HistoryWriter, - eventBusWriter: EventBusWriter, - ntabOnlyChannelSelector: NtabOnlyChannelSelector -)( - implicit statsReceiver: StatsReceiver) { - - private lazy val casLockPredicate = - CasLockPredicate(casLock, expiryDuration = 10.minutes)(statsReceiver) - private val candidateNotifierStats = statsReceiver.scope(this.getClass.getSimpleName) - private val historyWriteCounter = - candidateNotifierStats.counter("simply_notifier_history_write_num") - private val loggedOutHistoryWriteCounter = - candidateNotifierStats.counter("logged_out_simply_notifier_history_write_num") - private val notificationSenderLatency = - candidateNotifierStats.scope("notification_sender_send") - private val log = MRLogger("CandidateNotifier") - - private def mapIbisResponse(ibisResponse: IbisResponse): PushResponse = { - ibisResponse match { - case IbisResponse(Sent, _) => SentResponse - case IbisResponse(Fail, _) => SendFailedResponse - case IbisResponse(InvalidConfiguration, _) => InvalidConfigResponse - case IbisResponse(NoRequest, _) => NtabWriteOnlyResponse - } - } - - /** - * - write to history store - * - send the notification - * - scribe the notification - * - * final modifier is to signal that this function cannot be overriden. There's some critical logic - * in this function, and it's helpful to know that no sub-class overrides it. - */ - final def notify( - candidate: PushCandidate, - ): Future[PushResponse] = { - if (candidate.target.isDarkWrite) { - notificationSender.sendIbisDarkWrite(candidate).map(mapIbisResponse) - } else { - casLockPredicate(Seq(candidate)).flatMap { casLockResults => - if (casLockResults.head || candidate.target.pushContext - .exists(_.skipFilters.contains(true))) { - Future - .join( - candidate.target.isSilentPush, - OverrideNotificationUtil - .getOverrideInfo(candidate, candidateNotifierStats), - CopyUtil.getCopyFeatures(candidate, candidateNotifierStats) - ).flatMap { - case (isSilentPush, overrideInfoOpt, copyFeaturesMap) => - val channels = ntabOnlyChannelSelector.selectChannel(candidate) - channels.flatMap { channels => - candidate - .frigateNotificationForPersistence( - channels, - isSilentPush, - overrideInfoOpt, - copyFeaturesMap.keySet).flatMap { frigateNotificationForPersistence => - val result = if (candidate.target.isDarkWrite) { - candidateNotifierStats.counter("dark_write").incr() - Future.Unit - } else { - historyWriteCounter.incr() - historyWriter - .writeSendToHistory(candidate, frigateNotificationForPersistence) - } - result.flatMap { _ => - track(notificationSenderLatency)( - notificationSender - .notify(channels, candidate) - .map { ibisResponse => - eventBusWriter - .writeToEventBus(candidate, frigateNotificationForPersistence) - mapIbisResponse(ibisResponse) - }) - } - } - } - } - } else { - candidateNotifierStats.counter("filtered_by_cas_lock").incr() - Future.value(PushResponse(PushStatus.Filtered, Some(casLockPredicate.name))) - } - } - } - } - - final def loggedOutNotify( - candidate: PushCandidate, - ): Future[PushResponse] = { - if (candidate.target.isDarkWrite) { - notificationSender.sendIbisDarkWrite(candidate).map(mapIbisResponse) - } else { - casLockPredicate(Seq(candidate)).flatMap { casLockResults => - if (casLockResults.head || candidate.target.pushContext - .exists(_.skipFilters.contains(true))) { - val response = candidate.target.isSilentPush.flatMap { isSilentPush => - candidate - .frigateNotificationForPersistence( - Seq(ChannelName.PushNtab), - isSilentPush, - None, - Set.empty).flatMap { frigateNotificationForPersistence => - val result = if (candidate.target.isDarkWrite) { - candidateNotifierStats.counter("logged_out_dark_write").incr() - Future.Unit - } else { - loggedOutHistoryWriteCounter.incr() - historyWriter.writeSendToHistory(candidate, frigateNotificationForPersistence) - } - - result.flatMap { _ => - track(notificationSenderLatency)( - notificationSender - .loggedOutNotify(candidate) - .map { ibisResponse => - mapIbisResponse(ibisResponse) - }) - } - } - } - response - } else { - candidateNotifierStats.counter("filtered_by_cas_lock").incr() - Future.value(PushResponse(PushStatus.Filtered, Some(casLockPredicate.name))) - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.docx new file mode 100644 index 000000000..c5ddba31a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala deleted file mode 100644 index 07574f46f..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/LoggedOutRefreshForPushNotifier.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.config.CommonConstants -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.util.PushServiceUtil.FilteredLoggedOutResponseFut -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.refresh_handler.RFPHStatsRecorder -import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.util.Future -import com.twitter.util.JavaTimer -import com.twitter.util.Timer - -class LoggedOutRefreshForPushNotifier( - rfphStatsRecorder: RFPHStatsRecorder, - loCandidateNotifier: CandidateNotifier -)( - globalStats: StatsReceiver) { - private implicit val statsReceiver: StatsReceiver = - globalStats.scope("LoggedOutRefreshForPushHandler") - private val loPushStats: StatsReceiver = statsReceiver.scope("logged_out_push") - private val loSendLatency: StatsReceiver = statsReceiver.scope("logged_out_send") - private val processedCandidatesCounter: Counter = - statsReceiver.counter("processed_candidates_count") - private val validCandidatesCounter: Counter = statsReceiver.counter("valid_candidates_count") - private val okayCandidateCounter: Counter = statsReceiver.counter("ok_candidate_count") - private val nonOkayCandidateCounter: Counter = statsReceiver.counter("non_ok_candidate_count") - private val successNotifyCounter: Counter = statsReceiver.counter("success_notify_count") - private val notifyCandidate: Counter = statsReceiver.counter("notify_candidate") - private val noneCandidateResultCounter: Counter = statsReceiver.counter("none_candidate_count") - private val nonOkayPredsResult: Counter = statsReceiver.counter("non_okay_preds_result") - private val invalidResultCounter: Counter = statsReceiver.counter("invalid_result_count") - private val filteredLoggedOutResponse: Counter = statsReceiver.counter("filtered_response_count") - - implicit private val timer: Timer = new JavaTimer(true) - val log = MRLogger("LoggedOutRefreshForNotifier") - - private def notify( - candidatesResult: CandidateResult[PushCandidate, Result] - ): Future[LoggedOutResponse] = { - val candidate = candidatesResult.candidate - if (candidate != null) - notifyCandidate.incr() - val predsResult = candidatesResult.result - if (predsResult != OK) { - nonOkayPredsResult.incr() - val invalidResult = predsResult - invalidResult match { - case Invalid(Some(reason)) => - invalidResultCounter.incr() - Future.value(LoggedOutResponse(PushStatus.Filtered, Some(reason))) - case _ => - filteredLoggedOutResponse.incr() - Future.value(LoggedOutResponse(PushStatus.Filtered, None)) - } - } else { - track(loSendLatency)(loCandidateNotifier.loggedOutNotify(candidate).map { res => - LoggedOutResponse(res.status) - }) - } - } - - def checkResponseAndNotify( - response: Response[PushCandidate, Result] - ): Future[LoggedOutResponse] = { - val receivers = Seq(statsReceiver) - val loggedOutResponse = response match { - case Response(OK, processedCandidates) => - processedCandidatesCounter.incr(processedCandidates.size) - val validCandidates = processedCandidates.filter(_.result == OK) - validCandidatesCounter.incr(validCandidates.size) - - validCandidates.headOption match { - case Some(candidatesResult) => - candidatesResult.result match { - case OK => - okayCandidateCounter.incr() - notify(candidatesResult) - .onSuccess { nr => - successNotifyCounter.incr() - loPushStats.scope("lo_result").counter(nr.status.name).incr() - } - case _ => - nonOkayCandidateCounter.incr() - FilteredLoggedOutResponseFut - } - case _ => - noneCandidateResultCounter.incr() - FilteredLoggedOutResponseFut - } - - case Response(Invalid(reason), _) => - FilteredLoggedOutResponseFut.map(_.copy(filteredBy = reason)) - - case _ => - FilteredLoggedOutResponseFut - } - val bstats = BroadcastStatsReceiver(receivers) - Stat - .timeFuture(bstats.stat("logged_out_latency"))( - loggedOutResponse.raiseWithin(CommonConstants.maxPushRequestDuration) - ) - .onFailure { exception => - rfphStatsRecorder.loggedOutRequestExceptionStats(exception, bstats) - } - loggedOutResponse - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.docx new file mode 100644 index 000000000..2a2d35c7a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala deleted file mode 100644 index 70a695fb3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationSender.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.Sent -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.take.sender.Ibis2Sender -import com.twitter.frigate.pushservice.take.sender.NtabSender -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.util.Future -import com.twitter.frigate.thriftscala.ChannelName - -/** - * NotificationSender wraps up all the notification infra send logic, and serves as an abstract layer - * between CandidateNotifier and the respective senders including ntab, ibis, which is being - * gated with both a decider/feature switch - */ -class NotificationSender( - ibis2Sender: Ibis2Sender, - ntabSender: NtabSender, - statsReceiver: StatsReceiver, - notificationScribe: NotificationScribe => Unit) { - - private val notificationNotifierStats = statsReceiver.scope(this.getClass.getSimpleName) - private val ibis2SendLatency = notificationNotifierStats.scope("ibis2_send") - private val loggedOutIbis2SendLatency = notificationNotifierStats.scope("logged_out_ibis2_send") - private val ntabSendLatency = notificationNotifierStats.scope("ntab_send") - - private val ntabWriteThenSkipPushCounter = - notificationNotifierStats.counter("ntab_write_then_skip_push") - private val ntabWriteThenIbisSendCounter = - notificationNotifierStats.counter("ntab_write_then_ibis_send") - notificationNotifierStats.counter("ins_dark_traffic_send") - - private val ntabOnlyChannelSenderV3Counter = - notificationNotifierStats.counter("ntab_only_channel_send_v3") - - def sendIbisDarkWrite(candidate: PushCandidate): Future[IbisResponse] = { - ibis2Sender.sendAsDarkWrite(candidate) - } - - private def isNtabOnlySend( - channels: Seq[ChannelName] - ): Future[Boolean] = { - val isNtabOnlyChannel = channels.contains(ChannelName.NtabOnly) - if (isNtabOnlyChannel) ntabOnlyChannelSenderV3Counter.incr() - - Future.value(isNtabOnlyChannel) - } - - private def isPushOnly(channels: Seq[ChannelName], candidate: PushCandidate): Future[Boolean] = { - Future.value(channels.contains(ChannelName.PushOnly)) - } - - def notify( - channels: Seq[ChannelName], - candidate: PushCandidate - ): Future[IbisResponse] = { - Future - .join(isPushOnly(channels, candidate), isNtabOnlySend(channels)).map { - case (isPushOnly, isNtabOnly) => - if (isPushOnly) { - track(ibis2SendLatency)(ibis2Sender.send(channels, candidate, notificationScribe, None)) - } else { - track(ntabSendLatency)( - ntabSender - .send(candidate, isNtabOnly)) - .flatMap { ntabResponse => - if (isNtabOnly) { - ntabWriteThenSkipPushCounter.incr() - candidate - .scribeData(channels = channels).foreach(notificationScribe).map(_ => - IbisResponse(Sent)) - } else { - ntabWriteThenIbisSendCounter.incr() - track(ibis2SendLatency)( - ibis2Sender.send(channels, candidate, notificationScribe, ntabResponse)) - } - } - - } - }.flatten - } - - def loggedOutNotify( - candidate: PushCandidate - ): Future[IbisResponse] = { - val ibisResponse = { - track(loggedOutIbis2SendLatency)( - ibis2Sender.send(Seq(ChannelName.PushNtab), candidate, notificationScribe, None)) - } - ibisResponse - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.docx new file mode 100644 index 000000000..4aed0ac1a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala deleted file mode 100644 index c2f729115..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/NotificationServiceSender.scala +++ /dev/null @@ -1,273 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.ntab.InvalidNTABWriteRequestException -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.notificationservice.thriftscala._ -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.util.Future -import scala.util.control.NoStackTrace - -class NtabCopyIdNotFoundException(private val message: String) - extends Exception(message) - with NoStackTrace - -class InvalidNtabCopyIdException(private val message: String) - extends Exception(message) - with NoStackTrace - -object NotificationServiceSender { - - def generateSocialContextTextEntities( - ntabDisplayNamesAndIdsFut: Future[Seq[(String, Long)]], - otherCountFut: Future[Int] - ): Future[Seq[DisplayTextEntity]] = { - Future.join(ntabDisplayNamesAndIdsFut, otherCountFut).map { - case (namesWithIdInOrder, otherCount) => - val displays = namesWithIdInOrder.zipWithIndex.map { - case ((name, id), index) => - DisplayTextEntity( - name = "user" + s"${index + 1}", - value = TextValue.Text(name), - emphasis = true, - userId = Some(id) - ) - } ++ Seq( - DisplayTextEntity(name = "nameCount", value = TextValue.Number(namesWithIdInOrder.size)) - ) - - val otherDisplay = if (otherCount > 0) { - Some( - DisplayTextEntity( - name = "otherCount", - value = TextValue.Number(otherCount) - ) - ) - } else None - displays ++ otherDisplay - } - } - - def getDisplayTextEntityFromUser( - userOpt: Option[User], - fieldName: String, - isBold: Boolean - ): Option[DisplayTextEntity] = { - for { - user <- userOpt - profile <- user.profile - } yield { - DisplayTextEntity( - name = fieldName, - value = TextValue.Text(profile.name), - emphasis = isBold, - userId = Some(user.id) - ) - } - } - - def getDisplayTextEntityFromUser( - user: Future[Option[User]], - fieldName: String, - isBold: Boolean - ): Future[Option[DisplayTextEntity]] = { - user.map { getDisplayTextEntityFromUser(_, fieldName, isBold) } - } -} - -case class NotificationServiceRequest( - candidate: PushCandidate, - impressionId: String, - isBadgeUpdate: Boolean, - overrideId: Option[String] = None) - -class NotificationServiceSender( - send: (Target, CreateGenericNotificationRequest) => Future[CreateGenericNotificationResponse], - enableWritesParam: Param[Boolean], - enableForEmployeesParam: Param[Boolean], - enableForEveryoneParam: Param[Boolean] -)( - implicit globalStats: StatsReceiver) - extends ReadableStore[NotificationServiceRequest, CreateGenericNotificationResponse] { - - val log = MRLogger(this.getClass.getName) - - val stats = globalStats.scope("NotificationServiceSender") - val requestEmpty = stats.scope("request_empty") - val requestNonEmpty = stats.counter("request_non_empty") - - val requestBadgeCount = stats.counter("request_badge_count") - - val successfulWrite = stats.counter("successful_write") - val successfulWriteScope = stats.scope("successful_write") - val failedWriteScope = stats.scope("failed_write") - val gotNonSuccessResponse = stats.counter("got_non_success_response") - val gotEmptyResponse = stats.counter("got_empty_response") - val deciderTurnedOffResponse = stats.scope("decider_turned_off_response") - - val disabledByDeciderForCandidate = stats.scope("model/candidate").counter("disabled_by_decider") - val sentToAlphaUserForCandidate = - stats.scope("model/candidate").counter("send_to_employee_or_team") - val sentToNonBucketedUserForCandidate = - stats.scope("model/candidate").counter("send_to_non_bucketed_decidered_user") - val noSendForCandidate = stats.scope("model/candidate").counter("no_send") - - val ineligibleUsersForCandidate = stats.scope("model/candidate").counter("ineligible_users") - - val darkWriteRequestsForCandidate = stats.scope("model/candidate").counter("dark_write_traffic") - - val heavyUserForCandidateCounter = stats.scope("model/candidate").counter("target_heavy") - val nonHeavyUserForCandidateCounter = stats.scope("model/candidate").counter("target_non_heavy") - - val skipWritingToNTAB = stats.counter("skip_writing_to_ntab") - - val ntabWriteDisabledForCandidate = stats.scope("model/candidate").counter("ntab_write_disabled") - - val ntabOverrideEnabledForCandidate = stats.scope("model/candidate").counter("override_enabled") - val ntabTTLForCandidate = stats.scope("model/candidate").counter("ttl_enabled") - - override def get( - notifRequest: NotificationServiceRequest - ): Future[Option[CreateGenericNotificationResponse]] = { - notifRequest.candidate.target.deviceInfo.flatMap { deviceInfoOpt => - val disableWritingToNtab = - notifRequest.candidate.target.params(PushParams.DisableWritingToNTAB) - - if (disableWritingToNtab) { - skipWritingToNTAB.incr() - Future.None - } else { - if (notifRequest.overrideId.nonEmpty) { ntabOverrideEnabledForCandidate.incr() } - Future - .join( - notifRequest.candidate.ntabRequest, - ntabWritesEnabledForCandidate(notifRequest.candidate)).flatMap { - case (Some(ntabRequest), ntabWritesEnabled) if ntabWritesEnabled => - if (ntabRequest.expiryTimeMillis.nonEmpty) { ntabTTLForCandidate.incr() } - sendNTabRequest( - ntabRequest, - notifRequest.candidate.target, - notifRequest.isBadgeUpdate, - notifRequest.candidate.commonRecType, - isFromCandidate = true, - overrideId = notifRequest.overrideId - ) - case (Some(_), ntabWritesEnabled) if !ntabWritesEnabled => - ntabWriteDisabledForCandidate.incr() - Future.None - case (None, ntabWritesEnabled) => - if (!ntabWritesEnabled) ntabWriteDisabledForCandidate.incr() - requestEmpty.counter(s"candidate_${notifRequest.candidate.commonRecType}").incr() - Future.None - } - } - } - } - - private def sendNTabRequest( - genericNotificationRequest: CreateGenericNotificationRequest, - target: Target, - isBadgeUpdate: Boolean, - crt: CommonRecommendationType, - isFromCandidate: Boolean, - overrideId: Option[String] - ): Future[Option[CreateGenericNotificationResponse]] = { - requestNonEmpty.incr() - val notifSvcReq = - genericNotificationRequest.copy( - sendBadgeCountUpdate = isBadgeUpdate, - overrideId = overrideId - ) - requestBadgeCount.incr() - send(target, notifSvcReq) - .map { response => - if (response.responseType.equals(CreateGenericNotificationResponseType.DecideredOff)) { - deciderTurnedOffResponse.counter(s"$crt").incr() - deciderTurnedOffResponse.counter(s"${genericNotificationRequest.genericType}").incr() - throw InvalidNTABWriteRequestException("Decider is turned off") - } else { - Some(response) - } - } - .onFailure { ex => - stats.counter(s"error_${ex.getClass.getCanonicalName}").incr() - failedWriteScope.counter(s"${crt}").incr() - log - .error( - ex, - s"NTAB failure $notifSvcReq" - ) - } - .onSuccess { - case Some(response) => - successfulWrite.incr() - val successfulWriteScopeString = if (isFromCandidate) "model/candidate" else "envelope" - successfulWriteScope.scope(successfulWriteScopeString).counter(s"$crt").incr() - if (response.responseType != CreateGenericNotificationResponseType.Success) { - gotNonSuccessResponse.incr() - log.warning(s"NTAB dropped $notifSvcReq with response $response") - } - - case _ => - gotEmptyResponse.incr() - } - } - - private def ntabWritesEnabledForCandidate(cand: PushCandidate): Future[Boolean] = { - if (!cand.target.params(enableWritesParam)) { - disabledByDeciderForCandidate.incr() - Future.False - } else { - Future - .join( - cand.target.isAnEmployee, - cand.target.isInNotificationsServiceWhitelist, - cand.target.isTeamMember - ) - .flatMap { - case (isEmployee, isInNotificationsServiceWhitelist, isTeamMember) => - cand.target.deviceInfo.flatMap { deviceInfoOpt => - deviceInfoOpt - .map { deviceInfo => - cand.target.isHeavyUserState.map { isHeavyUser => - val isAlphaTester = (isEmployee && cand.target - .params(enableForEmployeesParam)) || isInNotificationsServiceWhitelist || isTeamMember - if (cand.target.isDarkWrite) { - stats - .scope("model/candidate").counter( - s"dark_write_${cand.commonRecType}").incr() - darkWriteRequestsForCandidate.incr() - false - } else if (isAlphaTester || deviceInfo.isMRinNTabEligible - || cand.target.insertMagicrecsIntoNTabForNonPushableUsers) { - if (isHeavyUser) heavyUserForCandidateCounter.incr() - else nonHeavyUserForCandidateCounter.incr() - - val enabledForDesiredUsers = cand.target.params(enableForEveryoneParam) - if (isAlphaTester) { - sentToAlphaUserForCandidate.incr() - true - } else if (enabledForDesiredUsers) { - sentToNonBucketedUserForCandidate.incr() - true - } else { - noSendForCandidate.incr() - false - } - } else { - ineligibleUsersForCandidate.incr() - false - } - } - }.getOrElse(Future.False) - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.docx new file mode 100644 index 000000000..3cc8a6f6d Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala deleted file mode 100644 index feb65dffe..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/SendHandlerNotifier.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Response -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.util.NotificationScribeUtil -import com.twitter.frigate.common.util.PushServiceUtil -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.util.Future - -class SendHandlerNotifier( - candidateNotifier: CandidateNotifier, - private val statsReceiver: StatsReceiver) { - - val missingResponseCounter = statsReceiver.counter("missing_response") - val filteredResponseCounter = statsReceiver.counter("filtered") - - /** - * - * @param isScribeInfoRequired: [[Boolean]] to indicate if scribe info is required - * @param candidate: [[PushCandidate]] to build the scribe data from - * @return: scribe response string - */ - private def scribeInfoForResponse( - isScribeInfoRequired: Boolean, - candidate: PushCandidate - ): Future[Option[String]] = { - if (isScribeInfoRequired) { - candidate.scribeData().map { scribedInfo => - Some(NotificationScribeUtil.convertToJsonString(scribedInfo)) - } - } else Future.None - } - - /** - * - * @param response: Candidate validation response - * @param responseWithScribedInfo: boolean indicating if scribe data is expected in push response - * @return: [[PushResponse]] containing final result of send request for [[com.twitter.frigate.pushservice.thriftscala.PushRequest]] - */ - final def checkResponseAndNotify( - response: Response[PushCandidate, Result], - responseWithScribedInfo: Boolean - ): Future[PushResponse] = { - - response match { - case Response(OK, processedCandidates) => - val (validCandidates, invalidCandidates) = processedCandidates.partition(_.result == OK) - validCandidates.headOption match { - case Some(candidateResult) => - val scribeInfo = - scribeInfoForResponse(responseWithScribedInfo, candidateResult.candidate) - scribeInfo.flatMap { scribedData => - val response: Future[PushResponse] = - candidateNotifier.notify(candidateResult.candidate) - response.map(_.copy(notifScribe = scribedData)) - } - - case None => - invalidCandidates.headOption match { - case Some(candidateResult) => - filteredResponseCounter.incr() - val response = candidateResult.result match { - case Invalid(reason) => PushResponse(PushStatus.Filtered, filteredBy = reason) - case _ => PushResponse(PushStatus.Filtered, filteredBy = Some("unknown")) - } - - val scribeInfo = - scribeInfoForResponse(responseWithScribedInfo, candidateResult.candidate) - scribeInfo.map(scribeData => response.copy(notifScribe = scribeData)) - - case None => - missingResponseCounter.incr() - PushServiceUtil.FilteredPushResponseFut - } - } - - case Response(Invalid(reason), _) => - throw new IllegalStateException(s"Unexpected target filtering in SendHandler: $reason") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.docx new file mode 100644 index 000000000..46c89cf2b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala deleted file mode 100644 index ee85ba590..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/CandidateValidator.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.take.predicates.TakeCommonPredicates -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.hermit.predicate.ConcurrentPredicate -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.SequentialPredicate -import com.twitter.util.Future - -trait CandidateValidator extends TakeCommonPredicates { - - override implicit val statsReceiver: StatsReceiver = config.statsReceiver - - protected val log = MRLogger("CandidateValidator") - - private lazy val skipFiltersCounter = statsReceiver.counter("enable_skip_filters") - private lazy val emailUserSkipFiltersCounter = - statsReceiver.counter("email_user_enable_skip_filters") - private lazy val enablePredicatesCounter = statsReceiver.counter("enable_predicates") - - protected def enabledPredicates[C <: PushCandidate]( - candidate: C, - predicates: List[NamedPredicate[C]] - ): List[NamedPredicate[C]] = { - val target = candidate.target - val skipFilters: Boolean = - target.pushContext.flatMap(_.skipFilters).getOrElse(false) || target.params( - PushFeatureSwitchParams.SkipPostRankingFilters) - - if (skipFilters) { - skipFiltersCounter.incr() - if (target.isEmailUser) emailUserSkipFiltersCounter.incr() - - val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil) - if (predicatesToEnable.nonEmpty) enablePredicatesCounter.incr() - - // if we skip predicates on pushContext, only enable the explicitly specified predicates - predicates.filter(predicatesToEnable.contains) - } else predicates - } - - protected def executeSequentialPredicates[C <: PushCandidate]( - candidate: C, - predicates: List[NamedPredicate[C]] - ): Future[Option[Predicate[C]]] = { - val predicatesEnabled = enabledPredicates(candidate, predicates) - val sequentialPredicate = new SequentialPredicate(predicatesEnabled) - - sequentialPredicate.track(Seq(candidate)).map(_.head) - } - - protected def executeConcurrentPredicates[C <: PushCandidate]( - candidate: C, - predicates: List[NamedPredicate[C]] - ): Future[List[Predicate[C]]] = { - val predicatesEnabled = enabledPredicates(candidate, predicates) - val concurrentPredicate: ConcurrentPredicate[C] = new ConcurrentPredicate[C](predicatesEnabled) - concurrentPredicate.track(Seq(candidate)).map(_.head) - } - - protected val candidatePredicatesMap: Map[CommonRecommendationType, List[ - NamedPredicate[_ <: PushCandidate] - ]] - - protected def getCRTPredicates[C <: PushCandidate]( - CRT: CommonRecommendationType - ): List[NamedPredicate[C]] = { - candidatePredicatesMap.get(CRT) match { - case Some(predicates) => - predicates.asInstanceOf[List[NamedPredicate[C]]] - case _ => - throw new IllegalStateException( - s"Unknown CommonRecommendationType for Predicates: ${CRT.name}") - } - } - - def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.docx new file mode 100644 index 000000000..24d2b1cb9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala deleted file mode 100644 index ecc99cc9e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/RFPHCandidateValidator.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.take.predicates.candidate_map.CandidatePredicatesMap -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -class RFPHCandidateValidator(override val config: Config) extends CandidateValidator { - private val rFPHCandidateValidatorStats = statsReceiver.scope(this.getClass.getSimpleName) - private val concurrentPredicateCount = rFPHCandidateValidatorStats.counter("concurrent") - private val sequentialPredicateCount = rFPHCandidateValidatorStats.counter("sequential") - - override protected val candidatePredicatesMap = CandidatePredicatesMap(config) - - override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { - val candidatePredicates = getCRTPredicates(candidate.commonRecType) - val predicates = rfphPrePredicates ++ candidatePredicates ++ postPredicates - if (candidate.target.isEmailUser) { - concurrentPredicateCount.incr() - executeConcurrentPredicates(candidate, predicates).map(_.headOption) - } else { - sequentialPredicateCount.incr() - executeSequentialPredicates(candidate, predicates) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.docx new file mode 100644 index 000000000..d9aa7e364 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala deleted file mode 100644 index 096e9f102..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPostCandidateValidator.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.take.predicates.candidate_map.SendHandlerCandidatePredicatesMap -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -class SendHandlerPostCandidateValidator(override val config: Config) extends CandidateValidator { - - override protected val candidatePredicatesMap = - SendHandlerCandidatePredicatesMap.postCandidatePredicates(config) - - private val sendHandlerPostCandidateValidatorStats = - statsReceiver.counter("sendHandlerPostCandidateValidator_stats") - - override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { - val candidatePredicates = getCRTPredicates(candidate.commonRecType) - val predicates = candidatePredicates ++ postPredicates - - sendHandlerPostCandidateValidatorStats.incr() - - executeConcurrentPredicates(candidate, predicates) - .map(_.headOption) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.docx new file mode 100644 index 000000000..155e9ba58 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala deleted file mode 100644 index eb0293017..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/candidate_validator/SendHandlerPreCandidateValidator.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.take.candidate_validator - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.take.predicates.candidate_map.SendHandlerCandidatePredicatesMap -import com.twitter.hermit.predicate.Predicate -import com.twitter.util.Future - -class SendHandlerPreCandidateValidator(override val config: Config) extends CandidateValidator { - - override protected val candidatePredicatesMap = - SendHandlerCandidatePredicatesMap.preCandidatePredicates(config) - - private val sendHandlerPreCandidateValidatorStats = - statsReceiver.counter("sendHandlerPreCandidateValidator_stats") - - override def validateCandidate[C <: PushCandidate](candidate: C): Future[Option[Predicate[C]]] = { - val candidatePredicates = getCRTPredicates(candidate.commonRecType) - val predicates = sendHandlerPrePredicates ++ candidatePredicates - - sendHandlerPreCandidateValidatorStats.incr() - executeSequentialPredicates(candidate, predicates) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.docx new file mode 100644 index 000000000..e83b785a4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala deleted file mode 100644 index a278ef237..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelCandidate.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future -import java.util.concurrent.ConcurrentHashMap -import scala.collection.concurrent -import scala.collection.convert.decorateAsScala._ - -/** - * A class to save all the channel related information - */ -trait ChannelForCandidate { - self: PushCandidate => - - // Cache of channel selection result - private[this] val selectedChannels: concurrent.Map[String, Future[Seq[ChannelName]]] = - new ConcurrentHashMap[String, Future[Seq[ChannelName]]]().asScala - - // Returns the channel information from all ChannelSelectors. - def getChannels(): Future[Seq[ChannelName]] = { - Future.collect(selectedChannels.values.toSeq).map { c => c.flatten } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.docx new file mode 100644 index 000000000..57c9856d9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala deleted file mode 100644 index 62378a4bc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/ChannelSelector.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future - -abstract class ChannelSelector { - - // Returns a map of channel name, and the candidates that can be sent on that channel. - def selectChannel( - candidate: PushCandidate - ): Future[Seq[ChannelName]] - - def getSelectorName(): String -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.docx new file mode 100644 index 000000000..9e447e06c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala deleted file mode 100644 index e999da9be..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/channel_selection/NtabOnlyChannelSelector.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.frigate.pushservice.take - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.util.Future - -class NtabOnlyChannelSelector extends ChannelSelector { - val SELECTOR_NAME = "NtabOnlyChannelSelector" - - def getSelectorName(): String = SELECTOR_NAME - - // Returns a map of channel name, and the candidates that can be sent on that channel - def selectChannel( - candidate: PushCandidate - ): Future[Seq[ChannelName]] = { - // Check candidate channel eligible (based on setting, push cap etc - // Decide which candidate can be sent on what channel - val channelName: Future[ChannelName] = Future.value(ChannelName.PushNtab) - channelName.map(channel => Seq(channel)) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.docx new file mode 100644 index 000000000..2bf15dabf Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala deleted file mode 100644 index 2bdf412ac..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/EventBusWriter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.take.history - -import com.twitter.eventbus.client.EventBusPublisher -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.NotificationScribeUtil -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala.FrigateNotification - -class EventBusWriter( - eventBusPublisher: EventBusPublisher[NotificationScribe], - stats: StatsReceiver) { - private def writeSendEventToEventBus( - target: PushTypes.Target, - notificationScribe: NotificationScribe - ): Unit = { - if (target.params(PushParams.EnablePushSendEventBus)) { - val result = eventBusPublisher.publish(notificationScribe) - result.onFailure { _ => stats.counter("push_send_eventbus_failure").incr() } - } - } - - def writeToEventBus( - candidate: PushCandidate, - frigateNotificationForPersistence: FrigateNotification - ): Unit = { - val notificationScribe = NotificationScribeUtil.getNotificationScribe( - targetId = candidate.target.targetId, - impressionId = candidate.impressionId, - frigateNotification = frigateNotificationForPersistence, - createdAt = candidate.createdAt - ) - writeSendEventToEventBus(candidate.target, notificationScribe) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.docx new file mode 100644 index 000000000..1e10e799e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala deleted file mode 100644 index ca9fe31bc..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/history/HistoryWriter.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.frigate.pushservice.take.history - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.history.HistoryStoreKeyContext -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.conversions.DurationOps._ - -class HistoryWriter(historyStore: PushServiceHistoryStore, stats: StatsReceiver) { - private lazy val historyWriterStats = stats.scope(this.getClass.getSimpleName) - private lazy val historyWriteCounter = historyWriterStats.counter("history_write_num") - private lazy val loggedOutHistoryWriteCounter = - historyWriterStats.counter("logged_out_history_write_num") - - private def writeTtlForHistory(candidate: PushCandidate): Duration = { - if (candidate.target.isLoggedOutUser) { - 60.days - } else if (RecTypes.isTweetType(candidate.commonRecType)) { - candidate.target.params(PushFeatureSwitchParams.FrigateHistoryTweetNotificationWriteTtl) - } else candidate.target.params(PushFeatureSwitchParams.FrigateHistoryOtherNotificationWriteTtl) - } - - def writeSendToHistory( - candidate: PushCandidate, - frigateNotificationForPersistence: FrigateNotification - ): Future[Unit] = { - val historyStoreKeyContext = HistoryStoreKeyContext( - candidate.target.targetId, - candidate.target.pushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) - ) - if (candidate.target.isLoggedOutUser) { - loggedOutHistoryWriteCounter.incr() - } else { - historyWriteCounter.incr() - } - historyStore - .put( - historyStoreKeyContext, - candidate.createdAt, - frigateNotificationForPersistence, - Some(writeTtlForHistory(candidate)) - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.docx new file mode 100644 index 000000000..0f8cad91b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala deleted file mode 100644 index 99719af99..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicRFPHPredicates.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait BasicRFPHPredicates[C <: PushCandidate] { - val predicates: List[NamedPredicate[C]] -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.docx new file mode 100644 index 000000000..d9e07dea4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala deleted file mode 100644 index 591a4df75..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicSendHandlerPredicates.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait BasicSendHandlerPredicates[C <: PushCandidate] { - - // specific predicates per candidate type before basic SendHandler predicates - val preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - // specific predicates per candidate type after basic SendHandler predicates, could be empty - val postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.docx new file mode 100644 index 000000000..61c435067 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala deleted file mode 100644 index 0750abd6e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicates.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.predicate.BqmlHealthModelPredicates -import com.twitter.frigate.pushservice.predicate.BqmlQualityModelPredicates -import com.twitter.frigate.pushservice.predicate.HealthPredicates -import com.twitter.frigate.pushservice.predicate.OONSpreadControlPredicate -import com.twitter.frigate.pushservice.predicate.OONTweetNegativeFeedbackBasedPredicate -import com.twitter.frigate.pushservice.predicate.OutOfNetworkCandidatesQualityPredicates -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.PNegMultimodalPredicates -import com.twitter.frigate.pushservice.predicate.TargetEngagementPredicate -import com.twitter.frigate.pushservice.predicate.TweetEngagementRatioPredicate -import com.twitter.frigate.pushservice.predicate.TweetLanguagePredicate -import com.twitter.frigate.pushservice.predicate.TweetWithheldContentPredicate - -trait BasicTweetPredicates { - - def config: Config - - implicit def statsReceiver: StatsReceiver - - final lazy val basicTweetPredicates = - List( - HealthPredicates.sensitiveMediaCategoryPredicate(), - HealthPredicates.profanityPredicate(), - PredicatesForCandidate.disableOutNetworkTweetPredicate(config.edgeStore), - TweetEngagementRatioPredicate.QTtoNtabClickBasedPredicate(), - TweetLanguagePredicate.oonTweeetLanguageMatch(), - HealthPredicates.userHealthSignalsPredicate(config.userHealthSignalStore), - HealthPredicates.authorSensitiveMediaPredicate(config.producerMediaRepresentationStore), - HealthPredicates.authorProfileBasedPredicate(), - PNegMultimodalPredicates.healthSignalScorePNegMultimodalPredicate( - config.tweetHealthScoreStore), - BqmlHealthModelPredicates.healthModelOonPredicate( - config.filteringModelScorer, - config.producerMediaRepresentationStore, - config.userHealthSignalStore, - config.tweetHealthScoreStore), - BqmlQualityModelPredicates.BqmlQualityModelOonPredicate(config.filteringModelScorer), - HealthPredicates.tweetHealthSignalScorePredicate(config.tweetHealthScoreStore), - HealthPredicates - .tweetHealthSignalScorePredicate(config.tweetHealthScoreStore, applyToQuoteTweet = true), - PredicatesForCandidate.nullCastF1ProtectedExperientPredicate( - config.cachedTweetyPieStoreV2 - ), - OONTweetNegativeFeedbackBasedPredicate.ntabDislikeBasedPredicate(), - OONSpreadControlPredicate.oonTweetSpreadControlPredicate(), - OONSpreadControlPredicate.oonAuthorSpreadControlPredicate(), - HealthPredicates.healthSignalScoreMultilingualPnsfwTweetTextPredicate( - config.tweetHealthScoreStore), - PredicatesForCandidate - .recommendedTweetAuthorAcceptableToTargetUser(config.edgeStore), - HealthPredicates.healthSignalScorePnsfwTweetTextPredicate(config.tweetHealthScoreStore), - HealthPredicates.healthSignalScoreSpammyTweetPredicate(config.tweetHealthScoreStore), - OutOfNetworkCandidatesQualityPredicates.NegativeKeywordsPredicate( - config.postRankingFeatureStoreClient), - PredicatesForCandidate.authorNotBeingDeviceFollowed(config.edgeStore), - TweetWithheldContentPredicate(), - PredicatesForCandidate.noOptoutFreeFormInterestPredicate, - PredicatesForCandidate.disableInNetworkTweetPredicate(config.edgeStore), - TweetEngagementRatioPredicate.TweetReplyLikeRatioPredicate(), - TargetEngagementPredicate( - config.userTweetPerspectiveStore, - defaultForMissing = true - ), - ) -} - -/** - * This trait is a new version of BasicTweetPredicates - * Difference from old version is that basicTweetPredicates are different - * basicTweetPredicates here don't include Social Graph Service related predicates - */ -trait BasicTweetPredicatesWithoutSGSPredicates { - - def config: Config - - implicit def statsReceiver: StatsReceiver - - final lazy val basicTweetPredicates = { - List( - HealthPredicates.healthSignalScoreSpammyTweetPredicate(config.tweetHealthScoreStore), - PredicatesForCandidate.nullCastF1ProtectedExperientPredicate( - config.cachedTweetyPieStoreV2 - ), - TweetWithheldContentPredicate(), - TargetEngagementPredicate( - config.userTweetPerspectiveStore, - defaultForMissing = true - ), - PredicatesForCandidate.noOptoutFreeFormInterestPredicate, - HealthPredicates.userHealthSignalsPredicate(config.userHealthSignalStore), - HealthPredicates.tweetHealthSignalScorePredicate(config.tweetHealthScoreStore), - BqmlQualityModelPredicates.BqmlQualityModelOonPredicate(config.filteringModelScorer), - BqmlHealthModelPredicates.healthModelOonPredicate( - config.filteringModelScorer, - config.producerMediaRepresentationStore, - config.userHealthSignalStore, - config.tweetHealthScoreStore), - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.docx new file mode 100644 index 000000000..04dcff8a7 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala deleted file mode 100644 index 7d660632a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/BasicTweetPredicatesForRFPH.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait BasicTweetPredicatesForRFPH[C <: PushCandidate with TweetCandidate with TweetDetails] - extends BasicTweetPredicates - with BasicRFPHPredicates[C] { - - // specific predicates per candidate type before basic tweet predicates - def preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - // specific predicates per candidate type after basic tweet predicates - def postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - override lazy val predicates: List[NamedPredicate[C]] = - preCandidateSpecificPredicates ++ basicTweetPredicates ++ postCandidateSpecificPredicates -} - -/** - * This trait is a new version of BasicTweetPredicatesForRFPH - * Difference from old version is that basicTweetPredicates are different - * basicTweetPredicates here don't include Social Graph Service related predicates - */ -trait BasicTweetPredicatesForRFPHWithoutSGSPredicates[ - C <: PushCandidate with TweetCandidate with TweetDetails] - extends BasicTweetPredicatesWithoutSGSPredicates - with BasicRFPHPredicates[C] { - - // specific predicates per candidate type before basic tweet predicates - def preCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - // specific predicates per candidate type after basic tweet predicates - def postCandidateSpecificPredicates: List[NamedPredicate[C]] = List.empty - - override lazy val predicates: List[NamedPredicate[C]] = - preCandidateSpecificPredicates ++ basicTweetPredicates ++ postCandidateSpecificPredicates - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.docx new file mode 100644 index 000000000..6f14d8385 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala deleted file mode 100644 index e85dc95f0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/OutOfNetworkTweetPredicates.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.hermit.predicate.NamedPredicate - -trait OutOfNetworkTweetPredicates[C <: PushCandidate with TweetCandidate with TweetDetails] - extends BasicTweetPredicatesForRFPH[C] { - - override lazy val preCandidateSpecificPredicates: List[NamedPredicate[C]] = - List( - PredicatesForCandidate.authorNotBeingFollowed(config.edgeStore) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.docx new file mode 100644 index 000000000..fc0d9fe48 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala deleted file mode 100644 index e921f3fcf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/TakeCommonPredicates.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.pushservice.predicate.CrtDeciderPredicate -import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate -import com.twitter.frigate.pushservice.predicate.ScarecrowPredicate -import com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue.NtabCaretClickFatiguePredicate -import com.twitter.hermit.predicate.NamedPredicate - -trait TakeCommonPredicates { - def config: Config - - implicit def statsReceiver: StatsReceiver - - lazy val rfphPrePredicates: List[NamedPredicate[PushCandidate]] = List( - CrtDeciderPredicate(config.decider), - PredicatesForCandidate.isChannelValidPredicate, - ) - - lazy val sendHandlerPrePredicates: List[NamedPredicate[PushCandidate]] = List( - CrtDeciderPredicate(config.decider), - PredicatesForCandidate.enableSendHandlerCandidates, - PredicatesForCandidate.mrWebHoldbackPredicate, - PredicatesForCandidate.targetUserExists, - PredicatesForCandidate.authorInSocialContext, - PredicatesForCandidate.recommendedTweetIsAuthoredBySelf, - PredicatesForCandidate.selfInSocialContext, - NtabCaretClickFatiguePredicate() - ) - - lazy val postPredicates: List[NamedPredicate[PushCandidate]] = List( - ScarecrowPredicate(config.scarecrowCheckEventStore) - ) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.docx new file mode 100644 index 000000000..910e78cba Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala deleted file mode 100644 index aa4642cea..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/CandidatePredicatesMap.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates.candidate_map - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ -import com.twitter.hermit.predicate.NamedPredicate - -object CandidatePredicatesMap { - - def apply( - implicit config: Config - ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { - - val trendTweetCandidatePredicates = TrendTweetPredicates(config).predicates - val tripTweetCandidatePredicates = TripTweetCandidatePredicates(config).predicates - val f1TweetCandidatePredicates = F1TweetCandidatePredicates(config).predicates - val oonTweetCandidatePredicates = OutOfNetworkTweetCandidatePredicates(config).predicates - val tweetActionCandidatePredicates = TweetActionCandidatePredicates(config).predicates - val topicProofTweetCandidatePredicates = TopicProofTweetCandidatePredicates(config).predicates - val addressBookPushPredicates = AddressBookPushCandidatePredicates(config).predicates - val completeOnboardingPushPredicates = CompleteOnboardingPushCandidatePredicates( - config).predicates - val popGeoTweetCandidatePredicate = PopGeoTweetCandidatePredicates(config).predicates - val topTweetImpressionsCandidatePredicates = TopTweetImpressionsPushCandidatePredicates( - config).predicates - val listCandidatePredicates = ListRecommendationPredicates(config).predicates - val subscribedSearchTweetCandidatePredicates = SubscribedSearchTweetCandidatePredicates( - config).predicates - - Map( - F1FirstdegreeTweet -> f1TweetCandidatePredicates, - F1FirstdegreePhoto -> f1TweetCandidatePredicates, - F1FirstdegreeVideo -> f1TweetCandidatePredicates, - ElasticTimelineTweet -> oonTweetCandidatePredicates, - ElasticTimelinePhoto -> oonTweetCandidatePredicates, - ElasticTimelineVideo -> oonTweetCandidatePredicates, - TwistlyTweet -> oonTweetCandidatePredicates, - TwistlyPhoto -> oonTweetCandidatePredicates, - TwistlyVideo -> oonTweetCandidatePredicates, - ExploreVideoTweet -> oonTweetCandidatePredicates, - UserInterestinTweet -> oonTweetCandidatePredicates, - UserInterestinPhoto -> oonTweetCandidatePredicates, - UserInterestinVideo -> oonTweetCandidatePredicates, - PastEmailEngagementTweet -> oonTweetCandidatePredicates, - PastEmailEngagementPhoto -> oonTweetCandidatePredicates, - PastEmailEngagementVideo -> oonTweetCandidatePredicates, - TagSpaceTweet -> oonTweetCandidatePredicates, - TwhinTweet -> oonTweetCandidatePredicates, - FrsTweet -> oonTweetCandidatePredicates, - MrModelingBasedTweet -> oonTweetCandidatePredicates, - TrendTweet -> trendTweetCandidatePredicates, - ReverseAddressbookTweet -> oonTweetCandidatePredicates, - ForwardAddressbookTweet -> oonTweetCandidatePredicates, - TripGeoTweet -> oonTweetCandidatePredicates, - TripHqTweet -> tripTweetCandidatePredicates, - DetopicTweet -> oonTweetCandidatePredicates, - CrowdSearchTweet -> oonTweetCandidatePredicates, - TweetFavorite -> tweetActionCandidatePredicates, - TweetFavoritePhoto -> tweetActionCandidatePredicates, - TweetFavoriteVideo -> tweetActionCandidatePredicates, - TweetRetweet -> tweetActionCandidatePredicates, - TweetRetweetPhoto -> tweetActionCandidatePredicates, - TweetRetweetVideo -> tweetActionCandidatePredicates, - TopicProofTweet -> topicProofTweetCandidatePredicates, - SubscribedSearch -> subscribedSearchTweetCandidatePredicates, - AddressBookUploadPush -> addressBookPushPredicates, - CompleteOnboardingPush -> completeOnboardingPushPredicates, - List -> listCandidatePredicates, - GeoPopTweet -> popGeoTweetCandidatePredicate, - TweetImpressions -> topTweetImpressionsCandidatePredicates - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.docx new file mode 100644 index 000000000..dc9db91dd Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala deleted file mode 100644 index e37a91044..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/predicates/candidate_map/SendHandlerCandidatePredicatesMap.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.twitter.frigate.pushservice.take.predicates.candidate_map - -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model._ -import com.twitter.frigate.pushservice.config.Config -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ -import com.twitter.hermit.predicate.NamedPredicate - -object SendHandlerCandidatePredicatesMap { - - def preCandidatePredicates( - implicit config: Config - ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { - val magicFanoutNewsEventCandidatePredicates = - MagicFanoutNewsEventCandidatePredicates(config).preCandidateSpecificPredicates - - val scheduledSpaceSubscriberPredicates = ScheduledSpaceSubscriberCandidatePredicates( - config).preCandidateSpecificPredicates - - val scheduledSpaceSpeakerPredicates = ScheduledSpaceSpeakerCandidatePredicates( - config).preCandidateSpecificPredicates - - val magicFanoutSportsEventCandidatePredicates = - MagicFanoutSportsEventCandidatePredicates(config).preCandidateSpecificPredicates - - val magicFanoutProductLaunchPredicates = MagicFanoutProductLaunchPushCandidatePredicates( - config).preCandidateSpecificPredicates - - val creatorSubscriptionFanoutPredicates = MagicFanouCreatorSubscriptionEventPushPredicates( - config).preCandidateSpecificPredicates - - val newCreatorFanoutPredicates = MagicFanoutNewCreatorEventPushPredicates( - config).preCandidateSpecificPredicates - - Map( - MagicFanoutNewsEvent -> magicFanoutNewsEventCandidatePredicates, - ScheduledSpaceSubscriber -> scheduledSpaceSubscriberPredicates, - ScheduledSpaceSpeaker -> scheduledSpaceSpeakerPredicates, - MagicFanoutSportsEvent -> magicFanoutSportsEventCandidatePredicates, - MagicFanoutProductLaunch -> magicFanoutProductLaunchPredicates, - NewCreator -> newCreatorFanoutPredicates, - CreatorSubscriber -> creatorSubscriptionFanoutPredicates - ) - } - - def postCandidatePredicates( - implicit config: Config - ): Map[CommonRecommendationType, List[NamedPredicate[_ <: PushCandidate]]] = { - val magicFanoutNewsEventCandidatePredicates = - MagicFanoutNewsEventCandidatePredicates(config).postCandidateSpecificPredicates - - val scheduledSpaceSubscriberPredicates = ScheduledSpaceSubscriberCandidatePredicates( - config).postCandidateSpecificPredicates - - val scheduledSpaceSpeakerPredicates = ScheduledSpaceSpeakerCandidatePredicates( - config).postCandidateSpecificPredicates - - val magicFanoutSportsEventCandidatePredicates = - MagicFanoutSportsEventCandidatePredicates(config).postCandidateSpecificPredicates - val magicFanoutProductLaunchPredicates = MagicFanoutProductLaunchPushCandidatePredicates( - config).postCandidateSpecificPredicates - val creatorSubscriptionFanoutPredicates = MagicFanouCreatorSubscriptionEventPushPredicates( - config).postCandidateSpecificPredicates - val newCreatorFanoutPredicates = MagicFanoutNewCreatorEventPushPredicates( - config).postCandidateSpecificPredicates - - Map( - MagicFanoutNewsEvent -> magicFanoutNewsEventCandidatePredicates, - ScheduledSpaceSubscriber -> scheduledSpaceSubscriberPredicates, - ScheduledSpaceSpeaker -> scheduledSpaceSpeakerPredicates, - MagicFanoutSportsEvent -> magicFanoutSportsEventCandidatePredicates, - MagicFanoutProductLaunch -> magicFanoutProductLaunchPredicates, - NewCreator -> newCreatorFanoutPredicates, - CreatorSubscriber -> creatorSubscriptionFanoutPredicates - ) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.docx new file mode 100644 index 000000000..229f83d53 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala deleted file mode 100644 index a17d7fe1e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/Ibis2Sender.scala +++ /dev/null @@ -1,185 +0,0 @@ -package com.twitter.frigate.pushservice.take.sender - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.common.base.TweetDetails -import com.twitter.frigate.common.store.IbisResponse -import com.twitter.frigate.common.store.InvalidConfiguration -import com.twitter.frigate.common.store.NoRequest -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.frigate.pushservice.store.Ibis2Store -import com.twitter.frigate.pushservice.store.TweetTranslationStore -import com.twitter.frigate.pushservice.util.CopyUtil -import com.twitter.frigate.pushservice.util.FunctionalUtil -import com.twitter.frigate.pushservice.util.InlineActionUtil -import com.twitter.frigate.pushservice.util.OverrideNotificationUtil -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.frigate.scribe.thriftscala.NotificationScribe -import com.twitter.frigate.thriftscala.ChannelName -import com.twitter.frigate.thriftscala.NotificationDisplayLocation -import com.twitter.ibis2.service.thriftscala.Ibis2Request -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class Ibis2Sender( - pushIbisV2Store: Ibis2Store, - tweetTranslationStore: ReadableStore[TweetTranslationStore.Key, TweetTranslationStore.Value], - statsReceiver: StatsReceiver) { - - private val stats = statsReceiver.scope(getClass.getSimpleName) - private val silentPushCounter = stats.counter("silent_push") - private val ibisSendFailureCounter = stats.scope("ibis_send_failure").counter("failures") - private val buggyAndroidReleaseCounter = stats.counter("is_buggy_android_release") - private val androidPrimaryCounter = stats.counter("android_primary_device") - private val addTranslationModelValuesCounter = stats.counter("with_translation_model_values") - private val patchNtabResponseEnabled = stats.scope("with_ntab_response") - private val noIbisPushStats = stats.counter("no_ibis_push") - - private def ibisSend( - candidate: PushCandidate, - translationModelValues: Option[Map[String, String]] = None, - ntabResponse: Option[CreateGenericNotificationResponse] = None - ): Future[IbisResponse] = { - if (candidate.frigateNotification.notificationDisplayLocation != NotificationDisplayLocation.PushToMobileDevice) { - Future.value(IbisResponse(InvalidConfiguration)) - } else { - candidate.ibis2Request.flatMap { - case Some(request) => - val requestWithTranslationMV = - addTranslationModelValues(request, translationModelValues) - val patchedIbisRequest = { - if (candidate.target.isLoggedOutUser) { - requestWithTranslationMV - } else { - patchNtabResponseToIbisRequest(requestWithTranslationMV, candidate, ntabResponse) - } - } - pushIbisV2Store.send(patchedIbisRequest, candidate) - case _ => - noIbisPushStats.incr() - Future.value(IbisResponse(sendStatus = NoRequest, ibis2Response = None)) - } - } - } - - def sendAsDarkWrite( - candidate: PushCandidate - ): Future[IbisResponse] = { - ibisSend(candidate) - } - - def send( - channels: Seq[ChannelName], - pushCandidate: PushCandidate, - notificationScribe: NotificationScribe => Unit, - ntabResponse: Option[CreateGenericNotificationResponse], - ): Future[IbisResponse] = pushCandidate.target.isSilentPush.flatMap { isSilentPush: Boolean => - if (isSilentPush) silentPushCounter.incr() - pushCandidate.target.deviceInfo.flatMap { deviceInfo => - if (deviceInfo.exists(_.isSim40AndroidVersion)) buggyAndroidReleaseCounter.incr() - if (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfo)) androidPrimaryCounter.incr() - Future - .join( - OverrideNotificationUtil - .getOverrideInfo(pushCandidate, stats), - CopyUtil.getCopyFeatures(pushCandidate, stats), - getTranslationModelValues(pushCandidate) - ).flatMap { - case (overrideInfoOpt, copyFeaturesMap, translationModelValues) => - ibisSend(pushCandidate, translationModelValues, ntabResponse) - .onSuccess { ibisResponse => - pushCandidate - .scribeData( - ibis2Response = ibisResponse.ibis2Response, - isSilentPush = isSilentPush, - overrideInfoOpt = overrideInfoOpt, - copyFeaturesList = copyFeaturesMap.keySet, - channels = channels - ).foreach(notificationScribe) - }.onFailure { _ => - pushCandidate - .scribeData(channels = channels).foreach { data => - ibisSendFailureCounter.incr() - notificationScribe(data) - } - } - } - } - } - - private def getTranslationModelValues( - candidate: PushCandidate - ): Future[Option[Map[String, String]]] = { - candidate match { - case tweetCandidate: TweetCandidate with TweetDetails => - val key = TweetTranslationStore.Key( - target = candidate.target, - tweetId = tweetCandidate.tweetId, - tweet = tweetCandidate.tweet, - crt = candidate.commonRecType - ) - - tweetTranslationStore - .get(key) - .map { - case Some(value) => - Some( - Map( - "translated_tweet_text" -> value.translatedTweetText, - "localized_source_language" -> value.localizedSourceLanguage - )) - case None => None - } - case _ => Future.None - } - } - - private def addTranslationModelValues( - ibisRequest: Ibis2Request, - translationModelValues: Option[Map[String, String]] - ): Ibis2Request = { - (translationModelValues, ibisRequest.modelValues) match { - case (Some(translationModelVal), Some(existingModelValues)) => - addTranslationModelValuesCounter.incr() - ibisRequest.copy(modelValues = Some(translationModelVal ++ existingModelValues)) - case (Some(translationModelVal), None) => - addTranslationModelValuesCounter.incr() - ibisRequest.copy(modelValues = Some(translationModelVal)) - case (None, _) => ibisRequest - } - } - - private def patchNtabResponseToIbisRequest( - ibis2Req: Ibis2Request, - candidate: PushCandidate, - ntabResponse: Option[CreateGenericNotificationResponse] - ): Ibis2Request = { - if (candidate.target.params(FS.EnableInlineFeedbackOnPush)) { - patchNtabResponseEnabled.counter().incr() - val dislikePosition = candidate.target.params(FS.InlineFeedbackSubstitutePosition) - val dislikeActionOption = ntabResponse - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("ntab_response_exist"))) - .flatMap(response => InlineActionUtil.getDislikeInlineAction(candidate, response)) - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("dislike_action_generated"))) - - // Only generate patch serialized inline action when original request has existing serialized_inline_actions_v2 - val patchedSerializedActionOption = ibis2Req.modelValues - .flatMap(model => model.get("serialized_inline_actions_v2")) - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("inline_action_v2_exists"))) - .map(serialized => - InlineActionUtil - .patchInlineActionAtPosition(serialized, dislikeActionOption, dislikePosition)) - .map(FunctionalUtil.incr(patchNtabResponseEnabled.counter("patch_inline_action_generated"))) - - (ibis2Req.modelValues, patchedSerializedActionOption) match { - case (Some(existingModelValue), Some(patchedActionV2)) => - patchNtabResponseEnabled.scope("patch_applied").counter().incr() - ibis2Req.copy(modelValues = - Some(existingModelValue ++ Map("serialized_inline_actions_v2" -> patchedActionV2))) - case _ => ibis2Req - } - } else ibis2Req - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.docx new file mode 100644 index 000000000..44022a0c9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala deleted file mode 100644 index 5019aa040..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/take/sender/NtabSender.scala +++ /dev/null @@ -1,237 +0,0 @@ -package com.twitter.frigate.pushservice.take.sender - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.ibis.PushOverrideInfo -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.frigate.pushservice.take.NotificationServiceRequest -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.hermit.store.common.ReadableWritableStore -import com.twitter.notificationservice.api.thriftscala.DeleteCurrentTimelineForUserRequest -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.notificationservice.thriftscala.DeleteGenericNotificationRequest -import com.twitter.notificationservice.thriftscala.GenericNotificationKey -import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object OverrideCandidate extends Enumeration { - val One: String = "overrideEntry1" -} - -class NtabSender( - notificationServiceSender: ReadableStore[ - NotificationServiceRequest, - CreateGenericNotificationResponse - ], - nTabHistoryStore: ReadableWritableStore[(Long, String), GenericNotificationOverrideKey], - nTabDelete: DeleteGenericNotificationRequest => Future[Unit], - nTabDeleteTimeline: DeleteCurrentTimelineForUserRequest => Future[Unit] -)( - implicit statsReceiver: StatsReceiver) { - - private[this] val nTabDeleteRequests = statsReceiver.counter("ntab_delete_request") - private[this] val nTabDeleteTimelineRequests = - statsReceiver.counter("ntab_delete_timeline_request") - private[this] val ntabOverrideImpressionNotFound = - statsReceiver.counter("ntab_impression_not_found") - private[this] val nTabOverrideOverriddenStat = - statsReceiver.counter("ntab_override_overridden") - private[this] val storeGenericNotifOverrideKey = - statsReceiver.counter("ntab_store_generic_notif_key") - private[this] val prevGenericNotifKeyNotFound = - statsReceiver.counter("ntab_prev_generic_notif_key_not_found") - - private[this] val ntabOverride = - statsReceiver.scope("ntab_override") - private[this] val ntabRequestWithOverrideId = - ntabOverride.counter("request") - private[this] val storeGenericNotifOverrideKeyWithOverrideId = - ntabOverride.counter("store_override_key") - - def send( - candidate: PushCandidate, - isNtabOnlyNotification: Boolean - ): Future[Option[CreateGenericNotificationResponse]] = { - if (candidate.target.params(FSParams.EnableOverrideIdNTabRequest)) { - ntabRequestWithOverrideId.incr() - overridePreviousEntry(candidate).flatMap { _ => - if (shouldDisableNtabOverride(candidate)) { - sendNewEntry(candidate, isNtabOnlyNotification, None) - } else { - sendNewEntry(candidate, isNtabOnlyNotification, Some(OverrideCandidate.One)) - } - } - } else { - for { - notificationOverwritten <- overrideNSlot(candidate) - _ <- deleteCachedApiTimeline(candidate, notificationOverwritten) - gnResponse <- sendNewEntry(candidate, isNtabOnlyNotification) - } yield gnResponse - } - } - - private def sendNewEntry( - candidate: PushCandidate, - isNtabOnlyNotif: Boolean, - overrideId: Option[String] = None - ): Future[Option[CreateGenericNotificationResponse]] = { - notificationServiceSender - .get( - NotificationServiceRequest( - candidate, - impressionId = candidate.impressionId, - isBadgeUpdate = isNtabOnlyNotif, - overrideId = overrideId - )).flatMap { - case Some(response) => - storeGenericNotifKey(candidate, response, overrideId).map { _ => Some(response) } - case _ => Future.None - } - } - - private def storeGenericNotifKey( - candidate: PushCandidate, - createGenericNotificationResponse: CreateGenericNotificationResponse, - overrideId: Option[String] - ): Future[Unit] = { - if (candidate.target.params(FSParams.EnableStoringNtabGenericNotifKey)) { - createGenericNotificationResponse.successKey match { - case Some(genericNotificationKey) => - val userId = genericNotificationKey.userId - if (overrideId.nonEmpty) { - storeGenericNotifOverrideKeyWithOverrideId.incr() - } - val gnOverrideKey = GenericNotificationOverrideKey( - userId = userId, - hashKey = genericNotificationKey.hashKey, - timestampMillis = genericNotificationKey.timestampMillis, - overrideId = overrideId - ) - val mhKeyVal = - ((userId, candidate.impressionId), gnOverrideKey) - storeGenericNotifOverrideKey.incr() - nTabHistoryStore.put(mhKeyVal) - case _ => Future.Unit - } - } else Future.Unit - } - - private def candidateEligibleForOverride( - targetHistory: History, - targetEntries: Seq[FrigateNotification], - ): FrigateNotification = { - val timestampToEntriesMap = - targetEntries.map { entry => - PushOverrideInfo - .getTimestampInMillisForFrigateNotification(entry, targetHistory, statsReceiver) - .getOrElse(PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) -> entry - }.toMap - - PushOverrideInfo.getOldestFrigateNotification(timestampToEntriesMap) - } - - private def overrideNSlot(candidate: PushCandidate): Future[Boolean] = { - if (candidate.target.params(FSParams.EnableNslotsForOverrideOnNtab)) { - val targetHistoryFut = candidate.target.history - targetHistoryFut.flatMap { targetHistory => - val nonEligibleOverrideTypes = - Seq(RecTypes.RecommendedSpaceFanoutTypes ++ RecTypes.ScheduledSpaceReminderTypes) - - val overrideNotifs = PushOverrideInfo - .getOverrideEligiblePushNotifications( - targetHistory, - candidate.target.params(FSParams.OverrideNotificationsLookbackDurationForNTab), - statsReceiver - ).filterNot { - case notification => - nonEligibleOverrideTypes.contains(notification.commonRecommendationType) - } - - val maxNumUnreadEntries = - candidate.target.params(FSParams.OverrideNotificationsMaxCountForNTab) - if (overrideNotifs.nonEmpty && overrideNotifs.size >= maxNumUnreadEntries) { - val eligibleOverrideCandidateOpt = candidateEligibleForOverride( - targetHistory, - overrideNotifs - ) - eligibleOverrideCandidateOpt match { - case overrideCandidate if overrideCandidate.impressionId.nonEmpty => - deleteNTabEntryFromGenericNotificationStore( - candidate.target.targetId, - eligibleOverrideCandidateOpt.impressionId.head) - case _ => - ntabOverrideImpressionNotFound.incr() - Future.False - } - } else Future.False - } - } else { - Future.False - } - } - - private def shouldDisableNtabOverride(candidate: PushCandidate): Boolean = - RecTypes.isSendHandlerType(candidate.commonRecType) - - private def overridePreviousEntry(candidate: PushCandidate): Future[Boolean] = { - - if (shouldDisableNtabOverride(candidate)) { - nTabOverrideOverriddenStat.incr() - Future.False - } else { - val targetHistoryFut = candidate.target.history - targetHistoryFut.flatMap { targetHistory => - val impressionIds = PushOverrideInfo.getImpressionIdsOfPrevEligiblePushNotif( - targetHistory, - candidate.target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), - statsReceiver) - - if (impressionIds.nonEmpty) { - deleteNTabEntryFromGenericNotificationStore(candidate.target.targetId, impressionIds.head) - } else { - ntabOverrideImpressionNotFound.incr() - Future.False // no deletes issued - } - } - } - } - - private def deleteCachedApiTimeline( - candidate: PushCandidate, - isNotificationOverridden: Boolean - ): Future[Unit] = { - if (isNotificationOverridden && candidate.target.params(FSParams.EnableDeletingNtabTimeline)) { - val deleteTimelineRequest = DeleteCurrentTimelineForUserRequest(candidate.target.targetId) - nTabDeleteTimelineRequests.incr() - nTabDeleteTimeline(deleteTimelineRequest) - } else { - Future.Unit - } - } - - private def deleteNTabEntryFromGenericNotificationStore( - targetUserId: Long, - targetImpressionId: String - ): Future[Boolean] = { - val mhKey = (targetUserId, targetImpressionId) - val genericNotificationKeyFut = nTabHistoryStore.get(mhKey) - genericNotificationKeyFut.flatMap { - case Some(genericNotifOverrideKey) => - val gnKey = GenericNotificationKey( - userId = genericNotifOverrideKey.userId, - hashKey = genericNotifOverrideKey.hashKey, - timestampMillis = genericNotifOverrideKey.timestampMillis - ) - val deleteEntryRequest = DeleteGenericNotificationRequest(gnKey) - nTabDeleteRequests.incr() - nTabDelete(deleteEntryRequest).map(_ => true) - case _ => - prevGenericNotifKeyNotFound.incr() - Future.False - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.docx new file mode 100644 index 000000000..d1a2a6510 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala deleted file mode 100644 index 9690e9bad..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/CustomFSFields.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.featureswitches.FSCustomMapInput -import com.twitter.featureswitches.parsing.DynMap -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.util.NsfwInfo -import com.twitter.gizmoduck.thriftscala.User - -object CustomFSFields { - private val IsReturningUser = "is_returning_user" - private val DaysSinceSignup = "days_since_signup" - private val DaysSinceLogin = "days_since_login" - private val DaysSinceReactivation = "days_since_reactivation" - private val ReactivationDate = "reactivation_date" - private val FollowGraphSize = "follow_graph_size" - private val GizmoduckUserType = "gizmoduck_user_type" - private val UserAge = "mr_user_age" - private val SensitiveOptIn = "sensitive_opt_in" - private val NsfwFollowRatio = "nsfw_follow_ratio" - private val TotalFollows = "follow_count" - private val NsfwRealGraphScore = "nsfw_real_graph_score" - private val NsfwProfileVisit = "nsfw_profile_visit" - private val TotalSearches = "total_searches" - private val NsfwSearchScore = "nsfw_search_score" - private val HasReportedNsfw = "nsfw_reported" - private val HasDislikedNsfw = "nsfw_disliked" - private val UserState = "user_state" - private val MrUserState = "mr_user_state" - private val NumDaysReceivedPushInLast30Days = - "num_days_received_push_in_last_30_days" - private val RecommendationsSetting = "recommendations_setting" - private val TopicsSetting = "topics_setting" - private val SpacesSetting = "spaces_setting" - private val NewsSetting = "news_setting" - private val LiveVideoSetting = "live_video_setting" - private val HasRecentPushableRebDevice = "has_recent_pushable_rweb_device" - private val RequestSource = "request_source" -} - -case class CustomFSFields( - isReactivatedUser: Boolean, - daysSinceSignup: Int, - numDaysReceivedPushInLast30Days: Int, - daysSinceLogin: Option[Int], - daysSinceReactivation: Option[Int], - user: Option[User], - userState: Option[String], - mrUserState: Option[String], - reactivationDate: Option[String], - requestSource: Option[String], - userAge: Option[Int], - nsfwInfo: Option[NsfwInfo], - deviceInfo: Option[DeviceInfo]) { - - import CustomFSFields._ - - private val keyValMap: Map[String, Any] = Map( - IsReturningUser -> isReactivatedUser, - DaysSinceSignup -> daysSinceSignup, - DaysSinceLogin -> daysSinceLogin, - NumDaysReceivedPushInLast30Days -> numDaysReceivedPushInLast30Days - ) ++ - daysSinceReactivation.map(DaysSinceReactivation -> _) ++ - reactivationDate.map(ReactivationDate -> _) ++ - user.flatMap(_.counts.map(counts => FollowGraphSize -> counts.following)) ++ - user.map(u => GizmoduckUserType -> u.userType.name) ++ - userState.map(UserState -> _) ++ - mrUserState.map(MrUserState -> _) ++ - requestSource.map(RequestSource -> _) ++ - userAge.map(UserAge -> _) ++ - nsfwInfo.flatMap(_.senstiveOptIn).map(SensitiveOptIn -> _) ++ - nsfwInfo - .map { nsInfo => - Map[String, Any]( - NsfwFollowRatio -> nsInfo.nsfwFollowRatio, - TotalFollows -> nsInfo.totalFollowCount, - NsfwRealGraphScore -> nsInfo.realGraphScore, - NsfwProfileVisit -> nsInfo.nsfwProfileVisits, - TotalSearches -> nsInfo.totalSearches, - NsfwSearchScore -> nsInfo.searchNsfwScore, - HasReportedNsfw -> nsInfo.hasReported, - HasDislikedNsfw -> nsInfo.hasDisliked - ) - }.getOrElse(Map.empty[String, Any]) ++ - deviceInfo - .map { deviceInfo => - Map[String, Boolean]( - RecommendationsSetting -> deviceInfo.isRecommendationsEligible, - TopicsSetting -> deviceInfo.isTopicsEligible, - SpacesSetting -> deviceInfo.isSpacesEligible, - LiveVideoSetting -> deviceInfo.isBroadcastsEligible, - NewsSetting -> deviceInfo.isNewsEligible, - HasRecentPushableRebDevice -> deviceInfo.hasRecentPushableRWebDevice - ) - }.getOrElse(Map.empty[String, Boolean]) - - val fsMap = FSCustomMapInput(DynMap(keyValMap)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.docx new file mode 100644 index 000000000..f68a21b80 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala deleted file mode 100644 index facbaede0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/LoggedOutPushTargetUserBuilder.scala +++ /dev/null @@ -1,182 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.history.HistoryStoreKeyContext -import com.twitter.frigate.common.history.MagicFanoutReasonHistory -import com.twitter.frigate.common.history.PushServiceHistoryStore -import com.twitter.frigate.common.history.RecItems -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.util.ABDeciderWithOverride -import com.twitter.frigate.common.util.LanguageLocaleUtil -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.frigate.thriftscala.UserForPushTargeting -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.interests.thriftscala.InterestId -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.service.metastore.gen.thriftscala.UserLanguages -import com.twitter.stitch.Stitch -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.columns.frigate.logged_out_web_notifications.thriftscala.LOWebNotificationMetadata -import com.twitter.timelines.configapi -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures - -case class LoggedOutPushTargetUserBuilder( - historyStore: PushServiceHistoryStore, - inputDecider: Decider, - inputAbDecider: LoggingABDecider, - loggedOutPushInfoStore: ReadableStore[Long, LOWebNotificationMetadata] -)( - globalStatsReceiver: StatsReceiver) { - private val stats = globalStatsReceiver.scope("LORefreshForPushHandler") - private val noHistoryCounter = stats.counter("no_logged_out_history") - private val historyFoundCounter = stats.counter("logged_out_history_counter") - private val noLoggedOutUserCounter = stats.counter("no_logged_out_user") - private val countryCodeCounter = stats.counter("country_counter") - private val noCountryCodeCounter = stats.counter("no_country_counter") - private val noLanguageCodeCounter = stats.counter("no_language_counter") - - def buildTarget( - guestId: Long, - inputPushContext: Option[PushContext] - ): Future[Target] = { - - val historyStoreKeyContext = HistoryStoreKeyContext( - guestId, - inputPushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) - ) - if (historyStore.get(historyStoreKeyContext, Some(30.days)) == Future.None) { - noHistoryCounter.incr() - } else { - historyFoundCounter.incr() - - } - if (loggedOutPushInfoStore.get(guestId) == Future.None) { - noLoggedOutUserCounter.incr() - } - Future - .join( - historyStore.get(historyStoreKeyContext, Some(30.days)), - loggedOutPushInfoStore.get(guestId) - ).map { - case (loNotifHistory, loggedOutUserPushInfo) => - new Target { - override lazy val stats: StatsReceiver = globalStatsReceiver - override val targetId: Long = guestId - override val targetGuestId = Some(guestId) - override lazy val decider: Decider = inputDecider - override lazy val loggedOutMetadata = Future.value(loggedOutUserPushInfo) - val rawLanguageFut = loggedOutMetadata.map { metadata => metadata.map(_.language) } - override val targetLanguage: Future[Option[String]] = rawLanguageFut.map { rawLang => - if (rawLang.isDefined) { - val lang = LanguageLocaleUtil.getStandardLanguageCode(rawLang.get) - if (lang.isEmpty) { - noLanguageCodeCounter.incr() - None - } else { - Option(lang) - } - } else None - } - val country = loggedOutMetadata.map(_.map(_.countryCode)) - if (country.isDefined) { - countryCodeCounter.incr() - } else { - noCountryCodeCounter.incr() - } - if (loNotifHistory == null) { - noHistoryCounter.incr() - } else { - historyFoundCounter.incr() - } - override lazy val location: Future[Option[Location]] = country.map { - case Some(code) => - Some( - Location( - city = "", - region = "", - countryCode = code, - confidence = 0.0, - lat = None, - lon = None, - metro = None, - placeIds = None, - weightedLocations = None, - createdAtMsec = None, - ip = None, - isSignupIp = None, - placeMap = None - )) - case _ => None - } - - override lazy val pushContext: Option[PushContext] = inputPushContext - override lazy val history: Future[History] = Future.value(loNotifHistory) - override lazy val magicFanoutReasonHistory30Days: Future[MagicFanoutReasonHistory] = - Future.value(null) - override lazy val globalStats: StatsReceiver = globalStatsReceiver - override lazy val pushTargeting: Future[Option[UserForPushTargeting]] = Future.None - override lazy val appPermissions: Future[Option[AppPermission]] = Future.None - override lazy val lastHTLVisitTimestamp: Future[Option[Long]] = Future.None - override lazy val pushRecItems: Future[RecItems] = Future.value(null) - - override lazy val isNewSignup: Boolean = false - override lazy val metastoreLanguages: Future[Option[UserLanguages]] = Future.None - override lazy val optOutUserInterests: Future[Option[Seq[InterestId]]] = Future.None - override lazy val mrRequestContextForFeatureStore: MrRequestContextForFeatureStore = - null - override lazy val targetUser: Future[Option[User]] = Future.None - override lazy val notificationFeedbacks: Future[Option[Seq[FeedbackPromptValue]]] = - Future.None - override lazy val promptFeedbacks: Stitch[Seq[FeedbackPromptValue]] = null - override lazy val seedsWithWeight: Future[Option[Map[Long, Double]]] = Future.None - override lazy val tweetImpressionResults: Future[Seq[Long]] = Future.Nil - override lazy val params: configapi.Params = Params.Empty - override lazy val deviceInfo: Future[Option[DeviceInfo]] = Future.None - override lazy val userFeatures: Future[Option[UserFeatures]] = Future.None - override lazy val isOpenAppExperimentUser: Future[Boolean] = Future.False - override lazy val featureMap: Future[FeatureMap] = Future.value(null) - override lazy val dauProbability: Future[Option[DauProbability]] = Future.None - override lazy val labeledPushRecsHydrated: Future[Option[UserHistoryValue]] = - Future.None - override lazy val onlineLabeledPushRecs: Future[Option[UserHistoryValue]] = Future.None - override lazy val realGraphFeatures: Future[Option[RealGraphFeatures]] = Future.None - override lazy val stpResult: Future[Option[STPResult]] = Future.None - override lazy val globalOptoutProbabilities: Seq[Future[Option[Double]]] = Seq.empty - override lazy val bucketOptoutProbability: Future[Option[Double]] = Future.None - override lazy val utcOffset: Future[Option[Duration]] = Future.None - override lazy val abDecider: ABDeciderWithOverride = - ABDeciderWithOverride(inputAbDecider, ddgOverrideOption)(globalStatsReceiver) - override lazy val resurrectionDate: Future[Option[String]] = Future.None - override lazy val isResurrectedUser: Boolean = false - override lazy val timeSinceResurrection: Option[Duration] = None - override lazy val inlineActionHistory: Future[Seq[(Long, String)]] = Future.Nil - override lazy val caretFeedbacks: Future[Option[Seq[CaretFeedbackDetails]]] = - Future.None - - override def targetHydrationContext: Future[HydrationContext] = Future.value(null) - override def isBlueVerified: Future[Option[Boolean]] = Future.None - override def isVerified: Future[Option[Boolean]] = Future.None - override def isSuperFollowCreator: Future[Option[Boolean]] = Future.None - } - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.docx new file mode 100644 index 000000000..bb2c2c9aa Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala deleted file mode 100644 index 8378500af..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/PushTargetUserBuilder.scala +++ /dev/null @@ -1,694 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.abdecider.LoggingABDecider -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.discovery.common.configapi.ConfigParamsBuilder -import com.twitter.discovery.common.configapi.ExperimentOverride -import com.twitter.featureswitches.Recipient -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.history._ -import com.twitter.frigate.common.logger.MRLogger -import com.twitter.frigate.common.store.FeedbackRequest -import com.twitter.frigate.common.store.PushRecItemsKey -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.store.interests.UserId -import com.twitter.frigate.common.util._ -import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.pushcap.thriftscala.PushcapInfo -import com.twitter.frigate.pushcap.thriftscala.PushcapUserHistory -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.ml.HydrationContextBuilder -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.store.LabeledPushRecsStoreKey -import com.twitter.frigate.pushservice.store.OnlineUserHistoryKey -import com.twitter.frigate.pushservice.util.NsfwInfo -import com.twitter.frigate.pushservice.util.NsfwPersonalizationUtil -import com.twitter.frigate.pushservice.util.PushAppPermissionUtil -import com.twitter.frigate.pushservice.util.PushCapUtil.getMinimumRestrictedPushcapInfo -import com.twitter.frigate.pushservice.thriftscala.PushContext -import com.twitter.frigate.pushservice.thriftscala.RequestSource -import com.twitter.frigate.thriftscala.SecondaryAccountsByUserState -import com.twitter.frigate.thriftscala.UserForPushTargeting -import com.twitter.frigate.user_states.thriftscala.MRUserHmmState -import com.twitter.frigate.user_states.thriftscala.{UserState => MrUserState} -import com.twitter.frontpage.stream.util.SnowflakeUtil -import com.twitter.geoduck.common.thriftscala.Place -import com.twitter.geoduck.service.thriftscala.LocationResponse -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.model.user_state.UserState -import com.twitter.hermit.model.user_state.UserState.UserState -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.ibis.thriftscala.ContentRecData -import com.twitter.interests.thriftscala.InterestId -import com.twitter.notificationservice.feedback.thriftscala.FeedbackInteraction -import com.twitter.notificationservice.genericfeedbackstore.FeedbackPromptValue -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStore -import com.twitter.notificationservice.genericfeedbackstore.GenericFeedbackStoreException -import com.twitter.notificationservice.model.service.DismissMenuFeedbackAction -import com.twitter.notificationservice.scribe.manhattan.GenericNotificationsFeedbackRequest -import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails -import com.twitter.nrel.heavyranker.FeatureHydrator -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.rux.common.strato.thriftscala.UserTargetingProperty -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation -import com.twitter.service.metastore.gen.thriftscala.Location -import com.twitter.service.metastore.gen.thriftscala.UserLanguages -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi -import com.twitter.timelines.real_graph.thriftscala.{RealGraphFeatures => RealGraphFeaturesUnion} -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures -import com.twitter.ubs.thriftscala.SellerApplicationState -import com.twitter.ubs.thriftscala.SellerTrack -import com.twitter.user_session_store.thriftscala.UserSession -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Time -import com.twitter.wtf.scalding.common.thriftscala.UserFeatures - -case class PushTargetUserBuilder( - historyStore: PushServiceHistoryStore, - emailHistoryStore: PushServiceHistoryStore, - labeledPushRecsStore: ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue], - onlineUserHistoryStore: ReadableStore[OnlineUserHistoryKey, UserHistoryValue], - pushRecItemsStore: ReadableStore[PushRecItemsKey, RecItems], - userStore: ReadableStore[Long, User], - pushInfoStore: ReadableStore[Long, UserForPushTargeting], - userCountryStore: ReadableStore[Long, Location], - userUtcOffsetStore: ReadableStore[Long, Duration], - dauProbabilityStore: ReadableStore[Long, DauProbability], - nsfwConsumerStore: ReadableStore[Long, NSFWUserSegmentation], - userFeatureStore: ReadableStore[Long, UserFeatures], - userTargetingPropertyStore: ReadableStore[Long, UserTargetingProperty], - mrUserStateStore: ReadableStore[Long, MRUserHmmState], - tweetImpressionStore: ReadableStore[Long, Seq[Long]], - ntabCaretFeedbackStore: ReadableStore[GenericNotificationsFeedbackRequest, Seq[ - CaretFeedbackDetails - ]], - genericFeedbackStore: ReadableStore[FeedbackRequest, Seq[FeedbackPromptValue]], - genericNotificationFeedbackStore: GenericFeedbackStore, - timelinesUserSessionStore: ReadableStore[Long, UserSession], - cachedTweetyPieStore: ReadableStore[Long, TweetyPieResult], - strongTiesStore: ReadableStore[Long, STPResult], - userHTLLastVisitStore: ReadableStore[Long, Seq[Long]], - userLanguagesStore: ReadableStore[Long, UserLanguages], - inputDecider: Decider, - inputAbDecider: LoggingABDecider, - realGraphScoresTop500InStore: ReadableStore[Long, Map[Long, Double]], - recentFollowsStore: ReadableStore[Long, Seq[Long]], - resurrectedUserStore: ReadableStore[Long, String], - configParamsBuilder: ConfigParamsBuilder, - optOutUserInterestsStore: ReadableStore[UserId, Seq[InterestId]], - deviceInfoStore: ReadableStore[Long, DeviceInfo], - pushcapDynamicPredictionStore: ReadableStore[Long, PushcapUserHistory], - appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission], - optoutModelScorer: PushMLModelScorer, - inlineActionHistoryStore: ReadableStore[Long, Seq[(Long, String)]], - featureHydrator: FeatureHydrator, - openAppUserStore: ReadableStore[Long, Boolean], - openedPushByHourAggregatedStore: ReadableStore[Long, Map[Int, Int]], - geoduckStoreV2: ReadableStore[Long, LocationResponse], - superFollowEligibilityUserStore: ReadableStore[Long, Boolean], - superFollowApplicationStatusStore: ReadableStore[(Long, SellerTrack), SellerApplicationState] -)( - globalStatsReceiver: StatsReceiver) { - - implicit val statsReceiver: StatsReceiver = globalStatsReceiver - - private val log = MRLogger("PushTargetUserBuilder") - private val recentFollowscounter = statsReceiver.counter("query_recent_follows") - private val isModelTrainingDataCounter = - statsReceiver.scope("TargetUserBuilder").counter("is_model_training") - private val feedbackStoreGenerationErr = statsReceiver.counter("feedback_store_generation_error") - private val newSignUpUserStats = statsReceiver.counter("new_signup_user") - private val pushcapSelectionStat = statsReceiver.scope("pushcap_modeling") - private val dormantUserCount = statsReceiver.counter("dormant_user_counter") - private val optoutModelStat = statsReceiver.scope("optout_modeling") - private val placeFoundStat = statsReceiver.scope("geoduck_v2").stat("places_found") - private val placesNotFound = statsReceiver.scope("geoduck_v2").counter("places_not_found") - // Email history store stats - private val emailHistoryStats = statsReceiver.scope("email_tweet_history") - private val emptyEmailHistoryCounter = emailHistoryStats.counter("empty") - private val nonEmptyEmailHistoryCounter = emailHistoryStats.counter("non_empty") - - private val MagicRecsCategory = "MagicRecs" - private val MomentsViaMagicRecsCategory = "MomentsViaMagicRecs" - private val MomentsCategory = "Moments" - - def buildTarget( - userId: Long, - inputPushContext: Option[PushContext], - forcedFeatureValues: Option[Map[String, configapi.FeatureValue]] = None - ): Future[Target] = { - val historyStoreKeyContext = HistoryStoreKeyContext( - userId, - inputPushContext.flatMap(_.useMemcacheForHistory).getOrElse(false) - ) - Future - .join( - userStore.get(userId), - deviceInfoStore.get(userId), - pushInfoStore.get(userId), - historyStore.get(historyStoreKeyContext, Some(30.days)), - emailHistoryStore.get( - HistoryStoreKeyContext(userId, useStoreB = false), - Some(7.days) // we only keep 7 days of email tweet history - ) - ).flatMap { - case (userOpt, deviceInfoOpt, userForPushTargetingInfoOpt, notifHistory, emailHistory) => - getCustomFSFields( - userId, - userOpt, - deviceInfoOpt, - userForPushTargetingInfoOpt, - notifHistory, - inputPushContext.flatMap(_.requestSource)).map { customFSField => - new Target { - - override lazy val stats: StatsReceiver = statsReceiver - - override val targetId: Long = userId - - override val targetUser: Future[Option[User]] = Future.value(userOpt) - - override val isEmailUser: Boolean = - inputPushContext.flatMap(_.requestSource) match { - case Some(source) if source == RequestSource.Email => true - case _ => false - } - - override val pushContext = inputPushContext - - override def globalStats: StatsReceiver = globalStatsReceiver - - override lazy val abDecider: ABDeciderWithOverride = - ABDeciderWithOverride(inputAbDecider, ddgOverrideOption) - - override lazy val pushRecItems: Future[RecItems] = - pushRecItemsStore - .get(PushRecItemsKey(historyStoreKeyContext, history)) - .map(_.getOrElse(RecItems.empty)) - - // List of past tweet candidates sent in the past through email with timestamp - override lazy val emailRecItems: Future[Seq[(Time, Long)]] = { - Future.value { - emailHistory.sortedEmailHistory.flatMap { - case (timeStamp, notification) => - notification.contentRecsNotification - .map { notification => - notification.recommendations.contentRecCollections.flatMap { - contentRecs => - contentRecs.contentRecModules.flatMap { contentRecModule => - contentRecModule.recData match { - case ContentRecData.TweetRec(tweetRec) => - nonEmptyEmailHistoryCounter.incr() - Seq(tweetRec.tweetId) - case _ => - emptyEmailHistoryCounter.incr() - Nil - } - } - } - }.getOrElse { - emptyEmailHistoryCounter.incr() - Nil - }.map(timeStamp -> _) - } - } - } - - override lazy val history: Future[History] = Future.value(notifHistory) - - override lazy val pushTargeting: Future[Option[UserForPushTargeting]] = - Future.value(userForPushTargetingInfoOpt) - - override lazy val decider: Decider = inputDecider - - override lazy val location: Future[Option[Location]] = - userCountryStore.get(userId) - - override lazy val deviceInfo: Future[Option[DeviceInfo]] = - Future.value(deviceInfoOpt) - - override lazy val targetLanguage: Future[Option[String]] = targetUser map { userOpt => - userOpt.flatMap(_.account.map(_.language)) - } - - override lazy val targetAgeInYears: Future[Option[Int]] = - Future.value(customFSField.userAge) - - override lazy val metastoreLanguages: Future[Option[UserLanguages]] = - userLanguagesStore.get(targetId) - - override lazy val utcOffset: Future[Option[Duration]] = - userUtcOffsetStore.get(targetId) - - override lazy val userFeatures: Future[Option[UserFeatures]] = - userFeatureStore.get(targetId) - - override lazy val targetUserState: Future[Option[UserState]] = - Future.value( - customFSField.userState - .flatMap(userState => UserState.valueOf(userState))) - - override lazy val targetMrUserState: Future[Option[MrUserState]] = - Future.value( - customFSField.mrUserState - .flatMap(mrUserState => MrUserState.valueOf(mrUserState))) - - override lazy val accountStateWithDeviceInfo: Future[ - Option[SecondaryAccountsByUserState] - ] = Future.None - - override lazy val dauProbability: Future[Option[DauProbability]] = { - dauProbabilityStore.get(targetId) - } - - override lazy val labeledPushRecsHydrated: Future[Option[UserHistoryValue]] = - labeledPushRecsStore.get(LabeledPushRecsStoreKey(this, historyStoreKeyContext)) - - override lazy val onlineLabeledPushRecs: Future[Option[UserHistoryValue]] = - labeledPushRecsHydrated.flatMap { labeledPushRecs => - history.flatMap { history => - onlineUserHistoryStore.get( - OnlineUserHistoryKey(targetId, labeledPushRecs, Some(history)) - ) - } - } - - override lazy val tweetImpressionResults: Future[Seq[Long]] = - tweetImpressionStore.get(targetId).map { - case Some(impressionList) => - impressionList - case _ => Nil - } - - override lazy val realGraphFeatures: Future[Option[RealGraphFeatures]] = - timelinesUserSessionStore.get(targetId).map { userSessionOpt => - userSessionOpt.flatMap { userSession => - userSession.realGraphFeatures.collect { - case RealGraphFeaturesUnion.V1(rGFeatures) => - rGFeatures - } - } - } - - override lazy val stpResult: Future[Option[STPResult]] = - strongTiesStore.get(targetId) - - override lazy val lastHTLVisitTimestamp: Future[Option[Long]] = - userHTLLastVisitStore.get(targetId).map { - case Some(lastVisitTimestamps) if lastVisitTimestamps.nonEmpty => - Some(lastVisitTimestamps.max) - case _ => None - } - - override lazy val caretFeedbacks: Future[Option[Seq[CaretFeedbackDetails]]] = { - val scribeHistoryLookbackPeriod = 365.days - val now = Time.now - val request = GenericNotificationsFeedbackRequest( - userId = targetId, - eventStartTimestamp = now - scribeHistoryLookbackPeriod, - eventEndTimestamp = now, - filterCategory = - Some(Set(MagicRecsCategory, MomentsViaMagicRecsCategory, MomentsCategory)), - filterFeedbackActionText = - Some(Set(DismissMenuFeedbackAction.FeedbackActionTextSeeLessOften)) - ) - ntabCaretFeedbackStore.get(request) - } - - override lazy val notificationFeedbacks: Future[ - Option[Seq[FeedbackPromptValue]] - ] = { - val scribeHistoryLookbackPeriod = 30.days - val now = Time.now - val request = FeedbackRequest( - userId = targetId, - oldestTimestamp = scribeHistoryLookbackPeriod.ago, - newestTimestamp = Time.now, - feedbackInteraction = FeedbackInteraction.Feedback - ) - genericFeedbackStore.get(request) - } - - // DEPRECATED: Use notificationFeedbacks instead. - // This method will increase latency dramatically. - override lazy val promptFeedbacks: Stitch[Seq[FeedbackPromptValue]] = { - val scribeHistoryLookbackPeriod = 7.days - - genericNotificationFeedbackStore - .getAll( - userId = targetId, - oldestTimestamp = scribeHistoryLookbackPeriod.ago, - newestTimestamp = Time.now, - feedbackInteraction = FeedbackInteraction.Feedback - ).handle { - case _: GenericFeedbackStoreException => { - feedbackStoreGenerationErr.incr() - Seq.empty[FeedbackPromptValue] - } - } - } - - override lazy val optOutUserInterests: Future[Option[Seq[InterestId]]] = { - optOutUserInterestsStore.get(targetId) - } - - private val experimentOverride = ddgOverrideOption.map { - case DDGOverride(Some(exp), Some(bucket)) => - Set(ExperimentOverride(exp, bucket)) - case _ => Set.empty[ExperimentOverride] - } - - override val signupCountryCode = - Future.value(userOpt.flatMap(_.safety.flatMap(_.signupCountryCode))) - - override lazy val params: configapi.Params = { - val fsRecipient = Recipient( - userId = Some(targetId), - userRoles = userOpt.flatMap(_.roles.map(_.roles.toSet)), - clientApplicationId = deviceInfoOpt.flatMap(_.guessedPrimaryClientAppId), - userAgent = deviceInfoOpt.flatMap(_.guessedPrimaryDeviceUserAgent), - countryCode = - userOpt.flatMap(_.account.flatMap(_.countryCode.map(_.toUpperCase))), - customFields = Some(customFSField.fsMap), - signupCountryCode = - userOpt.flatMap(_.safety.flatMap(_.signupCountryCode.map(_.toUpperCase))), - languageCode = deviceInfoOpt.flatMap { - _.deviceLanguages.flatMap(IbisAppPushDeviceSettingsUtil.inferredDeviceLanguage) - } - ) - - configParamsBuilder.build( - userId = Some(targetId), - experimentOverrides = experimentOverride, - featureRecipient = Some(fsRecipient), - forcedFeatureValues = forcedFeatureValues.getOrElse(Map.empty), - ) - } - - override lazy val mrRequestContextForFeatureStore = - MrRequestContextForFeatureStore(targetId, params, isModelTrainingData) - - override lazy val dynamicPushcap: Future[Option[PushcapInfo]] = { - // Get the pushcap from the pushcap model prediction store - if (params(PushParams.EnableModelBasedPushcapAssignments)) { - val originalPushcapInfoFut = - PushCapUtil.getPushcapFromUserHistory( - userId, - pushcapDynamicPredictionStore, - params(FeatureSwitchParams.PushcapModelType), - params(FeatureSwitchParams.PushcapModelPredictionVersion), - pushcapSelectionStat - ) - // Modify the push cap info if there is a restricted min value for predicted push caps. - val restrictedPushcap = params(PushFeatureSwitchParams.RestrictedMinModelPushcap) - originalPushcapInfoFut.map { - case Some(originalPushcapInfo) => - Some( - getMinimumRestrictedPushcapInfo( - restrictedPushcap, - originalPushcapInfo, - pushcapSelectionStat)) - case _ => None - } - } else Future.value(None) - } - - override lazy val targetHydrationContext: Future[HydrationContext] = - HydrationContextBuilder.build(this) - - override lazy val featureMap: Future[FeatureMap] = - targetHydrationContext.flatMap { hydrationContext => - featureHydrator.hydrateTarget( - hydrationContext, - this.params, - this.mrRequestContextForFeatureStore) - } - - override lazy val globalOptoutProbabilities: Seq[Future[Option[Double]]] = { - params(PushFeatureSwitchParams.GlobalOptoutModelParam).map { model_id => - optoutModelScorer - .singlePredictionForTargetLevel(model_id, targetId, featureMap) - } - } - - override lazy val bucketOptoutProbability: Future[Option[Double]] = { - Future - .collect(globalOptoutProbabilities).map { - _.zip(params(PushFeatureSwitchParams.GlobalOptoutThresholdParam)) - .exists { - case (Some(score), threshold) => score >= threshold - case _ => false - } - }.flatMap { - case true => - optoutModelScorer.singlePredictionForTargetLevel( - params(PushFeatureSwitchParams.BucketOptoutModelParam), - targetId, - featureMap) - case _ => Future.None - } - } - - override lazy val optoutAdjustedPushcap: Future[Option[Short]] = { - if (params(PushFeatureSwitchParams.EnableOptoutAdjustedPushcap)) { - bucketOptoutProbability.map { - case Some(score) => - val idx = params(PushFeatureSwitchParams.BucketOptoutSlotThresholdParam) - .indexWhere(score <= _) - if (idx >= 0) { - val pushcap = - params(PushFeatureSwitchParams.BucketOptoutSlotPushcapParam)(idx).toShort - optoutModelStat.scope("adjusted_pushcap").counter(f"$pushcap").incr() - if (pushcap >= 0) Some(pushcap) - else None - } else None - case _ => None - } - } else Future.None - } - - override lazy val seedsWithWeight: Future[Option[Map[Long, Double]]] = { - Future - .join( - realGraphScoresTop500InStore.get(userId), - targetUserState, - targetUser - ) - .flatMap { - case (seedSetOpt, userState, gizmoduckUser) => - val seedSet = seedSetOpt.getOrElse(Map.empty[Long, Double]) - - //If new sign_up or New user, combine recent_follows with real graph seedset - val isNewUserEnabled = { - val isNewerThan7days = customFSField.daysSinceSignup <= 7 - val isNewUserState = userState.contains(UserState.New) - isNewUserState || isNewSignup || isNewerThan7days - } - - val nonSeedSetFollowsFut = gizmoduckUser match { - case Some(user) if isNewUserEnabled => - recentFollowscounter.incr() - recentFollowsStore.get(user.id) - - case Some(user) if this.isModelTrainingData => - recentFollowscounter.incr() - isModelTrainingDataCounter.incr() - recentFollowsStore.get(user.id) - - case _ => Future.None - } - nonSeedSetFollowsFut.map { nonSeedSetFollows => - Some( - SeedsetUtil.combineRecentFollowsWithWeightedSeedset( - seedSet, - nonSeedSetFollows.getOrElse(Nil) - ) - ) - } - } - } - - override def magicFanoutReasonHistory30Days: Future[MagicFanoutReasonHistory] = - history.map(history => MagicFanoutReasonHistory(history)) - - override val isNewSignup: Boolean = - pushContext.flatMap(_.isFromNewUserLoopProcessor).getOrElse(false) - - override lazy val resurrectionDate: Future[Option[String]] = - Future.value(customFSField.reactivationDate) - - override lazy val isResurrectedUser: Boolean = - customFSField.daysSinceReactivation.isDefined - - override lazy val timeSinceResurrection: Option[Duration] = - customFSField.daysSinceReactivation.map(Duration.fromDays) - - override lazy val appPermissions: Future[Option[AppPermission]] = - PushAppPermissionUtil.getAppPermission( - userId, - PushAppPermissionUtil.AddressBookPermissionKey, - deviceInfo, - appPermissionStore) - - override lazy val inlineActionHistory: Future[Seq[(Long, String)]] = { - inlineActionHistoryStore - .get(userId).map { - case Some(sortedInlineActionHistory) => sortedInlineActionHistory - case _ => Seq.empty - } - } - - lazy val isOpenAppExperimentUser: Future[Boolean] = - openAppUserStore.get(userId).map(_.contains(true)) - - override lazy val openedPushByHourAggregated: Future[Option[Map[Int, Int]]] = - openedPushByHourAggregatedStore.get(userId) - - override lazy val places: Future[Seq[Place]] = { - geoduckStoreV2 - .get(targetId) - .map(_.flatMap(_.places)) - .map { - case Some(placeSeq) if placeSeq.nonEmpty => - placeFoundStat.add(placeSeq.size) - placeSeq - case _ => - placesNotFound.incr() - Seq.empty - } - } - - override val isBlueVerified: Future[Option[Boolean]] = - Future.value(userOpt.flatMap(_.safety.flatMap(_.isBlueVerified))) - - override val isVerified: Future[Option[Boolean]] = - Future.value(userOpt.flatMap(_.safety.map(_.verified))) - - override lazy val isSuperFollowCreator: Future[Option[Boolean]] = - superFollowEligibilityUserStore.get(targetId) - } - } - } - } - - /** - * Provide general way to add needed FS for target user, and package them in CustomFSFields. - * Custom Fields is a powerful feature that allows Feature Switch library users to define and - * match against any arbitrary fields. - **/ - private def getCustomFSFields( - userId: Long, - userOpt: Option[User], - deviceInfo: Option[DeviceInfo], - userForPushTargetingInfo: Option[UserForPushTargeting], - notifHistory: History, - requestSource: Option[RequestSource] - ): Future[CustomFSFields] = { - val reactivationDateFutOpt: Future[Option[String]] = resurrectedUserStore.get(userId) - val reactivationTimeFutOpt: Future[Option[Time]] = - reactivationDateFutOpt.map(_.map(dateStr => DateUtil.dateStrToTime(dateStr))) - - val isReactivatedUserFut: Future[Boolean] = reactivationTimeFutOpt.map { timeOpt => - timeOpt - .exists { time => Time.now - time < 30.days } - } - - val daysSinceReactivationFut: Future[Option[Int]] = - reactivationTimeFutOpt.map(_.map(time => Time.now.since(time).inDays)) - - val daysSinceSignup: Int = (Time.now - SnowflakeUtil.timeFromId(userId)).inDays - if (daysSinceSignup < 14) newSignUpUserStats.incr() - - val targetAgeInYears = userOpt.flatMap(_.extendedProfile.flatMap(_.ageInYears)) - - val lastLoginFut: Future[Option[Long]] = - userHTLLastVisitStore.get(userId).map { - case Some(lastHTLVisitTimes) => - val latestHTLVisitTime = lastHTLVisitTimes.max - userForPushTargetingInfo.flatMap( - _.lastActiveOnAppTimestamp - .map(_.max(latestHTLVisitTime)).orElse(Some(latestHTLVisitTime))) - case None => - userForPushTargetingInfo.flatMap(_.lastActiveOnAppTimestamp) - } - - val daysSinceLoginFut = lastLoginFut.map { - _.map { lastLoginTimestamp => - val timeSinceLogin = Time.now - Time.fromMilliseconds(lastLoginTimestamp) - if (timeSinceLogin.inDays > 21) { - dormantUserCount.incr() - } - timeSinceLogin.inDays - } - } - - /* Could add more custom FS here */ - val userNSFWInfoFut: Future[Option[NsfwInfo]] = - nsfwConsumerStore - .get(userId).map(_.map(nsfwUserSegmentation => NsfwInfo(nsfwUserSegmentation))) - - val userStateFut: Future[Option[String]] = userFeatureStore.get(userId).map { userFeaturesOpt => - userFeaturesOpt.flatMap { uFeats => - uFeats.userState.map(uState => uState.name) - } - } - - val mrUserStateFut: Future[Option[String]] = - mrUserStateStore.get(userId).map { mrUserStateOpt => - mrUserStateOpt.flatMap { mrUserState => - mrUserState.userState.map(_.name) - } - } - - Future - .join( - reactivationDateFutOpt, - isReactivatedUserFut, - userStateFut, - mrUserStateFut, - daysSinceLoginFut, - daysSinceReactivationFut, - userNSFWInfoFut - ).map { - case ( - reactivationDate, - isReactivatedUser, - userState, - mrUserState, - daysSinceLogin, - daysSinceReactivation, - userNSFWInfo) => - val numDaysReceivedPushInLast30Days: Int = - notifHistory.history.keys.map(_.inDays).toSet.size - - NsfwPersonalizationUtil.computeNsfwUserStats(userNSFWInfo) - - CustomFSFields( - isReactivatedUser, - daysSinceSignup, - numDaysReceivedPushInLast30Days, - daysSinceLogin, - daysSinceReactivation, - userOpt, - userState, - mrUserState, - reactivationDate, - requestSource.map(_.name), - targetAgeInYears, - userNSFWInfo, - deviceInfo - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.docx new file mode 100644 index 000000000..232974401 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala deleted file mode 100644 index 745e061fb..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/RFPHTargetPredicateGenerator.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.predicate.TargetPromptFeedbackFatiguePredicate -import com.twitter.frigate.common.predicate.TargetUserPredicates -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.predicate.TargetNtabCaretClickFatiguePredicate -import com.twitter.frigate.pushservice.predicate.TargetPredicates -import com.twitter.hermit.predicate.NamedPredicate - -class RFPHTargetPredicateGenerator(implicit statsReceiver: StatsReceiver) { - val predicates: List[NamedPredicate[Target]] = List( - TargetPredicates.magicRecsMinDurationSinceSent(), - TargetPredicates.targetHTLVisitPredicate(), - TargetPredicates.inlineActionFatiguePredicate(), - TargetPredicates.targetFatiguePredicate(), - TargetUserPredicates.secondaryDormantAccountPredicate(), - TargetPredicates.targetValidMobileSDKPredicate, - TargetPredicates.targetPushBitEnabledPredicate, - TargetUserPredicates.targetUserExists(), - TargetPredicates.paramPredicate(PushFeatureSwitchParams.EnablePushRecommendationsParam), - TargetPromptFeedbackFatiguePredicate.responseNoPredicate( - PushParams.EnablePromptFeedbackFatigueResponseNoPredicate, - PushConstants.AcceptableTimeSinceLastNegativeResponse), - TargetPredicates.teamExceptedPredicate(TargetNtabCaretClickFatiguePredicate.apply()), - TargetPredicates.optoutProbPredicate(), - TargetPredicates.webNotifsHoldback() - ) -} - -object RFPHTargetPredicates { - def apply(implicit statsReceiver: StatsReceiver): List[NamedPredicate[Target]] = - new RFPHTargetPredicateGenerator().predicates -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.docx new file mode 100644 index 000000000..ecf38082e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala deleted file mode 100644 index c84af1286..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetAppPermissions.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.util.Future - -trait TargetAppPermissions { - - def appPermissions: Future[Option[AppPermission]] - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.docx new file mode 100644 index 000000000..cc6f3a504 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala deleted file mode 100644 index 2a9c26e8b..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/target/TargetScoringDetails.scala +++ /dev/null @@ -1,121 +0,0 @@ -package com.twitter.frigate.pushservice.target - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.FeatureMap -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.TargetDecider -import com.twitter.frigate.common.candidate.UserDetails -import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue -import com.twitter.frigate.dau_model.thriftscala.DauProbability -import com.twitter.frigate.scribe.thriftscala.SkipModelInfo -import com.twitter.hermit.stp.thriftscala.STPResult -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphFeatures -import com.twitter.util.Future -import com.twitter.util.Time -import com.twitter.frigate.pushservice.params.DeciderKey -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.params.WeightedOpenOrNtabClickModel -import com.twitter.frigate.pushservice.util.PushDeviceUtil -import com.twitter.nrel.hydration.push.HydrationContext -import com.twitter.timelines.configapi.FSParam - -trait TargetScoringDetails { - tuc: TargetUser with TargetDecider with TargetABDecider with UserDetails => - - def stats: StatsReceiver - - /* - * We have 3 types of model training data: - * 1, skip ranker and model predicates - * controlled by decider frigate_notifier_quality_model_training_data - * the data distribution is same to the distribution in ranking - * 2, skip model predicates only - * controlled by decider skip_ml_model_predicate - * the data distribution is same to the distribution in filtering - * 3, no skip, only scribe features - * controlled by decider scribe_model_features - * the data distribution is same to production traffic - * The "miscellaneous" is used to store all misc information for selecting the data offline (e.g., ddg-bucket information) - * */ - lazy val skipModelInfo: Option[SkipModelInfo] = { - val trainingDataDeciderKey = DeciderKey.trainingDataDeciderKey.toString - val skipMlModelPredicateDeciderKey = DeciderKey.skipMlModelPredicateDeciderKey.toString - val scribeModelFeaturesDeciderKey = DeciderKey.scribeModelFeaturesDeciderKey.toString - val miscellaneous = None - - if (isDeciderEnabled(trainingDataDeciderKey, stats, useRandomRecipient = true)) { - Some( - SkipModelInfo( - skipPushOpenPredicate = Some(true), - skipPushRanker = Some(true), - miscellaneous = miscellaneous)) - } else if (isDeciderEnabled(skipMlModelPredicateDeciderKey, stats, useRandomRecipient = true)) { - Some( - SkipModelInfo( - skipPushOpenPredicate = Some(true), - skipPushRanker = Some(false), - miscellaneous = miscellaneous)) - } else if (isDeciderEnabled(scribeModelFeaturesDeciderKey, stats, useRandomRecipient = true)) { - Some(SkipModelInfo(noSkipButScribeFeatures = Some(true), miscellaneous = miscellaneous)) - } else { - Some(SkipModelInfo(miscellaneous = miscellaneous)) - } - } - - lazy val scribeFeatureForRequestScribe = - isDeciderEnabled( - DeciderKey.scribeModelFeaturesForRequestScribe.toString, - stats, - useRandomRecipient = true) - - lazy val rankingModelParam: Future[FSParam[WeightedOpenOrNtabClickModel.ModelNameType]] = - tuc.deviceInfo.map { deviceInfoOpt => - if (PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) && - tuc.params(PushParams.AndroidOnlyRankingExperimentParam)) { - PushFeatureSwitchParams.WeightedOpenOrNtabClickRankingModelForAndroidParam - } else { - PushFeatureSwitchParams.WeightedOpenOrNtabClickRankingModelParam - } - } - - lazy val filteringModelParam: FSParam[WeightedOpenOrNtabClickModel.ModelNameType] = - PushFeatureSwitchParams.WeightedOpenOrNtabClickFilteringModelParam - - def skipMlRanker: Boolean = skipModelInfo.exists(_.skipPushRanker.contains(true)) - - def skipModelPredicate: Boolean = skipModelInfo.exists(_.skipPushOpenPredicate.contains(true)) - - def noSkipButScribeFeatures: Boolean = - skipModelInfo.exists(_.noSkipButScribeFeatures.contains(true)) - - def isModelTrainingData: Boolean = skipMlRanker || skipModelPredicate || noSkipButScribeFeatures - - def scribeFeatureWithoutHydratingNewFeatures: Boolean = - isDeciderEnabled( - DeciderKey.scribeModelFeaturesWithoutHydratingNewFeaturesDeciderKey.toString, - stats, - useRandomRecipient = true - ) - - def targetHydrationContext: Future[HydrationContext] - - def featureMap: Future[FeatureMap] - - def dauProbability: Future[Option[DauProbability]] - - def labeledPushRecsHydrated: Future[Option[UserHistoryValue]] - - def onlineLabeledPushRecs: Future[Option[UserHistoryValue]] - - def realGraphFeatures: Future[Option[RealGraphFeatures]] - - def stpResult: Future[Option[STPResult]] - - def globalOptoutProbabilities: Seq[Future[Option[Double]]] - - def bucketOptoutProbability: Future[Option[Double]] - - val sendTime: Long = Time.now.inMillis -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.docx new file mode 100644 index 000000000..dad4ee39f Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala deleted file mode 100644 index dad918023..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdaptorUtils.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object AdaptorUtils { - def getTweetyPieResults( - tweetIds: Set[Long], - tweetyPieStore: ReadableStore[Long, TweetyPieResult], - ): Future[Map[Long, Option[TweetyPieResult]]] = - FutureOps - .mapCollect(tweetyPieStore.multiGet(tweetIds)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.docx new file mode 100644 index 000000000..4c6b96016 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala deleted file mode 100644 index 1d3ff461e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/AdhocStatsUtil.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.common.base.TweetCandidate -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.ListOfAdhocIdsForStatsTracking - -class AdhocStatsUtil(stats: StatsReceiver) { - - private def getAdhocIds(candidate: PushCandidate): Set[Long] = - candidate.target.params(ListOfAdhocIdsForStatsTracking) - - private def isAdhocTweetCandidate(candidate: PushCandidate): Boolean = { - candidate match { - case tweetCandidate: RawCandidate with TweetCandidate with TweetAuthor => - tweetCandidate.authorId.exists(id => getAdhocIds(candidate).contains(id)) - case _ => false - } - } - - def getCandidateSourceStats(hydratedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - hydratedCandidates.foreach { hydratedCandidate => - if (isAdhocTweetCandidate(hydratedCandidate.candidate)) { - stats.scope("candidate_source").counter(hydratedCandidate.source).incr() - } - } - } - - def getPreRankingFilterStats( - preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]] - ): Unit = { - preRankingFilteredCandidates.foreach { filteredCandidate => - if (isAdhocTweetCandidate(filteredCandidate.candidate)) { - filteredCandidate.result match { - case Invalid(reason) => - stats.scope("preranking_filter").counter(reason.getOrElse("unknown_reason")).incr() - case _ => - } - } - } - } - - def getLightRankingStats(lightRankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - lightRankedCandidates.foreach { lightRankedCandidate => - if (isAdhocTweetCandidate(lightRankedCandidate.candidate)) { - stats.scope("light_ranker").counter("passed_light_ranking").incr() - } - } - } - - def getRankingStats(rankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - rankedCandidates.zipWithIndex.foreach { - case (rankedCandidate, index) => - val rankerStats = stats.scope("heavy_ranker") - if (isAdhocTweetCandidate(rankedCandidate.candidate)) { - rankerStats.counter("ranked_candidates").incr() - rankerStats.stat("rank").add(index.toFloat) - rankedCandidate.candidate.modelScores.map { modelScores => - modelScores.foreach { - case (modelName, score) => - // mutiply score by 1000 to not lose precision while converting to Float - val precisionScore = (score * 100000).toFloat - rankerStats.stat(modelName).add(precisionScore) - } - } - } - } - } - def getReRankingStats(rankedCandidates: Seq[CandidateDetails[PushCandidate]]): Unit = { - rankedCandidates.zipWithIndex.foreach { - case (rankedCandidate, index) => - val rankerStats = stats.scope("re_ranking") - if (isAdhocTweetCandidate(rankedCandidate.candidate)) { - rankerStats.counter("re_ranked_candidates").incr() - rankerStats.stat("re_rank").add(index.toFloat) - } - } - } - - def getTakeCandidateResultStats( - allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]] - ): Unit = { - val takeStats = stats.scope("take_step") - allTakeCandidateResults.foreach { candidateResult => - if (isAdhocTweetCandidate(candidateResult.candidate)) { - candidateResult.result match { - case OK => - takeStats.counter("sent").incr() - case Invalid(reason) => - takeStats.counter(reason.getOrElse("unknown_reason")).incr() - case _ => - takeStats.counter("unknown_filter").incr() - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.docx new file mode 100644 index 000000000..0ce6c8bb6 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala deleted file mode 100644 index 6fa0bc288..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/Candidate2FrigateNotification.scala +++ /dev/null @@ -1,119 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.thriftscala.FrigateNotification -import com.twitter.frigate.thriftscala.NotificationDisplayLocation - -object Candidate2FrigateNotification { - - def getFrigateNotification( - candidate: PushCandidate - )( - implicit statsReceiver: StatsReceiver - ): FrigateNotification = { - candidate match { - - case topicTweetCandidate: PushCandidate with BaseTopicTweetCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - crt = topicTweetCandidate.commonRecType, - tweetId = topicTweetCandidate.tweetId, - scActions = Nil, - authorIdOpt = topicTweetCandidate.authorId, - pushCopyId = topicTweetCandidate.pushCopyId, - ntabCopyId = topicTweetCandidate.ntabCopyId, - simclusterId = None, - semanticCoreEntityIds = topicTweetCandidate.semanticCoreEntityId.map(List(_)), - candidateContent = topicTweetCandidate.content, - trendId = None - ) - - case trendTweetCandidate: PushCandidate with TrendTweetCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - trendTweetCandidate.commonRecType, - trendTweetCandidate.tweetId, - Nil, - trendTweetCandidate.authorId, - trendTweetCandidate.pushCopyId, - trendTweetCandidate.ntabCopyId, - None, - None, - trendTweetCandidate.content, - Some(trendTweetCandidate.trendId) - ) - - case tripTweetCandidate: PushCandidate with OutOfNetworkTweetCandidate with TripCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - crt = tripTweetCandidate.commonRecType, - tweetId = tripTweetCandidate.tweetId, - scActions = Nil, - authorIdOpt = tripTweetCandidate.authorId, - pushCopyId = tripTweetCandidate.pushCopyId, - ntabCopyId = tripTweetCandidate.ntabCopyId, - simclusterId = None, - semanticCoreEntityIds = None, - candidateContent = tripTweetCandidate.content, - trendId = None, - tweetTripDomain = tripTweetCandidate.tripDomain - ) - - case outOfNetworkTweetCandidate: PushCandidate with OutOfNetworkTweetCandidate => - PushAdaptorUtil.getFrigateNotificationForTweet( - crt = outOfNetworkTweetCandidate.commonRecType, - tweetId = outOfNetworkTweetCandidate.tweetId, - scActions = Nil, - authorIdOpt = outOfNetworkTweetCandidate.authorId, - pushCopyId = outOfNetworkTweetCandidate.pushCopyId, - ntabCopyId = outOfNetworkTweetCandidate.ntabCopyId, - simclusterId = None, - semanticCoreEntityIds = None, - candidateContent = outOfNetworkTweetCandidate.content, - trendId = None - ) - - case userCandidate: PushCandidate with UserCandidate with SocialContextActions => - PushAdaptorUtil.getFrigateNotificationForUser( - userCandidate.commonRecType, - userCandidate.userId, - userCandidate.socialContextActions, - userCandidate.pushCopyId, - userCandidate.ntabCopyId - ) - - case userCandidate: PushCandidate with UserCandidate => - PushAdaptorUtil.getFrigateNotificationForUser( - userCandidate.commonRecType, - userCandidate.userId, - Nil, - userCandidate.pushCopyId, - userCandidate.ntabCopyId - ) - - case tweetCandidate: PushCandidate with TweetCandidate with TweetDetails with SocialContextActions => - PushAdaptorUtil.getFrigateNotificationForTweetWithSocialContextActions( - tweetCandidate.commonRecType, - tweetCandidate.tweetId, - tweetCandidate.socialContextActions, - tweetCandidate.authorId, - tweetCandidate.pushCopyId, - tweetCandidate.ntabCopyId, - candidateContent = tweetCandidate.content, - semanticCoreEntityIds = None, - trendId = None - ) - case pushCandidate: PushCandidate => - FrigateNotification( - commonRecommendationType = pushCandidate.commonRecType, - notificationDisplayLocation = NotificationDisplayLocation.PushToMobileDevice, - pushCopyId = pushCandidate.pushCopyId, - ntabCopyId = pushCandidate.ntabCopyId - ) - - case _ => - statsReceiver - .scope(s"${candidate.commonRecType}").counter("frigate_notification_error").incr() - throw new IllegalStateException("Incorrect candidate type when create FrigateNotification") - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.docx new file mode 100644 index 000000000..92c39b257 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala deleted file mode 100644 index 8b737fe67..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateHydrationUtil.scala +++ /dev/null @@ -1,439 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.channels.common.thriftscala.ApiList -import com.twitter.escherbird.common.thriftscala.Domains -import com.twitter.escherbird.metadata.thriftscala.EntityMegadata -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base._ -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.magic_events.thriftscala.FanoutEvent -import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason -import com.twitter.frigate.magic_events.thriftscala.TargetID -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model._ -import com.twitter.frigate.pushservice.model.FanoutReasonEntities -import com.twitter.frigate.pushservice.ml.PushMLModelScorer -import com.twitter.frigate.pushservice.model.candidate.CopyIds -import com.twitter.frigate.pushservice.store.EventRequest -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent} -import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities -import com.twitter.storehaus.FutureOps -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.UserId -import com.twitter.ubs.thriftscala.AudioSpace -import com.twitter.util.Future - -object CandidateHydrationUtil { - - def getAuthorIdFromTweetCandidate(tweetCandidate: TweetCandidate): Option[Long] = { - tweetCandidate match { - case candidate: TweetCandidate with TweetAuthor => - candidate.authorId - case _ => None - } - } - - private def getCandidateAuthorFromUserMap( - tweetCandidate: TweetCandidate, - userMap: Map[Long, User] - ): Option[User] = { - getAuthorIdFromTweetCandidate(tweetCandidate) match { - case Some(id) => - userMap.get(id) - case _ => - None - } - } - - private def getRelationshipMapForInNetworkCandidate( - candidate: RawCandidate with TweetAuthor, - relationshipMap: Map[RelationEdge, Boolean] - ): Map[RelationEdge, Boolean] = { - val relationEdges = - RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(candidate).toSet - relationEdges.map { relationEdge => - (relationEdge, relationshipMap(relationEdge)) - }.toMap - } - - private def getTweetCandidateSocialContextUsers( - candidate: RawCandidate with SocialContextActions, - userMap: Map[Long, User] - ): Map[Long, Option[User]] = { - candidate.socialContextUserIds.map { userId => userId -> userMap.get(userId) }.toMap - } - - type TweetWithSocialContextTraits = TweetCandidate with TweetDetails with SocialContextActions - - def getHydratedCandidateForTweetRetweet( - candidate: RawCandidate with TweetWithSocialContextTraits, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TweetRetweetPushCandidate = { - new TweetRetweetPushCandidate( - candidate = candidate, - socialContextUserMap = Future.value(getTweetCandidateSocialContextUsers(candidate, userMap)), - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds: CopyIds - ) - } - - def getHydratedCandidateForTweetFavorite( - candidate: RawCandidate with TweetWithSocialContextTraits, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TweetFavoritePushCandidate = { - new TweetFavoritePushCandidate( - candidate = candidate, - socialContextUserMap = Future.value(getTweetCandidateSocialContextUsers(candidate, userMap)), - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds = copyIds - ) - } - - def getHydratedCandidateForF1FirstDegreeTweet( - candidate: RawCandidate with F1FirstDegree, - userMap: Map[Long, User], - relationshipMap: Map[RelationEdge, Boolean], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): F1TweetPushCandidate = { - new F1TweetPushCandidate( - candidate = candidate, - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - socialGraphServiceResultMap = - getRelationshipMapForInNetworkCandidate(candidate, relationshipMap), - copyIds = copyIds - ) - } - def getHydratedTopicProofTweetCandidate( - candidate: RawCandidate with TopicProofTweetCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushMLModelScorer: PushMLModelScorer - ): TopicProofTweetPushCandidate = - new TopicProofTweetPushCandidate( - candidate, - getCandidateAuthorFromUserMap(candidate, userMap), - copyIds - ) - - def getHydratedSubscribedSearchTweetCandidate( - candidate: RawCandidate with SubscribedSearchTweetCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushMLModelScorer: PushMLModelScorer - ): SubscribedSearchTweetPushCandidate = - new SubscribedSearchTweetPushCandidate( - candidate, - getCandidateAuthorFromUserMap(candidate, userMap), - copyIds) - - def getHydratedListCandidate( - apiListStore: ReadableStore[Long, ApiList], - candidate: RawCandidate with ListPushCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushMLModelScorer: PushMLModelScorer - ): ListRecommendationPushCandidate = { - new ListRecommendationPushCandidate(apiListStore, candidate, copyIds) - } - - def getHydratedCandidateForOutOfNetworkTweetCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): OutOfNetworkTweetPushCandidate = { - new OutOfNetworkTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TopicCandidate, - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds: CopyIds - ) - } - - def getHydratedCandidateForTripTweetCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, - userMap: Map[Long, User], - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TripTweetPushCandidate = { - new TripTweetPushCandidate( - candidate: RawCandidate with OutOfNetworkTweetCandidate with TripCandidate, - author = Future.value(getCandidateAuthorFromUserMap(candidate, userMap)), - copyIds: CopyIds - ) - } - - def getHydratedCandidateForDiscoverTwitterCandidate( - candidate: RawCandidate with DiscoverTwitterCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): DiscoverTwitterPushCandidate = { - new DiscoverTwitterPushCandidate( - candidate = candidate, - copyIds = copyIds - ) - } - - /** - * /* - * This method can be reusable for hydrating event candidates - **/ - * @param candidate - * @param fanoutMetadataStore - * @param semanticCoreMegadataStore - * @return (hydratedEvent, hydratedFanoutEvent, hydratedSemanticEntityResults, hydratedSemanticCoreMegadata) - */ - private def hydrateMagicFanoutEventCandidate( - candidate: RawCandidate with MagicFanoutEventCandidate, - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata] - ): Future[MagicFanoutEventHydratedInfo] = { - - val fanoutEventFut = fanoutMetadataStore.get((candidate.eventId, candidate.pushId)) - - val semanticEntityForQueries: Seq[SemanticEntityForQuery] = { - val semanticCoreEntityIdQueries = candidate.candidateMagicEventsReasons match { - case magicEventsReasons: Seq[MagicEventsReason] => - magicEventsReasons.map(_.reason).collect { - case TargetID.SemanticCoreID(scInterest) => - SemanticEntityForQuery(domainId = scInterest.domainId, entityId = scInterest.entityId) - } - case _ => Seq.empty - } - val eventEntityQuery = SemanticEntityForQuery( - domainId = Domains.EventsEntityService.value, - entityId = candidate.eventId) - semanticCoreEntityIdQueries :+ eventEntityQuery - } - - val semanticEntityResultsFut = FutureOps.mapCollect( - semanticCoreMegadataStore.multiGet(semanticEntityForQueries.toSet) - ) - - Future - .join(fanoutEventFut, semanticEntityResultsFut).map { - case (fanoutEvent, semanticEntityResults) => - MagicFanoutEventHydratedInfo( - fanoutEvent, - semanticEntityResults - ) - case _ => - throw new IllegalArgumentException( - "event candidate hydration errors" + candidate.frigateNotification.toString) - } - } - - def getHydratedCandidateForMagicFanoutNewsEvent( - candidate: RawCandidate with MagicFanoutNewsEventCandidate, - copyIds: CopyIds, - lexServiceStore: ReadableStore[EventRequest, LiveEvent], - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutNewsEventPushCandidate] = { - val magicFanoutEventHydratedInfoFut = hydrateMagicFanoutEventCandidate( - candidate, - fanoutMetadataStore, - semanticCoreMegadataStore - ) - - lazy val simClusterToEntityMappingFut: Future[Map[Int, Option[SimClustersInferredEntities]]] = - Future.collect { - simClusterToEntityStore.multiGet( - FanoutReasonEntities - .from(candidate.candidateMagicEventsReasons.map(_.reason)).simclusterIds.map( - _.clusterId) - ) - } - - Future - .join( - magicFanoutEventHydratedInfoFut, - simClusterToEntityMappingFut - ).map { - case (magicFanoutEventHydratedInfo, simClusterToEntityMapping) => - new MagicFanoutNewsEventPushCandidate( - candidate = candidate, - copyIds = copyIds, - fanoutEvent = magicFanoutEventHydratedInfo.fanoutEvent, - semanticEntityResults = magicFanoutEventHydratedInfo.semanticEntityResults, - simClusterToEntities = simClusterToEntityMapping, - lexServiceStore = lexServiceStore, - interestsLookupStore = interestsLookupStore, - uttEntityHydrationStore = uttEntityHydrationStore - ) - } - } - - def getHydratedCandidateForMagicFanoutSportsEvent( - candidate: RawCandidate - with MagicFanoutSportsEventCandidate - with MagicFanoutSportsScoreInformation, - copyIds: CopyIds, - lexServiceStore: ReadableStore[EventRequest, LiveEvent], - fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent], - semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata], - interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests], - uttEntityHydrationStore: UttEntityHydrationStore - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutSportsPushCandidate] = { - val magicFanoutEventHydratedInfoFut = hydrateMagicFanoutEventCandidate( - candidate, - fanoutMetadataStore, - semanticCoreMegadataStore - ) - - magicFanoutEventHydratedInfoFut.map { magicFanoutEventHydratedInfo => - new MagicFanoutSportsPushCandidate( - candidate = candidate, - copyIds = copyIds, - fanoutEvent = magicFanoutEventHydratedInfo.fanoutEvent, - semanticEntityResults = magicFanoutEventHydratedInfo.semanticEntityResults, - simClusterToEntities = Map.empty, - lexServiceStore = lexServiceStore, - interestsLookupStore = interestsLookupStore, - uttEntityHydrationStore = uttEntityHydrationStore - ) - } - } - - def getHydratedCandidateForMagicFanoutProductLaunch( - candidate: RawCandidate with MagicFanoutProductLaunchCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutProductLaunchPushCandidate] = - Future.value(new MagicFanoutProductLaunchPushCandidate(candidate, copyIds)) - - def getHydratedCandidateForMagicFanoutCreatorEvent( - candidate: RawCandidate with MagicFanoutCreatorEventCandidate, - safeUserStore: ReadableStore[Long, User], - copyIds: CopyIds, - creatorTweetCountStore: ReadableStore[UserId, Int] - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[MagicFanoutCreatorEventPushCandidate] = { - safeUserStore.get(candidate.creatorId).map { hydratedCreatorUser => - new MagicFanoutCreatorEventPushCandidate( - candidate, - hydratedCreatorUser, - copyIds, - creatorTweetCountStore) - } - } - - def getHydratedCandidateForScheduledSpaceSubscriber( - candidate: RawCandidate with ScheduledSpaceSubscriberCandidate, - safeUserStore: ReadableStore[Long, User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[ScheduledSpaceSubscriberPushCandidate] = { - - candidate.hostId match { - case Some(spaceHostId) => - safeUserStore.get(spaceHostId).map { hydratedHost => - new ScheduledSpaceSubscriberPushCandidate( - candidate = candidate, - hostUser = hydratedHost, - copyIds = copyIds, - audioSpaceStore = audioSpaceStore - ) - } - case _ => - Future.exception( - new IllegalStateException( - "Missing Space Host Id for hydrating ScheduledSpaceSubscriberCandidate")) - } - } - - def getHydratedCandidateForScheduledSpaceSpeaker( - candidate: RawCandidate with ScheduledSpaceSpeakerCandidate, - safeUserStore: ReadableStore[Long, User], - copyIds: CopyIds, - audioSpaceStore: ReadableStore[String, AudioSpace] - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): Future[ScheduledSpaceSpeakerPushCandidate] = { - - candidate.hostId match { - case Some(spaceHostId) => - safeUserStore.get(spaceHostId).map { hydratedHost => - new ScheduledSpaceSpeakerPushCandidate( - candidate = candidate, - hostUser = hydratedHost, - copyIds = copyIds, - audioSpaceStore = audioSpaceStore - ) - } - case _ => - Future.exception( - new RuntimeException( - "Missing Space Host Id for hydrating ScheduledSpaceSpeakerCandidate")) - } - } - - def getHydratedCandidateForTopTweetImpressionsCandidate( - candidate: RawCandidate with TopTweetImpressionsCandidate, - copyIds: CopyIds - )( - implicit stats: StatsReceiver, - pushModelScorer: PushMLModelScorer - ): TopTweetImpressionsPushCandidate = { - new TopTweetImpressionsPushCandidate( - candidate = candidate, - copyIds = copyIds - ) - } - - def isNsfwAccount(user: User, nsfwTokens: Seq[String]): Boolean = { - def hasNsfwToken(str: String): Boolean = nsfwTokens.exists(str.toLowerCase().contains(_)) - - val name = user.profile.map(_.name).getOrElse("") - val screenName = user.profile.map(_.screenName).getOrElse("") - val location = user.profile.map(_.location).getOrElse("") - val description = user.profile.map(_.description).getOrElse("") - val hasNsfwFlag = - user.safety.map(safety => safety.nsfwUser || safety.nsfwAdmin).getOrElse(false) - hasNsfwToken(name) || hasNsfwToken(screenName) || hasNsfwToken(location) || hasNsfwToken( - description) || hasNsfwFlag - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.docx new file mode 100644 index 000000000..72f709d0a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala deleted file mode 100644 index a4802da45..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CandidateUtil.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.SocialContextActions -import com.twitter.frigate.common.base.TargetInfo -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.base.TopicProofTweetCandidate -import com.twitter.frigate.common.base.TweetAuthorDetails -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.params.CrtGroupEnum -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType.TripGeoTweet -import com.twitter.frigate.thriftscala.CommonRecommendationType.TripHqTweet -import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} -import com.twitter.util.Future - -object CandidateUtil { - private val mrTwistlyMetricTags = - Seq(MetricTag.PushOpenOrNtabClick, MetricTag.RequestHealthFilterPushOpenBasedTweetEmbedding) - - def getSocialContextActionsFromCandidate(candidate: RawCandidate): Seq[TSocialContextAction] = { - candidate match { - case candidateWithSocialContex: RawCandidate with SocialContextActions => - candidateWithSocialContex.socialContextActions.map { scAction => - TSocialContextAction( - scAction.userId, - scAction.timestampInMillis, - scAction.tweetId - ) - } - case _ => Seq.empty - } - } - - /** - * Ranking Social Context based on the Real Graph weight - * @param socialContextActions Sequence of Social Context Actions - * @param seedsWithWeight Real Graph map consisting of User ID as key and RG weight as the value - * @param defaultToRecency Boolean to represent if we should use the timestamp of the SC to rank - * @return Returns the ranked sequence of SC Actions - */ - def getRankedSocialContext( - socialContextActions: Seq[SocialContextAction], - seedsWithWeight: Future[Option[Map[Long, Double]]], - defaultToRecency: Boolean - ): Future[Seq[SocialContextAction]] = { - seedsWithWeight.map { - case Some(followingsMap) => - socialContextActions.sortBy { action => -followingsMap.getOrElse(action.userId, 0.0) } - case _ => - if (defaultToRecency) socialContextActions.sortBy(-_.timestampInMillis) - else socialContextActions - } - } - - def shouldApplyHealthQualityFiltersForPrerankingPredicates( - candidate: TweetAuthorDetails with TargetInfo[TargetUser with TargetABDecider] - )( - implicit stats: StatsReceiver - ): Future[Boolean] = { - candidate.tweetAuthor.map { - case Some(user) => - val numFollowers: Double = user.counts.map(_.followers.toDouble).getOrElse(0.0) - numFollowers < candidate.target - .params(PushFeatureSwitchParams.NumFollowerThresholdForHealthAndQualityFiltersPreranking) - case _ => true - } - } - - def shouldApplyHealthQualityFilters( - candidate: PushCandidate - )( - implicit stats: StatsReceiver - ): Boolean = { - val numFollowers = - candidate.numericFeatures.getOrElse("RecTweetAuthor.User.ActiveFollowers", 0.0) - numFollowers < candidate.target - .params(PushFeatureSwitchParams.NumFollowerThresholdForHealthAndQualityFilters) - } - - def useAggressiveHealthThresholds(cand: PushCandidate): Boolean = - isMrTwistlyCandidate(cand) || - (cand.commonRecType == CommonRecommendationType.GeoPopTweet && cand.target.params( - PushFeatureSwitchParams.PopGeoTweetEnableAggressiveThresholds)) - - def isMrTwistlyCandidate(cand: PushCandidate): Boolean = - cand match { - case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => - oonCandidate.tagsCR - .getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty && oonCandidate.tagsCR - .map(_.toSet.size).getOrElse(0) == 1 - case oonCandidate: PushCandidate with TopicProofTweetCandidate - if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => - oonCandidate.tagsCR - .getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty && oonCandidate.tagsCR - .map(_.toSet.size).getOrElse(0) == 1 - case _ => false - } - - def getTagsCRCount(cand: PushCandidate): Double = - cand match { - case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => - oonCandidate.tagsCR.map(_.toSet.size).getOrElse(0).toDouble - case oonCandidate: PushCandidate with TopicProofTweetCandidate - if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => - oonCandidate.tagsCR.map(_.toSet.size).getOrElse(0).toDouble - case _ => 0.0 - } - - def isRelatedToMrTwistlyCandidate(cand: PushCandidate): Boolean = - cand match { - case oonCandidate: PushCandidate with OutOfNetworkTweetCandidate => - oonCandidate.tagsCR.getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty - case oonCandidate: PushCandidate with TopicProofTweetCandidate - if cand.target.params(PushFeatureSwitchParams.EnableHealthFiltersForTopicProofTweet) => - oonCandidate.tagsCR.getOrElse(Seq.empty).intersect(mrTwistlyMetricTags).nonEmpty - case _ => false - } - - def getCrtGroup(commonRecType: CommonRecommendationType): CrtGroupEnum.Value = { - commonRecType match { - case crt if RecTypes.twistlyTweets(crt) => CrtGroupEnum.Twistly - case crt if RecTypes.frsTypes(crt) => CrtGroupEnum.Frs - case crt if RecTypes.f1RecTypes(crt) => CrtGroupEnum.F1 - case crt if crt == TripGeoTweet || crt == TripHqTweet => CrtGroupEnum.Trip - case crt if RecTypes.TopicTweetTypes(crt) => CrtGroupEnum.Topic - case crt if RecTypes.isGeoPopTweetType(crt) => CrtGroupEnum.GeoPop - case _ => CrtGroupEnum.Other - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.docx new file mode 100644 index 000000000..a161d65f4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala deleted file mode 100644 index 95208c35e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/CopyUtil.scala +++ /dev/null @@ -1,448 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FS} -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.util.Future -import com.twitter.util.Time - -object CopyUtil { - - /** - * Get a list of history feature copy alone with metadata in the look back period, the metadata - * can be used to calculate number of copy pushed after the current feature copy - * @param candidate the candidate to be pushed to the user - * @return Future[Seq((..,))], which is a seq of the history FEATURE copy along with - * metadata within the look back period. In the tuple, the 4 elements represents: - * 1. Timestamp of the past feature copy - * 2. Option[Seq()] of copy feature names of the past copy - * 3. Index of the particular feature copy in look back history if normal copy presents - */ - private def getPastCopyFeaturesList( - candidate: PushCandidate - ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { - val target = candidate.target - - target.history.map { targetHistory => - val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) - val notificationHistoryInLookbackDuration = targetHistory.sortedHistory - .takeWhile { - case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp - } - notificationHistoryInLookbackDuration.zipWithIndex - .filter { - case ((_, notification), _) => - notification.copyFeatures match { - case Some(copyFeatures) => copyFeatures.nonEmpty - case _ => false - } - } - .collect { - case ((timestamp, notification), notificationIndex) => - (timestamp, notification.copyFeatures, notificationIndex) - } - } - } - - private def getPastCopyFeaturesListForF1( - candidate: PushCandidate - ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { - val target = candidate.target - target.history.map { targetHistory => - val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) - val notificationHistoryInLookbackDuration = targetHistory.sortedHistory - .takeWhile { - case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp - } - notificationHistoryInLookbackDuration.zipWithIndex - .filter { - case ((_, notification), _) => - notification.copyFeatures match { - case Some(copyFeatures) => - RecTypes.isF1Type(notification.commonRecommendationType) && copyFeatures.nonEmpty - case _ => false - } - } - .collect { - case ((timestamp, notification), notificationIndex) => - (timestamp, notification.copyFeatures, notificationIndex) - } - } - } - - private def getPastCopyFeaturesListForOON( - candidate: PushCandidate - ): Future[Seq[(Time, Option[Seq[String]], Int)]] = { - val target = candidate.target - target.history.map { targetHistory => - val historyLookbackDuration = target.params(FS.CopyFeaturesHistoryLookbackDuration) - val notificationHistoryInLookbackDuration = targetHistory.sortedHistory - .takeWhile { - case (notifTimestamp, _) => historyLookbackDuration.ago < notifTimestamp - } - notificationHistoryInLookbackDuration.zipWithIndex - .filter { - case ((_, notification), _) => - notification.copyFeatures match { - case Some(copyFeatures) => - !RecTypes.isF1Type(notification.commonRecommendationType) && copyFeatures.nonEmpty - - case _ => false - } - } - .collect { - case ((timestamp, notification), notificationIndex) => - (timestamp, notification.copyFeatures, notificationIndex) - } - } - } - private def getEmojiFeaturesMap( - candidate: PushCandidate, - copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], - lastHTLVisitTimestamp: Option[Long], - stats: StatsReceiver - ): Map[String, String] = { - val (emojiFatigueDuration, emojiFatigueNumOfPushes) = { - if (RecTypes.isF1Type(candidate.commonRecType)) { - ( - candidate.target.params(FS.F1EmojiCopyFatigueDuration), - candidate.target.params(FS.F1EmojiCopyNumOfPushesFatigue)) - } else { - ( - candidate.target.params(FS.OonEmojiCopyFatigueDuration), - candidate.target.params(FS.OonEmojiCopyNumOfPushesFatigue)) - } - } - - val scopedStats = stats - .scope("getEmojiFeaturesMap").scope(candidate.commonRecType.toString).scope( - emojiFatigueDuration.toString) - val addedEmojiCopyFeature = scopedStats.counter("added_emoji") - val fatiguedEmojiCopyFeature = scopedStats.counter("no_emoji") - - val copyFeatureType = PushConstants.EmojiFeatureNameForIbis2ModelValues - - val durationFatigueCarryFunc = () => - isUnderDurationFatigue(copyFeatureHistory, copyFeatureType, emojiFatigueDuration) - - val enableHTLBasedFatigueBasicRule = candidate.target.params(FS.EnableHTLBasedFatigueBasicRule) - val minDuration = candidate.target.params(FS.MinFatigueDurationSinceLastHTLVisit) - val lastHTLVisitBasedNonFatigueWindow = - candidate.target.params(FS.LastHTLVisitBasedNonFatigueWindow) - val htlBasedCopyFatigueCarryFunc = () => - isUnderHTLBasedFatigue(lastHTLVisitTimestamp, minDuration, lastHTLVisitBasedNonFatigueWindow) - - val isUnderFatigue = getIsUnderFatigue( - Seq( - (durationFatigueCarryFunc, true), - (htlBasedCopyFatigueCarryFunc, enableHTLBasedFatigueBasicRule), - ), - scopedStats - ) - - if (!isUnderFatigue) { - addedEmojiCopyFeature.incr() - Map(PushConstants.EmojiFeatureNameForIbis2ModelValues -> "true") - } else { - fatiguedEmojiCopyFeature.incr() - Map.empty[String, String] - } - } - - private def getTargetFeaturesMap( - candidate: PushCandidate, - copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], - lastHTLVisitTimestamp: Option[Long], - stats: StatsReceiver - ): Map[String, String] = { - val targetFatigueDuration = { - if (RecTypes.isF1Type(candidate.commonRecType)) { - candidate.target.params(FS.F1TargetCopyFatigueDuration) - } else { - candidate.target.params(FS.OonTargetCopyFatigueDuration) - } - } - - val scopedStats = stats - .scope("getTargetFeaturesMap").scope(candidate.commonRecType.toString).scope( - targetFatigueDuration.toString) - val addedTargetCopyFeature = scopedStats.counter("added_target") - val fatiguedTargetCopyFeature = scopedStats.counter("no_target") - - val featureCopyType = PushConstants.TargetFeatureNameForIbis2ModelValues - val durationFatigueCarryFunc = () => - isUnderDurationFatigue(copyFeatureHistory, featureCopyType, targetFatigueDuration) - - val enableHTLBasedFatigueBasicRule = candidate.target.params(FS.EnableHTLBasedFatigueBasicRule) - val minDuration = candidate.target.params(FS.MinFatigueDurationSinceLastHTLVisit) - val lastHTLVisitBasedNonFatigueWindow = - candidate.target.params(FS.LastHTLVisitBasedNonFatigueWindow) - val htlBasedCopyFatigueCarryFunc = () => - isUnderHTLBasedFatigue(lastHTLVisitTimestamp, minDuration, lastHTLVisitBasedNonFatigueWindow) - - val isUnderFatigue = getIsUnderFatigue( - Seq( - (durationFatigueCarryFunc, true), - (htlBasedCopyFatigueCarryFunc, enableHTLBasedFatigueBasicRule), - ), - scopedStats - ) - - if (!isUnderFatigue) { - addedTargetCopyFeature.incr() - Map(PushConstants.TargetFeatureNameForIbis2ModelValues -> "true") - } else { - - fatiguedTargetCopyFeature.incr() - Map.empty[String, String] - } - } - - type FatigueRuleFlag = Boolean - type FatigueRuleFunc = () => Boolean - - def getIsUnderFatigue( - fatigueRulesWithFlags: Seq[(FatigueRuleFunc, FatigueRuleFlag)], - statsReceiver: StatsReceiver, - ): Boolean = { - val defaultFatigue = true - val finalFatigueRes = - fatigueRulesWithFlags.zipWithIndex.foldLeft(defaultFatigue)( - (fatigueSoFar, fatigueRuleFuncWithFlagAndIndex) => { - val ((fatigueRuleFunc, flag), index) = fatigueRuleFuncWithFlagAndIndex - val funcScopedStats = statsReceiver.scope(s"fatigueFunction${index}") - if (flag) { - val shouldFatigueForTheRule = fatigueRuleFunc() - funcScopedStats.scope(s"eval_${shouldFatigueForTheRule}").counter().incr() - val f = fatigueSoFar && shouldFatigueForTheRule - f - } else { - fatigueSoFar - } - }) - statsReceiver.scope(s"final_fatigue_${finalFatigueRes}").counter().incr() - finalFatigueRes - } - - private def isUnderDurationFatigue( - copyFeatureHistory: Seq[(Time, Option[Seq[String]], Int)], - copyFeatureType: String, - fatigueDuration: com.twitter.util.Duration, - ): Boolean = { - copyFeatureHistory.exists { - case (notifTimestamp, Some(copyFeatures), _) if copyFeatures.contains(copyFeatureType) => - notifTimestamp > fatigueDuration.ago - case _ => false - } - } - - private def isUnderHTLBasedFatigue( - lastHTLVisitTimestamp: Option[Long], - minDurationSinceLastHTLVisit: com.twitter.util.Duration, - lastHTLVisitBasedNonFatigueWindow: com.twitter.util.Duration, - ): Boolean = { - val lastHTLVisit = lastHTLVisitTimestamp.map(t => Time.fromMilliseconds(t)).getOrElse(Time.now) - val first = Time.now < (lastHTLVisit + minDurationSinceLastHTLVisit) - val second = - Time.now > (lastHTLVisit + minDurationSinceLastHTLVisit + lastHTLVisitBasedNonFatigueWindow) - first || second - } - - def getOONCBasedFeature( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Map[String, String]] = { - val target = candidate.target - val metric = stats.scope("getOONCBasedFeature") - if (target.params(FS.EnableOONCBasedCopy)) { - candidate.mrWeightedOpenOrNtabClickRankingProbability.map { - case Some(score) if score >= target.params(FS.HighOONCThresholdForCopy) => - metric.counter("high_OONC").incr() - metric.counter(FS.HighOONCTweetFormat.toString).incr() - Map( - "whole_template" -> JsonMarshal.toJson( - Map( - target.params(FS.HighOONCTweetFormat).toString -> true - ))) - case Some(score) if score <= target.params(FS.LowOONCThresholdForCopy) => - metric.counter("low_OONC").incr() - metric.counter(FS.LowOONCThresholdForCopy.toString).incr() - Map( - "whole_template" -> JsonMarshal.toJson( - Map( - target.params(FS.LowOONCTweetFormat).toString -> true - ))) - case _ => - metric.counter("not_in_OONC_range").incr() - Map.empty[String, String] - } - } else { - Future.value(Map.empty[String, String]) - } - } - - def getCopyFeatures( - candidate: PushCandidate, - stats: StatsReceiver, - ): Future[Map[String, String]] = { - if (candidate.target.isLoggedOutUser) { - Future.value(Map.empty[String, String]) - } else { - val featureMaps = getCopyBodyFeatures(candidate, stats) - for { - titleFeat <- getCopyTitleFeatures(candidate, stats) - nsfwFeat <- getNsfwCopyFeatures(candidate, stats) - ooncBasedFeature <- getOONCBasedFeature(candidate, stats) - } yield { - titleFeat ++ featureMaps ++ nsfwFeat ++ ooncBasedFeature - } - } - } - - private def getCopyTitleFeatures( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Map[String, String]] = { - val scopedStats = stats.scope("CopyUtil").scope("getCopyTitleFeatures") - - val target = candidate.target - - if ((RecTypes.isSimClusterBasedType(candidate.commonRecType) && target.params( - FS.EnableCopyFeaturesForOon)) || (RecTypes.isF1Type(candidate.commonRecType) && target - .params(FS.EnableCopyFeaturesForF1))) { - - val enableTargetAndEmojiSplitFatigue = target.params(FS.EnableTargetAndEmojiSplitFatigue) - val isTargetF1Type = RecTypes.isF1Type(candidate.commonRecType) - - val copyFeatureHistoryFuture = if (enableTargetAndEmojiSplitFatigue && isTargetF1Type) { - getPastCopyFeaturesListForF1(candidate) - } else if (enableTargetAndEmojiSplitFatigue && !isTargetF1Type) { - getPastCopyFeaturesListForOON(candidate) - } else { - getPastCopyFeaturesList(candidate) - } - - Future - .join( - copyFeatureHistoryFuture, - target.lastHTLVisitTimestamp, - ).map { - case (copyFeatureHistory, lastHTLVisitTimestamp) => - val emojiFeatures = { - if ((RecTypes.isF1Type(candidate.commonRecType) && target.params( - FS.EnableEmojiInF1Copy)) - || RecTypes.isSimClusterBasedType(candidate.commonRecType) && target.params( - FS.EnableEmojiInOonCopy)) { - getEmojiFeaturesMap( - candidate, - copyFeatureHistory, - lastHTLVisitTimestamp, - scopedStats) - } else Map.empty[String, String] - } - - val targetFeatures = { - if ((RecTypes.isF1Type(candidate.commonRecType) && target.params( - FS.EnableTargetInF1Copy)) || (RecTypes.isSimClusterBasedType( - candidate.commonRecType) && target.params(FS.EnableTargetInOonCopy))) { - getTargetFeaturesMap( - candidate, - copyFeatureHistory, - lastHTLVisitTimestamp, - scopedStats) - } else Map.empty[String, String] - } - - val baseCopyFeaturesMap = - if (emojiFeatures.nonEmpty || targetFeatures.nonEmpty) - Map(PushConstants.EnableCopyFeaturesForIbis2ModelValues -> "true") - else Map.empty[String, String] - baseCopyFeaturesMap ++ emojiFeatures ++ targetFeatures - case _ => - Map.empty[String, String] - } - } else Future.value(Map.empty[String, String]) - } - - private def getCopyBodyTruncateFeatures( - candidate: PushCandidate, - ): Map[String, String] = { - if (candidate.target.params(FS.EnableIosCopyBodyTruncate)) { - Map("enable_body_truncate_ios" -> "true") - } else { - Map.empty[String, String] - } - } - - private def getNsfwCopyFeatures( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Map[String, String]] = { - val scopedStats = stats.scope("CopyUtil").scope("getNsfwCopyBodyFeatures") - val hasNsfwScoreF1Counter = scopedStats.counter("f1_has_nsfw_score") - val hasNsfwScoreOonCounter = scopedStats.counter("oon_has_nsfw_score") - val noNsfwScoreCounter = scopedStats.counter("no_nsfw_score") - val nsfwScoreF1 = scopedStats.stat("f1_nsfw_score") - val nsfwScoreOon = scopedStats.stat("oon_nsfw_score") - val isNsfwF1Counter = scopedStats.counter("is_f1_nsfw") - val isNsfwOonCounter = scopedStats.counter("is_oon_nsfw") - - val target = candidate.target - val nsfwScoreFut = if (target.params(FS.EnableNsfwCopy)) { - candidate.mrNsfwScore - } else Future.None - - nsfwScoreFut.map { - case Some(nsfwScore) => - if (RecTypes.isF1Type(candidate.commonRecType)) { - hasNsfwScoreF1Counter.incr() - nsfwScoreF1.add(nsfwScore.toFloat * 10000) - if (nsfwScore > target.params(FS.NsfwScoreThresholdForF1Copy)) { - isNsfwF1Counter.incr() - Map("is_f1_nsfw" -> "true") - } else { - Map.empty[String, String] - } - } else if (RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType)) { - nsfwScoreOon.add(nsfwScore.toFloat * 10000) - hasNsfwScoreOonCounter.incr() - if (nsfwScore > target.params(FS.NsfwScoreThresholdForOONCopy)) { - isNsfwOonCounter.incr() - Map("is_oon_nsfw" -> "true") - } else { - Map.empty[String, String] - } - } else { - Map.empty[String, String] - } - case _ => - noNsfwScoreCounter.incr() - Map.empty[String, String] - } - } - - private def getCopyBodyFeatures( - candidate: PushCandidate, - stats: StatsReceiver - ): Map[String, String] = { - val target = candidate.target - val scopedStats = stats.scope("CopyUtil").scope("getCopyBodyFeatures") - - val copyBodyFeatures = { - if (RecTypes.isF1Type(candidate.commonRecType) && target.params(FS.EnableF1CopyBody)) { - scopedStats.counter("f1BodyExpEnabled").incr() - Map(PushConstants.CopyBodyExpIbisModelValues -> "true") - } else if (RecTypes.isOutOfNetworkTweetRecType(candidate.commonRecType) && target.params( - FS.EnableOONCopyBody)) { - scopedStats.counter("oonBodyExpEnabled").incr() - Map(PushConstants.CopyBodyExpIbisModelValues -> "true") - } else - Map.empty[String, String] - } - val copyBodyTruncateFeatures = getCopyBodyTruncateFeatures(candidate) - copyBodyFeatures ++ copyBodyTruncateFeatures - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.docx new file mode 100644 index 000000000..62e73e955 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala deleted file mode 100644 index 34d5b9bae..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/EmailLandingPageExperimentUtil.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableRuxLandingPage -import com.twitter.frigate.pushservice.params.PushParams.EnableRuxLandingPageAndroidParam -import com.twitter.frigate.pushservice.params.PushParams.EnableRuxLandingPageIOSParam -import com.twitter.frigate.pushservice.params.PushParams.RuxLandingPageExperimentKeyAndroidParam -import com.twitter.frigate.pushservice.params.PushParams.RuxLandingPageExperimentKeyIOSParam -import com.twitter.frigate.pushservice.params.PushParams.ShowRuxLandingPageAsModalOnIOS -import com.twitter.rux.common.context.thriftscala.MagicRecsNTabTweet -import com.twitter.rux.common.context.thriftscala.MagicRecsPushTweet -import com.twitter.rux.common.context.thriftscala.RuxContext -import com.twitter.rux.common.context.thriftscala.Source -import com.twitter.rux.common.encode.RuxContextEncoder - -/** - * This class provides utility functions for email landing page for push - */ -object EmailLandingPageExperimentUtil { - val ruxCxtEncoder = new RuxContextEncoder() - - def getIbis2ModelValue( - deviceInfoOpt: Option[DeviceInfo], - target: Target, - tweetId: Long - ): Map[String, String] = { - val enable = enablePushEmailLanding(deviceInfoOpt, target) - if (enable) { - val ruxCxt = if (deviceInfoOpt.exists(_.isRuxLandingPageEligible)) { - val encodedCxt = getRuxContext(tweetId, target, deviceInfoOpt) - Map("rux_cxt" -> encodedCxt) - } else Map.empty[String, String] - val enableModal = if (showModalForIOS(deviceInfoOpt, target)) { - Map("enable_modal" -> "true") - } else Map.empty[String, String] - - Map("land_on_email_landing_page" -> "true") ++ ruxCxt ++ enableModal - } else Map.empty[String, String] - } - - def createNTabRuxLandingURI(screenName: String, tweetId: Long): String = { - val encodedCxt = - ruxCxtEncoder.encode(RuxContext(Some(Source.MagicRecsNTabTweet(MagicRecsNTabTweet(tweetId))))) - s"$screenName/status/${tweetId.toString}?cxt=$encodedCxt" - } - - private def getRuxContext( - tweetId: Long, - target: Target, - deviceInfoOpt: Option[DeviceInfo] - ): String = { - val isDeviceIOS = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - val isDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - val keyOpt = if (isDeviceIOS) { - target.params(RuxLandingPageExperimentKeyIOSParam) - } else if (isDeviceAndroid) { - target.params(RuxLandingPageExperimentKeyAndroidParam) - } else None - val context = RuxContext(Some(Source.MagicRecsTweet(MagicRecsPushTweet(tweetId))), None, keyOpt) - ruxCxtEncoder.encode(context) - } - - private def enablePushEmailLanding( - deviceInfoOpt: Option[DeviceInfo], - target: Target - ): Boolean = - deviceInfoOpt.exists(deviceInfo => - if (deviceInfo.isEmailLandingPageEligible) { - val isRuxLandingPageEnabled = target.params(EnableRuxLandingPage) - isRuxLandingPageEnabled && isRuxLandingEnabledBasedOnDeviceInfo(deviceInfoOpt, target) - } else false) - - private def showModalForIOS(deviceInfoOpt: Option[DeviceInfo], target: Target): Boolean = { - deviceInfoOpt.exists { deviceInfo => - deviceInfo.isRuxLandingPageAsModalEligible && target.params(ShowRuxLandingPageAsModalOnIOS) - } - } - - private def isRuxLandingEnabledBasedOnDeviceInfo( - deviceInfoOpt: Option[DeviceInfo], - target: Target - ): Boolean = { - val isDeviceIOS = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - val isDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - if (isDeviceIOS) { - target.params(EnableRuxLandingPageIOSParam) - } else if (isDeviceAndroid) { - target.params(EnableRuxLandingPageAndroidParam) - } else true - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.docx new file mode 100644 index 000000000..bf5ec84b9 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala deleted file mode 100644 index a2721873e..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/FunctionalUtil.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.Counter - -object FunctionalUtil { - def incr[T](counter: Counter): T => T = { x => - { - counter.incr() - x - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.docx new file mode 100644 index 000000000..71cabc82e Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala deleted file mode 100644 index de1108f56..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/IbisScribeTargets.scala +++ /dev/null @@ -1,55 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType._ - -object IbisScribeTargets { - val User2 = "magic_rec_user_2" - val User4 = "magic_rec_user_4" - val Tweet2 = "magic_rec_tweet_2" - val Tweet4 = "magic_rec_tweet_4" - val Tweet5 = "magic_rec_tweet_5" - val Tweet9 = "magic_rec_tweet_9" - val Tweet10 = "magic_rec_tweet_10" - val Tweet11 = "magic_rec_tweet_11" - val Tweet12 = "magic_rec_tweet_12" - val Tweet16 = "magic_rec_tweet_16" - val Hashtag = "magic_rec_hashtag" - val UnreadBadgeCount17 = "magic_rec_unread_badge_count_17" - val Highlights = "highlights" - val TweetAnalytics = "magic_rec_tweet_analytics" - val Untracked = "untracked" - - def crtToScribeTarget(crt: CommonRecommendationType): String = crt match { - case UserFollow => - User2 - case HermitUser => - User4 - case TweetRetweet | TweetFavorite => - Tweet2 - case TweetRetweetPhoto | TweetFavoritePhoto => - Tweet4 - case TweetRetweetVideo | TweetFavoriteVideo => - Tweet5 - case UrlTweetLanding => - Tweet9 - case F1FirstdegreeTweet | F1FirstdegreePhoto | F1FirstdegreeVideo => - Tweet10 - case AuthorTargetingTweet => - Tweet11 - case PeriscopeShare => - Tweet12 - case CommonRecommendationType.Highlights => - Highlights - case HashtagTweet | HashtagTweetRetweet => - Hashtag - case PinnedTweet => - Tweet16 - case UnreadBadgeCount => - UnreadBadgeCount17 - case TweetImpressions => - TweetAnalytics - case _ => - Untracked - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.docx new file mode 100644 index 000000000..6e48d9bde Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala deleted file mode 100644 index 2900b7418..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/InlineActionUtil.scala +++ /dev/null @@ -1,219 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.google.common.io.BaseEncoding -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.InlineActionsEnum -import com.twitter.frigate.pushservice.params.PushParams -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.ibis2.lib.util.JsonMarshal -import com.twitter.notifications.platform.thriftscala._ -import com.twitter.notificationservice.thriftscala.CreateGenericNotificationResponse -import com.twitter.scrooge.BinaryThriftStructSerializer -import com.twitter.util.Future - -/** - * This class provides utility functions for inline action for push - */ -object InlineActionUtil { - - def scopedStats(statsReceiver: StatsReceiver): StatsReceiver = - statsReceiver.scope(getClass.getSimpleName) - - /** - * Util function to build web inline actions for Ibis - * @param actions list of inline actions to be hydrated depending on the CRT - * @param enableForDesktopWeb if web inline actions should be shown on desktop RWeb, for experimentation purpose - * @param enableForMobileWeb if web inline actions should be shwon on mobile RWeb, for experimentation purpose - * @return Params for web inline actions to be consumed by `smart.inline.actions.web.mustache` in Ibis - */ - def getGeneratedTweetInlineActionsForWeb( - actions: Seq[InlineActionsEnum.Value], - enableForDesktopWeb: Boolean, - enableForMobileWeb: Boolean - ): Map[String, String] = { - if (!enableForDesktopWeb && !enableForMobileWeb) { - Map.empty - } else { - val inlineActions = buildEnrichedInlineActionsMap(actions) ++ Map( - "enable_for_desktop_web" -> enableForDesktopWeb.toString, - "enable_for_mobile_web" -> enableForMobileWeb.toString - ) - Map( - "inline_action_details_web" -> JsonMarshal.toJson(inlineActions), - ) - } - } - - def getGeneratedTweetInlineActionsV1( - actions: Seq[InlineActionsEnum.Value] - ): Map[String, String] = { - val inlineActions = buildEnrichedInlineActionsMap(actions) - Map( - "inline_action_details" -> JsonMarshal.toJson(inlineActions) - ) - } - - private def buildEnrichedInlineActionsMap( - actions: Seq[InlineActionsEnum.Value] - ): Map[String, Seq[Map[String, Any]]] = { - Map( - "actions" -> actions - .map(_.toString.toLowerCase) - .zipWithIndex - .map { - case (a: String, i: Int) => - Map("action" -> a) ++ Map( - s"use_${a}_stringcenter_key" -> true, - "last" -> (i == (actions.length - 1)) - ) - }.seq - ) - } - - def getGeneratedTweetInlineActionsV2( - actions: Seq[InlineActionsEnum.Value] - ): Map[String, String] = { - val v2CustomActions = actions - .map { - case InlineActionsEnum.Favorite => - NotificationCustomAction( - Some("mr_inline_favorite_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Favorite)) - ) - case InlineActionsEnum.Follow => - NotificationCustomAction( - Some("mr_inline_follow_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Follow))) - case InlineActionsEnum.Reply => - NotificationCustomAction( - Some("mr_inline_reply_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Reply))) - case InlineActionsEnum.Retweet => - NotificationCustomAction( - Some("mr_inline_retweet_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Retweet))) - case _ => - NotificationCustomAction( - Some("mr_inline_favorite_title"), - CustomActionData.LegacyAction(LegacyAction(ActionIdentifier.Favorite)) - ) - } - val notifications = NotificationCustomActions(v2CustomActions) - Map("serialized_inline_actions_v2" -> serializeActionsToBase64(notifications)) - } - - def getDislikeInlineAction( - candidate: PushCandidate, - ntabResponse: CreateGenericNotificationResponse - ): Option[NotificationCustomAction] = { - ntabResponse.successKey.map(successKey => { - val urlParams = Map[String, String]( - "answer" -> "dislike", - "notification_hash" -> successKey.hashKey.toString, - "upstream_uid" -> candidate.impressionId, - "notification_timestamp" -> successKey.timestampMillis.toString - ) - val urlParamsString = urlParams.map(kvp => f"${kvp._1}=${kvp._2}").mkString("&") - - val httpPostRequest = HttpRequest.PostRequest( - PostRequest(url = f"/2/notifications/feedback.json?$urlParamsString", bodyParams = None)) - val httpRequestAction = HttpRequestAction( - httpRequest = httpPostRequest, - scribeAction = Option("dislike_scribe_action"), - isAuthorizationRequired = Option(true), - isDestructive = Option(false), - undoable = None - ) - val dislikeAction = CustomActionData.HttpRequestAction(httpRequestAction) - NotificationCustomAction(title = Option("mr_inline_dislike_title"), action = dislikeAction) - }) - } - - /** - * Given a serialized inline action v2, update the action at index to the given new action. - * If given index is bigger than current action length, append the given inline action at the end. - * @param serialized_inline_actions_v2 the original action in serialized version - * @param actionOption an Option of the new action to replace the old one - * @param index the position where the old action will be replaced - * @return a new serialized inline action v2 - */ - def patchInlineActionAtPosition( - serialized_inline_actions_v2: String, - actionOption: Option[NotificationCustomAction], - index: Int - ): String = { - val originalActions: Seq[NotificationCustomAction] = deserializeActionsFromString( - serialized_inline_actions_v2).actions - val newActions = actionOption match { - case Some(action) if index >= originalActions.size => originalActions ++ Seq(action) - case Some(action) => originalActions.updated(index, action) - case _ => originalActions - } - serializeActionsToBase64(NotificationCustomActions(newActions)) - } - - /** - * Return list of available inline actions for ibis2 model - */ - def getGeneratedTweetInlineActions( - target: Target, - statsReceiver: StatsReceiver, - actions: Seq[InlineActionsEnum.Value], - ): Map[String, String] = { - val scopedStatsReceiver = scopedStats(statsReceiver) - val useV1 = target.params(PushFeatureSwitchParams.UseInlineActionsV1) - val useV2 = target.params(PushFeatureSwitchParams.UseInlineActionsV2) - if (useV1 && useV2) { - scopedStatsReceiver.counter("use_v1_and_use_v2").incr() - getGeneratedTweetInlineActionsV1(actions) ++ getGeneratedTweetInlineActionsV2(actions) - } else if (useV1 && !useV2) { - scopedStatsReceiver.counter("only_use_v1").incr() - getGeneratedTweetInlineActionsV1(actions) - } else if (!useV1 && useV2) { - scopedStatsReceiver.counter("only_use_v2").incr() - getGeneratedTweetInlineActionsV2(actions) - } else { - scopedStatsReceiver.counter("use_neither_v1_nor_v2").incr() - Map.empty[String, String] - } - } - - /** - * Return Tweet inline action ibis2 model values after applying experiment logic - */ - def getTweetInlineActionValue(target: Target): Future[Map[String, String]] = { - if (target.isLoggedOutUser) { - Future( - Map( - "show_inline_action" -> "false" - ) - ) - } else { - val showInlineAction: Boolean = target.params(PushParams.MRAndroidInlineActionOnPushCopyParam) - Future( - Map( - "show_inline_action" -> s"$showInlineAction" - ) - ) - } - } - - private val binaryThriftStructSerializer: BinaryThriftStructSerializer[ - NotificationCustomActions - ] = BinaryThriftStructSerializer.apply(NotificationCustomActions) - private val base64Encoding = BaseEncoding.base64() - - def serializeActionsToBase64(notificationCustomActions: NotificationCustomActions): String = { - val actionsAsByteArray: Array[Byte] = - binaryThriftStructSerializer.toBytes(notificationCustomActions) - base64Encoding.encode(actionsAsByteArray) - } - - def deserializeActionsFromString(serializedInlineActionV2: String): NotificationCustomActions = { - val actionAsByteArray = base64Encoding.decode(serializedInlineActionV2) - binaryThriftStructSerializer.fromBytes(actionAsByteArray) - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.docx new file mode 100644 index 000000000..bea25ea1c Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala deleted file mode 100644 index a3a3ecf50..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MediaAnnotationsUtil.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate - -object MediaAnnotationsUtil { - - val mediaIdToCategoryMapping = Map("0" -> "0") - - val nudityCategoryId = "0" - val beautyCategoryId = "0" - val singlePersonCategoryId = "0" - val sensitiveMediaCategoryFeatureName = - "tweet.mediaunderstanding.tweet_annotations.sensitive_category_probabilities" - - def updateMediaCategoryStats( - candidates: Seq[CandidateDetails[PushCandidate]] - )( - implicit statsReceiver: StatsReceiver - ) = { - - val statScope = statsReceiver.scope("mediaStats") - val filteredCandidates = candidates.filter { candidate => - !candidate.candidate.sparseContinuousFeatures - .getOrElse(sensitiveMediaCategoryFeatureName, Map.empty[String, Double]).contains( - nudityCategoryId) - } - - if (filteredCandidates.isEmpty) - statScope.counter("emptyCandidateListAfterNudityFilter").incr() - else - statScope.counter("nonEmptyCandidateListAfterNudityFilter").incr() - candidates.foreach { candidate => - statScope.counter("totalCandidates").incr() - val mediaFeature = candidate.candidate.sparseContinuousFeatures - .getOrElse(sensitiveMediaCategoryFeatureName, Map.empty[String, Double]) - if (mediaFeature.nonEmpty) { - val mediaCategoryByMaxScore = mediaFeature.maxBy(_._2)._1 - statScope - .scope("mediaCategoryByMaxScore").counter(mediaIdToCategoryMapping - .getOrElse(mediaCategoryByMaxScore, "undefined")).incr() - - mediaFeature.keys.map { feature => - statScope - .scope("mediaCategory").counter(mediaIdToCategoryMapping - .getOrElse(feature, "undefined")).incr() - } - } - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.docx new file mode 100644 index 000000000..6ddd9bf63 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala deleted file mode 100644 index e96ecb817..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MinDurationModifierCalculator.scala +++ /dev/null @@ -1,187 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.TimeUtil -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.util.Future -import com.twitter.util.Time -import java.util.Calendar -import java.util.TimeZone - -case class MinDurationModifierCalculator() { - - private def mapCountryCodeToTimeZone( - countryCode: String, - stats: StatsReceiver - ): Option[Calendar] = { - PushConstants.countryCodeToTimeZoneMap - .get(countryCode.toUpperCase).map(timezone => - Calendar.getInstance(TimeZone.getTimeZone(timezone))) - } - - private def transformToHour( - dayOfHour: Int - ): Int = { - if (dayOfHour < 0) dayOfHour + 24 - else dayOfHour - } - - private def getMinDurationByHourOfDay( - hourOfDay: Int, - startTimeList: Seq[Int], - endTimeList: Seq[Int], - minDurationTimeModifierConst: Seq[Int], - stats: StatsReceiver - ): Option[Int] = { - val scopedStats = stats.scope("getMinDurationByHourOfDay") - scopedStats.counter("request").incr() - val durationOpt = (startTimeList, endTimeList, minDurationTimeModifierConst).zipped.toList - .filter { - case (startTime, endTime, _) => - if (startTime <= endTime) hourOfDay >= startTime && hourOfDay < endTime - else (hourOfDay >= startTime) || hourOfDay < endTime - case _ => false - }.map { - case (_, _, modifier) => modifier - }.headOption - durationOpt match { - case Some(duration) => scopedStats.counter(s"$duration.minutes").incr() - case _ => scopedStats.counter("none").incr() - } - durationOpt - } - - def getMinDurationModifier( - target: Target, - calendar: Calendar, - stats: StatsReceiver - ): Option[Int] = { - val startTimeList = target.params(FSParams.MinDurationModifierStartHourList) - val endTimeList = target.params(FSParams.MinDurationModifierEndHourList) - val minDurationTimeModifierConst = target.params(FSParams.MinDurationTimeModifierConst) - if (startTimeList.length != endTimeList.length || minDurationTimeModifierConst.length != startTimeList.length) { - None - } else { - val hourOfDay = calendar.get(Calendar.HOUR_OF_DAY) - getMinDurationByHourOfDay( - hourOfDay, - startTimeList, - endTimeList, - minDurationTimeModifierConst, - stats) - } - } - - def getMinDurationModifier( - target: Target, - countryCodeOpt: Option[String], - stats: StatsReceiver - ): Option[Int] = { - val scopedStats = stats - .scope("getMinDurationModifier") - scopedStats.counter("total_requests").incr() - - countryCodeOpt match { - case Some(countryCode) => - scopedStats - .counter("country_code_exists").incr() - val calendarOpt = mapCountryCodeToTimeZone(countryCode, scopedStats) - calendarOpt.flatMap(calendar => getMinDurationModifier(target, calendar, scopedStats)) - case _ => None - } - } - - def getMinDurationModifier(target: Target, stats: StatsReceiver): Future[Option[Int]] = { - val scopedStats = stats - .scope("getMinDurationModifier") - scopedStats.counter("total_requests").incr() - - val startTimeList = target.params(FSParams.MinDurationModifierStartHourList) - val endTimeList = target.params(FSParams.MinDurationModifierEndHourList) - val minDurationTimeModifierConst = target.params(FSParams.MinDurationTimeModifierConst) - if (startTimeList.length != endTimeList.length || minDurationTimeModifierConst.length != startTimeList.length) { - Future.value(None) - } else { - target.localTimeInHHMM.map { - case (hourOfDay, _) => - getMinDurationByHourOfDay( - hourOfDay, - startTimeList, - endTimeList, - minDurationTimeModifierConst, - scopedStats) - case _ => None - } - } - } - - def getMinDurationModifierByUserOpenedHistory( - target: Target, - openedPushByHourAggregatedOpt: Option[Map[Int, Int]], - stats: StatsReceiver - ): Option[Int] = { - val scopedStats = stats - .scope("getMinDurationModifierByUserOpenedHistory") - scopedStats.counter("total_requests").incr() - openedPushByHourAggregatedOpt match { - case Some(openedPushByHourAggregated) => - if (openedPushByHourAggregated.isEmpty) { - scopedStats.counter("openedPushByHourAggregated_empty").incr() - None - } else { - val currentUTCHour = TimeUtil.hourOfDay(Time.now) - val utcHourWithMaxOpened = if (target.params(FSParams.EnableRandomHourForQuickSend)) { - (target.targetId % 24).toInt - } else { - openedPushByHourAggregated.maxBy(_._2)._1 - } - val numOfMaxOpened = openedPushByHourAggregated.maxBy(_._2)._2 - if (numOfMaxOpened >= target.params(FSParams.SendTimeByUserHistoryMaxOpenedThreshold)) { - scopedStats.counter("pass_experiment_bucket_threshold").incr() - if (numOfMaxOpened >= target - .params(FSParams.SendTimeByUserHistoryMaxOpenedThreshold)) { // only update if number of opened pushes meet threshold - scopedStats.counter("pass_max_threshold").incr() - val quickSendBeforeHours = - target.params(FSParams.SendTimeByUserHistoryQuickSendBeforeHours) - val quickSendAfterHours = - target.params(FSParams.SendTimeByUserHistoryQuickSendAfterHours) - - val hoursToLessSend = target.params(FSParams.SendTimeByUserHistoryNoSendsHours) - - val quickSendTimeMinDurationInMinute = - target.params(FSParams.SendTimeByUserHistoryQuickSendMinDurationInMinute) - val noSendTimeMinDuration = - target.params(FSParams.SendTimeByUserHistoryNoSendMinDuration) - - val startTimeForNoSend = transformToHour( - utcHourWithMaxOpened - quickSendBeforeHours - hoursToLessSend) - val startTimeForQuickSend = transformToHour( - utcHourWithMaxOpened - quickSendBeforeHours) - val endTimeForNoSend = - transformToHour(utcHourWithMaxOpened - quickSendBeforeHours) - val endTimeForQuickSend = - transformToHour(utcHourWithMaxOpened + quickSendAfterHours) + 1 - - val startTimeList = Seq(startTimeForNoSend, startTimeForQuickSend) - val endTimeList = Seq(endTimeForNoSend, endTimeForQuickSend) - val minDurationTimeModifierConst = - Seq(noSendTimeMinDuration, quickSendTimeMinDurationInMinute) - - getMinDurationByHourOfDay( - currentUTCHour, - startTimeList, - endTimeList, - minDurationTimeModifierConst, - scopedStats) - - } else None - } else None - } - case _ => - None - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.docx new file mode 100644 index 000000000..34f037d54 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala deleted file mode 100644 index 33333c4c4..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/MrUserStateUtil.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser - -object MrUserStateUtil { - def updateMrUserStateStats(target: TargetUser)(implicit statsReceiver: StatsReceiver) = { - statsReceiver.counter("AllUserStates").incr() - target.targetMrUserState.map { - case Some(state) => - statsReceiver.counter(state.name).incr() - case _ => - statsReceiver.counter("UnknownUserState").incr() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.docx new file mode 100644 index 000000000..278a8cfbf Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala deleted file mode 100644 index 01c8d0a72..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/NsfwPersonalizationUtil.scala +++ /dev/null @@ -1,126 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.scio.nsfw_user_segmentation.thriftscala.NSFWUserSegmentation - -object NsfwPersonalizationUtil { - def computeNsfwUserStats( - targetNsfwInfo: Option[NsfwInfo] - )( - implicit statsReceiver: StatsReceiver - ): Unit = { - - def computeNsfwProfileVisitStats(sReceiver: StatsReceiver, nsfwProfileVisits: Long): Unit = { - if (nsfwProfileVisits >= 1) - sReceiver.counter("nsfwProfileVisits_gt_1").incr() - if (nsfwProfileVisits >= 2) - sReceiver.counter("nsfwProfileVisits_gt_2").incr() - if (nsfwProfileVisits >= 3) - sReceiver.counter("nsfwProfileVisits_gt_3").incr() - if (nsfwProfileVisits >= 5) - sReceiver.counter("nsfwProfileVisits_gt_5").incr() - if (nsfwProfileVisits >= 8) - sReceiver.counter("nsfwProfileVisits_gt_8").incr() - } - - def computeRatioStats( - sReceiver: StatsReceiver, - ratio: Int, - statName: String, - intervals: List[Double] = List(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9) - ): Unit = { - intervals.foreach { i => - if (ratio > i * 10000) - sReceiver.counter(f"${statName}_greater_than_${i}").incr() - } - } - val sReceiver = statsReceiver.scope("nsfw_personalization") - sReceiver.counter("AllUsers").incr() - - (targetNsfwInfo) match { - case (Some(nsfwInfo)) => - val sensitive = nsfwInfo.senstiveOptIn.getOrElse(false) - val nsfwFollowRatio = - nsfwInfo.nsfwFollowRatio - val totalFollows = nsfwInfo.totalFollowCount - val numNsfwProfileVisits = nsfwInfo.nsfwProfileVisits - val nsfwRealGraphScore = nsfwInfo.realGraphScore - val nsfwSearchScore = nsfwInfo.searchNsfwScore - val totalSearches = nsfwInfo.totalSearches - val realGraphScore = nsfwInfo.realGraphScore - val searchScore = nsfwInfo.searchNsfwScore - - if (sensitive) - sReceiver.counter("sensitiveOptInEnabled").incr() - else - sReceiver.counter("sensitiveOptInDisabled").incr() - - computeRatioStats(sReceiver, nsfwFollowRatio, "nsfwRatio") - computeNsfwProfileVisitStats(sReceiver, numNsfwProfileVisits) - computeRatioStats(sReceiver, nsfwRealGraphScore.toInt, "nsfwRealGraphScore") - - if (totalSearches >= 10) - computeRatioStats(sReceiver, nsfwSearchScore.toInt, "nsfwSearchScore") - if (searchScore == 0) - sReceiver.counter("lowSearchScore").incr() - if (realGraphScore < 500) - sReceiver.counter("lowRealScore").incr() - if (numNsfwProfileVisits == 0) - sReceiver.counter("lowProfileVisit").incr() - if (nsfwFollowRatio == 0) - sReceiver.counter("lowFollowScore").incr() - - if (totalSearches > 10 && searchScore > 5000) - sReceiver.counter("highSearchScore").incr() - if (realGraphScore > 7000) - sReceiver.counter("highRealScore").incr() - if (numNsfwProfileVisits > 5) - sReceiver.counter("highProfileVisit").incr() - if (totalFollows > 10 && nsfwFollowRatio > 7000) - sReceiver.counter("highFollowScore").incr() - - if (searchScore == 0 && realGraphScore <= 500 && numNsfwProfileVisits == 0 && nsfwFollowRatio == 0) - sReceiver.counter("lowIntent").incr() - if ((totalSearches > 10 && searchScore > 5000) || realGraphScore > 7000 || numNsfwProfileVisits > 5 || (totalFollows > 10 && nsfwFollowRatio > 7000)) - sReceiver.counter("highIntent").incr() - case _ => - } - } -} - -case class NsfwInfo(nsfwUserSegmentation: NSFWUserSegmentation) { - - val scalingFactor = 10000 // to convert float to int as custom fields cannot be float - val senstiveOptIn: Option[Boolean] = nsfwUserSegmentation.nsfwView - val totalFollowCount: Long = nsfwUserSegmentation.totalFollowCnt.getOrElse(0L) - val nsfwFollowCnt: Long = - nsfwUserSegmentation.nsfwAdminOrHighprecOrAgathaGtP98FollowsCnt.getOrElse(0L) - val nsfwFollowRatio: Int = { - if (totalFollowCount != 0) { - (nsfwFollowCnt * scalingFactor / totalFollowCount).toInt - } else 0 - } - val nsfwProfileVisits: Long = - nsfwUserSegmentation.nsfwAdminOrHighPrecOrAgathaGtP98Visits - .map(_.numProfilesInLast14Days).getOrElse(0L) - val realGraphScore: Int = - nsfwUserSegmentation.realGraphMetrics - .map { rm => - if (rm.totalOutboundRGScore != 0) - rm.totalNsfwAdmHPAgthGtP98OutboundRGScore * scalingFactor / rm.totalOutboundRGScore - else 0d - }.getOrElse(0d).toInt - val totalSearches: Long = - nsfwUserSegmentation.searchMetrics.map(_.numNonTrndSrchInLast14Days).getOrElse(0L) - val searchNsfwScore: Int = nsfwUserSegmentation.searchMetrics - .map { sm => - if (sm.numNonTrndNonHshtgSrchInLast14Days != 0) - sm.numNonTrndNonHshtgGlobalNsfwSrchInLast14Days.toDouble * scalingFactor / sm.numNonTrndNonHshtgSrchInLast14Days - else 0 - }.getOrElse(0d).toInt - val hasReported: Boolean = - nsfwUserSegmentation.notifFeedbackMetrics.exists(_.notifReportMetrics.exists(_.countTotal != 0)) - val hasDisliked: Boolean = - nsfwUserSegmentation.notifFeedbackMetrics - .exists(_.notifDislikeMetrics.exists(_.countTotal != 0)) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.docx new file mode 100644 index 000000000..e347ef3b4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala deleted file mode 100644 index ac4aba8a7..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/OverrideNotificationUtil.scala +++ /dev/null @@ -1,230 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.MagicFanoutEventCandidate -import com.twitter.frigate.common.history.History -import com.twitter.frigate.common.rec_types.RecTypes -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.model.ibis.PushOverrideInfo -import com.twitter.frigate.pushservice.params.PushConstants -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.pushservice.params.{PushFeatureSwitchParams => FSParams} -import com.twitter.frigate.thriftscala.CollapseInfo -import com.twitter.frigate.thriftscala.CommonRecommendationType -import com.twitter.frigate.thriftscala.CommonRecommendationType.MagicFanoutSportsEvent -import com.twitter.frigate.thriftscala.OverrideInfo -import com.twitter.util.Future -import java.util.UUID - -object OverrideNotificationUtil { - - /** - * Gets Override Info for the current notification. - * @param candidate [[PushCandidate]] object representing the recommendation candidate - * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called - * @return Returns OverrideInfo if CollapseInfo exists, else None - */ - - def getOverrideInfo( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Option[OverrideInfo]] = { - if (candidate.target.isLoggedOutUser) { - Future.None - } else if (isOverrideEnabledForCandidate(candidate)) - getCollapseInfo(candidate, stats).map(_.map(OverrideInfo(_))) - else Future.None - } - - private def getCollapseInfo( - candidate: PushCandidate, - stats: StatsReceiver - ): Future[Option[CollapseInfo]] = { - val target = candidate.target - for { - targetHistory <- target.history - deviceInfo <- target.deviceInfo - } yield getCollapseInfo(target, targetHistory, deviceInfo, stats) - } - - /** - * Get Collapse Info for the current notification. - * @param target Push Target - recipient of the notification - * @param targetHistory Target's History - * @param deviceInfoOpt `Option` of the Target's Device Info - * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called - * @return Returns CollapseInfo if the Target is eligible for Override Notifs, else None - */ - def getCollapseInfo( - target: PushTypes.Target, - targetHistory: History, - deviceInfoOpt: Option[DeviceInfo], - stats: StatsReceiver - ): Option[CollapseInfo] = { - val overrideInfoOfLastNotif = - PushOverrideInfo.getOverrideInfoOfLastEligiblePushNotif( - targetHistory, - target.params(FSParams.OverrideNotificationsLookbackDurationForOverrideInfo), - stats) - overrideInfoOfLastNotif match { - case Some(prevOverrideInfo) if isOverrideEnabled(target, deviceInfoOpt, stats) => - val notifsInLastOverrideChain = - PushOverrideInfo.getMrPushNotificationsInOverrideChain( - targetHistory, - prevOverrideInfo.collapseInfo.overrideChainId, - stats) - val numNotifsInLastOverrideChain = notifsInLastOverrideChain.size - val timestampOfFirstNotifInOverrideChain = - PushOverrideInfo - .getTimestampInMillisForFrigateNotification( - notifsInLastOverrideChain.last, - targetHistory, - stats).getOrElse(PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) - if (numNotifsInLastOverrideChain < target.params(FSParams.MaxMrPushSends24HoursParam) && - timestampOfFirstNotifInOverrideChain > PushConstants.DefaultLookBackForHistory.ago.inMilliseconds) { - Some(prevOverrideInfo.collapseInfo) - } else { - val prevCollapseId = prevOverrideInfo.collapseInfo.collapseId - val newOverrideChainId = UUID.randomUUID.toString.replaceAll("-", "") - Some(CollapseInfo(prevCollapseId, newOverrideChainId)) - } - case None if isOverrideEnabled(target, deviceInfoOpt, stats) => - val newOverrideChainId = UUID.randomUUID.toString.replaceAll("-", "") - Some(CollapseInfo("", newOverrideChainId)) - case _ => None // Override is disabled for everything else - } - } - - /** - * Gets the collapse and impression identifier for the current override notification - * @param target Push Target - recipient of the notification - * @param stats StatsReceiver to track stats for this function as well as the subsequent funcs. called - * @return A Future of Collapse ID as well as the Impression ID. - */ - def getCollapseAndImpressionIdForOverride( - candidate: PushCandidate - ): Future[Option[(String, Seq[String])]] = { - if (isOverrideEnabledForCandidate(candidate)) { - val target = candidate.target - val stats = candidate.statsReceiver - Future.join(target.history, target.deviceInfo).map { - case (targetHistory, deviceInfoOpt) => - val collapseInfoOpt = getCollapseInfo(target, targetHistory, deviceInfoOpt, stats) - - val impressionIds = candidate.commonRecType match { - case MagicFanoutSportsEvent - if target.params(FSParams.EnableEventIdBasedOverrideForSportsCandidates) => - PushOverrideInfo.getImpressionIdsForPrevEligibleMagicFanoutEventCandidates( - targetHistory, - target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), - stats, - MagicFanoutSportsEvent, - candidate - .asInstanceOf[RawCandidate with MagicFanoutEventCandidate].eventId - ) - case _ => - PushOverrideInfo.getImpressionIdsOfPrevEligiblePushNotif( - targetHistory, - target.params(FSParams.OverrideNotificationsLookbackDurationForImpressionId), - stats) - } - - collapseInfoOpt match { - case Some(collapseInfo) if impressionIds.nonEmpty => - val notifsInLastOverrideChain = - PushOverrideInfo.getMrPushNotificationsInOverrideChain( - targetHistory, - collapseInfo.overrideChainId, - stats) - stats - .scope("OverrideNotificationUtil").stat("number_of_notifications_sent").add( - notifsInLastOverrideChain.size + 1) - Some((collapseInfo.collapseId, impressionIds)) - case _ => None - } - case _ => None - } - } else Future.None - } - - /** - * Checks to see if override notifications are enabled based on the Target's Device Info and Params - * @param target Push Target - recipient of the notification - * @param deviceInfoOpt `Option` of the Target's Device Info - * @param stats StatsReceiver to track stats for this function - * @return Returns True if Override Notifications are enabled for the provided - * Target, else False. - */ - private def isOverrideEnabled( - target: PushTypes.Target, - deviceInfoOpt: Option[DeviceInfo], - stats: StatsReceiver - ): Boolean = { - val scopedStats = stats.scope("OverrideNotificationUtil").scope("isOverrideEnabled") - val enabledForAndroidCounter = scopedStats.counter("android_enabled") - val disabledForAndroidCounter = scopedStats.counter("android_disabled") - val enabledForIosCounter = scopedStats.counter("ios_enabled") - val disabledForIosCounter = scopedStats.counter("ios_disabled") - val disabledForOtherDevicesCounter = scopedStats.counter("other_disabled") - - val isPrimaryDeviceAndroid = PushDeviceUtil.isPrimaryDeviceAndroid(deviceInfoOpt) - val isPrimaryDeviceIos = PushDeviceUtil.isPrimaryDeviceIOS(deviceInfoOpt) - - lazy val validAndroidDevice = - isPrimaryDeviceAndroid && target.params(FSParams.EnableOverrideNotificationsForAndroid) - lazy val validIosDevice = - isPrimaryDeviceIos && target.params(FSParams.EnableOverrideNotificationsForIos) - - if (isPrimaryDeviceAndroid) { - if (validAndroidDevice) enabledForAndroidCounter.incr() else disabledForAndroidCounter.incr() - } else if (isPrimaryDeviceIos) { - if (validIosDevice) enabledForIosCounter.incr() else disabledForIosCounter.incr() - } else { - disabledForOtherDevicesCounter.incr() - } - - validAndroidDevice || validIosDevice - } - - /** - * Checks if override is enabled for the currently supported types for SendHandler or not. - * This method is package private for unit testing. - * @param candidate [[PushCandidate]] - * @param stats StatsReceiver to track statistics for this function - * @return Returns True if override notifications are enabled for the current type, otherwise False. - */ - private def isOverrideEnabledForSendHandlerCandidate( - candidate: PushCandidate - ): Boolean = { - val scopedStats = candidate.statsReceiver - .scope("OverrideNotificationUtil").scope("isOverrideEnabledForSendHandlerType") - - val overrideSupportedTypesForSpaces: Set[CommonRecommendationType] = Set( - CommonRecommendationType.SpaceSpeaker, - CommonRecommendationType.SpaceHost - ) - - val isOverrideSupportedForSpaces = { - overrideSupportedTypesForSpaces.contains(candidate.commonRecType) && - candidate.target.params(FSParams.EnableOverrideForSpaces) - } - - val isOverrideSupportedForSports = { - candidate.commonRecType == CommonRecommendationType.MagicFanoutSportsEvent && - candidate.target - .params(PushFeatureSwitchParams.EnableOverrideForSportsCandidates) - } - - val isOverrideSupported = isOverrideSupportedForSpaces || isOverrideSupportedForSports - - scopedStats.counter(s"$isOverrideSupported").incr() - isOverrideSupported - } - - private[util] def isOverrideEnabledForCandidate(candidate: PushCandidate) = - !RecTypes.isSendHandlerType( - candidate.commonRecType) || isOverrideEnabledForSendHandlerCandidate(candidate) -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.docx new file mode 100644 index 000000000..e94d87b07 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala deleted file mode 100644 index 7bc29cf01..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAdaptorUtil.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.contentrecommender.thriftscala.MetricTag -import com.twitter.frigate.common.base.AlgorithmScore -import com.twitter.frigate.common.base.OutOfNetworkTweetCandidate -import com.twitter.frigate.common.base.SocialContextAction -import com.twitter.frigate.common.base.TopicCandidate -import com.twitter.frigate.common.base.TripCandidate -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction} -import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT} -import com.twitter.frigate.thriftscala._ -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.trends.trip_v1.trip_tweets.thriftscala.TripDomain -import scala.collection.Seq - -case class MediaCRT( - crt: CRT, - photoCRT: CRT, - videoCRT: CRT) - -object PushAdaptorUtil { - - def getFrigateNotificationForUser( - crt: CRT, - userId: Long, - scActions: Seq[SocialContextAction], - pushCopyId: Option[Int], - ntabCopyId: Option[Int] - ): FrigateNotification = { - - val thriftSCActions = scActions.map { scAction => - TSocialContextAction( - scAction.userId, - scAction.timestampInMillis, - scAction.tweetId - ) - } - FrigateNotification( - crt, - NotificationDisplayLocation.PushToMobileDevice, - userNotification = Some(UserNotification(userId, thriftSCActions)), - pushCopyId = pushCopyId, - ntabCopyId = ntabCopyId - ) - } - - def getFrigateNotificationForTweet( - crt: CRT, - tweetId: Long, - scActions: Seq[TSocialContextAction], - authorIdOpt: Option[Long], - pushCopyId: Option[Int], - ntabCopyId: Option[Int], - simclusterId: Option[Int], - semanticCoreEntityIds: Option[List[Long]], - candidateContent: Option[CandidateContent], - trendId: Option[String], - tweetTripDomain: Option[scala.collection.Set[TripDomain]] = None - ): FrigateNotification = { - FrigateNotification( - crt, - NotificationDisplayLocation.PushToMobileDevice, - tweetNotification = Some( - TweetNotification( - tweetId, - scActions, - authorIdOpt, - simclusterId, - semanticCoreEntityIds, - trendId, - tripDomain = tweetTripDomain) - ), - pushCopyId = pushCopyId, - ntabCopyId = ntabCopyId, - candidateContent = candidateContent - ) - } - - def getFrigateNotificationForTweetWithSocialContextActions( - crt: CRT, - tweetId: Long, - scActions: Seq[SocialContextAction], - authorIdOpt: Option[Long], - pushCopyId: Option[Int], - ntabCopyId: Option[Int], - candidateContent: Option[CandidateContent], - semanticCoreEntityIds: Option[List[Long]], - trendId: Option[String] - ): FrigateNotification = { - - val thriftSCActions = scActions.map { scAction => - TSocialContextAction( - scAction.userId, - scAction.timestampInMillis, - scAction.tweetId - ) - } - - getFrigateNotificationForTweet( - crt = crt, - tweetId = tweetId, - scActions = thriftSCActions, - authorIdOpt = authorIdOpt, - pushCopyId = pushCopyId, - ntabCopyId = ntabCopyId, - simclusterId = None, - candidateContent = candidateContent, - semanticCoreEntityIds = semanticCoreEntityIds, - trendId = trendId - ) - } - - def generateOutOfNetworkTweetCandidates( - inputTarget: Target, - id: Long, - mediaCRT: MediaCRT, - result: Option[TweetyPieResult], - localizedEntity: Option[LocalizedEntity] = None, - isMrBackfillFromCR: Option[Boolean] = None, - tagsFromCR: Option[Seq[MetricTag]] = None, - score: Option[Double] = None, - algorithmTypeCR: Option[String] = None, - tripTweetDomain: Option[scala.collection.Set[TripDomain]] = None - ): RawCandidate - with OutOfNetworkTweetCandidate - with TopicCandidate - with TripCandidate - with AlgorithmScore = { - new RawCandidate - with OutOfNetworkTweetCandidate - with TopicCandidate - with TripCandidate - with AlgorithmScore { - override val tweetId: Long = id - override val target: Target = inputTarget - override val tweetyPieResult: Option[TweetyPieResult] = result - override val localizedUttEntity: Option[LocalizedEntity] = localizedEntity - override val semanticCoreEntityId: Option[Long] = localizedEntity.map(_.entityId) - override def commonRecType: CRT = - getMediaBasedCRT(mediaCRT.crt, mediaCRT.photoCRT, mediaCRT.videoCRT) - override def isMrBackfillCR: Option[Boolean] = isMrBackfillFromCR - override def tagsCR: Option[Seq[MetricTag]] = tagsFromCR - override def algorithmScore: Option[Double] = score - override def algorithmCR: Option[String] = algorithmTypeCR - override def tripDomain: Option[collection.Set[TripDomain]] = tripTweetDomain - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.docx new file mode 100644 index 000000000..750eef535 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala deleted file mode 100644 index 0afa90fed..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushAppPermissionUtil.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.onboarding.task.service.models.external.PermissionState -import com.twitter.permissions_storage.thriftscala.AppPermission -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -object PushAppPermissionUtil { - - final val AddressBookPermissionKey = "addressBook" - final val SyncStateKey = "syncState" - final val SyncStateOnValue = "on" - - /** - * Obtains the specified target's App Permissions, based on their primary device. - * @param targetId Target's Identifier - * @param permissionName The permission type we are querying for (address book, geolocation, etc.) - * @param deviceInfoFut Device info of the Target, presented as a Future - * @param appPermissionStore Readable Store which allows us to query the App Permission Strato Column - * @return Returns the AppPermission of the Target, presented as a Future - */ - def getAppPermission( - targetId: Long, - permissionName: String, - deviceInfoFut: Future[Option[DeviceInfo]], - appPermissionStore: ReadableStore[(Long, (String, String)), AppPermission] - ): Future[Option[AppPermission]] = { - deviceInfoFut.flatMap { deviceInfoOpt => - val primaryDeviceIdOpt = deviceInfoOpt.flatMap(_.primaryDeviceId) - primaryDeviceIdOpt match { - case Some(primaryDeviceId) => - val queryKey = (targetId, (primaryDeviceId, permissionName)) - appPermissionStore.get(queryKey) - case _ => Future.None - } - } - } - - def hasTargetUploadedAddressBook( - appPermissionOpt: Option[AppPermission] - ): Boolean = { - appPermissionOpt.exists { appPermission => - val syncState = appPermission.metadata.get(SyncStateKey) - appPermission.systemPermissionState == PermissionState.On && syncState - .exists(_.equalsIgnoreCase(SyncStateOnValue)) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.docx new file mode 100644 index 000000000..73824247b Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala deleted file mode 100644 index d5d79c4fd..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushCapUtil.scala +++ /dev/null @@ -1,184 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.candidate.FrigateHistory -import com.twitter.frigate.common.candidate.ResurrectedUserDetails -import com.twitter.frigate.common.candidate.TargetABDecider -import com.twitter.frigate.common.candidate.UserDetails -import com.twitter.frigate.pushcap.thriftscala.ModelType -import com.twitter.frigate.pushcap.thriftscala.PushcapInfo -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -import com.twitter.frigate.scribe.thriftscala.PushCapInfo -import com.twitter.util.Duration -import com.twitter.util.Future - -case class PushCapFatigueInfo( - pushcap: Int, - fatigueInterval: Duration) {} - -object PushCapUtil { - - def getDefaultPushCap(target: Target): Future[Int] = { - Future.value(target.params(PushFeatureSwitchParams.MaxMrPushSends24HoursParam)) - } - - def getMinimumRestrictedPushcapInfo( - restrictedPushcap: Int, - originalPushcapInfo: PushcapInfo, - statsReceiver: StatsReceiver - ): PushcapInfo = { - if (originalPushcapInfo.pushcap < restrictedPushcap) { - statsReceiver - .scope("minModelPushcapRestrictions").counter( - f"num_users_adjusted_from_${originalPushcapInfo.pushcap}_to_${restrictedPushcap}").incr() - PushcapInfo( - pushcap = restrictedPushcap.toShort, - modelType = ModelType.NoModel, - timestamp = 0L, - fatigueMinutes = Some((24L / restrictedPushcap) * 60L) - ) - } else originalPushcapInfo - } - - def getPushCapFatigue( - target: Target, - statsReceiver: StatsReceiver - ): Future[PushCapFatigueInfo] = { - val pushCapStats = statsReceiver.scope("pushcap_stats") - target.dynamicPushcap - .map { dynamicPushcapOpt => - val pushCap: Int = dynamicPushcapOpt match { - case Some(pushcapInfo) => pushcapInfo.pushcap - case _ => target.params(PushFeatureSwitchParams.MaxMrPushSends24HoursParam) - } - - pushCapStats.stat("pushCapValueStats").add(pushCap) - pushCapStats - .scope("pushCapValueCount").counter(f"num_users_with_pushcap_$pushCap").incr() - - target.finalPushcapAndFatigue += "pushPushCap" -> PushCapInfo("pushPushCap", pushCap.toByte) - - PushCapFatigueInfo(pushCap, 24.hours) - } - } - - def getMinDurationsSincePushWithoutUsingPushCap( - target: TargetUser - with TargetABDecider - with FrigateHistory - with UserDetails - with ResurrectedUserDetails - )( - implicit statsReceiver: StatsReceiver - ): Duration = { - val minDurationSincePush = - if (target.params(PushFeatureSwitchParams.EnableGraduallyRampUpNotification)) { - val daysInterval = - target.params(PushFeatureSwitchParams.GraduallyRampUpPhaseDurationDays).inDays.toDouble - val daysSinceActivation = - if (target.isResurrectedUser && target.timeSinceResurrection.isDefined) { - target.timeSinceResurrection.map(_.inDays.toDouble).get - } else { - target.timeElapsedAfterSignup.inDays.toDouble - } - val phaseInterval = - Math.max( - 1, - Math.ceil(daysSinceActivation / daysInterval).toInt - ) - val minDuration = 24 / phaseInterval - val finalMinDuration = - Math.max(4, minDuration).hours - statsReceiver - .scope("GraduallyRampUpFinalMinDuration").counter(s"$finalMinDuration.hours").incr() - finalMinDuration - } else { - target.params(PushFeatureSwitchParams.MinDurationSincePushParam) - } - statsReceiver - .scope("minDurationsSincePushWithoutUsingPushCap").counter( - s"$minDurationSincePush.hours").incr() - minDurationSincePush - } - - def getMinDurationSincePush( - target: Target, - statsReceiver: StatsReceiver - ): Future[Duration] = { - val minDurationStats: StatsReceiver = statsReceiver.scope("pushcapMinDuration_stats") - val minDurationModifierCalculator = - MinDurationModifierCalculator() - val openedPushByHourAggregatedFut = - if (target.params(PushFeatureSwitchParams.EnableQueryUserOpenedHistory)) - target.openedPushByHourAggregated - else Future.None - Future - .join( - target.dynamicPushcap, - target.accountCountryCode, - openedPushByHourAggregatedFut - ) - .map { - case (dynamicPushcapOpt, countryCodeOpt, openedPushByHourAggregated) => - val minDurationSincePush: Duration = { - val isGraduallyRampingUpResurrected = target.isResurrectedUser && target.params( - PushFeatureSwitchParams.EnableGraduallyRampUpNotification) - if (isGraduallyRampingUpResurrected || target.params( - PushFeatureSwitchParams.EnableExplicitPushCap)) { - getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) - } else { - dynamicPushcapOpt match { - case Some(pushcapInfo) => - pushcapInfo.fatigueMinutes match { - case Some(fatigueMinutes) => (fatigueMinutes / 60).hours - case _ if pushcapInfo.pushcap > 0 => (24 / pushcapInfo.pushcap).hours - case _ => getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) - } - case _ => - getMinDurationsSincePushWithoutUsingPushCap(target)(minDurationStats) - } - } - } - - val modifiedMinDurationSincePush = - if (target.params(PushFeatureSwitchParams.EnableMinDurationModifier)) { - val modifierHourOpt = - minDurationModifierCalculator.getMinDurationModifier( - target, - countryCodeOpt, - statsReceiver.scope("MinDuration")) - modifierHourOpt match { - case Some(modifierHour) => modifierHour.hours - case _ => minDurationSincePush - } - } else if (target.params( - PushFeatureSwitchParams.EnableMinDurationModifierByUserHistory)) { - val modifierMinuteOpt = - minDurationModifierCalculator.getMinDurationModifierByUserOpenedHistory( - target, - openedPushByHourAggregated, - statsReceiver.scope("MinDuration")) - - modifierMinuteOpt match { - case Some(modifierMinute) => modifierMinute.minutes - case _ => minDurationSincePush - } - } else minDurationSincePush - - target.finalPushcapAndFatigue += "pushFatigue" -> PushCapInfo( - "pushFatigue", - modifiedMinDurationSincePush.inHours.toByte) - - minDurationStats - .stat("minDurationSincePushValueStats").add(modifiedMinDurationSincePush.inHours) - minDurationStats - .scope("minDurationSincePushValueCount").counter( - s"$modifiedMinDurationSincePush").incr() - - modifiedMinDurationSincePush - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.docx new file mode 100644 index 000000000..0d4d5d180 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala deleted file mode 100644 index d191d742a..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushDeviceUtil.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.common.store.deviceinfo.MobileClientType -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.util.Future -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.finagle.stats.StatsReceiver - -object PushDeviceUtil { - - def isPrimaryDeviceAndroid(deviceInfoOpt: Option[DeviceInfo]): Boolean = { - deviceInfoOpt.exists { - _.guessedPrimaryClient.exists { clientType => - (clientType == MobileClientType.Android) || (clientType == MobileClientType.AndroidLite) - } - } - } - - def isPrimaryDeviceIOS(deviceInfoOpt: Option[DeviceInfo]): Boolean = { - deviceInfoOpt.exists { - _.guessedPrimaryClient.exists { clientType => - (clientType == MobileClientType.Iphone) || (clientType == MobileClientType.Ipad) - } - } - } - - def isPushRecommendationsEligible(target: Target): Future[Boolean] = - target.deviceInfo.map(_.exists(_.isRecommendationsEligible)) - - def isTopicsEligible( - target: Target, - statsReceiver: StatsReceiver = NullStatsReceiver - ): Future[Boolean] = { - val isTopicsSkipFatigue = Future.True - - Future.join(isTopicsSkipFatigue, target.deviceInfo.map(_.exists(_.isTopicsEligible))).map { - case (isTopicsNotFatigue, isTopicsEligibleSetting) => - isTopicsNotFatigue && isTopicsEligibleSetting - } - } - - def isSpacesEligible(target: Target): Future[Boolean] = - target.deviceInfo.map(_.exists(_.isSpacesEligible)) - - def isNtabOnlyEligible: Future[Boolean] = { - Future.False - } - - def isRecommendationsEligible(target: Target): Future[Boolean] = { - Future.join(isPushRecommendationsEligible(target), isNtabOnlyEligible).map { - case (isPushRecommendation, isNtabOnly) => isPushRecommendation || isNtabOnly - case _ => false - } - } - -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.docx new file mode 100644 index 000000000..10265f810 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala deleted file mode 100644 index 7567726bf..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushIbisUtil.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.util.Future - -object PushIbisUtil { - - def getSocialContextModelValues(socialContextUserIds: Seq[Long]): Map[String, String] = { - - val socialContextSize = socialContextUserIds.size - - val (displaySocialContexts, otherCount) = { - if (socialContextSize < 3) (socialContextUserIds, 0) - else (socialContextUserIds.take(1), socialContextSize - 1) - } - - val usersValue = displaySocialContexts.map(_.toString).mkString(",") - - if (otherCount > 0) Map("social_users" -> s"$usersValue+$otherCount") - else Map("social_users" -> usersValue) - } - - def mergeFutModelValues( - mvFut1: Future[Map[String, String]], - mvFut2: Future[Map[String, String]] - ): Future[Map[String, String]] = { - Future.join(mvFut1, mvFut2).map { - case (mv1, mv2) => mv1 ++ mv2 - } - } - - def mergeModelValues( - mvFut1: Future[Map[String, String]], - mv2: Map[String, String] - ): Future[Map[String, String]] = - mvFut1.map { mv1 => mv1 ++ mv2 } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.docx new file mode 100644 index 000000000..7ccb9eb2d Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala deleted file mode 100644 index 8f9fd63c3..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/PushToHomeUtil.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.deviceinfo.DeviceInfo -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams -object PushToHomeUtil { - def getIbis2ModelValue( - deviceInfoOpt: Option[DeviceInfo], - target: Target, - stats: StatsReceiver - ): Option[Map[String, String]] = { - deviceInfoOpt.flatMap { deviceInfo => - val isAndroidEnabled = deviceInfo.isLandOnHomeAndroid && target.params( - PushFeatureSwitchParams.EnableTweetPushToHomeAndroid) - val isIOSEnabled = deviceInfo.isLandOnHomeiOS && target.params( - PushFeatureSwitchParams.EnableTweetPushToHomeiOS) - if (isAndroidEnabled || isIOSEnabled) { - stats.counter("enable_push_to_home").incr() - Some(Map("is_land_on_home" -> "true")) - } else None - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.docx new file mode 100644 index 000000000..b70028515 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala deleted file mode 100644 index 015e065ec..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RFPHTakeStepUtil.scala +++ /dev/null @@ -1,114 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.Counter -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.refresh_handler.ResultWithDebugInfo -import com.twitter.frigate.pushservice.predicate.BigFilteringEpsilonGreedyExplorationPredicate -import com.twitter.frigate.pushservice.predicate.MlModelsHoldbackExperimentPredicate -import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.util.Future - -class RFPHTakeStepUtil()(globalStats: StatsReceiver) { - - implicit val statsReceiver: StatsReceiver = - globalStats.scope("RefreshForPushHandler") - private val takeStats: StatsReceiver = statsReceiver.scope("take") - private val notifierStats = takeStats.scope("notifier") - private val validatorStats = takeStats.scope("validator") - private val validatorLatency: Stat = validatorStats.stat("latency") - - private val executedPredicatesInTandem: Counter = - takeStats.counter("predicates_executed_in_tandem") - - private val bigFilteringEpsGreedyPredicate: NamedPredicate[PushCandidate] = - BigFilteringEpsilonGreedyExplorationPredicate()(takeStats) - private val bigFilteringEpsGreedyStats: StatsReceiver = - takeStats.scope("big_filtering_eps_greedy_predicate") - - private val modelPredicate: NamedPredicate[PushCandidate] = - MlModelsHoldbackExperimentPredicate()(takeStats) - private val mlPredicateStats: StatsReceiver = takeStats.scope("ml_predicate") - - private def updateFilteredStatusExptStats(candidate: PushCandidate, predName: String): Unit = { - - val recTypeStat = globalStats.scope( - candidate.commonRecType.toString - ) - - recTypeStat.counter(PushStatus.Filtered.toString).incr() - recTypeStat - .scope(PushStatus.Filtered.toString) - .counter(predName) - .incr() - } - - def isCandidateValid( - candidate: PushCandidate, - candidateValidator: RFPHCandidateValidator - ): Future[ResultWithDebugInfo] = { - val predResultFuture = Stat.timeFuture(validatorLatency) { - Future - .join( - bigFilteringEpsGreedyPredicate.apply(Seq(candidate)), - modelPredicate.apply(Seq(candidate)) - ).flatMap { - case (Seq(true), Seq(true)) => - executedPredicatesInTandem.incr() - - bigFilteringEpsGreedyStats - .scope(candidate.commonRecType.toString) - .counter("passed") - .incr() - - mlPredicateStats - .scope(candidate.commonRecType.toString) - .counter("passed") - .incr() - candidateValidator.validateCandidate(candidate).map((_, Nil)) - case (Seq(false), _) => - bigFilteringEpsGreedyStats - .scope(candidate.commonRecType.toString) - .counter("filtered") - .incr() - Future.value((Some(bigFilteringEpsGreedyPredicate), Nil)) - case (_, _) => - mlPredicateStats - .scope(candidate.commonRecType.toString) - .counter("filtered") - .incr() - Future.value((Some(modelPredicate), Nil)) - } - } - - predResultFuture.map { - case (Some(pred: NamedPredicate[_]), candPredicateResults) => - takeStats.counter("filtered_by_named_general_predicate").incr() - updateFilteredStatusExptStats(candidate, pred.name) - ResultWithDebugInfo( - Invalid(Some(pred.name)), - candPredicateResults - ) - - case (Some(_), candPredicateResults) => - takeStats.counter("filtered_by_unnamed_general_predicate").incr() - updateFilteredStatusExptStats(candidate, predName = "unk") - ResultWithDebugInfo( - Invalid(Some("unnamed_candidate_predicate")), - candPredicateResults - ) - - case (None, candPredicateResults) => - takeStats.counter("accepted_push_ok").incr() - ResultWithDebugInfo( - OK, - candPredicateResults - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.docx new file mode 100644 index 000000000..79629f59a Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala deleted file mode 100644 index 8f24756ae..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/RelationshipUtil.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.frigate.common.base.TweetAuthor -import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate -import com.twitter.hermit.predicate.socialgraph.Edge -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.socialgraph.thriftscala.RelationshipType - -/** - * This class provides utility functions for relationshipEdge for each Candidate type. - */ -object RelationshipUtil { - - /** - * Form relationEdges - * @param candidate PushCandidate - * @param relationship relationshipTypes for different candidate types - * @return relationEdges for different candidate types - */ - private def formRelationEdgeWithTargetIdAndAuthorId( - candidate: RawCandidate, - relationship: List[RelationshipType with Product] - ): List[RelationEdge] = { - candidate match { - case candidate: RawCandidate with TweetAuthor => - candidate.authorId match { - case Some(authorId) => - val edge = Edge(candidate.target.targetId, authorId) - for { - r <- relationship - } yield RelationEdge(edge, r) - case _ => List.empty[RelationEdge] - } - case _ => List.empty[RelationEdge] - } - } - - /** - * Form all relationshipEdges for basicTweetRelationShips - * @param candidate PushCandidate - * @return List of relationEdges for basicTweetRelationShips - */ - def getBasicTweetRelationships(candidate: RawCandidate): List[RelationEdge] = { - val relationship = List( - RelationshipType.DeviceFollowing, - RelationshipType.Blocking, - RelationshipType.BlockedBy, - RelationshipType.HideRecommendations, - RelationshipType.Muting) - formRelationEdgeWithTargetIdAndAuthorId(candidate, relationship) - } - - /** - * Form all relationshipEdges for F1tweetsRelationships - * @param candidate PushCandidate - * @return List of relationEdges for F1tweetsRelationships - */ - def getPreCandidateRelationshipsForInNetworkTweets( - candidate: RawCandidate - ): List[RelationEdge] = { - val relationship = List(RelationshipType.Following) - getBasicTweetRelationships(candidate) ++ formRelationEdgeWithTargetIdAndAuthorId( - candidate, - relationship) - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.docx new file mode 100644 index 000000000..9a4c0a4ec Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala deleted file mode 100644 index 1b16ec8c0..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/ResponseStatsTrackUtils.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.BroadcastStatsReceiver -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.thriftscala.PushResponse -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.frigate.thriftscala.CommonRecommendationType - -object ResponseStatsTrackUtils { - def trackStatsForResponseToRequest( - crt: CommonRecommendationType, - target: Target, - response: PushResponse, - receivers: Seq[StatsReceiver] - )( - originalStats: StatsReceiver - ): Unit = { - val newReceivers = Seq( - originalStats - .scope("is_model_training_data") - .scope(target.isModelTrainingData.toString), - originalStats.scope("scribe_target").scope(IbisScribeTargets.crtToScribeTarget(crt)) - ) - - val broadcastStats = BroadcastStatsReceiver(receivers) - val broadcastStatsWithExpts = BroadcastStatsReceiver(newReceivers ++ receivers) - - if (response.status == PushStatus.Sent) { - if (target.isModelTrainingData) { - broadcastStats.counter("num_training_data_recs_sent").incr() - } - } - broadcastStatsWithExpts.counter(response.status.toString).incr() - if (response.status == PushStatus.Filtered) { - broadcastStats - .scope(response.status.toString) - .counter(response.filteredBy.getOrElse("None")) - .incr() - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.docx new file mode 100644 index 000000000..7b11634a4 Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala deleted file mode 100644 index 4174fa21c..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/SendHandlerPredicateUtil.scala +++ /dev/null @@ -1,129 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.CandidateDetails -import com.twitter.frigate.common.base.CandidateResult -import com.twitter.frigate.common.base.Invalid -import com.twitter.frigate.common.base.OK -import com.twitter.frigate.common.base.Result -import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate -import com.twitter.frigate.pushservice.refresh_handler.ResultWithDebugInfo -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator -import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator -import com.twitter.frigate.pushservice.thriftscala.PushStatus -import com.twitter.hermit.predicate.NamedPredicate -import com.twitter.util.Future - -class SendHandlerPredicateUtil()(globalStats: StatsReceiver) { - implicit val statsReceiver: StatsReceiver = - globalStats.scope("SendHandler") - private val validateStats: StatsReceiver = statsReceiver.scope("validate") - - private def updateFilteredStatusExptStats(candidate: PushCandidate, predName: String): Unit = { - - val recTypeStat = globalStats.scope( - candidate.commonRecType.toString - ) - - recTypeStat.counter(PushStatus.Filtered.toString).incr() - recTypeStat - .scope(PushStatus.Filtered.toString) - .counter(predName) - .incr() - } - - /** - * Parsing the candidateValidtor result into desired format for preValidation before ml filtering - * @param hydratedCandidates - * @param candidateValidator - * @return - */ - def preValidationForCandidate( - hydratedCandidates: Seq[CandidateDetails[PushCandidate]], - candidateValidator: SendHandlerPreCandidateValidator - ): Future[ - (Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]]) - ] = { - val predResultFuture = - Future.collect( - hydratedCandidates.map(hydratedCandidate => - candidateValidator.validateCandidate(hydratedCandidate.candidate)) - ) - - predResultFuture.map { results => - results - .zip(hydratedCandidates) - .foldLeft( - ( - Seq.empty[CandidateDetails[PushCandidate]], - Seq.empty[CandidateResult[PushCandidate, Result]] - ) - ) { - case ((goodCandidates, filteredCandidates), (result, candidateDetails)) => - result match { - case None => - (goodCandidates :+ candidateDetails, filteredCandidates) - case Some(pred: NamedPredicate[_]) => - val r = Invalid(Some(pred.name)) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - case Some(_) => - val r = Invalid(Some("Filtered by un-named predicate")) - ( - goodCandidates, - filteredCandidates :+ CandidateResult[PushCandidate, Result]( - candidateDetails.candidate, - candidateDetails.source, - r - ) - ) - } - } - } - } - - /** - * Parsing the candidateValidtor result into desired format for postValidation including and after ml filtering - * @param candidate - * @param candidateValidator - * @return - */ - def postValidationForCandidate( - candidate: PushCandidate, - candidateValidator: SendHandlerPostCandidateValidator - ): Future[ResultWithDebugInfo] = { - val predResultFuture = - candidateValidator.validateCandidate(candidate) - - predResultFuture.map { - case (Some(pred: NamedPredicate[_])) => - validateStats.counter("filtered_by_named_general_predicate").incr() - updateFilteredStatusExptStats(candidate, pred.name) - ResultWithDebugInfo( - Invalid(Some(pred.name)), - Nil - ) - - case Some(_) => - validateStats.counter("filtered_by_unnamed_general_predicate").incr() - updateFilteredStatusExptStats(candidate, predName = "unk") - ResultWithDebugInfo( - Invalid(Some("unnamed_candidate_predicate")), - Nil - ) - - case _ => - validateStats.counter("accepted_push_ok").incr() - ResultWithDebugInfo( - OK, - Nil - ) - } - } -} diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.docx b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.docx new file mode 100644 index 000000000..942e914db Binary files /dev/null and b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.docx differ diff --git a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala b/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala deleted file mode 100644 index a5b7cf2c5..000000000 --- a/pushservice/src/main/scala/com/twitter/frigate/pushservice/util/TopicsUtil.scala +++ /dev/null @@ -1,340 +0,0 @@ -package com.twitter.frigate.pushservice.util - -import com.twitter.contentrecommender.thriftscala.DisplayLocation -import com.twitter.finagle.stats.Stat -import com.twitter.frigate.common.base.TargetUser -import com.twitter.frigate.common.predicate.CommonOutNetworkTweetCandidatesSourcePredicates.authorNotBeingFollowedPredicate -import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext -import com.twitter.frigate.pushservice.model.PushTypes.Target -import com.twitter.frigate.pushservice.model.PushTypes -import com.twitter.frigate.pushservice.store.UttEntityHydrationQuery -import com.twitter.frigate.pushservice.store.UttEntityHydrationStore -import com.twitter.hermit.predicate.Predicate -import com.twitter.hermit.predicate.socialgraph.RelationEdge -import com.twitter.interests.thriftscala.InterestRelationType -import com.twitter.interests.thriftscala.InterestRelationship -import com.twitter.interests.thriftscala.InterestedInInterestLookupContext -import com.twitter.interests.thriftscala.InterestedInInterestModel -import com.twitter.interests.thriftscala.ProductId -import com.twitter.interests.thriftscala.UserInterest -import com.twitter.interests.thriftscala.UserInterestData -import com.twitter.interests.thriftscala.UserInterests -import com.twitter.interests.thriftscala.{TopicListingViewerContext => TopicListingViewerContextCR} -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.timelines.configapi.Param -import com.twitter.topiclisting.TopicListingViewerContext -import com.twitter.topiclisting.utt.LocalizedEntity -import com.twitter.tsp.thriftscala.TopicListingSetting -import com.twitter.tsp.thriftscala.TopicSocialProofRequest -import com.twitter.tsp.thriftscala.TopicSocialProofResponse -import com.twitter.tsp.thriftscala.TopicWithScore -import com.twitter.util.Future -import scala.collection.Map - -case class TweetWithTopicProof( - tweetId: Long, - topicId: Long, - authorId: Option[Long], - score: Double, - tweetyPieResult: TweetyPieResult, - topicListingSetting: String, - algorithmCR: Option[String], - isOON: Boolean) - -object TopicsUtil { - - /** - * Obtains the Localized Entities for the provided SC Entity IDs - * @param target The target user for which we're obtaining candidates - * @param semanticCoreEntityIds The seq. of entity ids for which we would like to obtain the Localized Entities - * @param uttEntityHydrationStore Store to query the actual LocalizedEntities - * @return A Future Map consisting of the entity id as the key and LocalizedEntity as the value - */ - def getLocalizedEntityMap( - target: Target, - semanticCoreEntityIds: Set[Long], - uttEntityHydrationStore: UttEntityHydrationStore - ): Future[Map[Long, LocalizedEntity]] = { - buildTopicListingViewerContext(target) - .flatMap { topicListingViewerContext => - val query = UttEntityHydrationQuery(topicListingViewerContext, semanticCoreEntityIds.toSeq) - val localizedTopicEntitiesFut = - uttEntityHydrationStore.getLocalizedTopicEntities(query).map(_.flatten) - localizedTopicEntitiesFut.map { localizedTopicEntities => - localizedTopicEntities.map { localizedTopicEntity => - localizedTopicEntity.entityId -> localizedTopicEntity - }.toMap - } - } - } - - /** - * Fetch explict followed interests i.e Topics for targetUser - * - * @param targetUser: [[Target]] object representing a user eligible for MagicRecs notification - * @return: list of all Topics(Interests) Followed by targetUser - */ - def getTopicsFollowedByUser( - targetUser: Target, - interestsWithLookupContextStore: ReadableStore[ - InterestsLookupRequestWithContext, - UserInterests - ], - followedTopicsStats: Stat - ): Future[Option[Seq[UserInterest]]] = { - buildTopicListingViewerContext(targetUser).flatMap { topicListingViewerContext => - // explicit interests relation query - val explicitInterestsLookupRequest = InterestsLookupRequestWithContext( - targetUser.targetId, - Some( - InterestedInInterestLookupContext( - explicitContext = None, - inferredContext = None, - productId = Some(ProductId.Followable), - topicListingViewerContext = Some(topicListingViewerContext.toThrift), - disableExplicit = None, - disableImplicit = Some(true) - ) - ) - ) - - // filter explicit follow relationships from response - interestsWithLookupContextStore.get(explicitInterestsLookupRequest).map { - _.flatMap { userInterests => - val followedTopics = userInterests.interests.map { - _.filter { - case UserInterest(_, Some(interestData)) => - interestData match { - case UserInterestData.InterestedIn(interestedIn) => - interestedIn.exists { - case InterestedInInterestModel.ExplicitModel(explicitModel) => - explicitModel match { - case InterestRelationship.V1(v1) => - v1.relation == InterestRelationType.Followed - - case _ => false - } - - case _ => false - } - - case _ => false - } - - case _ => false // interestData unavailable - } - } - followedTopicsStats.add(followedTopics.getOrElse(Seq.empty[UserInterest]).size) - followedTopics - } - } - } - } - - /** - * - * @param target : [[Target]] object respresenting MagicRecs user - * - * @return: [[TopicListingViewerContext]] for querying topics - */ - def buildTopicListingViewerContext(target: Target): Future[TopicListingViewerContext] = { - Future.join(target.inferredUserDeviceLanguage, target.countryCode, target.targetUser).map { - case (inferredLanguage, countryCode, userInfo) => - TopicListingViewerContext( - userId = Some(target.targetId), - guestId = None, - deviceId = None, - clientApplicationId = None, - userAgent = None, - languageCode = inferredLanguage, - countryCode = countryCode, - userRoles = userInfo.flatMap(_.roles.map(_.roles.toSet)) - ) - } - } - - /** - * - * @param target : [[Target]] object respresenting MagicRecs user - * - * @return: [[TopicListingViewerContext]] for querying topics - */ - def buildTopicListingViewerContextForCR(target: Target): Future[TopicListingViewerContextCR] = { - TopicsUtil.buildTopicListingViewerContext(target).map(_.toThrift) - } - - /** - * - * @param target : [[Target]] object respresenting MagicRecs user - * @param tweets : [[Seq[TweetyPieResult]]] object representing Tweets to get TSP for - * @param topicSocialProofServiceStore: [[ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse]]] - * @param edgeStore: [[ReadableStore[RelationEdge, Boolean]]]] - * - * @return: [[Future[Seq[TweetWithTopicProof]]]] Tweets with topic proof - */ - def getTopicSocialProofs( - inputTarget: Target, - tweets: Seq[TweetyPieResult], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - edgeStore: ReadableStore[RelationEdge, Boolean], - scoreThresholdParam: Param[Double] - ): Future[Seq[TweetWithTopicProof]] = { - buildTopicListingViewerContextForCR(inputTarget).flatMap { topicListingContext => - val tweetIds: Set[Long] = tweets.map(_.tweet.id).toSet - val tweetIdsToTweetyPie = tweets.map(tp => tp.tweet.id -> tp).toMap - val topicSocialProofRequest = - TopicSocialProofRequest( - inputTarget.targetId, - tweetIds, - DisplayLocation.MagicRecsRecommendTopicTweets, - TopicListingSetting.Followable, - topicListingContext) - - topicSocialProofServiceStore - .get(topicSocialProofRequest).flatMap { - case Some(topicSocialProofResponse) => - val topicProofCandidates = topicSocialProofResponse.socialProofs.collect { - case (tweetId, topicsWithScore) - if topicsWithScore.nonEmpty && topicsWithScore - .maxBy(_.score).score >= inputTarget - .params(scoreThresholdParam) => - // Get the topic with max score if there are any topics returned - val topicWithScore = topicsWithScore.maxBy(_.score) - TweetWithTopicProof( - tweetId, - topicWithScore.topicId, - tweetIdsToTweetyPie(tweetId).tweet.coreData.map(_.userId), - topicWithScore.score, - tweetIdsToTweetyPie(tweetId), - topicWithScore.topicFollowType.map(_.name).getOrElse(""), - topicWithScore.algorithmType.map(_.name), - isOON = true - ) - }.toSeq - - hydrateTopicProofCandidatesWithEdgeStore(inputTarget, topicProofCandidates, edgeStore) - case _ => Future.value(Seq.empty[TweetWithTopicProof]) - } - } - } - - /** - * Obtain TopicWithScores for provided tweet candidates and target - * @param target target user - * @param Tweets tweet candidates represented in a (tweetId, TweetyPieResult) map - * @param topicSocialProofServiceStore store to query topic social proof - * @param enableTopicAnnotation whether to enable topic annotation - * @param topicScoreThreshold threshold for topic score - * @return a (tweetId, TopicWithScore) map where the topic with highest topic score (if exists) is chosen - */ - def getTopicsWithScoreMap( - target: PushTypes.Target, - Tweets: Map[Long, Option[TweetyPieResult]], - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - enableTopicAnnotation: Boolean, - topicScoreThreshold: Double - ): Future[Option[Map[Long, TopicWithScore]]] = { - - if (enableTopicAnnotation) { - TopicsUtil - .buildTopicListingViewerContextForCR(target).flatMap { topicListingContext => - val tweetIds = Tweets.keySet - val topicSocialProofRequest = - TopicSocialProofRequest( - target.targetId, - tweetIds, - DisplayLocation.MagicRecsRecommendTopicTweets, - TopicListingSetting.Followable, - topicListingContext) - - topicSocialProofServiceStore - .get(topicSocialProofRequest).map { - _.map { topicSocialProofResponse => - topicSocialProofResponse.socialProofs - .collect { - case (tweetId, topicsWithScore) - if topicsWithScore.nonEmpty && Tweets(tweetId).nonEmpty - && topicsWithScore.maxBy(_.score).score >= topicScoreThreshold => - tweetId -> topicsWithScore.maxBy(_.score) - } - - } - } - } - } else { - Future.None - } - - } - - /** - * Obtain LocalizedEntities for provided tweet candidates and target - * @param target target user - * @param Tweets tweet candidates represented in a (tweetId, TweetyPieResult) map - * @param uttEntityHydrationStore store to query the actual LocalizedEntities - * @param topicSocialProofServiceStore store to query topic social proof - * @param enableTopicAnnotation whether to enable topic annotation - * @param topicScoreThreshold threshold for topic score - * @return a (tweetId, LocalizedEntity Option) Future map that stores Localized Entity (can be empty) for given tweetId - */ - def getTweetIdLocalizedEntityMap( - target: PushTypes.Target, - Tweets: Map[Long, Option[TweetyPieResult]], - uttEntityHydrationStore: UttEntityHydrationStore, - topicSocialProofServiceStore: ReadableStore[TopicSocialProofRequest, TopicSocialProofResponse], - enableTopicAnnotation: Boolean, - topicScoreThreshold: Double - ): Future[Map[Long, Option[LocalizedEntity]]] = { - - val topicWithScoreMap = getTopicsWithScoreMap( - target, - Tweets, - topicSocialProofServiceStore, - enableTopicAnnotation, - topicScoreThreshold) - - topicWithScoreMap.flatMap { topicWithScores => - topicWithScores match { - case Some(topics) => - val topicIds = topics.collect { case (_, topic) => topic.topicId }.toSet - val LocalizedEntityMapFut = - getLocalizedEntityMap(target, topicIds, uttEntityHydrationStore) - - LocalizedEntityMapFut.map { LocalizedEntityMap => - topics.map { - case (tweetId, topic) => - tweetId -> LocalizedEntityMap.get(topic.topicId) - } - } - case _ => Future.value(Map[Long, Option[LocalizedEntity]]()) - } - } - - } - - /** - * Hydrate TweetWithTopicProof candidates with isOON field info, - * based on the following relationship between target user and candidate author in edgeStore - * @return TweetWithTopicProof candidates with isOON field populated - */ - def hydrateTopicProofCandidatesWithEdgeStore( - inputTarget: TargetUser, - topicProofCandidates: Seq[TweetWithTopicProof], - edgeStore: ReadableStore[RelationEdge, Boolean], - ): Future[Seq[TweetWithTopicProof]] = { - // IDs of all authors of TopicProof candidates that are OON with respect to inputTarget - val validOONAuthorIdsFut = - Predicate.filter( - topicProofCandidates.flatMap(_.authorId).distinct, - authorNotBeingFollowedPredicate(inputTarget, edgeStore)) - - validOONAuthorIdsFut.map { validOONAuthorIds => - topicProofCandidates.map(candidate => { - candidate.copy(isOON = - candidate.authorId.isDefined && validOONAuthorIds.contains(candidate.authorId.get)) - }) - } - } - -} diff --git a/recos-injector/BUILD.bazel b/recos-injector/BUILD.bazel deleted file mode 100644 index dbd3d0619..000000000 --- a/recos-injector/BUILD.bazel +++ /dev/null @@ -1 +0,0 @@ -# This prevents SQ query from grabbing //:all since it traverses up once to find a BUILD (DPB-14048) diff --git a/recos-injector/BUILD.docx b/recos-injector/BUILD.docx new file mode 100644 index 000000000..8841c8c66 Binary files /dev/null and b/recos-injector/BUILD.docx differ diff --git a/recos-injector/CONFIG.docx b/recos-injector/CONFIG.docx new file mode 100644 index 000000000..7206e30aa Binary files /dev/null and b/recos-injector/CONFIG.docx differ diff --git a/recos-injector/CONFIG.ini b/recos-injector/CONFIG.ini deleted file mode 100644 index b3f176acb..000000000 --- a/recos-injector/CONFIG.ini +++ /dev/null @@ -1,10 +0,0 @@ -; See http://go/CONFIG.ini - -[jira] -project: SD - -[docbird] -project_name = recos-injector - -[kite] -project: recos-injector diff --git a/recos-injector/README.docx b/recos-injector/README.docx new file mode 100644 index 000000000..b0ebe619f Binary files /dev/null and b/recos-injector/README.docx differ diff --git a/recos-injector/README.md b/recos-injector/README.md deleted file mode 100644 index a391578c2..000000000 --- a/recos-injector/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Recos-Injector - -Recos-Injector is a streaming event processor used to build input streams for GraphJet-based services. It is a general-purpose tool that consumes arbitrary incoming event streams (e.g., Fav, RT, Follow, client_events, etc.), applies filtering, and combines and publishes cleaned up events to corresponding GraphJet services. Each GraphJet-based service subscribes to a dedicated Kafka topic, and Recos-Injector enables GraphJet-based services to consume any event they want. - -## How to run Recos-Injector server tests - -You can run tests by using the following command from your project's root directory: - - $ bazel build recos-injector/... - $ bazel test recos-injector/... - -## How to run recos-injector-server in development on a local machine - -The simplest way to stand up a service is to run it locally. To run -recos-injector-server in development mode, compile the project and then -execute it with `bazel run`: - - $ bazel build recos-injector/server:bin - $ bazel run recos-injector/server:bin - -A tunnel can be set up in order for downstream queries to work properly. -Upon successful server startup, try to `curl` its admin endpoint in another -terminal: - - $ curl -s localhost:9990/admin/ping - pong - -Run `curl -s localhost:9990/admin` to see a list of all available admin endpoints. - -## Querying Recos-Injector server from a Scala console - -Recos-Injector does not have a Thrift endpoint. Instead, it reads Event Bus and Kafka queues and writes to the Recos-Injector Kafka. - -## Generating a package for deployment - -To package your service into a zip file for deployment, run: - - $ bazel bundle recos-injector/server:bin --bundle-jvm-archive=zip - -If the command is successful, a file named `dist/recos-injector-server.zip` will be created. diff --git a/recos-injector/server/BUILD b/recos-injector/server/BUILD deleted file mode 100644 index 55e2ab35b..000000000 --- a/recos-injector/server/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -target( - name = "server", - tags = ["bazel-compatible"], - dependencies = [ - "recos-injector/server/src/main/scala/com/twitter/recosinjector", - ], -) - -test_suite( - name = "tests", - tags = ["bazel-compatible"], - dependencies = [ - "recos-injector/server/src/test/scala/com/twitter/recosinjector", - ], -) - -jvm_binary( - name = "bin", - basename = "recos-injector-server", - main = "com.twitter.recosinjector.Main", - platform = "java11", - runtime_platform = "java11", - tags = [ - "bazel-compatible:migrated", - ], - dependencies = [ - ":server", - "3rdparty/jvm/org/slf4j:slf4j-jdk14", - ], -) - -jvm_app( - name = "bundle", - basename = "recos-injector", - binary = ":bin", - bundles = [bundle( - fileset = ["config/*"], - owning_target = "recos-injector/server/config:files", - )], - tags = [ - "bazel-compatible:migrated", - ], -) diff --git a/recos-injector/server/BUILD.docx b/recos-injector/server/BUILD.docx new file mode 100644 index 000000000..743e970fd Binary files /dev/null and b/recos-injector/server/BUILD.docx differ diff --git a/recos-injector/server/config/BUILD b/recos-injector/server/config/BUILD deleted file mode 100644 index 68161409f..000000000 --- a/recos-injector/server/config/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -resources( - sources = [ - "!*.pyc", - "!BUILD*", - "*", - ], - tags = ["bazel-compatible"], -) - -# Created for Bazel compatibility. -# In Bazel, loose files must be part of a target to be included into a bundle. -# See also http://go/bazel-compatibility/bundle_does_not_match_any_files -files( - name = "files", - sources = [ - "!BUILD", - "**/*", - ], - tags = ["bazel-compatible"], -) diff --git a/recos-injector/server/config/BUILD.docx b/recos-injector/server/config/BUILD.docx new file mode 100644 index 000000000..b2bd85d50 Binary files /dev/null and b/recos-injector/server/config/BUILD.docx differ diff --git a/recos-injector/server/config/change_log_config.docx b/recos-injector/server/config/change_log_config.docx new file mode 100644 index 000000000..52bdff926 Binary files /dev/null and b/recos-injector/server/config/change_log_config.docx differ diff --git a/recos-injector/server/config/change_log_config.ini b/recos-injector/server/config/change_log_config.ini deleted file mode 100755 index 6a708e921..000000000 --- a/recos-injector/server/config/change_log_config.ini +++ /dev/null @@ -1,7 +0,0 @@ -[Configs] -DCS = all -ROLE = recos-injector -JOB = recos-injector -ENV = prod -PACKAGE = recos-injector-release -PATH = recos-injector diff --git a/recos-injector/server/config/decider.docx b/recos-injector/server/config/decider.docx new file mode 100644 index 000000000..50058708c Binary files /dev/null and b/recos-injector/server/config/decider.docx differ diff --git a/recos-injector/server/config/decider.yml b/recos-injector/server/config/decider.yml deleted file mode 100644 index 13cb5262d..000000000 --- a/recos-injector/server/config/decider.yml +++ /dev/null @@ -1,11 +0,0 @@ -tweet_event_transformer_user_tweet_entity_edges: - comment: "Enables the generation of UserTweetEntity edges in tweet event transformer" - default_availability: 0 - -enable_emit_tweet_edge_from_reply: - comment: "Decides when processing a Reply edge, whether to generate a Tweet edge for it as well" - default_availability: 0 - -enable_unfavorite_edge: - comment: "Decides when processing a UnfavoriteEvent from Timeline events, whether to process unfav edges" - default_availability: 0 diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/BUILD deleted file mode 100644 index ec46c6a41..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = [ - "bazel-compatible", - ], - dependencies = [ - "3rdparty/jvm/io/netty:netty4-tcnative-boringssl-static", - "3rdparty/jvm/org/apache/thrift:libthrift", - "eventbus/client", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client", - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/server", - "finagle/finagle-core/src/main", - "finagle/finagle-http/src/main/scala", - "finagle/finagle-stats", - "finagle/finagle-thriftmux", - "recos-injector/server/config", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/clients", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/config", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/decider", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/edges", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:events-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "thrift-web-forms", - "twitter-server-internal", - "twitter-server/server/src/main/scala", - "twitter-server/slf4j-jdk14/src/main/scala/com/twitter/server/logging", - "util/util-app", - "util/util-logging/src/main/scala", - "util/util-stats/src/main/scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/BUILD.docx new file mode 100644 index 000000000..6c663a9bc Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/Main.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/Main.docx new file mode 100644 index 000000000..4493b8838 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/Main.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/Main.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/Main.scala deleted file mode 100644 index be8d147b1..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/Main.scala +++ /dev/null @@ -1,213 +0,0 @@ -package com.twitter.recosinjector - -import com.twitter.app.Flag -import com.twitter.finagle.http.HttpMuxer -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.ElfOwlFilter -import com.twitter.recosinjector.clients.Gizmoduck -import com.twitter.recosinjector.clients.RecosHoseEntitiesCache -import com.twitter.recosinjector.clients.SocialGraph -import com.twitter.recosinjector.clients.Tweetypie -import com.twitter.recosinjector.clients.UrlResolver -import com.twitter.recosinjector.config._ -import com.twitter.recosinjector.edges.SocialWriteEventToUserUserGraphBuilder -import com.twitter.recosinjector.edges.TimelineEventToUserTweetEntityGraphBuilder -import com.twitter.recosinjector.edges.TweetEventToUserTweetEntityGraphBuilder -import com.twitter.recosinjector.edges.TweetEventToUserUserGraphBuilder -import com.twitter.recosinjector.edges.UnifiedUserActionToUserVideoGraphBuilder -import com.twitter.recosinjector.edges.UnifiedUserActionToUserAdGraphBuilder -import com.twitter.recosinjector.edges.UnifiedUserActionToUserTweetGraphPlusBuilder -import com.twitter.recosinjector.edges.UserTweetEntityEdgeBuilder -import com.twitter.recosinjector.event_processors.SocialWriteEventProcessor -import com.twitter.recosinjector.event_processors.TimelineEventProcessor -import com.twitter.recosinjector.event_processors.TweetEventProcessor -import com.twitter.recosinjector.publishers.KafkaEventPublisher -import com.twitter.recosinjector.uua_processors.UnifiedUserActionProcessor -import com.twitter.recosinjector.uua_processors.UnifiedUserActionsConsumer -import com.twitter.server.logging.{Logging => JDK14Logging} -import com.twitter.server.Deciderable -import com.twitter.server.TwitterServer -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.timelineservice.thriftscala.{Event => TimelineEvent} -import com.twitter.tweetypie.thriftscala.TweetEvent -import com.twitter.util.Await -import com.twitter.util.Duration -import java.util.concurrent.TimeUnit - -object Main extends TwitterServer with JDK14Logging with Deciderable { self => - - implicit val stats: StatsReceiver = statsReceiver - - private val dataCenter: Flag[String] = flag("service.cluster", "atla", "Data Center") - private val serviceRole: Flag[String] = flag("service.role", "Service Role") - private val serviceEnv: Flag[String] = flag("service.env", "Service Env") - private val serviceName: Flag[String] = flag("service.name", "Service Name") - private val shardId = flag("shardId", 0, "Shard ID") - private val numShards = flag("numShards", 1, "Number of shards for this service") - private val truststoreLocation = - flag[String]("truststore_location", "", "Truststore file location") - - def main(): Unit = { - val serviceIdentifier = ServiceIdentifier( - role = serviceRole(), - service = serviceName(), - environment = serviceEnv(), - zone = dataCenter() - ) - println("ServiceIdentifier = " + serviceIdentifier.toString) - log.info("ServiceIdentifier = " + serviceIdentifier.toString) - - val shard = shardId() - val numOfShards = numShards() - val environment = serviceEnv() - - implicit val config: DeployConfig = { - environment match { - case "prod" => ProdConfig(serviceIdentifier)(stats) - case "staging" | "devel" => StagingConfig(serviceIdentifier) - case env => throw new Exception(s"Unknown environment $env") - } - } - - // Initialize the config and wait for initialization to finish - Await.ready(config.init()) - - log.info( - "Starting Recos Injector: environment %s, clientId %s", - environment, - config.recosInjectorThriftClientId - ) - log.info("Starting shard Id: %d of %d shards...".format(shard, numOfShards)) - - // Client wrappers - val cache = new RecosHoseEntitiesCache(config.recosInjectorCoreSvcsCacheClient) - val gizmoduck = new Gizmoduck(config.userStore) - val socialGraph = new SocialGraph(config.socialGraphIdStore) - val tweetypie = new Tweetypie(config.tweetyPieStore) - val urlResolver = new UrlResolver(config.urlInfoStore) - - // Edge builders - val userTweetEntityEdgeBuilder = new UserTweetEntityEdgeBuilder(cache, urlResolver) - - // Publishers - val kafkaEventPublisher = KafkaEventPublisher( - "/s/kafka/recommendations:kafka-tls", - config.outputKafkaTopicPrefix, - config.recosInjectorThriftClientId, - truststoreLocation()) - - // Message Builders - val socialWriteToUserUserMessageBuilder = - new SocialWriteEventToUserUserGraphBuilder()( - statsReceiver.scope("SocialWriteEventToUserUserGraphBuilder") - ) - - val timelineToUserTweetEntityMessageBuilder = new TimelineEventToUserTweetEntityGraphBuilder( - userTweetEntityEdgeBuilder = userTweetEntityEdgeBuilder - )(statsReceiver.scope("TimelineEventToUserTweetEntityGraphBuilder")) - - val tweetEventToUserTweetEntityGraphBuilder = new TweetEventToUserTweetEntityGraphBuilder( - userTweetEntityEdgeBuilder = userTweetEntityEdgeBuilder, - tweetCreationStore = config.tweetCreationStore, - decider = config.recosInjectorDecider - )(statsReceiver.scope("TweetEventToUserTweetEntityGraphBuilder")) - - val socialWriteEventProcessor = new SocialWriteEventProcessor( - eventBusStreamName = s"recos_injector_social_write_event_$environment", - thriftStruct = WriteEvent, - serviceIdentifier = serviceIdentifier, - kafkaEventPublisher = kafkaEventPublisher, - userUserGraphTopic = KafkaEventPublisher.UserUserTopic, - userUserGraphMessageBuilder = socialWriteToUserUserMessageBuilder - )(statsReceiver.scope("SocialWriteEventProcessor")) - - val tweetToUserUserMessageBuilder = new TweetEventToUserUserGraphBuilder()( - statsReceiver.scope("TweetEventToUserUserGraphBuilder") - ) - - val unifiedUserActionToUserVideoGraphBuilder = new UnifiedUserActionToUserVideoGraphBuilder( - userTweetEntityEdgeBuilder = userTweetEntityEdgeBuilder - )(statsReceiver.scope("UnifiedUserActionToUserVideoGraphBuilder")) - - val unifiedUserActionToUserAdGraphBuilder = new UnifiedUserActionToUserAdGraphBuilder( - userTweetEntityEdgeBuilder = userTweetEntityEdgeBuilder - )(statsReceiver.scope("UnifiedUserActionToUserAdGraphBuilder")) - - val unifiedUserActionToUserTweetGraphPlusBuilder = - new UnifiedUserActionToUserTweetGraphPlusBuilder( - userTweetEntityEdgeBuilder = userTweetEntityEdgeBuilder - )(statsReceiver.scope("UnifiedUserActionToUserTweetGraphPlusBuilder")) - - // Processors - val tweetEventProcessor = new TweetEventProcessor( - eventBusStreamName = s"recos_injector_tweet_events_$environment", - thriftStruct = TweetEvent, - serviceIdentifier = serviceIdentifier, - userUserGraphMessageBuilder = tweetToUserUserMessageBuilder, - userUserGraphTopic = KafkaEventPublisher.UserUserTopic, - userTweetEntityGraphMessageBuilder = tweetEventToUserTweetEntityGraphBuilder, - userTweetEntityGraphTopic = KafkaEventPublisher.UserTweetEntityTopic, - kafkaEventPublisher = kafkaEventPublisher, - socialGraph = socialGraph, - tweetypie = tweetypie, - gizmoduck = gizmoduck - )(statsReceiver.scope("TweetEventProcessor")) - - val timelineEventProcessor = new TimelineEventProcessor( - eventBusStreamName = s"recos_injector_timeline_events_prototype_$environment", - thriftStruct = TimelineEvent, - serviceIdentifier = serviceIdentifier, - kafkaEventPublisher = kafkaEventPublisher, - userTweetEntityGraphTopic = KafkaEventPublisher.UserTweetEntityTopic, - userTweetEntityGraphMessageBuilder = timelineToUserTweetEntityMessageBuilder, - decider = config.recosInjectorDecider, - gizmoduck = gizmoduck, - tweetypie = tweetypie - )(statsReceiver.scope("TimelineEventProcessor")) - - val eventBusProcessors = Seq( - timelineEventProcessor, - socialWriteEventProcessor, - tweetEventProcessor - ) - - val uuaProcessor = new UnifiedUserActionProcessor( - gizmoduck = gizmoduck, - tweetypie = tweetypie, - kafkaEventPublisher = kafkaEventPublisher, - userVideoGraphTopic = KafkaEventPublisher.UserVideoTopic, - userVideoGraphBuilder = unifiedUserActionToUserVideoGraphBuilder, - userAdGraphTopic = KafkaEventPublisher.UserAdTopic, - userAdGraphBuilder = unifiedUserActionToUserAdGraphBuilder, - userTweetGraphPlusTopic = KafkaEventPublisher.UserTweetPlusTopic, - userTweetGraphPlusBuilder = unifiedUserActionToUserTweetGraphPlusBuilder)( - statsReceiver.scope("UnifiedUserActionProcessor")) - - val uuaConsumer = new UnifiedUserActionsConsumer(uuaProcessor, truststoreLocation()) - - // Start-up init and graceful shutdown setup - - // wait a bit for services to be ready - Thread.sleep(5000L) - - log.info("Starting the event processors") - eventBusProcessors.foreach(_.start()) - - log.info("Starting the uua processors") - uuaConsumer.atLeastOnceProcessor.start() - - this.addAdminRoute(ElfOwlFilter.getPostbackRoute()) - - onExit { - log.info("Shutting down the event processors") - eventBusProcessors.foreach(_.stop()) - log.info("Shutting down the uua processors") - uuaConsumer.atLeastOnceProcessor.close() - log.info("done exit") - } - - // Wait on the thriftServer so that shutdownTimeout is respected. - Await.result(adminHttpServer) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/BUILD deleted file mode 100644 index 9b0d8be74..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-stats", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/base", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", - "servo/repo/src/main/scala", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/recos:recos-internal-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "stitch/stitch-tweetypie/src/main/scala", - "util/util-logging/src/main/scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/BUILD.docx new file mode 100644 index 000000000..2750e55a2 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Gizmoduck.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Gizmoduck.docx new file mode 100644 index 000000000..0c3473f7c Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Gizmoduck.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Gizmoduck.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Gizmoduck.scala deleted file mode 100644 index f6806a7f5..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Gizmoduck.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.recosinjector.clients - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.logging.Logger -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class Gizmoduck( - userStore: ReadableStore[Long, User] -)( - implicit statsReceiver: StatsReceiver) { - private val log = Logger() - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - - def getUser(userId: Long): Future[Option[User]] = { - userStore - .get(userId) - .rescue { - case e => - stats.scope("getUserFailure").counter(e.getClass.getSimpleName).incr() - log.error(s"Failed with message ${e.toString}") - Future.None - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/RecosHoseEntitiesCache.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/RecosHoseEntitiesCache.docx new file mode 100644 index 000000000..6190ac73e Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/RecosHoseEntitiesCache.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/RecosHoseEntitiesCache.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/RecosHoseEntitiesCache.scala deleted file mode 100644 index c90ac5cc7..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/RecosHoseEntitiesCache.scala +++ /dev/null @@ -1,137 +0,0 @@ -package com.twitter.recosinjector.clients - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.io.Buf -import com.twitter.recos.internal.thriftscala.{RecosHoseEntities, RecosHoseEntity} -import com.twitter.servo.cache.ThriftSerializer -import com.twitter.util.{Duration, Future, Time} -import org.apache.thrift.protocol.TBinaryProtocol - -case class CacheEntityEntry( - cachePrefix: String, - hashedEntityId: Int, - entity: String) { - val fullKey: String = cachePrefix + hashedEntityId -} - -object RecosHoseEntitiesCache { - val EntityTTL: Duration = 30.hours - val EntitiesSerializer = - new ThriftSerializer[RecosHoseEntities](RecosHoseEntities, new TBinaryProtocol.Factory()) - - val HashtagPrefix: String = "h" - val UrlPrefix: String = "u" -} - -/** - * A cache layer to store entities. - * Graph services like user_tweet_entity_graph and user_url_graph store user interactions with - * entities in a tweet, such as HashTags and URLs. These entities are string values that can be - * potentially very big. Therefore, we instead store a hashed id in the graph edge, and keep a - * (hashedId -> entity) mapping in this cache. The actual entity values can be recovered - * by the graph service at serving time using this cache. - */ -class RecosHoseEntitiesCache(client: Client) { - import RecosHoseEntitiesCache._ - - private def isEntityWithinTTL(entity: RecosHoseEntity, ttlInMillis: Long): Boolean = { - entity.timestamp.exists(timestamp => Time.now.inMilliseconds - timestamp <= ttlInMillis) - } - - /** - * Add a new RecosHoseEntity into RecosHoseEntities - */ - private def updateRecosHoseEntities( - existingEntitiesOpt: Option[RecosHoseEntities], - newEntityString: String, - stats: StatsReceiver - ): RecosHoseEntities = { - val existingEntities = existingEntitiesOpt.map(_.entities).getOrElse(Nil) - - // Discard expired and duplicate existing entities - val validExistingEntities = existingEntities - .filter(entity => isEntityWithinTTL(entity, EntityTTL.inMillis)) - .filter(_.entity != newEntityString) - - val newRecosHoseEntity = RecosHoseEntity(newEntityString, Some(Time.now.inMilliseconds)) - RecosHoseEntities(validExistingEntities :+ newRecosHoseEntity) - } - - private def getRecosHoseEntitiesCache( - cacheEntries: Seq[CacheEntityEntry], - stats: StatsReceiver - ): Future[Map[String, Option[RecosHoseEntities]]] = { - client - .get(cacheEntries.map(_.fullKey)) - .map(_.map { - case (cacheKey, buf) => - val recosHoseEntitiesTry = EntitiesSerializer.from(Buf.ByteArray.Owned.extract(buf)) - if (recosHoseEntitiesTry.isThrow) { - stats.counter("cache_get_deserialization_failure").incr() - } - cacheKey -> recosHoseEntitiesTry.toOption - }) - .onSuccess { _ => stats.counter("get_cache_success").incr() } - .onFailure { ex => - stats.scope("get_cache_failure").counter(ex.getClass.getSimpleName).incr() - } - } - - private def putRecosHoseEntitiesCache( - cacheKey: String, - recosHoseEntities: RecosHoseEntities, - stats: StatsReceiver - ): Unit = { - val serialized = EntitiesSerializer.to(recosHoseEntities) - if (serialized.isThrow) { - stats.counter("cache_put_serialization_failure").incr() - } - serialized.toOption.map { bytes => - client - .set(cacheKey, 0, EntityTTL.fromNow, Buf.ByteArray.Owned(bytes)) - .onSuccess { _ => stats.counter("put_cache_success").incr() } - .onFailure { ex => - stats.scope("put_cache_failure").counter(ex.getClass.getSimpleName).incr() - } - } - } - - /** - * Store a list of new entities into the cache by their cacheKeys, and remove expired/invalid - * values in the existing cache entries at the same time - */ - def updateEntitiesCache( - newCacheEntries: Seq[CacheEntityEntry], - stats: StatsReceiver - ): Future[Unit] = { - stats.counter("update_cache_request").incr() - getRecosHoseEntitiesCache(newCacheEntries, stats) - .map { existingCacheEntries => - newCacheEntries.foreach { newCacheEntry => - val fullKey = newCacheEntry.fullKey - val existingRecosHoseEntities = existingCacheEntries.get(fullKey).flatten - stats.stat("num_existing_entities").add(existingRecosHoseEntities.size) - if (existingRecosHoseEntities.isEmpty) { - stats.counter("existing_entities_empty").incr() - } - - val updatedRecosHoseEntities = updateRecosHoseEntities( - existingRecosHoseEntities, - newCacheEntry.entity, - stats - ) - stats.stat("num_updated_entities").add(updatedRecosHoseEntities.entities.size) - - if (updatedRecosHoseEntities.entities.nonEmpty) { - putRecosHoseEntitiesCache(fullKey, updatedRecosHoseEntities, stats) - } - } - } - .onSuccess { _ => stats.counter("update_cache_success").incr() } - .onFailure { ex => - stats.scope("update_cache_failure").counter(ex.getClass.getSimpleName).incr() - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/SocialGraph.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/SocialGraph.docx new file mode 100644 index 000000000..a3729e03f Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/SocialGraph.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/SocialGraph.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/SocialGraph.scala deleted file mode 100644 index f721db2e7..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/SocialGraph.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.recosinjector.clients - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.socialgraph.thriftscala._ -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -class SocialGraph( - socialGraphIdStore: ReadableStore[IdsRequest, IdsResult] -)( - implicit statsReceiver: StatsReceiver) { - import SocialGraph._ - private val log = Logger() - - private val followedByNotMutedByStats = statsReceiver.scope("followedByNotMutedBy") - - private def fetchIdsFromSocialGraph( - userId: Long, - ids: Seq[Long], - relationshipTypes: Map[RelationshipType, Boolean], - lookupContext: Option[LookupContext] = IncludeInactiveUnionLookupContext, - stats: StatsReceiver - ): Future[Seq[Long]] = { - if (ids.isEmpty) { - stats.counter("fetchIdsEmpty").incr() - Future.Nil - } else { - val relationships = relationshipTypes.map { - case (relationshipType, hasRelationship) => - SrcRelationship( - source = userId, - relationshipType = relationshipType, - hasRelationship = hasRelationship, - targets = Some(ids) - ) - }.toSeq - val idsRequest = IdsRequest( - relationships = relationships, - pageRequest = SelectAllPageRequest, - context = lookupContext - ) - socialGraphIdStore - .get(idsRequest) - .map { _.map(_.ids).getOrElse(Nil) } - .rescue { - case e => - stats.scope("fetchIdsFailure").counter(e.getClass.getSimpleName).incr() - log.error(s"Failed with message ${e.toString}") - Future.Nil - } - } - } - - // which of the users in candidates follow userId and have not muted userId - def followedByNotMutedBy(userId: Long, candidates: Seq[Long]): Future[Seq[Long]] = { - fetchIdsFromSocialGraph( - userId, - candidates, - FollowedByNotMutedRelationships, - IncludeInactiveLookupContext, - followedByNotMutedByStats - ) - } - -} - -object SocialGraph { - val SelectAllPageRequest = Some(PageRequest(selectAll = Some(true))) - - val IncludeInactiveLookupContext = Some(LookupContext(includeInactive = true)) - val IncludeInactiveUnionLookupContext = Some( - LookupContext(includeInactive = true, performUnion = Some(true)) - ) - - val FollowedByNotMutedRelationships: Map[RelationshipType, Boolean] = Map( - RelationshipType.FollowedBy -> true, - RelationshipType.MutedBy -> false - ) -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Tweetypie.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Tweetypie.docx new file mode 100644 index 000000000..d1a180e60 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Tweetypie.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Tweetypie.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Tweetypie.scala deleted file mode 100644 index 9e81f6121..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/Tweetypie.scala +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.recosinjector.clients - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.stitch.tweetypie.TweetyPie.{TweetyPieException, TweetyPieResult} -import com.twitter.storehaus.ReadableStore -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.util.Future - -class Tweetypie( - tweetyPieStore: ReadableStore[Long, TweetyPieResult] -)( - implicit statsReceiver: StatsReceiver) { - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val failureStats = stats.scope("getTweetFailure") - - def getTweet(tweetId: Long): Future[Option[Tweet]] = { - tweetyPieStore - .get(tweetId) - .map { _.map(_.tweet) } - .rescue { - case e: TweetyPieException => - // Usually results from trying to query a protected or unsafe tweet - failureStats.scope("TweetyPieException").counter(e.result.tweetState.toString).incr() - Future.None - case e => - failureStats.counter(e.getClass.getSimpleName).incr() - Future.None - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/UrlResolver.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/UrlResolver.docx new file mode 100644 index 000000000..bf2147a3b Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/UrlResolver.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/UrlResolver.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/UrlResolver.scala deleted file mode 100644 index 95817181d..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/clients/UrlResolver.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.twitter.recosinjector.clients - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.util.DefaultTimer -import com.twitter.frigate.common.util.{SnowflakeUtils, UrlInfo} -import com.twitter.storehaus.{FutureOps, ReadableStore} -import com.twitter.util.{Duration, Future, Timer} - -class UrlResolver( - urlInfoStore: ReadableStore[String, UrlInfo] -)( - implicit statsReceiver: StatsReceiver) { - private val EmptyFutureMap = Future.value(Map.empty[String, String]) - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val twitterResolvedUrlCounter = stats.counter("twitterResolvedUrl") - private val resolvedUrlCounter = stats.counter("resolvedUrl") - private val noResolvedUrlCounter = stats.counter("noResolvedUrl") - - private val numNoDelayCounter = stats.counter("urlResolver_no_delay") - private val numDelayCounter = stats.counter("urlResolver_delay") - - implicit val timer: Timer = DefaultTimer - - /** - * Get the resolved URL map of the input raw URLs - * - * @param rawUrls list of raw URLs to query - * @return map of raw URL to resolved URL - */ - def getResolvedUrls(rawUrls: Set[String]): Future[Map[String, String]] = { - FutureOps - .mapCollect(urlInfoStore.multiGet[String](rawUrls)) - .map { resolvedUrlsMap => - resolvedUrlsMap.flatMap { - case ( - url, - Some( - UrlInfo( - Some(resolvedUrl), - Some(_), - Some(domain), - _, - _, - _, - _, - Some(_), - _, - _, - _, - _))) => - if (domain == "Twitter") { // Filter out Twitter based URLs - twitterResolvedUrlCounter.incr() - None - } else { - resolvedUrlCounter.incr() - Some(url -> resolvedUrl) - } - case _ => - noResolvedUrlCounter.incr() - None - } - } - } - - /** - * Get resolved url maps given a list of urls, grouping urls that point to the same webpage - */ - def getResolvedUrls(urls: Seq[String], tweetId: Long): Future[Map[String, String]] = { - if (urls.isEmpty) { - EmptyFutureMap - } else { - Future - .sleep(getUrlResolverDelayDuration(tweetId)) - .before(getResolvedUrls(urls.toSet)) - } - } - - /** - * Given a tweet, return the amount of delay needed before attempting to resolve the Urls - */ - private def getUrlResolverDelayDuration( - tweetId: Long - ): Duration = { - val urlResolverDelaySinceCreation = 12.seconds - val urlResolverDelayDuration = 4.seconds - val noDelay = 0.seconds - - // Check whether the tweet was created more than the specified delay duration before now. - // If the tweet ID is not based on Snowflake, this is false, and the delay is applied. - val isCreatedBeforeDelayThreshold = SnowflakeUtils - .tweetCreationTime(tweetId) - .map(_.untilNow) - .exists(_ > urlResolverDelaySinceCreation) - - if (isCreatedBeforeDelayThreshold) { - numNoDelayCounter.incr() - noDelay - } else { - numDelayCounter.incr() - urlResolverDelayDuration - } - } - -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/BUILD deleted file mode 100644 index 90dfe6868..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/BUILD +++ /dev/null @@ -1,36 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/bijection:scrooge", - "3rdparty/jvm/com/twitter/storehaus:core", - "abdecider", - "decider/src/main/scala", - "finagle/finagle-memcached/src/main/scala", - "finagle/finagle-stats", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", - "hermit/hermit-core:store", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/gizmoduck", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/tweetypie", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/decider", - "scribelib/validators/src/main/scala/com/twitter/scribelib/validators", - "src/scala/com/twitter/storehaus_internal/memcache", - "src/scala/com/twitter/storehaus_internal/memcache/config", - "src/scala/com/twitter/storehaus_internal/util", - "src/thrift/com/twitter/gizmoduck:thrift-java", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/spam/rtf:safety-level-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "stitch/stitch-core", - "stitch/stitch-socialgraph", - "stitch/stitch-storehaus/src/main/scala", - "stitch/stitch-tweetypie/src/main/scala", - "util/util-hashing/src/main/scala", - "util/util-logging/src/main/scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/BUILD.docx new file mode 100644 index 000000000..e4d1a9e63 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/CacheConfig.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/CacheConfig.docx new file mode 100644 index 000000000..9462215bb Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/CacheConfig.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/CacheConfig.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/CacheConfig.scala deleted file mode 100644 index dae1aba12..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/CacheConfig.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.recosinjector.config - -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus_internal.memcache.MemcacheStore -import com.twitter.storehaus_internal.util.{ClientName, ZkEndPoint} - -trait CacheConfig { - implicit def statsReceiver: StatsReceiver - - def serviceIdentifier: ServiceIdentifier - - def recosInjectorCoreSvcsCacheDest: String - - val recosInjectorCoreSvcsCacheClient: Client = MemcacheStore.memcachedClient( - name = ClientName("memcache-recos-injector"), - dest = ZkEndPoint(recosInjectorCoreSvcsCacheDest), - statsReceiver = statsReceiver, - serviceIdentifier = serviceIdentifier - ) - -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/Config.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/Config.docx new file mode 100644 index 000000000..5e6e21dc9 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/Config.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/Config.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/Config.scala deleted file mode 100644 index c2aa209a8..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/Config.scala +++ /dev/null @@ -1,41 +0,0 @@ -package com.twitter.recosinjector.config - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.frigate.common.store.TweetCreationTimeMHStore -import com.twitter.frigate.common.util.UrlInfo -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.recosinjector.decider.RecosInjectorDecider -import com.twitter.socialgraph.thriftscala.{IdsRequest, IdsResult} -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Future - -trait Config { self => - implicit def statsReceiver: StatsReceiver - - // ReadableStores - def tweetyPieStore: ReadableStore[Long, TweetyPieResult] - - def userStore: ReadableStore[Long, User] - - def socialGraphIdStore: ReadableStore[IdsRequest, IdsResult] - - def urlInfoStore: ReadableStore[String, UrlInfo] - - // Manhattan stores - def tweetCreationStore: TweetCreationTimeMHStore - - // Decider - def recosInjectorDecider: RecosInjectorDecider - - // Constants - def recosInjectorThriftClientId: ClientId - - def serviceIdentifier: ServiceIdentifier - - def outputKafkaTopicPrefix: String - - def init(): Future[Unit] = Future.Done -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/DeployConfig.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/DeployConfig.docx new file mode 100644 index 000000000..75237edbd Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/DeployConfig.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/DeployConfig.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/DeployConfig.scala deleted file mode 100644 index 1e547681b..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/DeployConfig.scala +++ /dev/null @@ -1,215 +0,0 @@ -package com.twitter.recosinjector.config - -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.client.ClientRegistry -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.TweetCreationTimeMHStore -import com.twitter.frigate.common.util.Finagle._ -import com.twitter.frigate.common.util.{UrlInfo, UrlInfoInjection, UrlResolver} -import com.twitter.gizmoduck.thriftscala.{LookupContext, QueryFields, User, UserService} -import com.twitter.hermit.store.common.{ObservedCachedReadableStore, ObservedMemcachedReadableStore} -import com.twitter.hermit.store.gizmoduck.GizmoduckUserStore -import com.twitter.hermit.store.tweetypie.TweetyPieStore -import com.twitter.logging.Logger -import com.twitter.pink_floyd.thriftscala.{ClientIdentifier, Storer} -import com.twitter.socialgraph.thriftscala.{IdsRequest, SocialGraphService} -import com.twitter.spam.rtf.thriftscala.SafetyLevel -import com.twitter.stitch.socialgraph.SocialGraph -import com.twitter.stitch.storehaus.ReadableStoreOfStitch -import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult -import com.twitter.storage.client.manhattan.kv.{ - ManhattanKVClient, - ManhattanKVClientMtlsParams, - ManhattanKVEndpointBuilder -} -import com.twitter.storehaus.ReadableStore -import com.twitter.tweetypie.thriftscala.{GetTweetOptions, TweetService} -import com.twitter.util.Future - -/* - * Any finagle clients should not be defined as lazy. If defined lazy, - * ClientRegistry.expAllRegisteredClientsResolved() call in init will not ensure that the clients - * are active before thrift endpoint is active. We want the clients to be active, because zookeeper - * resolution triggered by first request(s) might result in the request(s) failing. - */ -trait DeployConfig extends Config with CacheConfig { - implicit def statsReceiver: StatsReceiver - - def log: Logger - - // Clients - val gizmoduckClient = new UserService.FinagledClient( - readOnlyThriftService( - "gizmoduck", - "/s/gizmoduck/gizmoduck", - statsReceiver, - recosInjectorThriftClientId, - requestTimeout = 450.milliseconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - val tweetyPieClient = new TweetService.FinagledClient( - readOnlyThriftService( - "tweetypie", - "/s/tweetypie/tweetypie", - statsReceiver, - recosInjectorThriftClientId, - requestTimeout = 450.milliseconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - - val sgsClient = new SocialGraphService.FinagledClient( - readOnlyThriftService( - "socialgraph", - "/s/socialgraph/socialgraph", - statsReceiver, - recosInjectorThriftClientId, - requestTimeout = 450.milliseconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - - val pinkStoreClient = new Storer.FinagledClient( - readOnlyThriftService( - "pink_store", - "/s/spiderduck/pink-store", - statsReceiver, - recosInjectorThriftClientId, - requestTimeout = 450.milliseconds, - mTLSServiceIdentifier = Some(serviceIdentifier) - ) - ) - - // Stores - private val _gizmoduckStore = { - val queryFields: Set[QueryFields] = Set( - QueryFields.Discoverability, - QueryFields.Labels, - QueryFields.Safety - ) - val context: LookupContext = LookupContext( - includeDeactivated = true, - safetyLevel = Some(SafetyLevel.Recommendations) - ) - - GizmoduckUserStore( - client = gizmoduckClient, - queryFields = queryFields, - context = context, - statsReceiver = statsReceiver - ) - } - - override val userStore: ReadableStore[Long, User] = { - // memcache based cache - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = _gizmoduckStore, - cacheClient = recosInjectorCoreSvcsCacheClient, - ttl = 2.hours - )( - valueInjection = BinaryScalaCodec(User), - statsReceiver = statsReceiver.scope("UserStore"), - keyToString = { k: Long => - s"usri/$k" - } - ) - } - - /** - * TweetyPie store, used to fetch tweet objects when unavailable, and also as a source of - * tweet SafetyLevel filtering. - * Note: we do NOT cache TweetyPie calls, as it makes tweet SafetyLevel filtering less accurate. - * TweetyPie QPS is < 20K/cluster. - * More info is here: - * https://cgit.twitter.biz/source/tree/src/thrift/com/twitter/spam/rtf/safety_level.thrift - */ - override val tweetyPieStore: ReadableStore[Long, TweetyPieResult] = { - val getTweetOptions = Some( - GetTweetOptions( - includeCards = true, - safetyLevel = Some(SafetyLevel.RecosWritePath) - ) - ) - TweetyPieStore( - tweetyPieClient, - getTweetOptions, - convertExceptionsToNotFound = false // Do not suppress TweetyPie errors. Leave it to caller - ) - } - - private val _urlInfoStore = { - //Initialize pink store client, for parsing url - UrlResolver( - pinkStoreClient, - statsReceiver.scope("urlFetcher"), - clientId = ClientIdentifier.Recoshose - ) - } - - override val urlInfoStore: ReadableStore[String, UrlInfo] = { - // memcache based cache - val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = _urlInfoStore, - cacheClient = recosInjectorCoreSvcsCacheClient, - ttl = 2.hours - )( - valueInjection = UrlInfoInjection, - statsReceiver = statsReceiver.scope("UrlInfoStore"), - keyToString = { k: String => - s"uisri/$k" - } - ) - - ObservedCachedReadableStore.from( - memcachedStore, - ttl = 1.minutes, - maxKeys = 1e5.toInt, - windowSize = 10000L, - cacheName = "url_store_in_proc_cache" - )(statsReceiver.scope("url_store_in_proc_cache")) - } - - override val socialGraphIdStore = ReadableStoreOfStitch { idsRequest: IdsRequest => - SocialGraph(sgsClient).ids(idsRequest) - } - - /** - * MH Store for updating the last time user created a tweet - */ - val tweetCreationStore: TweetCreationTimeMHStore = { - val client = ManhattanKVClient( - appId = "recos_tweet_creation_info", - dest = "/s/manhattan/omega.native-thrift", - mtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier) - ) - - val endpoint = ManhattanKVEndpointBuilder(client) - .defaultMaxTimeout(700.milliseconds) - .statsReceiver( - statsReceiver - .scope(serviceIdentifier.zone) - .scope(serviceIdentifier.environment) - .scope("recos_injector_tweet_creation_info_store") - ) - .build() - - val dataset = if (serviceIdentifier.environment == "prod") { - "recos_injector_tweet_creation_info" - } else { - "recos_injector_tweet_creation_info_staging" - } - - new TweetCreationTimeMHStore( - cluster = serviceIdentifier.zone, - endpoint = endpoint, - dataset = dataset, - writeTtl = Some(14.days), - statsReceiver.scope("recos_injector_tweet_creation_info_store") - ) - } - - // wait for all serversets to populate - override def init(): Future[Unit] = ClientRegistry.expAllRegisteredClientsResolved().unit -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/ProdConfig.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/ProdConfig.docx new file mode 100644 index 000000000..1dac7217b Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/ProdConfig.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/ProdConfig.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/ProdConfig.scala deleted file mode 100644 index 7ead5c34d..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/ProdConfig.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.recosinjector.config - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.logging.Logger -import com.twitter.recosinjector.decider.RecosInjectorDecider - -case class ProdConfig( - override val serviceIdentifier: ServiceIdentifier -)(implicit val statsReceiver: StatsReceiver) extends { - // Due to trait initialization logic in Scala, any abstract members declared in Config or - // DeployConfig should be declared in this block. Otherwise the abstract member might initialize - // to null if invoked before before object creation finishing. - - val recosInjectorThriftClientId = ClientId("recos-injector.prod") - - val outputKafkaTopicPrefix = "recos_injector" - - val log = Logger("ProdConfig") - - val recosInjectorCoreSvcsCacheDest = "/srv#/prod/local/cache/recos_metadata" - - val recosInjectorDecider = RecosInjectorDecider( - isProd = true, - dataCenter = serviceIdentifier.zone - ) - -} with DeployConfig diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/StagingConfig.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/StagingConfig.docx new file mode 100644 index 000000000..bf5c1529e Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/StagingConfig.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/StagingConfig.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/StagingConfig.scala deleted file mode 100644 index 1caabea8e..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/config/StagingConfig.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.recosinjector.config - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.logging.Logger -import com.twitter.recosinjector.decider.RecosInjectorDecider - -case class StagingConfig( - override val serviceIdentifier: ServiceIdentifier -)( - implicit val statsReceiver: StatsReceiver) - extends { - // Due to trait initialization logic in Scala, any abstract members declared in Config or - // DeployConfig should be declared in this block. Otherwise the abstract member might initialize - // to null if invoked before before object creation finishing. - - val recosInjectorThriftClientId = ClientId("recos-injector.staging") - - val outputKafkaTopicPrefix = "staging_recos_injector" - - val log = Logger("StagingConfig") - - val recosInjectorCoreSvcsCacheDest = "/srv#/test/local/cache/twemcache_recos" - - val recosInjectorDecider = RecosInjectorDecider( - isProd = false, - dataCenter = serviceIdentifier.zone - ) - - val abDeciderLoggerNode = "staging_abdecider_scribe" - -} with DeployConfig diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/BUILD deleted file mode 100644 index 0a728c357..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/BUILD +++ /dev/null @@ -1,7 +0,0 @@ -scala_library( - platform = "java11", - tags = ["bazel-compatible"], - dependencies = [ - "decider/src/main/scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/BUILD.docx new file mode 100644 index 000000000..caa8bb615 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/RecosInjectorDecider.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/RecosInjectorDecider.docx new file mode 100644 index 000000000..24b28e8fa Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/RecosInjectorDecider.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/RecosInjectorDecider.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/RecosInjectorDecider.scala deleted file mode 100644 index d9bb86bf1..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/decider/RecosInjectorDecider.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.recosinjector.decider - -import com.twitter.decider.{Decider, DeciderFactory, RandomRecipient, Recipient} - -case class RecosInjectorDecider(isProd: Boolean, dataCenter: String) { - lazy val decider: Decider = DeciderFactory( - Some("config/decider.yml"), - Some(getOverlayPath(isProd, dataCenter)) - )() - - private def getOverlayPath(isProd: Boolean, dataCenter: String): String = { - if (isProd) { - s"/usr/local/config/overlays/recos-injector/recos-injector/prod/$dataCenter/decider_overlay.yml" - } else { - s"/usr/local/config/overlays/recos-injector/recos-injector/staging/$dataCenter/decider_overlay.yml" - } - } - - def getDecider: Decider = decider - - def isAvailable(feature: String, recipient: Option[Recipient]): Boolean = { - decider.isAvailable(feature, recipient) - } - - def isAvailable(feature: String): Boolean = isAvailable(feature, Some(RandomRecipient)) -} - -object RecosInjectorDeciderConstants { - val TweetEventTransformerUserTweetEntityEdgesDecider = - "tweet_event_transformer_user_tweet_entity_edges" - val EnableEmitTweetEdgeFromReply = "enable_emit_tweet_edge_from_reply" - val EnableUnfavoriteEdge = "enable_unfavorite_edge" -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/BUILD deleted file mode 100644 index c19b2d96e..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/storehaus:core", - "finagle/finagle-stats", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/predicate/socialgraph", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/clients", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/decider", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/filters", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/util", - "src/scala/com/twitter/recos/util:recos-util", - "src/thrift/com/twitter/recos:recos-injector-scala", - "src/thrift/com/twitter/recos:recos-internal-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:events-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/BUILD.docx new file mode 100644 index 000000000..658148858 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/Edges.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/Edges.docx new file mode 100644 index 000000000..cc49621ca Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/Edges.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/Edges.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/Edges.scala deleted file mode 100644 index e7ccd4fb0..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/Edges.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.recos.internal.thriftscala.RecosHoseMessage -import com.twitter.recos.recos_injector.thriftscala.{Features, UserTweetAuthorGraphMessage} -import com.twitter.recos.util.Action.Action -import com.twitter.recosinjector.util.TweetDetails -import scala.collection.Map - -trait Edge { - // RecosHoseMessage is the thrift struct that the graphs consume. - def convertToRecosHoseMessage: RecosHoseMessage - - // UserTweetAuthorGraphMessage is the thrift struct that user_tweet_author_graph consumes. - def convertToUserTweetAuthorGraphMessage: UserTweetAuthorGraphMessage -} - -/** - * Edge corresponding to UserTweetEntityEdge. - * It captures user-tweet interactions: Create, Like, Retweet, Reply etc. - */ -case class UserTweetEntityEdge( - sourceUser: Long, - targetTweet: Long, - action: Action, - cardInfo: Option[Byte], - metadata: Option[Long], - entitiesMap: Option[Map[Byte, Seq[Int]]], - tweetDetails: Option[TweetDetails]) - extends Edge { - - override def convertToRecosHoseMessage: RecosHoseMessage = { - RecosHoseMessage( - leftId = sourceUser, - rightId = targetTweet, - action = action.id.toByte, - card = cardInfo, - entities = entitiesMap, - edgeMetadata = metadata - ) - } - - private def getFeatures(tweetDetails: TweetDetails): Features = { - Features( - hasPhoto = Some(tweetDetails.hasPhoto), - hasVideo = Some(tweetDetails.hasVideo), - hasUrl = Some(tweetDetails.hasUrl), - hasHashtag = Some(tweetDetails.hasHashtag) - ) - } - - override def convertToUserTweetAuthorGraphMessage: UserTweetAuthorGraphMessage = { - UserTweetAuthorGraphMessage( - leftId = sourceUser, - rightId = targetTweet, - action = action.id.toByte, - card = cardInfo, - authorId = tweetDetails.flatMap(_.authorId), - features = tweetDetails.map(getFeatures) - ) - } -} - -/** - * Edge corresponding to UserUserGraph. - * It captures user-user interactions: Follow, Mention, Mediatag. - */ -case class UserUserEdge( - sourceUser: Long, - targetUser: Long, - action: Action, - metadata: Option[Long]) - extends Edge { - override def convertToRecosHoseMessage: RecosHoseMessage = { - RecosHoseMessage( - leftId = sourceUser, - rightId = targetUser, - action = action.id.toByte, - edgeMetadata = metadata - ) - } - - override def convertToUserTweetAuthorGraphMessage: UserTweetAuthorGraphMessage = { - throw new RuntimeException( - "convertToUserTweetAuthorGraphMessage not implemented in UserUserEdge.") - } - -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/EventToMessageBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/EventToMessageBuilder.docx new file mode 100644 index 000000000..325ff5bfe Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/EventToMessageBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/EventToMessageBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/EventToMessageBuilder.scala deleted file mode 100644 index 933bec61b..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/EventToMessageBuilder.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.base.Stats.track -import com.twitter.util.Future - -/** - * This is the generic interface that converts incoming Events (ex. TweetEvent, FavEvent, etc) - * into Edge for a specific output graph. It applies the following flow: - * - * event -> update event stats -> build edges -> filter edges - * - * Top-level statistics are provided for each step, such as latency and number of events - */ -trait EventToMessageBuilder[Event, E <: Edge] { - implicit val statsReceiver: StatsReceiver - - private lazy val processEventStats = statsReceiver.scope("process_event") - private lazy val numEventsStats = statsReceiver.counter("num_process_event") - private lazy val rejectEventStats = statsReceiver.counter("num_reject_event") - private lazy val buildEdgesStats = statsReceiver.scope("build") - private lazy val numAllEdgesStats = buildEdgesStats.counter("num_all_edges") - private lazy val filterEdgesStats = statsReceiver.scope("filter") - private lazy val numValidEdgesStats = statsReceiver.counter("num_valid_edges") - private lazy val numRecosHoseMessageStats = statsReceiver.counter("num_RecosHoseMessage") - - /** - * Given an incoming event, process and convert it into a sequence of RecosHoseMessages - * @param event - * @return - */ - def processEvent(event: Event): Future[Seq[Edge]] = { - track(processEventStats) { - shouldProcessEvent(event).flatMap { - case true => - numEventsStats.incr() - updateEventStatus(event) - for { - allEdges <- track(buildEdgesStats)(buildEdges(event)) - filteredEdges <- track(filterEdgesStats)(filterEdges(event, allEdges)) - } yield { - numAllEdgesStats.incr(allEdges.size) - numValidEdgesStats.incr(filteredEdges.size) - numRecosHoseMessageStats.incr(filteredEdges.size) - filteredEdges - } - case false => - rejectEventStats.incr() - Future.Nil - } - } - } - - /** - * Pre-process filter that determines whether the given event should be used to build edges. - * @param event - * @return - */ - def shouldProcessEvent(event: Event): Future[Boolean] - - /** - * Update cache/event logging related to the specific event. - * By default, no action will be taken. Override when necessary - * @param event - */ - def updateEventStatus(event: Event): Unit = {} - - /** - * Given an event, extract info and build a sequence of edges - * @param event - * @return - */ - def buildEdges(event: Event): Future[Seq[E]] - - /** - * Given a sequence of edges, filter and return the valid edges - * @param event - * @param edges - * @return - */ - def filterEdges(event: Event, edges: Seq[E]): Future[Seq[E]] -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/SocialWriteEventToUserUserGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/SocialWriteEventToUserUserGraphBuilder.docx new file mode 100644 index 000000000..237d327d6 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/SocialWriteEventToUserUserGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/SocialWriteEventToUserUserGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/SocialWriteEventToUserUserGraphBuilder.scala deleted file mode 100644 index edebbada1..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/SocialWriteEventToUserUserGraphBuilder.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.socialgraph.thriftscala.{ - Action => SocialGraphAction, - FollowGraphEvent, - FollowType, - WriteEvent -} -import com.twitter.util.Future - -/** - * Converts a WriteEvent to UserUserGraph's messages, including Mention and Mediatag messages - */ -class SocialWriteEventToUserUserGraphBuilder()(override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[WriteEvent, UserUserEdge] { - private val followOrFrictionlessFollowCounter = - statsReceiver.counter("num_follow_or_frictionless") - private val notFollowOrFrictionlessFollowCounter = - statsReceiver.counter("num_not_follow_or_frictionless") - private val followEdgeCounter = statsReceiver.counter("num_follow_edge") - - /** - * For now, we are only interested in Follow events - */ - override def shouldProcessEvent(event: WriteEvent): Future[Boolean] = { - event.action match { - case SocialGraphAction.Follow | SocialGraphAction.FrictionlessFollow => - followOrFrictionlessFollowCounter.incr() - Future(true) - case _ => - notFollowOrFrictionlessFollowCounter.incr() - Future(false) - } - } - - /** - * Determine whether a Follow event is valid/error free. - */ - private def isValidFollowEvent(followEvent: FollowGraphEvent): Boolean = { - followEvent.followType match { - case Some(FollowType.NormalFollow) | Some(FollowType.FrictionlessFollow) => - followEvent.result.validationError.isEmpty - case _ => - false - } - } - - override def buildEdges(event: WriteEvent): Future[Seq[UserUserEdge]] = { - val userUserEdges = event.follow - .map(_.collect { - case followEvent if isValidFollowEvent(followEvent) => - val sourceUserId = followEvent.result.request.source - val targetUserId = followEvent.result.request.target - followEdgeCounter.incr() - UserUserEdge( - sourceUserId, - targetUserId, - Action.Follow, - Some(System.currentTimeMillis()) - ) - }).getOrElse(Nil) - Future(userUserEdges) - } - - override def filterEdges( - event: WriteEvent, - edges: Seq[UserUserEdge] - ): Future[Seq[UserUserEdge]] = { - Future(edges) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetEntityGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetEntityGraphBuilder.docx new file mode 100644 index 000000000..e7c03dbc2 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetEntityGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetEntityGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetEntityGraphBuilder.scala deleted file mode 100644 index 73411b510..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetEntityGraphBuilder.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recosinjector.util.TweetFavoriteEventDetails -import com.twitter.util.Future - -class TimelineEventToUserTweetEntityGraphBuilder( - userTweetEntityEdgeBuilder: UserTweetEntityEdgeBuilder -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[TweetFavoriteEventDetails, UserTweetEntityEdge] { - - private val numFavEdgeCounter = statsReceiver.counter("num_favorite_edge") - private val numUnfavEdgeCounter = statsReceiver.counter("num_unfavorite_edge") - - override def shouldProcessEvent(event: TweetFavoriteEventDetails): Future[Boolean] = { - Future(true) - } - - override def buildEdges(details: TweetFavoriteEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val engagement = details.userTweetEngagement - val tweetDetails = engagement.tweetDetails - - val entitiesMapFut = userTweetEntityEdgeBuilder.getEntitiesMapAndUpdateCache( - tweetId = engagement.tweetId, - tweetDetails = tweetDetails - ) - - entitiesMapFut - .map { entitiesMap => - UserTweetEntityEdge( - sourceUser = engagement.engageUserId, - targetTweet = engagement.tweetId, - action = engagement.action, - metadata = engagement.engagementTimeMillis, - cardInfo = engagement.tweetDetails.map(_.cardInfo.toByte), - entitiesMap = entitiesMap, - tweetDetails = tweetDetails - ) - } - .map { edge => - edge match { - case fav if fav.action == Action.Favorite => - numFavEdgeCounter.incr() - case unfav if unfav.action == Action.Unfavorite => - numUnfavEdgeCounter.incr() - case _ => - } - Seq(edge) - } - } - - override def filterEdges( - event: TweetFavoriteEventDetails, - edges: Seq[UserTweetEntityEdge] - ): Future[Seq[UserTweetEntityEdge]] = { - Future(edges) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetGraphBuilder.docx new file mode 100644 index 000000000..f693a1e67 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetGraphBuilder.scala deleted file mode 100644 index 5d5572a54..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TimelineEventToUserTweetGraphBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recosinjector.util.TweetFavoriteEventDetails -import com.twitter.util.Future - -class TimelineEventToUserTweetGraphBuilder( - userTweetEntityEdgeBuilder: UserTweetEntityEdgeBuilder -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[TweetFavoriteEventDetails, UserTweetEntityEdge] { - - override def shouldProcessEvent(event: TweetFavoriteEventDetails): Future[Boolean] = { - Future(true) - } - - override def buildEdges(details: TweetFavoriteEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val engagement = details.userTweetEngagement - - engagement.action match { - case Action.Favorite => - val tweetDetails = engagement.tweetDetails - - val entitiesMapFut = userTweetEntityEdgeBuilder.getEntitiesMapAndUpdateCache( - tweetId = engagement.tweetId, - tweetDetails = tweetDetails - ) - - entitiesMapFut - .map { entitiesMap => - UserTweetEntityEdge( - sourceUser = engagement.engageUserId, - targetTweet = engagement.tweetId, - action = engagement.action, - metadata = engagement.engagementTimeMillis, - cardInfo = engagement.tweetDetails.map(_.cardInfo.toByte), - entitiesMap = entitiesMap, - tweetDetails = tweetDetails - ) - } - .map(Seq(_)) - - case _ => Future.Nil - } - } - - override def filterEdges( - event: TweetFavoriteEventDetails, - edges: Seq[UserTweetEntityEdge] - ): Future[Seq[UserTweetEntityEdge]] = { - Future(edges) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetEntityGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetEntityGraphBuilder.docx new file mode 100644 index 000000000..6e10fea9d Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetEntityGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetEntityGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetEntityGraphBuilder.scala deleted file mode 100644 index ef4a200ab..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetEntityGraphBuilder.scala +++ /dev/null @@ -1,343 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.TweetCreationTimeMHStore -import com.twitter.frigate.common.util.SnowflakeUtils -import com.twitter.recos.internal.thriftscala.{RecosUserTweetInfo, TweetType} -import com.twitter.recos.util.Action -import com.twitter.recosinjector.decider.RecosInjectorDecider -import com.twitter.recosinjector.decider.RecosInjectorDeciderConstants -import com.twitter.recosinjector.util.TweetCreateEventDetails -import com.twitter.util.{Future, Time} - -class TweetEventToUserTweetEntityGraphBuilder( - userTweetEntityEdgeBuilder: UserTweetEntityEdgeBuilder, - tweetCreationStore: TweetCreationTimeMHStore, - decider: RecosInjectorDecider -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[TweetCreateEventDetails, UserTweetEntityEdge] { - - // TweetCreationStore counters - private val lastTweetTimeNotInMh = statsReceiver.counter("last_tweet_time_not_in_mh") - private val tweetCreationStoreInserts = statsReceiver.counter("tweet_creation_store_inserts") - - private val numInvalidActionCounter = statsReceiver.counter("num_invalid_tweet_action") - - private val numTweetEdgesCounter = statsReceiver.counter("num_tweet_edge") - private val numRetweetEdgesCounter = statsReceiver.counter("num_retweet_edge") - private val numReplyEdgesCounter = statsReceiver.counter("num_reply_edge") - private val numQuoteEdgesCounter = statsReceiver.counter("num_quote_edge") - private val numIsMentionedEdgesCounter = statsReceiver.counter("num_isMentioned_edge") - private val numIsMediataggedEdgesCounter = statsReceiver.counter("num_isMediatagged_edge") - - private val numIsDecider = statsReceiver.counter("num_decider_enabled") - private val numIsNotDecider = statsReceiver.counter("num_decider_not_enabled") - - override def shouldProcessEvent(event: TweetCreateEventDetails): Future[Boolean] = { - val isDecider = decider.isAvailable( - RecosInjectorDeciderConstants.TweetEventTransformerUserTweetEntityEdgesDecider - ) - if (isDecider) { - numIsDecider.incr() - Future(true) - } else { - numIsNotDecider.incr() - Future(false) - } - } - - /** - * Build edges Reply event. Reply event emits 2 edges: - * author -> Reply -> SourceTweetId - * author -> Tweet -> ReplyId - * Do not associate entities in reply tweet to the source tweet - */ - private def buildReplyEdge(event: TweetCreateEventDetails) = { - val userTweetEngagement = event.userTweetEngagement - val authorId = userTweetEngagement.engageUserId - - val replyEdgeFut = event.sourceTweetDetails - .map { sourceTweetDetails => - val sourceTweetId = sourceTweetDetails.tweet.id - val sourceTweetEntitiesMapFut = userTweetEntityEdgeBuilder.getEntitiesMapAndUpdateCache( - tweetId = sourceTweetId, - tweetDetails = Some(sourceTweetDetails) - ) - - sourceTweetEntitiesMapFut.map { sourceTweetEntitiesMap => - val replyEdge = UserTweetEntityEdge( - sourceUser = authorId, - targetTweet = sourceTweetId, - action = Action.Reply, - metadata = Some(userTweetEngagement.tweetId), - cardInfo = Some(sourceTweetDetails.cardInfo.toByte), - entitiesMap = sourceTweetEntitiesMap, - tweetDetails = Some(sourceTweetDetails) - ) - numReplyEdgesCounter.incr() - Some(replyEdge) - } - }.getOrElse(Future.None) - - val tweetCreationEdgeFut = - if (decider.isAvailable(RecosInjectorDeciderConstants.EnableEmitTweetEdgeFromReply)) { - getAndUpdateLastTweetCreationTime( - authorId = authorId, - tweetId = userTweetEngagement.tweetId, - tweetType = TweetType.Reply - ).map { lastTweetTime => - val edge = UserTweetEntityEdge( - sourceUser = authorId, - targetTweet = userTweetEngagement.tweetId, - action = Action.Tweet, - metadata = lastTweetTime, - cardInfo = userTweetEngagement.tweetDetails.map(_.cardInfo.toByte), - entitiesMap = None, - tweetDetails = userTweetEngagement.tweetDetails - ) - numTweetEdgesCounter.incr() - Some(edge) - } - } else { - Future.None - } - - Future.join(replyEdgeFut, tweetCreationEdgeFut).map { - case (replyEdgeOpt, tweetCreationEdgeOpt) => - tweetCreationEdgeOpt.toSeq ++ replyEdgeOpt.toSeq - } - } - - /** - * Build a Retweet UTEG edge: author -> RT -> SourceTweetId. - */ - private def buildRetweetEdge(event: TweetCreateEventDetails) = { - val userTweetEngagement = event.userTweetEngagement - val tweetId = userTweetEngagement.tweetId - - event.sourceTweetDetails - .map { sourceTweetDetails => - val sourceTweetId = sourceTweetDetails.tweet.id // Id of the tweet being Retweeted - val sourceTweetEntitiesMapFut = userTweetEntityEdgeBuilder.getEntitiesMapAndUpdateCache( - tweetId = sourceTweetId, - tweetDetails = Some(sourceTweetDetails) - ) - - sourceTweetEntitiesMapFut.map { sourceTweetEntitiesMap => - val edge = UserTweetEntityEdge( - sourceUser = userTweetEngagement.engageUserId, - targetTweet = sourceTweetId, - action = Action.Retweet, - metadata = Some(tweetId), // metadata is the tweetId - cardInfo = Some(sourceTweetDetails.cardInfo.toByte), - entitiesMap = sourceTweetEntitiesMap, - tweetDetails = Some(sourceTweetDetails) - ) - numRetweetEdgesCounter.incr() - Seq(edge) - } - }.getOrElse(Future.Nil) - } - - /** - * Build edges for a Quote event. Quote tweet emits 2 edges: - * 1. A quote social proof: author -> Quote -> SourceTweetId - * 2. A tweet creation edge: author -> Tweet -> QuoteTweetId - */ - private def buildQuoteEdges( - event: TweetCreateEventDetails - ): Future[Seq[UserTweetEntityEdge]] = { - val userTweetEngagement = event.userTweetEngagement - val tweetId = userTweetEngagement.tweetId - val authorId = userTweetEngagement.engageUserId - - // do not associate entities in quote tweet to the source tweet, - // but associate entities to quote tweet in tweet creation event - val quoteTweetEdgeFut = event.sourceTweetDetails - .map { sourceTweetDetails => - val sourceTweetId = sourceTweetDetails.tweet.id // Id of the tweet being quoted - val sourceTweetEntitiesMapFut = userTweetEntityEdgeBuilder.getEntitiesMapAndUpdateCache( - tweetId = sourceTweetId, - tweetDetails = event.sourceTweetDetails - ) - - sourceTweetEntitiesMapFut.map { sourceTweetEntitiesMap => - val edge = UserTweetEntityEdge( - sourceUser = authorId, - targetTweet = sourceTweetId, - action = Action.Quote, - metadata = Some(tweetId), // metadata is tweetId - cardInfo = Some(sourceTweetDetails.cardInfo.toByte), // cardInfo of the source tweet - entitiesMap = sourceTweetEntitiesMap, - tweetDetails = Some(sourceTweetDetails) - ) - numQuoteEdgesCounter.incr() - Seq(edge) - } - }.getOrElse(Future.Nil) - - val tweetCreationEdgeFut = getAndUpdateLastTweetCreationTime( - authorId = authorId, - tweetId = tweetId, - tweetType = TweetType.Quote - ).map { lastTweetTime => - val metadata = lastTweetTime - val cardInfo = userTweetEngagement.tweetDetails.map(_.cardInfo.toByte) - val edge = UserTweetEntityEdge( - sourceUser = authorId, - targetTweet = tweetId, - action = Action.Tweet, - metadata = metadata, - cardInfo = cardInfo, - entitiesMap = None, - tweetDetails = userTweetEngagement.tweetDetails - ) - numTweetEdgesCounter.incr() - Seq(edge) - } - - Future.join(quoteTweetEdgeFut, tweetCreationEdgeFut).map { - case (quoteEdge, creationEdge) => - quoteEdge ++ creationEdge - } - } - - /** - * Build edges for a Tweet event. A Tweet emits 3 tyes edges: - * 1. A tweet creation edge: author -> Tweet -> TweetId - * 2. IsMentioned edges: mentionedUserId -> IsMentioned -> TweetId - * 3. IsMediatagged edges: mediataggedUserId -> IsMediatagged -> TweetId - */ - private def buildTweetEdges(event: TweetCreateEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val userTweetEngagement = event.userTweetEngagement - val tweetDetails = userTweetEngagement.tweetDetails - val tweetId = userTweetEngagement.tweetId - val authorId = userTweetEngagement.engageUserId - - val cardInfo = tweetDetails.map(_.cardInfo.toByte) - - val entitiesMapFut = userTweetEntityEdgeBuilder.getEntitiesMapAndUpdateCache( - tweetId = tweetId, - tweetDetails = tweetDetails - ) - - val lastTweetTimeFut = getAndUpdateLastTweetCreationTime( - authorId = authorId, - tweetId = tweetId, - tweetType = TweetType.Tweet - ) - - Future.join(entitiesMapFut, lastTweetTimeFut).map { - case (entitiesMap, lastTweetTime) => - val tweetCreationEdge = UserTweetEntityEdge( - sourceUser = authorId, - targetTweet = tweetId, - action = Action.Tweet, - metadata = lastTweetTime, - cardInfo = cardInfo, - entitiesMap = entitiesMap, - tweetDetails = userTweetEngagement.tweetDetails - ) - numTweetEdgesCounter.incr() - - val isMentionedEdges = event.validMentionUserIds - .map(_.map { mentionedUserId => - UserTweetEntityEdge( - sourceUser = mentionedUserId, - targetTweet = tweetId, - action = Action.IsMentioned, - metadata = Some(tweetId), - cardInfo = cardInfo, - entitiesMap = entitiesMap, - tweetDetails = userTweetEngagement.tweetDetails - ) - }).getOrElse(Nil) - numIsMentionedEdgesCounter.incr(isMentionedEdges.size) - - val isMediataggedEdges = event.validMediatagUserIds - .map(_.map { mediataggedUserId => - UserTweetEntityEdge( - sourceUser = mediataggedUserId, - targetTweet = tweetId, - action = Action.IsMediaTagged, - metadata = Some(tweetId), - cardInfo = cardInfo, - entitiesMap = entitiesMap, - tweetDetails = userTweetEngagement.tweetDetails - ) - }).getOrElse(Nil) - numIsMediataggedEdgesCounter.incr(isMediataggedEdges.size) - - Seq(tweetCreationEdge) ++ isMentionedEdges ++ isMediataggedEdges - } - } - - /** - * For a given user, read the user's last time tweeted from the MH store, and - * write the new tweet time into the MH store before returning. - * Note this function is async, so the MH write operations will continue to execute on its own. - * This might create a read/write race condition, but it's expected. - */ - private def getAndUpdateLastTweetCreationTime( - authorId: Long, - tweetId: Long, - tweetType: TweetType - ): Future[Option[Long]] = { - val newTweetInfo = RecosUserTweetInfo( - authorId, - tweetId, - tweetType, - SnowflakeUtils.tweetCreationTime(tweetId).map(_.inMillis).getOrElse(Time.now.inMillis) - ) - - tweetCreationStore - .get(authorId) - .map(_.map { previousTweetInfoSeq => - val lastTweetTime = previousTweetInfoSeq - .filter(info => info.tweetType == TweetType.Tweet || info.tweetType == TweetType.Quote) - .map(_.tweetTimestamp) - .sortBy(-_) - .headOption // Fetch the latest time user Tweeted or Quoted - .getOrElse( - Time.Bottom.inMillis - ) // Last tweet time never recorded in MH, default to oldest point in time - - if (lastTweetTime == Time.Bottom.inMillis) lastTweetTimeNotInMh.incr() - lastTweetTime - }) - .ensure { - tweetCreationStore - .put(authorId, newTweetInfo) - .onSuccess(_ => tweetCreationStoreInserts.incr()) - .onFailure { e => - statsReceiver.counter("write_failed_with_ex:" + e.getClass.getName).incr() - } - } - } - - override def buildEdges(event: TweetCreateEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val userTweetEngagement = event.userTweetEngagement - userTweetEngagement.action match { - case Action.Reply => - buildReplyEdge(event) - case Action.Retweet => - buildRetweetEdge(event) - case Action.Tweet => - buildTweetEdges(event) - case Action.Quote => - buildQuoteEdges(event) - case _ => - numInvalidActionCounter.incr() - Future.Nil - } - - } - - override def filterEdges( - event: TweetCreateEventDetails, - edges: Seq[UserTweetEntityEdge] - ): Future[Seq[UserTweetEntityEdge]] = { - Future(edges) // No filtering for now. Add more if needed - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetGraphBuilder.docx new file mode 100644 index 000000000..f29a38ca8 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetGraphBuilder.scala deleted file mode 100644 index c7f223800..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserTweetGraphBuilder.scala +++ /dev/null @@ -1,88 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.TweetCreationTimeMHStore -import com.twitter.frigate.common.util.SnowflakeUtils -import com.twitter.recos.internal.thriftscala.RecosUserTweetInfo -import com.twitter.recos.internal.thriftscala.TweetType -import com.twitter.recos.util.Action -import com.twitter.recosinjector.decider.RecosInjectorDecider -import com.twitter.recosinjector.decider.RecosInjectorDeciderConstants -import com.twitter.recosinjector.util.TweetCreateEventDetails -import com.twitter.util.Future -import com.twitter.util.Time - -class TweetEventToUserTweetGraphBuilder( - userTweetEntityEdgeBuilder: UserTweetEntityEdgeBuilder, - tweetCreationStore: TweetCreationTimeMHStore, - decider: RecosInjectorDecider -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[TweetCreateEventDetails, UserTweetEntityEdge] { - - private val numRetweetEdgesCounter = statsReceiver.counter("num_retweet_edge") - private val numIsDecider = statsReceiver.counter("num_decider_enabled") - private val numIsNotDecider = statsReceiver.counter("num_decider_not_enabled") - - override def shouldProcessEvent(event: TweetCreateEventDetails): Future[Boolean] = { - val isDecider = decider.isAvailable( - RecosInjectorDeciderConstants.TweetEventTransformerUserTweetEntityEdgesDecider - ) - if (isDecider) { - numIsDecider.incr() - Future(true) - } else { - numIsNotDecider.incr() - Future(false) - } - } - - /** - * Build a Retweet edge: author -> RT -> SourceTweetId. - */ - private def buildRetweetEdge(event: TweetCreateEventDetails) = { - val userTweetEngagement = event.userTweetEngagement - val tweetId = userTweetEngagement.tweetId - - event.sourceTweetDetails - .map { sourceTweetDetails => - val sourceTweetId = sourceTweetDetails.tweet.id // Id of the tweet being Retweeted - val sourceTweetEntitiesMapFut = userTweetEntityEdgeBuilder.getEntitiesMapAndUpdateCache( - tweetId = sourceTweetId, - tweetDetails = Some(sourceTweetDetails) - ) - - sourceTweetEntitiesMapFut.map { sourceTweetEntitiesMap => - val edge = UserTweetEntityEdge( - sourceUser = userTweetEngagement.engageUserId, - targetTweet = sourceTweetId, - action = Action.Retweet, - metadata = Some(tweetId), // metadata is the tweetId - cardInfo = Some(sourceTweetDetails.cardInfo.toByte), - entitiesMap = sourceTweetEntitiesMap, - tweetDetails = Some(sourceTweetDetails) - ) - numRetweetEdgesCounter.incr() - Seq(edge) - } - }.getOrElse(Future.Nil) - } - - override def buildEdges(event: TweetCreateEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val userTweetEngagement = event.userTweetEngagement - userTweetEngagement.action match { - case Action.Retweet => - buildRetweetEdge(event) - case _ => - Future.Nil - } - - } - - override def filterEdges( - event: TweetCreateEventDetails, - edges: Seq[UserTweetEntityEdge] - ): Future[Seq[UserTweetEntityEdge]] = { - Future(edges) // No filtering for now. Add more if needed - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserUserGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserUserGraphBuilder.docx new file mode 100644 index 000000000..5a15960f8 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserUserGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserUserGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserUserGraphBuilder.scala deleted file mode 100644 index 3eec7f55c..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/TweetEventToUserUserGraphBuilder.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recosinjector.util.TweetCreateEventDetails -import com.twitter.util.Future - -/** - * Given a tweet creation event, parse for UserUserGraph edges. Specifically, when a new tweet is - * created, extract the valid mentioned and mediatagged users in the tweet and create edges for them - */ -class TweetEventToUserUserGraphBuilder( -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[TweetCreateEventDetails, UserUserEdge] { - private val tweetOrQuoteEventCounter = statsReceiver.counter("num_tweet_or_quote_event") - private val nonTweetOrQuoteEventCounter = statsReceiver.counter("num_non_tweet_or_quote_event") - private val mentionEdgeCounter = statsReceiver.counter("num_mention_edge") - private val mediatagEdgeCounter = statsReceiver.counter("num_mediatag_edge") - - override def shouldProcessEvent(event: TweetCreateEventDetails): Future[Boolean] = { - // For user interactions, only new tweets and quotes are considered (no replies or retweets) - event.userTweetEngagement.action match { - case Action.Tweet | Action.Quote => - tweetOrQuoteEventCounter.incr() - Future(true) - case _ => - nonTweetOrQuoteEventCounter.incr() - Future(false) - } - } - - override def buildEdges(event: TweetCreateEventDetails): Future[Seq[UserUserEdge]] = { - val mentionEdges = event.validMentionUserIds - .map(_.map { mentionUserId => - UserUserEdge( - sourceUser = event.userTweetEngagement.engageUserId, - targetUser = mentionUserId, - action = Action.Mention, - metadata = Some(System.currentTimeMillis()) - ) - }).getOrElse(Nil) - - val mediatagEdges = event.validMediatagUserIds - .map(_.map { mediatagUserId => - UserUserEdge( - sourceUser = event.userTweetEngagement.engageUserId, - targetUser = mediatagUserId, - action = Action.MediaTag, - metadata = Some(System.currentTimeMillis()) - ) - }).getOrElse(Nil) - - mentionEdgeCounter.incr(mentionEdges.size) - mediatagEdgeCounter.incr(mediatagEdges.size) - Future(mentionEdges ++ mediatagEdges) - } - - override def filterEdges( - event: TweetCreateEventDetails, - edges: Seq[UserUserEdge] - ): Future[Seq[UserUserEdge]] = { - Future(edges) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserAdGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserAdGraphBuilder.docx new file mode 100644 index 000000000..24af57d1c Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserAdGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserAdGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserAdGraphBuilder.scala deleted file mode 100644 index e562ff2d1..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserAdGraphBuilder.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recosinjector.util.UuaEngagementEventDetails -import com.twitter.util.Future - -class UnifiedUserActionToUserAdGraphBuilder( - userTweetEntityEdgeBuilder: UserTweetEntityEdgeBuilder -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[UuaEngagementEventDetails, UserTweetEntityEdge] { - - override def shouldProcessEvent(event: UuaEngagementEventDetails): Future[Boolean] = { - event.userTweetEngagement.action match { - case Action.Click | Action.VideoPlayback75 | Action.Favorite => Future(true) - case _ => Future(false) - } - } - - override def buildEdges(details: UuaEngagementEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val engagement = details.userTweetEngagement - val tweetDetails = engagement.tweetDetails - - Future.value( - Seq( - UserTweetEntityEdge( - sourceUser = engagement.engageUserId, - targetTweet = engagement.tweetId, - action = engagement.action, - metadata = engagement.engagementTimeMillis, - cardInfo = engagement.tweetDetails.map(_.cardInfo.toByte), - entitiesMap = None, - tweetDetails = tweetDetails - ))) - } - - override def filterEdges( - event: UuaEngagementEventDetails, - edges: Seq[UserTweetEntityEdge] - ): Future[Seq[UserTweetEntityEdge]] = { - Future(edges) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserTweetGraphPlusBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserTweetGraphPlusBuilder.docx new file mode 100644 index 000000000..1eb09b94c Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserTweetGraphPlusBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserTweetGraphPlusBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserTweetGraphPlusBuilder.scala deleted file mode 100644 index 8f5ccb2b2..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserTweetGraphPlusBuilder.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recosinjector.util.UuaEngagementEventDetails -import com.twitter.util.Future - -class UnifiedUserActionToUserTweetGraphPlusBuilder( - userTweetEntityEdgeBuilder: UserTweetEntityEdgeBuilder -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[UuaEngagementEventDetails, UserTweetEntityEdge] { - - override def shouldProcessEvent(event: UuaEngagementEventDetails): Future[Boolean] = { - event.userTweetEngagement.action match { - case Action.Click | Action.VideoQualityView => Future(true) - case Action.Favorite | Action.Retweet | Action.Share => Future(true) - case Action.NotificationOpen | Action.EmailClick => Future(true) - case Action.Quote | Action.Reply => Future(true) - case Action.TweetNotInterestedIn | Action.TweetNotRelevant | Action.TweetSeeFewer | - Action.TweetReport | Action.TweetMuteAuthor | Action.TweetBlockAuthor => - Future(true) - case _ => Future(false) - } - } - - override def buildEdges(details: UuaEngagementEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val engagement = details.userTweetEngagement - val tweetDetails = engagement.tweetDetails - - Future - .value( - UserTweetEntityEdge( - sourceUser = engagement.engageUserId, - targetTweet = engagement.tweetId, - action = engagement.action, - metadata = engagement.engagementTimeMillis, - cardInfo = engagement.tweetDetails.map(_.cardInfo.toByte), - entitiesMap = None, - tweetDetails = tweetDetails - ) - ).map(Seq(_)) - } - - override def filterEdges( - event: UuaEngagementEventDetails, - edges: Seq[UserTweetEntityEdge] - ): Future[Seq[UserTweetEntityEdge]] = { - Future(edges) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserVideoGraphBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserVideoGraphBuilder.docx new file mode 100644 index 000000000..f32a071c6 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserVideoGraphBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserVideoGraphBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserVideoGraphBuilder.scala deleted file mode 100644 index 496c05dbd..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UnifiedUserActionToUserVideoGraphBuilder.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recosinjector.util.UuaEngagementEventDetails -import com.twitter.util.Future - -class UnifiedUserActionToUserVideoGraphBuilder( - userTweetEntityEdgeBuilder: UserTweetEntityEdgeBuilder -)( - override implicit val statsReceiver: StatsReceiver) - extends EventToMessageBuilder[UuaEngagementEventDetails, UserTweetEntityEdge] { - - private val numVideoPlayback50EdgeCounter = statsReceiver.counter("num_video_playback50_edge") - private val numUnVideoPlayback50Counter = statsReceiver.counter("num_non_video_playback50_edge") - - override def shouldProcessEvent(event: UuaEngagementEventDetails): Future[Boolean] = { - event.userTweetEngagement.action match { - case Action.VideoPlayback50 => Future(true) - case _ => Future(false) - } - } - - override def buildEdges(details: UuaEngagementEventDetails): Future[Seq[UserTweetEntityEdge]] = { - val engagement = details.userTweetEngagement - val tweetDetails = engagement.tweetDetails - - Future - .value( - UserTweetEntityEdge( - sourceUser = engagement.engageUserId, - targetTweet = engagement.tweetId, - action = engagement.action, - metadata = engagement.engagementTimeMillis, - cardInfo = engagement.tweetDetails.map(_.cardInfo.toByte), - entitiesMap = None, - tweetDetails = tweetDetails - ) - ).map { edge => - edge match { - case videoPlayback50 if videoPlayback50.action == Action.VideoPlayback50 => - numVideoPlayback50EdgeCounter.incr() - case _ => - numUnVideoPlayback50Counter.incr() - } - Seq(edge) - } - } - - override def filterEdges( - event: UuaEngagementEventDetails, - edges: Seq[UserTweetEntityEdge] - ): Future[Seq[UserTweetEntityEdge]] = { - Future(edges) - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UserTweetEntityEdgeBuilder.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UserTweetEntityEdgeBuilder.docx new file mode 100644 index 000000000..fb373ac58 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UserTweetEntityEdgeBuilder.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UserTweetEntityEdgeBuilder.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UserTweetEntityEdgeBuilder.scala deleted file mode 100644 index 283378b03..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/edges/UserTweetEntityEdgeBuilder.scala +++ /dev/null @@ -1,80 +0,0 @@ -package com.twitter.recosinjector.edges - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.graphjet.algorithms.RecommendationType -import com.twitter.recosinjector.clients.CacheEntityEntry -import com.twitter.recosinjector.clients.RecosHoseEntitiesCache -import com.twitter.recosinjector.clients.UrlResolver -import com.twitter.recosinjector.util.TweetDetails -import com.twitter.util.Future -import scala.collection.Map -import scala.util.hashing.MurmurHash3 - -class UserTweetEntityEdgeBuilder( - cache: RecosHoseEntitiesCache, - urlResolver: UrlResolver -)( - implicit val stats: StatsReceiver) { - - def getHashedEntities(entities: Seq[String]): Seq[Int] = { - entities.map(MurmurHash3.stringHash) - } - - /** - * Given the entities and their corresponding hashedIds, store the hashId->entity mapping into a - * cache. - * This is because UTEG edges only store the hashIds, and relies on the cache values to - * recover the actual entities. This allows us to store integer values instead of string in the - * edges to save space. - */ - private def storeEntitiesInCache( - urlEntities: Seq[String], - urlHashIds: Seq[Int] - ): Future[Unit] = { - val urlCacheEntries = urlHashIds.zip(urlEntities).map { - case (hashId, url) => - CacheEntityEntry(RecosHoseEntitiesCache.UrlPrefix, hashId, url) - } - cache.updateEntitiesCache( - newCacheEntries = urlCacheEntries, - stats = stats.scope("urlCache") - ) - } - - /** - * Return an entity mapping from GraphJet recType -> hash(entity) - */ - private def getEntitiesMap( - urlHashIds: Seq[Int] - ) = { - val entitiesMap = Seq( - RecommendationType.URL.getValue.toByte -> urlHashIds - ).collect { - case (keys, ids) if ids.nonEmpty => keys -> ids - }.toMap - if (entitiesMap.isEmpty) None else Some(entitiesMap) - } - - def getEntitiesMapAndUpdateCache( - tweetId: Long, - tweetDetails: Option[TweetDetails] - ): Future[Option[Map[Byte, Seq[Int]]]] = { - val resolvedUrlFut = urlResolver - .getResolvedUrls( - urls = tweetDetails.flatMap(_.urls).getOrElse(Nil), - tweetId = tweetId - ).map(_.values.toSeq) - - resolvedUrlFut.map { resolvedUrls => - val urlEntities = resolvedUrls - val urlHashIds = getHashedEntities(urlEntities) - - // Async call to cache - storeEntitiesInCache( - urlEntities = urlEntities, - urlHashIds = urlHashIds - ) - getEntitiesMap(urlHashIds) - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/BUILD deleted file mode 100644 index e7f3cdf50..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "eventbus/client", - "recos-injector/server/config", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/clients", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/config", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/decider", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/edges", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:events-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/BUILD.docx new file mode 100644 index 000000000..8c47b6531 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/EventBusProcessor.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/EventBusProcessor.docx new file mode 100644 index 000000000..41cc475ff Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/EventBusProcessor.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/EventBusProcessor.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/EventBusProcessor.scala deleted file mode 100644 index 9597b0bb8..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/EventBusProcessor.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.recosinjector.event_processors - -import com.twitter.eventbus.client.{EventBusSubscriber, EventBusSubscriberBuilder} -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.logging.Logger -import com.twitter.scrooge.{ThriftStruct, ThriftStructCodec} -import com.twitter.util.Future - -/** - * Main processor class that handles incoming EventBus events, which take forms of a ThriftStruct. - * This class is responsible for setting up the EventBus streams, and provides a processEvent() - * where child classes can decide what to do with incoming events - */ -trait EventBusProcessor[Event <: ThriftStruct] { - private val log = Logger() - - implicit def statsReceiver: StatsReceiver - - /** - * Full name of the EventBus stream this processor listens to - */ - val eventBusStreamName: String - - /** - * the thriftStruct definition of the objects passed in from the EventBus streams, such as - * TweetEvent, WriteEvent, etc. - */ - val thriftStruct: ThriftStructCodec[Event] - - val serviceIdentifier: ServiceIdentifier - - def processEvent(event: Event): Future[Unit] - - private def getEventBusSubscriberBuilder: EventBusSubscriberBuilder[Event] = - EventBusSubscriberBuilder() - .subscriberId(eventBusStreamName) - .serviceIdentifier(serviceIdentifier) - .thriftStruct(thriftStruct) - .numThreads(8) - .fromAllZones(true) // Receives traffic from all data centers - .skipToLatest(false) // Ensures we don't miss out on events during restart - .statsReceiver(statsReceiver) - - // lazy val ensures the subscriber is only initialized when start() is called - private lazy val eventBusSubscriber = getEventBusSubscriberBuilder.build(processEvent) - - def start(): EventBusSubscriber[Event] = eventBusSubscriber - - def stop(): Unit = { - eventBusSubscriber - .close() - .onSuccess { _ => - log.info(s"EventBus processor ${this.getClass.getSimpleName} is stopped") - } - .onFailure { ex: Throwable => - log.error(ex, s"Exception while stopping EventBus processor ${this.getClass.getSimpleName}") - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/SocialWriteEventProcessor.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/SocialWriteEventProcessor.docx new file mode 100644 index 000000000..eef4272f0 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/SocialWriteEventProcessor.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/SocialWriteEventProcessor.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/SocialWriteEventProcessor.scala deleted file mode 100644 index 550199f00..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/SocialWriteEventProcessor.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.twitter.recosinjector.event_processors - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recosinjector.edges.{EventToMessageBuilder, UserUserEdge} -import com.twitter.recosinjector.publishers.KafkaEventPublisher -import com.twitter.scrooge.ThriftStructCodec -import com.twitter.socialgraph.thriftscala.WriteEvent -import com.twitter.util.Future - -/** - * This processor listens to events from social graphs services. In particular, a major use case is - * to listen to user-user follow events. - */ -class SocialWriteEventProcessor( - override val eventBusStreamName: String, - override val thriftStruct: ThriftStructCodec[WriteEvent], - override val serviceIdentifier: ServiceIdentifier, - kafkaEventPublisher: KafkaEventPublisher, - userUserGraphTopic: String, - userUserGraphMessageBuilder: EventToMessageBuilder[WriteEvent, UserUserEdge] -)( - override implicit val statsReceiver: StatsReceiver) - extends EventBusProcessor[WriteEvent] { - - override def processEvent(event: WriteEvent): Future[Unit] = { - userUserGraphMessageBuilder.processEvent(event).map { edges => - edges.foreach { edge => - kafkaEventPublisher.publish(edge.convertToRecosHoseMessage, userUserGraphTopic) - } - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TimelineEventProcessor.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TimelineEventProcessor.docx new file mode 100644 index 000000000..93fc2839f Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TimelineEventProcessor.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TimelineEventProcessor.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TimelineEventProcessor.scala deleted file mode 100644 index 372da8e6f..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TimelineEventProcessor.scala +++ /dev/null @@ -1,150 +0,0 @@ -package com.twitter.recosinjector.event_processors - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recosinjector.clients.Gizmoduck -import com.twitter.recosinjector.clients.Tweetypie -import com.twitter.recosinjector.decider.RecosInjectorDecider -import com.twitter.recosinjector.decider.RecosInjectorDeciderConstants -import com.twitter.recosinjector.edges.TimelineEventToUserTweetEntityGraphBuilder -import com.twitter.recosinjector.filters.TweetFilter -import com.twitter.recosinjector.filters.UserFilter -import com.twitter.recosinjector.publishers.KafkaEventPublisher -import com.twitter.recosinjector.util.TweetDetails -import com.twitter.recosinjector.util.TweetFavoriteEventDetails -import com.twitter.recosinjector.util.UserTweetEngagement -import com.twitter.scrooge.ThriftStructCodec -import com.twitter.timelineservice.thriftscala.FavoriteEvent -import com.twitter.timelineservice.thriftscala.UnfavoriteEvent -import com.twitter.timelineservice.thriftscala.{Event => TimelineEvent} -import com.twitter.util.Future - -/** - * Processor for Timeline events, such as Favorite (liking) tweets - */ -class TimelineEventProcessor( - override val eventBusStreamName: String, - override val thriftStruct: ThriftStructCodec[TimelineEvent], - override val serviceIdentifier: ServiceIdentifier, - kafkaEventPublisher: KafkaEventPublisher, - userTweetEntityGraphTopic: String, - userTweetEntityGraphMessageBuilder: TimelineEventToUserTweetEntityGraphBuilder, - decider: RecosInjectorDecider, - gizmoduck: Gizmoduck, - tweetypie: Tweetypie -)( - override implicit val statsReceiver: StatsReceiver) - extends EventBusProcessor[TimelineEvent] { - - private val processEventDeciderCounter = statsReceiver.counter("num_process_timeline_event") - private val numFavoriteEventCounter = statsReceiver.counter("num_favorite_event") - private val numUnFavoriteEventCounter = statsReceiver.counter("num_unfavorite_event") - private val numNotFavoriteEventCounter = statsReceiver.counter("num_not_favorite_event") - - private val numSelfFavoriteCounter = statsReceiver.counter("num_self_favorite_event") - private val numNullCastTweetCounter = statsReceiver.counter("num_null_cast_tweet") - private val numTweetFailSafetyLevelCounter = statsReceiver.counter("num_fail_tweetypie_safety") - private val numFavoriteUserUnsafeCounter = statsReceiver.counter("num_favorite_user_unsafe") - private val engageUserFilter = new UserFilter(gizmoduck)(statsReceiver.scope("engage_user")) - private val tweetFilter = new TweetFilter(tweetypie) - - private val numProcessFavorite = statsReceiver.counter("num_process_favorite") - private val numNoProcessFavorite = statsReceiver.counter("num_no_process_favorite") - - private def getFavoriteEventDetails( - favoriteEvent: FavoriteEvent - ): TweetFavoriteEventDetails = { - - val engagement = UserTweetEngagement( - engageUserId = favoriteEvent.userId, - engageUser = favoriteEvent.user, - action = Action.Favorite, - engagementTimeMillis = Some(favoriteEvent.eventTimeMs), - tweetId = favoriteEvent.tweetId, // the tweet, or source tweet if target tweet is a retweet - tweetDetails = favoriteEvent.tweet.map(TweetDetails) // tweet always exists - ) - TweetFavoriteEventDetails(userTweetEngagement = engagement) - } - - private def getUnfavoriteEventDetails( - unfavoriteEvent: UnfavoriteEvent - ): TweetFavoriteEventDetails = { - val engagement = UserTweetEngagement( - engageUserId = unfavoriteEvent.userId, - engageUser = unfavoriteEvent.user, - action = Action.Unfavorite, - engagementTimeMillis = Some(unfavoriteEvent.eventTimeMs), - tweetId = unfavoriteEvent.tweetId, // the tweet, or source tweet if target tweet is a retweet - tweetDetails = unfavoriteEvent.tweet.map(TweetDetails) // tweet always exists - ) - TweetFavoriteEventDetails(userTweetEngagement = engagement) - } - - private def shouldProcessFavoriteEvent(event: TweetFavoriteEventDetails): Future[Boolean] = { - val engagement = event.userTweetEngagement - val engageUserId = engagement.engageUserId - val tweetId = engagement.tweetId - val authorIdOpt = engagement.tweetDetails.flatMap(_.authorId) - - val isSelfFavorite = authorIdOpt.contains(engageUserId) - val isNullCastTweet = engagement.tweetDetails.forall(_.isNullCastTweet) - val isEngageUserSafeFut = engageUserFilter.filterByUserId(engageUserId) - val isTweetPassSafetyFut = tweetFilter.filterForTweetypieSafetyLevel(tweetId) - - Future.join(isEngageUserSafeFut, isTweetPassSafetyFut).map { - case (isEngageUserSafe, isTweetPassSafety) => - if (isSelfFavorite) numSelfFavoriteCounter.incr() - if (isNullCastTweet) numNullCastTweetCounter.incr() - if (!isEngageUserSafe) numFavoriteUserUnsafeCounter.incr() - if (!isTweetPassSafety) numTweetFailSafetyLevelCounter.incr() - - !isSelfFavorite && !isNullCastTweet && isEngageUserSafe && isTweetPassSafety - } - } - - private def processFavoriteEvent(favoriteEvent: FavoriteEvent): Future[Unit] = { - val eventDetails = getFavoriteEventDetails(favoriteEvent) - shouldProcessFavoriteEvent(eventDetails).map { - case true => - numProcessFavorite.incr() - // Convert the event for UserTweetEntityGraph - userTweetEntityGraphMessageBuilder.processEvent(eventDetails).map { edges => - edges.foreach { edge => - kafkaEventPublisher.publish(edge.convertToRecosHoseMessage, userTweetEntityGraphTopic) - } - } - case false => - numNoProcessFavorite.incr() - } - } - - private def processUnFavoriteEvent(unFavoriteEvent: UnfavoriteEvent): Future[Unit] = { - if (decider.isAvailable(RecosInjectorDeciderConstants.EnableUnfavoriteEdge)) { - val eventDetails = getUnfavoriteEventDetails(unFavoriteEvent) - // Convert the event for UserTweetEntityGraph - userTweetEntityGraphMessageBuilder.processEvent(eventDetails).map { edges => - edges.foreach { edge => - kafkaEventPublisher.publish(edge.convertToRecosHoseMessage, userTweetEntityGraphTopic) - } - } - } else { - Future.Unit - } - } - - override def processEvent(event: TimelineEvent): Future[Unit] = { - processEventDeciderCounter.incr() - event match { - case TimelineEvent.Favorite(favoriteEvent: FavoriteEvent) => - numFavoriteEventCounter.incr() - processFavoriteEvent(favoriteEvent) - case TimelineEvent.Unfavorite(unFavoriteEvent: UnfavoriteEvent) => - numUnFavoriteEventCounter.incr() - processUnFavoriteEvent(unFavoriteEvent) - case _ => - numNotFavoriteEventCounter.incr() - Future.Unit - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TweetEventProcessor.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TweetEventProcessor.docx new file mode 100644 index 000000000..a8c8598d2 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TweetEventProcessor.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TweetEventProcessor.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TweetEventProcessor.scala deleted file mode 100644 index 0d9d16fa5..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/event_processors/TweetEventProcessor.scala +++ /dev/null @@ -1,256 +0,0 @@ -package com.twitter.recosinjector.event_processors - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.util.SnowflakeUtils -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.recos.util.Action -import com.twitter.recos.util.Action.Action -import com.twitter.recosinjector.clients.Gizmoduck -import com.twitter.recosinjector.clients.SocialGraph -import com.twitter.recosinjector.clients.Tweetypie -import com.twitter.recosinjector.edges.TweetEventToUserTweetEntityGraphBuilder -import com.twitter.recosinjector.edges.TweetEventToUserUserGraphBuilder -import com.twitter.recosinjector.filters.TweetFilter -import com.twitter.recosinjector.filters.UserFilter -import com.twitter.recosinjector.publishers.KafkaEventPublisher -import com.twitter.recosinjector.util.TweetCreateEventDetails -import com.twitter.recosinjector.util.TweetDetails -import com.twitter.recosinjector.util.UserTweetEngagement -import com.twitter.scrooge.ThriftStructCodec -import com.twitter.tweetypie.thriftscala.Tweet -import com.twitter.tweetypie.thriftscala.TweetCreateEvent -import com.twitter.tweetypie.thriftscala.TweetEvent -import com.twitter.tweetypie.thriftscala.TweetEventData -import com.twitter.util.Future - -/** - * Event processor for tweet_events EventBus stream from Tweetypie. This stream provides all the - * key events related to a new tweet, like Creation, Retweet, Quote Tweet, and Replying. - * It also carries the entities/metadata information in a tweet, including - * @ Mention, HashTag, MediaTag, URL, etc. - */ -class TweetEventProcessor( - override val eventBusStreamName: String, - override val thriftStruct: ThriftStructCodec[TweetEvent], - override val serviceIdentifier: ServiceIdentifier, - userUserGraphMessageBuilder: TweetEventToUserUserGraphBuilder, - userUserGraphTopic: String, - userTweetEntityGraphMessageBuilder: TweetEventToUserTweetEntityGraphBuilder, - userTweetEntityGraphTopic: String, - kafkaEventPublisher: KafkaEventPublisher, - socialGraph: SocialGraph, - gizmoduck: Gizmoduck, - tweetypie: Tweetypie -)( - override implicit val statsReceiver: StatsReceiver) - extends EventBusProcessor[TweetEvent] { - - private val tweetCreateEventCounter = statsReceiver.counter("num_tweet_create_events") - private val nonTweetCreateEventCounter = statsReceiver.counter("num_non_tweet_create_events") - - private val tweetActionStats = statsReceiver.scope("tweet_action") - private val numUrlCounter = statsReceiver.counter("num_tweet_url") - private val numMediaUrlCounter = statsReceiver.counter("num_tweet_media_url") - private val numHashTagCounter = statsReceiver.counter("num_tweet_hashtag") - - private val numMentionsCounter = statsReceiver.counter("num_tweet_mention") - private val numMediatagCounter = statsReceiver.counter("num_tweet_mediatag") - private val numValidMentionsCounter = statsReceiver.counter("num_tweet_valid_mention") - private val numValidMediatagCounter = statsReceiver.counter("num_tweet_valid_mediatag") - - private val numNullCastTweetCounter = statsReceiver.counter("num_null_cast_tweet") - private val numNullCastSourceTweetCounter = statsReceiver.counter("num_null_cast_source_tweet") - private val numTweetFailSafetyLevelCounter = statsReceiver.counter("num_fail_tweetypie_safety") - private val numAuthorUnsafeCounter = statsReceiver.counter("num_author_unsafe") - private val numProcessTweetCounter = statsReceiver.counter("num_process_tweet") - private val numNoProcessTweetCounter = statsReceiver.counter("num_no_process_tweet") - - private val selfRetweetCounter = statsReceiver.counter("num_retweets_self") - - private val engageUserFilter = new UserFilter(gizmoduck)(statsReceiver.scope("author_user")) - private val tweetFilter = new TweetFilter(tweetypie) - - private def trackTweetCreateEventStats(details: TweetCreateEventDetails): Unit = { - tweetActionStats.counter(details.userTweetEngagement.action.toString).incr() - - details.userTweetEngagement.tweetDetails.foreach { tweetDetails => - tweetDetails.mentionUserIds.foreach(mention => numMentionsCounter.incr(mention.size)) - tweetDetails.mediatagUserIds.foreach(mediatag => numMediatagCounter.incr(mediatag.size)) - tweetDetails.urls.foreach(urls => numUrlCounter.incr(urls.size)) - tweetDetails.mediaUrls.foreach(mediaUrls => numMediaUrlCounter.incr(mediaUrls.size)) - tweetDetails.hashtags.foreach(hashtags => numHashTagCounter.incr(hashtags.size)) - } - - details.validMentionUserIds.foreach(mentions => numValidMentionsCounter.incr(mentions.size)) - details.validMediatagUserIds.foreach(mediatags => numValidMediatagCounter.incr(mediatags.size)) - } - - /** - * Given a created tweet, return what type of tweet it is, i.e. Tweet, Retweet, Quote, or Reply。 - * Retweet, Quote, or Reply are responsive actions to a source tweet, so for these tweets, - * we also return the tweet id and author of the source tweet (ex. the tweet being retweeted). - */ - private def getTweetAction(tweetDetails: TweetDetails): Action = { - (tweetDetails.replySourceId, tweetDetails.retweetSourceId, tweetDetails.quoteSourceId) match { - case (Some(_), _, _) => - Action.Reply - case (_, Some(_), _) => - Action.Retweet - case (_, _, Some(_)) => - Action.Quote - case _ => - Action.Tweet - } - } - - /** - * Given a list of mentioned users and mediatagged users in the tweet, return the users who - * actually follow the source user. - */ - private def getFollowedByIds( - sourceUserId: Long, - mentionUserIds: Option[Seq[Long]], - mediatagUserIds: Option[Seq[Long]] - ): Future[Seq[Long]] = { - val uniqueEntityUserIds = - (mentionUserIds.getOrElse(Nil) ++ mediatagUserIds.getOrElse(Nil)).distinct - if (uniqueEntityUserIds.isEmpty) { - Future.Nil - } else { - socialGraph.followedByNotMutedBy(sourceUserId, uniqueEntityUserIds) - } - } - - private def getSourceTweet(tweetDetails: TweetDetails): Future[Option[Tweet]] = { - tweetDetails.sourceTweetId match { - case Some(sourceTweetId) => - tweetypie.getTweet(sourceTweetId) - case _ => - Future.None - } - } - - /** - * Extract and return the details when the source user created a new tweet. - */ - private def getTweetDetails( - tweet: Tweet, - engageUser: User - ): Future[TweetCreateEventDetails] = { - val tweetDetails = TweetDetails(tweet) - - val action = getTweetAction(tweetDetails) - val tweetCreationTimeMillis = SnowflakeUtils.tweetCreationTime(tweet.id).map(_.inMilliseconds) - val engageUserId = engageUser.id - val userTweetEngagement = UserTweetEngagement( - engageUserId = engageUserId, - engageUser = Some(engageUser), - action = action, - engagementTimeMillis = tweetCreationTimeMillis, - tweetId = tweet.id, - tweetDetails = Some(tweetDetails) - ) - - val sourceTweetFut = getSourceTweet(tweetDetails) - val followedByIdsFut = getFollowedByIds( - engageUserId, - tweetDetails.mentionUserIds, - tweetDetails.mediatagUserIds - ) - - Future.join(followedByIdsFut, sourceTweetFut).map { - case (followedByIds, sourceTweet) => - TweetCreateEventDetails( - userTweetEngagement = userTweetEngagement, - validEntityUserIds = followedByIds, - sourceTweetDetails = sourceTweet.map(TweetDetails) - ) - } - } - - /** - * Exclude any Retweets of one's own tweets - */ - private def isEventSelfRetweet(tweetEvent: TweetCreateEventDetails): Boolean = { - (tweetEvent.userTweetEngagement.action == Action.Retweet) && - tweetEvent.userTweetEngagement.tweetDetails.exists( - _.sourceTweetUserId.contains( - tweetEvent.userTweetEngagement.engageUserId - )) - } - - private def isTweetPassSafetyFilter(tweetEvent: TweetCreateEventDetails): Future[Boolean] = { - tweetEvent.userTweetEngagement.action match { - case Action.Reply | Action.Retweet | Action.Quote => - tweetEvent.userTweetEngagement.tweetDetails - .flatMap(_.sourceTweetId).map { sourceTweetId => - tweetFilter.filterForTweetypieSafetyLevel(sourceTweetId) - }.getOrElse(Future(false)) - case Action.Tweet => - tweetFilter.filterForTweetypieSafetyLevel(tweetEvent.userTweetEngagement.tweetId) - } - } - - private def shouldProcessTweetEvent(event: TweetCreateEventDetails): Future[Boolean] = { - val engagement = event.userTweetEngagement - val engageUserId = engagement.engageUserId - - val isNullCastTweet = engagement.tweetDetails.forall(_.isNullCastTweet) - val isNullCastSourceTweet = event.sourceTweetDetails.exists(_.isNullCastTweet) - val isSelfRetweet = isEventSelfRetweet(event) - val isEngageUserSafeFut = engageUserFilter.filterByUserId(engageUserId) - val isTweetPassSafetyFut = isTweetPassSafetyFilter(event) - - Future.join(isEngageUserSafeFut, isTweetPassSafetyFut).map { - case (isEngageUserSafe, isTweetPassSafety) => - if (isNullCastTweet) numNullCastTweetCounter.incr() - if (isNullCastSourceTweet) numNullCastSourceTweetCounter.incr() - if (!isEngageUserSafe) numAuthorUnsafeCounter.incr() - if (isSelfRetweet) selfRetweetCounter.incr() - if (!isTweetPassSafety) numTweetFailSafetyLevelCounter.incr() - - !isNullCastTweet && - !isNullCastSourceTweet && - !isSelfRetweet && - isEngageUserSafe && - isTweetPassSafety - } - } - - override def processEvent(event: TweetEvent): Future[Unit] = { - event.data match { - case TweetEventData.TweetCreateEvent(event: TweetCreateEvent) => - getTweetDetails( - tweet = event.tweet, - engageUser = event.user - ).flatMap { eventWithDetails => - tweetCreateEventCounter.incr() - - shouldProcessTweetEvent(eventWithDetails).map { - case true => - numProcessTweetCounter.incr() - trackTweetCreateEventStats(eventWithDetails) - // Convert the event for UserUserGraph - userUserGraphMessageBuilder.processEvent(eventWithDetails).map { edges => - edges.foreach { edge => - kafkaEventPublisher.publish(edge.convertToRecosHoseMessage, userUserGraphTopic) - } - } - // Convert the event for UserTweetEntityGraph - userTweetEntityGraphMessageBuilder.processEvent(eventWithDetails).map { edges => - edges.foreach { edge => - kafkaEventPublisher - .publish(edge.convertToRecosHoseMessage, userTweetEntityGraphTopic) - } - } - case false => - numNoProcessTweetCounter.incr() - } - } - case _ => - nonTweetCreateEventCounter.incr() - Future.Unit - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/BUILD deleted file mode 100644 index 9963bbe44..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "finagle/finagle-stats", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/clients", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/BUILD.docx new file mode 100644 index 000000000..b8261e27f Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/NullCastTweetFilter.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/NullCastTweetFilter.docx new file mode 100644 index 000000000..f0fba2252 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/NullCastTweetFilter.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/NullCastTweetFilter.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/NullCastTweetFilter.scala deleted file mode 100644 index 9a19efa75..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/NullCastTweetFilter.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.recosinjector.filters - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recosinjector.clients.Tweetypie -import com.twitter.util.Future - -/** - * Filters tweets that are null cast, i.e. tweet is not delivered to a user's followers, - * not shown in the user's timeline, and does not appear in search results. - * They are mainly ads tweets. - */ -class NullCastTweetFilter( - tweetypie: Tweetypie -)( - implicit statsReceiver: StatsReceiver) { - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val requests = stats.counter("requests") - private val filtered = stats.counter("filtered") - - // Return Future(True) to keep the Tweet. - def filter(tweetId: Long): Future[Boolean] = { - requests.incr() - tweetypie - .getTweet(tweetId) - .map { tweetOpt => - // If the null cast bit is Some(true), drop the tweet. - val isNullCastTweet = tweetOpt.flatMap(_.coreData).exists(_.nullcast) - if (isNullCastTweet) { - filtered.incr() - } - !isNullCastTweet - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/TweetFilter.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/TweetFilter.docx new file mode 100644 index 000000000..209f53267 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/TweetFilter.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/TweetFilter.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/TweetFilter.scala deleted file mode 100644 index aecc21515..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/TweetFilter.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.recosinjector.filters - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recosinjector.clients.Tweetypie -import com.twitter.util.Future - -class TweetFilter( - tweetypie: Tweetypie -)( - implicit statsReceiver: StatsReceiver) { - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val requests = stats.counter("requests") - private val filtered = stats.counter("filtered") - - /** - * Query Tweetypie to see if we can fetch a tweet object successfully. TweetyPie applies a safety - * filter and will not return the tweet object if the filter does not pass. - */ - def filterForTweetypieSafetyLevel(tweetId: Long): Future[Boolean] = { - requests.incr() - tweetypie - .getTweet(tweetId) - .map { - case Some(_) => - true - case _ => - filtered.incr() - false - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/UserFilter.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/UserFilter.docx new file mode 100644 index 000000000..bc94545b0 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/UserFilter.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/UserFilter.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/UserFilter.scala deleted file mode 100644 index b2e51f647..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/filters/UserFilter.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.recosinjector.filters - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.gizmoduck.thriftscala.{LabelValue, User} -import com.twitter.recosinjector.clients.Gizmoduck -import com.twitter.util.Future - -class UserFilter( - gizmoduck: Gizmoduck -)( - implicit statsReceiver: StatsReceiver) { - private val stats = statsReceiver.scope(this.getClass.getSimpleName) - private val requests = stats.counter("requests") - private val filtered = stats.counter("filtered") - - private def isUnsafe(user: User): Boolean = - user.safety.exists { s => - s.deactivated || s.suspended || s.restricted || s.nsfwUser || s.nsfwAdmin || s.isProtected - } - - private def hasNsfwHighPrecisionLabel(user: User): Boolean = - user.labels.exists { - _.labels.exists(_.labelValue == LabelValue.NsfwHighPrecision) - } - - /** - * NOTE: This will by-pass Gizmoduck's safety level, and might allow invalid users to pass filter. - * Consider using filterByUserId instead. - * Return true if the user is valid, otherwise return false. - * It will first attempt to use the user object provided by the caller, and will call Gizmoduck - * to back fill if the caller does not provide it. This helps reduce Gizmoduck traffic. - */ - def filterByUser( - userId: Long, - userOpt: Option[User] = None - ): Future[Boolean] = { - requests.incr() - val userFut = userOpt match { - case Some(user) => Future(Some(user)) - case _ => gizmoduck.getUser(userId) - } - - userFut.map(_.exists { user => - val isValidUser = !isUnsafe(user) && !hasNsfwHighPrecisionLabel(user) - if (!isValidUser) filtered.incr() - isValidUser - }) - } - - /** - * Given a userId, return true if the user is valid. This id done in 2 steps: - * 1. Applying Gizmoduck's safety level while querying for the user from Gizmoduck - * 2. If a user passes Gizmoduck's safety level, check its specific user status - */ - def filterByUserId(userId: Long): Future[Boolean] = { - requests.incr() - gizmoduck - .getUser(userId) - .map { userOpt => - val isValidUser = userOpt.exists { user => - !(isUnsafe(user) || hasNsfwHighPrecisionLabel(user)) - } - if (!isValidUser) { - filtered.incr() - } - isValidUser - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/BUILD deleted file mode 100644 index 20ea2ba3a..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/apache/thrift:libthrift", - "finatra-internal/messaging/kafka/src/main/scala", - "servo/repo/src/main/scala", - "src/thrift/com/twitter/recos:recos-injector-scala", - "src/thrift/com/twitter/recos:recos-internal-scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/BUILD.docx new file mode 100644 index 000000000..22d0eb8cf Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/KafkaEventPublisher.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/KafkaEventPublisher.docx new file mode 100644 index 000000000..428f73cf1 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/KafkaEventPublisher.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/KafkaEventPublisher.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/KafkaEventPublisher.scala deleted file mode 100644 index 1c1ad8207..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers/KafkaEventPublisher.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.recosinjector.publishers - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.recos.internal.thriftscala.RecosHoseMessage -import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.config.SaslConfigs -import org.apache.kafka.common.config.SslConfigs -import org.apache.kafka.common.security.auth.SecurityProtocol -import org.apache.kafka.common.serialization.StringSerializer - -case class KafkaEventPublisher( - kafkaDest: String, - outputKafkaTopicPrefix: String, - clientId: ClientId, - truststoreLocation: String) { - - private val producer = FinagleKafkaProducerBuilder[String, RecosHoseMessage]() - .dest(kafkaDest) - .clientId(clientId.name) - .keySerializer(new StringSerializer) - .valueSerializer(ScalaSerdes.Thrift[RecosHoseMessage].serializer) - .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) - .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststoreLocation) - .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) - .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") - .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") - // Use Native Kafka Client - .buildClient() - - def publish( - message: RecosHoseMessage, - topic: String - )( - implicit statsReceiver: StatsReceiver - ): Unit = { - val topicName = s"${outputKafkaTopicPrefix}_$topic" - // Kafka Producer is thread-safe. No extra Future-pool protect. - producer.send(new ProducerRecord(topicName, message)) - statsReceiver.counter(topicName + "_written_msg_success").incr() - } -} - -object KafkaEventPublisher { - // Kafka topics available for publishing - val UserVideoTopic = "user_video" - val UserTweetEntityTopic = "user_tweet_entity" - val UserUserTopic = "user_user" - val UserAdTopic = "user_tweet" - val UserTweetPlusTopic = "user_tweet_plus" -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/BUILD deleted file mode 100644 index f39ec2f4a..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/twitter/graphjet", - "finagle/finagle-stats", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/clients", - "src/scala/com/twitter/recos/util:recos-util", - "src/thrift/com/twitter/tweetypie:tweet-scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/BUILD.docx new file mode 100644 index 000000000..d63161136 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/EventDetails.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/EventDetails.docx new file mode 100644 index 000000000..d254fa321 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/EventDetails.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/EventDetails.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/EventDetails.scala deleted file mode 100644 index 9df3e752b..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/util/EventDetails.scala +++ /dev/null @@ -1,126 +0,0 @@ -package com.twitter.recosinjector.util - -import com.twitter.frigate.common.base.TweetUtil -import com.twitter.gizmoduck.thriftscala.User -import com.twitter.recos.util.Action.Action -import com.twitter.tweetypie.thriftscala.Tweet - -/** - * This is used to store information about a newly created tweet - * @param validEntityUserIds For users mentioned or mediatagged in the tweet, these follow the - * engage user and only they are are considered valid - * @param sourceTweetDetails For Reply, Quote, or RT, source tweet is the tweet being actioned on - */ -case class TweetCreateEventDetails( - userTweetEngagement: UserTweetEngagement, - validEntityUserIds: Seq[Long], - sourceTweetDetails: Option[TweetDetails]) { - // A mention is only valid if the mentioned user follows the source user - val validMentionUserIds: Option[Seq[Long]] = { - userTweetEngagement.tweetDetails.flatMap(_.mentionUserIds.map(_.intersect(validEntityUserIds))) - } - - // A mediatag is only valid if the mediatagged user follows the source user - val validMediatagUserIds: Option[Seq[Long]] = { - userTweetEngagement.tweetDetails.flatMap(_.mediatagUserIds.map(_.intersect(validEntityUserIds))) - } -} - -/** - * Stores information about a favorite/unfav engagement. - * NOTE: This could either be Likes, or UNLIKEs (i.e. when user cancels the Like) - * @param userTweetEngagement the engagement details - */ -case class TweetFavoriteEventDetails( - userTweetEngagement: UserTweetEngagement) - -/** - * Stores information about a unified user action engagement. - * @param userTweetEngagement the engagement details - */ -case class UuaEngagementEventDetails( - userTweetEngagement: UserTweetEngagement) - -/** - * Details about a user-tweet engagement, like when a user tweeted/liked a tweet - * @param engageUserId User that engaged with the tweet - * @param action The action the user took on the tweet - * @param tweetId The type of engagement the user took on the tweet - */ -case class UserTweetEngagement( - engageUserId: Long, - engageUser: Option[User], - action: Action, - engagementTimeMillis: Option[Long], - tweetId: Long, - tweetDetails: Option[TweetDetails]) - -/** - * Helper class that decomposes a tweet object and provides related details about this tweet - */ -case class TweetDetails(tweet: Tweet) { - val authorId: Option[Long] = tweet.coreData.map(_.userId) - - val urls: Option[Seq[String]] = tweet.urls.map(_.map(_.url)) - - val mediaUrls: Option[Seq[String]] = tweet.media.map(_.map(_.expandedUrl)) - - val hashtags: Option[Seq[String]] = tweet.hashtags.map(_.map(_.text)) - - // mentionUserIds include reply user ids at the beginning of a tweet - val mentionUserIds: Option[Seq[Long]] = tweet.mentions.map(_.flatMap(_.userId)) - - val mediatagUserIds: Option[Seq[Long]] = tweet.mediaTags.map { - _.tagMap.flatMap { - case (_, mediaTag) => mediaTag.flatMap(_.userId) - }.toSeq - } - - val replySourceId: Option[Long] = tweet.coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)) - val replyUserId: Option[Long] = tweet.coreData.flatMap(_.reply.map(_.inReplyToUserId)) - - val retweetSourceId: Option[Long] = tweet.coreData.flatMap(_.share.map(_.sourceStatusId)) - val retweetUserId: Option[Long] = tweet.coreData.flatMap(_.share.map(_.sourceUserId)) - - val quoteSourceId: Option[Long] = tweet.quotedTweet.map(_.tweetId) - val quoteUserId: Option[Long] = tweet.quotedTweet.map(_.userId) - val quoteTweetUrl: Option[String] = tweet.quotedTweet.flatMap(_.permalink.map(_.shortUrl)) - - //If the tweet is retweet/reply/quote, this is the tweet that the new tweet responds to - val (sourceTweetId, sourceTweetUserId) = { - (replySourceId, retweetSourceId, quoteSourceId) match { - case (Some(replyId), _, _) => - (Some(replyId), replyUserId) - case (_, Some(retweetId), _) => - (Some(retweetId), retweetUserId) - case (_, _, Some(quoteId)) => - (Some(quoteId), quoteUserId) - case _ => - (None, None) - } - } - - // Boolean information - val hasPhoto: Boolean = TweetUtil.containsPhotoTweet(tweet) - - val hasVideo: Boolean = TweetUtil.containsVideoTweet(tweet) - - // TweetyPie does not populate url fields in a quote tweet create event, even though we - // consider quote tweets as url tweets. This boolean helps make up for it. - // Details: https://groups.google.com/a/twitter.com/d/msg/eng/BhK1XAcSSWE/F8Gc4_5uDwAJ - val hasQuoteTweetUrl: Boolean = tweet.quotedTweet.exists(_.permalink.isDefined) - - val hasUrl: Boolean = this.urls.exists(_.nonEmpty) || hasQuoteTweetUrl - - val hasHashtag: Boolean = this.hashtags.exists(_.nonEmpty) - - val isCard: Boolean = hasUrl | hasPhoto | hasVideo - - implicit def bool2Long(b: Boolean): Long = if (b) 1L else 0L - - // Return a hashed long that contains card type information of the tweet - val cardInfo: Long = isCard | (hasUrl << 1) | (hasPhoto << 2) | (hasVideo << 3) - - // nullcast tweet is one that is purposefully not broadcast to followers, ex. an ad tweet. - val isNullCastTweet: Boolean = tweet.coreData.exists(_.nullcast) -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/BUILD b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/BUILD deleted file mode 100644 index c1370fe90..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/BUILD +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - platform = "java11", - strict_deps = False, - tags = ["bazel-compatible"], - dependencies = [ - "eventbus/client", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "kafka/libs/src/main/scala/com/twitter/kafka/client/headers", - "kafka/libs/src/main/scala/com/twitter/kafka/client/processor", - "recos-injector/server/config", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/clients", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/config", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/decider", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/edges", - "recos-injector/server/src/main/scala/com/twitter/recosinjector/publishers", - "src/thrift/com/twitter/clientapp/gen:clientapp-scala", - "src/thrift/com/twitter/gizmoduck:user-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:events-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala", - ], -) diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/BUILD.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/BUILD.docx new file mode 100644 index 000000000..e279d2d14 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/BUILD.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionProcessor.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionProcessor.docx new file mode 100644 index 000000000..f71da7589 Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionProcessor.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionProcessor.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionProcessor.scala deleted file mode 100644 index 731fa343b..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionProcessor.scala +++ /dev/null @@ -1,181 +0,0 @@ -package com.twitter.recosinjector.uua_processors - -import org.apache.kafka.clients.consumer.ConsumerRecord -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.recos.util.Action -import com.twitter.recos.util.Action.Action -import com.twitter.recosinjector.clients.Gizmoduck -import com.twitter.recosinjector.clients.Tweetypie -import com.twitter.recosinjector.edges.UnifiedUserActionToUserVideoGraphBuilder -import com.twitter.recosinjector.edges.UnifiedUserActionToUserAdGraphBuilder -import com.twitter.recosinjector.edges.UnifiedUserActionToUserTweetGraphPlusBuilder -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.unified_user_actions.thriftscala.ActionType -import com.twitter.unified_user_actions.thriftscala.Item -import com.twitter.recosinjector.filters.UserFilter -import com.twitter.recosinjector.publishers.KafkaEventPublisher -import com.twitter.recosinjector.util.TweetDetails -import com.twitter.recosinjector.util.UserTweetEngagement -import com.twitter.recosinjector.util.UuaEngagementEventDetails -import com.twitter.unified_user_actions.thriftscala.NotificationContent -import com.twitter.unified_user_actions.thriftscala.NotificationInfo -import com.twitter.util.Future - -class UnifiedUserActionProcessor( - gizmoduck: Gizmoduck, - tweetypie: Tweetypie, - kafkaEventPublisher: KafkaEventPublisher, - userVideoGraphTopic: String, - userVideoGraphBuilder: UnifiedUserActionToUserVideoGraphBuilder, - userAdGraphTopic: String, - userAdGraphBuilder: UnifiedUserActionToUserAdGraphBuilder, - userTweetGraphPlusTopic: String, - userTweetGraphPlusBuilder: UnifiedUserActionToUserTweetGraphPlusBuilder -)( - implicit statsReceiver: StatsReceiver) { - - val messagesProcessedCount = statsReceiver.counter("messages_processed") - - val eventsByTypeCounts = statsReceiver.scope("events_by_type") - private val numSelfEngageCounter = statsReceiver.counter("num_self_engage_event") - private val numTweetFailSafetyLevelCounter = statsReceiver.counter("num_fail_tweetypie_safety") - private val numNullCastTweetCounter = statsReceiver.counter("num_null_cast_tweet") - private val numEngageUserUnsafeCounter = statsReceiver.counter("num_engage_user_unsafe") - private val engageUserFilter = new UserFilter(gizmoduck)(statsReceiver.scope("engage_user")) - private val numNoProcessTweetCounter = statsReceiver.counter("num_no_process_tweet") - private val numProcessTweetCounter = statsReceiver.counter("num_process_tweet") - - private def getUuaEngagementEventDetails( - unifiedUserAction: UnifiedUserAction - ): Option[Future[UuaEngagementEventDetails]] = { - val userIdOpt = unifiedUserAction.userIdentifier.userId - val tweetIdOpt = unifiedUserAction.item match { - case Item.TweetInfo(tweetInfo) => Some(tweetInfo.actionTweetId) - case Item.NotificationInfo( - NotificationInfo(_, NotificationContent.TweetNotification(notification))) => - Some(notification.tweetId) - case _ => None - } - val timestamp = unifiedUserAction.eventMetadata.sourceTimestampMs - val action = getTweetAction(unifiedUserAction.actionType) - - tweetIdOpt - .flatMap { tweetId => - userIdOpt.map { engageUserId => - val tweetFut = tweetypie.getTweet(tweetId) - tweetFut.map { tweetOpt => - val tweetDetailsOpt = tweetOpt.map(TweetDetails) - val engagement = UserTweetEngagement( - engageUserId = engageUserId, - action = action, - engagementTimeMillis = Some(timestamp), - tweetId = tweetId, - engageUser = None, - tweetDetails = tweetDetailsOpt - ) - UuaEngagementEventDetails(engagement) - } - } - } - } - - private def getTweetAction(action: ActionType): Action = { - action match { - case ActionType.ClientTweetVideoPlayback50 => Action.VideoPlayback50 - case ActionType.ClientTweetClick => Action.Click - case ActionType.ClientTweetVideoPlayback75 => Action.VideoPlayback75 - case ActionType.ClientTweetVideoQualityView => Action.VideoQualityView - case ActionType.ServerTweetFav => Action.Favorite - case ActionType.ServerTweetReply => Action.Reply - case ActionType.ServerTweetRetweet => Action.Retweet - case ActionType.ClientTweetQuote => Action.Quote - case ActionType.ClientNotificationOpen => Action.NotificationOpen - case ActionType.ClientTweetEmailClick => Action.EmailClick - case ActionType.ClientTweetShareViaBookmark => Action.Share - case ActionType.ClientTweetShareViaCopyLink => Action.Share - case ActionType.ClientTweetSeeFewer => Action.TweetSeeFewer - case ActionType.ClientTweetNotRelevant => Action.TweetNotRelevant - case ActionType.ClientTweetNotInterestedIn => Action.TweetNotInterestedIn - case ActionType.ServerTweetReport => Action.TweetReport - case ActionType.ClientTweetMuteAuthor => Action.TweetMuteAuthor - case ActionType.ClientTweetBlockAuthor => Action.TweetBlockAuthor - case _ => Action.UnDefined - } - } - private def shouldProcessTweetEngagement( - event: UuaEngagementEventDetails, - isAdsUseCase: Boolean = false - ): Future[Boolean] = { - val engagement = event.userTweetEngagement - val engageUserId = engagement.engageUserId - val authorIdOpt = engagement.tweetDetails.flatMap(_.authorId) - - val isSelfEngage = authorIdOpt.contains(engageUserId) - val isNullCastTweet = engagement.tweetDetails.forall(_.isNullCastTweet) - val isEngageUserSafeFut = engageUserFilter.filterByUserId(engageUserId) - val isTweetPassSafety = - engagement.tweetDetails.isDefined // Tweetypie can fetch a tweet object successfully - - isEngageUserSafeFut.map { isEngageUserSafe => - if (isSelfEngage) numSelfEngageCounter.incr() - if (isNullCastTweet) numNullCastTweetCounter.incr() - if (!isEngageUserSafe) numEngageUserUnsafeCounter.incr() - if (!isTweetPassSafety) numTweetFailSafetyLevelCounter.incr() - - !isSelfEngage && (!isNullCastTweet && !isAdsUseCase || isNullCastTweet && isAdsUseCase) && isEngageUserSafe && isTweetPassSafety - } - } - - def apply(record: ConsumerRecord[UnKeyed, UnifiedUserAction]): Future[Unit] = { - - messagesProcessedCount.incr() - val unifiedUserAction = record.value - eventsByTypeCounts.counter(unifiedUserAction.actionType.toString).incr() - - getTweetAction(unifiedUserAction.actionType) match { - case Action.UnDefined => - numNoProcessTweetCounter.incr() - Future.Unit - case action => - getUuaEngagementEventDetails(unifiedUserAction) - .map { - _.flatMap { detail => - // The following cases are set up specifically for an ads relevance demo. - val actionForAds = Set(Action.Click, Action.Favorite, Action.VideoPlayback75) - if (actionForAds.contains(action)) - shouldProcessTweetEngagement(detail, isAdsUseCase = true).map { - case true => - userAdGraphBuilder.processEvent(detail).map { edges => - edges.foreach { edge => - kafkaEventPublisher - .publish(edge.convertToRecosHoseMessage, userAdGraphTopic) - } - } - numProcessTweetCounter.incr() - case _ => - } - - shouldProcessTweetEngagement(detail).map { - case true => - userVideoGraphBuilder.processEvent(detail).map { edges => - edges.foreach { edge => - kafkaEventPublisher - .publish(edge.convertToRecosHoseMessage, userVideoGraphTopic) - } - } - - userTweetGraphPlusBuilder.processEvent(detail).map { edges => - edges.foreach { edge => - kafkaEventPublisher - .publish(edge.convertToRecosHoseMessage, userTweetGraphPlusTopic) - } - } - numProcessTweetCounter.incr() - case _ => - } - } - }.getOrElse(Future.Unit) - } - } -} diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionsConsumer.docx b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionsConsumer.docx new file mode 100644 index 000000000..66274a70c Binary files /dev/null and b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionsConsumer.docx differ diff --git a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionsConsumer.scala b/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionsConsumer.scala deleted file mode 100644 index 022343cea..000000000 --- a/recos-injector/server/src/main/scala/com/twitter/recosinjector/uua_processors/UnifiedUserActionsConsumer.scala +++ /dev/null @@ -1,71 +0,0 @@ -package com.twitter.recosinjector.uua_processors - -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder -import com.twitter.finatra.kafka.domain.KafkaGroupId -import com.twitter.finatra.kafka.domain.SeekStrategy -import com.twitter.finatra.kafka.serde.ScalaSerdes -import com.twitter.finatra.kafka.serde.UnKeyed -import com.twitter.finatra.kafka.serde.UnKeyedSerde -import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.common.config.SaslConfigs -import org.apache.kafka.common.config.SslConfigs -import org.apache.kafka.common.security.auth.SecurityProtocol -import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction -import com.twitter.kafka.client.processor.AtLeastOnceProcessor -import com.twitter.kafka.client.processor.ThreadSafeKafkaConsumerClient -import com.twitter.conversions.StorageUnitOps._ - -class UnifiedUserActionsConsumer( - processor: UnifiedUserActionProcessor, - truststoreLocation: String -)( - implicit statsReceiver: StatsReceiver) { - import UnifiedUserActionsConsumer._ - - private val kafkaClient = new ThreadSafeKafkaConsumerClient[UnKeyed, UnifiedUserAction]( - FinagleKafkaConsumerBuilder[UnKeyed, UnifiedUserAction]() - .groupId(KafkaGroupId(uuaRecosInjectorGroupId)) - .keyDeserializer(UnKeyedSerde.deserializer) - .valueDeserializer(ScalaSerdes.Thrift[UnifiedUserAction].deserializer) - .dest(uuaDest) - .maxPollRecords(maxPollRecords) - .maxPollInterval(maxPollInterval) - .fetchMax(fetchMax) - .seekStrategy(SeekStrategy.END) - .enableAutoCommit(false) // AtLeastOnceProcessor performs commits manually - .withConfig(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SecurityProtocol.SASL_SSL.toString) - .withConfig(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststoreLocation) - .withConfig(SaslConfigs.SASL_MECHANISM, SaslConfigs.GSSAPI_MECHANISM) - .withConfig(SaslConfigs.SASL_KERBEROS_SERVICE_NAME, "kafka") - .withConfig(SaslConfigs.SASL_KERBEROS_SERVER_NAME, "kafka") - .config) - - val atLeastOnceProcessor: AtLeastOnceProcessor[UnKeyed, UnifiedUserAction] = { - AtLeastOnceProcessor[UnKeyed, UnifiedUserAction]( - name = processorName, - topic = uuaTopic, - consumer = kafkaClient, - processor = processor.apply, - maxPendingRequests = maxPendingRequests, - workerThreads = workerThreads, - commitIntervalMs = commitIntervalMs, - statsReceiver = statsReceiver.scope(processorName) - ) - } - -} - -object UnifiedUserActionsConsumer { - val maxPollRecords = 1000 - val maxPollInterval = 5.minutes - val fetchMax = 1.megabytes - val maxPendingRequests = 1000 - val workerThreads = 16 - val commitIntervalMs = 10.seconds.inMilliseconds - val processorName = "unified_user_actions_processor" - val uuaTopic = "unified_user_actions_engagements" - val uuaDest = "/s/kafka/bluebird-1:kafka-tls" - val uuaRecosInjectorGroupId = "recos-injector" -} diff --git a/representation-manager/BUILD.bazel b/representation-manager/BUILD.bazel deleted file mode 100644 index 1624a57d4..000000000 --- a/representation-manager/BUILD.bazel +++ /dev/null @@ -1 +0,0 @@ -# This prevents SQ query from grabbing //:all since it traverses up once to find a BUILD diff --git a/representation-manager/BUILD.docx b/representation-manager/BUILD.docx new file mode 100644 index 000000000..078f9bb66 Binary files /dev/null and b/representation-manager/BUILD.docx differ diff --git a/representation-manager/README.docx b/representation-manager/README.docx new file mode 100644 index 000000000..79322b2cb Binary files /dev/null and b/representation-manager/README.docx differ diff --git a/representation-manager/README.md b/representation-manager/README.md deleted file mode 100644 index 44cd25ee7..000000000 --- a/representation-manager/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Representation Manager # - -**Representation Manager** (RMS) serves as a centralized embedding management system, providing SimClusters or other embeddings as facade of the underlying storage or services. - diff --git a/representation-manager/bin/deploy.docx b/representation-manager/bin/deploy.docx new file mode 100644 index 000000000..ffaaa78aa Binary files /dev/null and b/representation-manager/bin/deploy.docx differ diff --git a/representation-manager/bin/deploy.sh b/representation-manager/bin/deploy.sh deleted file mode 100755 index 5729d9903..000000000 --- a/representation-manager/bin/deploy.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -JOB=representation-manager bazel run --ui_event_filters=-info,-stdout,-stderr --noshow_progress \ - //relevance-platform/src/main/python/deploy -- "$@" diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/BUILD b/representation-manager/client/src/main/scala/com/twitter/representation_manager/BUILD deleted file mode 100644 index 1f69a2176..000000000 --- a/representation-manager/client/src/main/scala/com/twitter/representation_manager/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-thrift-client", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/readablestore", - "representation-manager/client/src/main/scala/com/twitter/representation_manager/config", - "representation-manager/server/src/main/thrift:thrift-scala", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "stitch/stitch-storehaus", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/BUILD.docx b/representation-manager/client/src/main/scala/com/twitter/representation_manager/BUILD.docx new file mode 100644 index 000000000..a992cd6ba Binary files /dev/null and b/representation-manager/client/src/main/scala/com/twitter/representation_manager/BUILD.docx differ diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/StoreBuilder.docx b/representation-manager/client/src/main/scala/com/twitter/representation_manager/StoreBuilder.docx new file mode 100644 index 000000000..3d9d7e751 Binary files /dev/null and b/representation-manager/client/src/main/scala/com/twitter/representation_manager/StoreBuilder.docx differ diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/StoreBuilder.scala b/representation-manager/client/src/main/scala/com/twitter/representation_manager/StoreBuilder.scala deleted file mode 100644 index 2314a8254..000000000 --- a/representation-manager/client/src/main/scala/com/twitter/representation_manager/StoreBuilder.scala +++ /dev/null @@ -1,208 +0,0 @@ -package com.twitter.representation_manager - -import com.twitter.finagle.memcached.{Client => MemcachedClient} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.hermit.store.common.ObservedCachedReadableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.representation_manager.config.ClientConfig -import com.twitter.representation_manager.config.DisabledInMemoryCacheParams -import com.twitter.representation_manager.config.EnabledInMemoryCacheParams -import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.LocaleEntityId -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} -import com.twitter.storehaus.ReadableStore -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.strato.thrift.ScroogeConvImplicits._ - -/** - * This is the class that offers features to build readable stores for a given - * SimClustersEmbeddingView (i.e. embeddingType and modelVersion). It applies ClientConfig - * for a particular service and build ReadableStores which implement that config. - */ -class StoreBuilder( - clientConfig: ClientConfig, - stratoClient: StratoClient, - memCachedClient: MemcachedClient, - globalStats: StatsReceiver, -) { - private val stats = - globalStats.scope("representation_manager_client").scope(this.getClass.getSimpleName) - - // Column consts - private val ColPathPrefix = "recommendations/representation_manager/" - private val SimclustersTweetColPath = ColPathPrefix + "simClustersEmbedding.Tweet" - private val SimclustersUserColPath = ColPathPrefix + "simClustersEmbedding.User" - private val SimclustersTopicIdColPath = ColPathPrefix + "simClustersEmbedding.TopicId" - private val SimclustersLocaleEntityIdColPath = - ColPathPrefix + "simClustersEmbedding.LocaleEntityId" - - def buildSimclustersTweetEmbeddingStore( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[Long, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersTweetColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - - addCacheLayer(rawStore, embeddingColumnView) - } - - def buildSimclustersUserEmbeddingStore( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[Long, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersUserColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - - addCacheLayer(rawStore, embeddingColumnView) - } - - def buildSimclustersTopicIdEmbeddingStore( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[TopicId, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[TopicId, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersTopicIdColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - - addCacheLayer(rawStore, embeddingColumnView) - } - - def buildSimclustersLocaleEntityIdEmbeddingStore( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[LocaleEntityId, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[LocaleEntityId, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersLocaleEntityIdColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - - addCacheLayer(rawStore, embeddingColumnView) - } - - def buildSimclustersTweetEmbeddingStoreWithEmbeddingIdAsKey( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersTweetColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId(_, _, InternalId.TweetId(tweetId)) => - tweetId - } - - addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView) - } - - def buildSimclustersUserEmbeddingStoreWithEmbeddingIdAsKey( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersUserColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId(_, _, InternalId.UserId(userId)) => - userId - } - - addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView) - } - - def buildSimclustersTopicEmbeddingStoreWithEmbeddingIdAsKey( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[TopicId, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersTopicIdColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId(_, _, InternalId.TopicId(topicId)) => - topicId - } - - addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView) - } - - def buildSimclustersTopicIdEmbeddingStoreWithEmbeddingIdAsKey( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[TopicId, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersTopicIdColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId(_, _, InternalId.TopicId(topicId)) => - topicId - } - - addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView) - } - - def buildSimclustersLocaleEntityIdEmbeddingStoreWithEmbeddingIdAsKey( - embeddingColumnView: SimClustersEmbeddingView - ): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - val rawStore = StratoFetchableStore - .withView[LocaleEntityId, SimClustersEmbeddingView, ThriftSimClustersEmbedding]( - stratoClient, - SimclustersLocaleEntityIdColPath, - embeddingColumnView) - .mapValues(SimClustersEmbedding(_)) - val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId(_, _, InternalId.LocaleEntityId(localeEntityId)) => - localeEntityId - } - - addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView) - } - - private def addCacheLayer[K]( - rawStore: ReadableStore[K, SimClustersEmbedding], - embeddingColumnView: SimClustersEmbeddingView, - ): ReadableStore[K, SimClustersEmbedding] = { - // Add in-memory caching based on ClientConfig - val inMemCacheParams = clientConfig.inMemoryCacheConfig - .getCacheSetup(embeddingColumnView.embeddingType, embeddingColumnView.modelVersion) - - val statsPerStore = stats - .scope(embeddingColumnView.embeddingType.name).scope(embeddingColumnView.modelVersion.name) - - inMemCacheParams match { - case DisabledInMemoryCacheParams => - ObservedReadableStore( - store = rawStore - )(statsPerStore) - case EnabledInMemoryCacheParams(ttl, maxKeys, cacheName) => - ObservedCachedReadableStore.from[K, SimClustersEmbedding]( - rawStore, - ttl = ttl, - maxKeys = maxKeys, - cacheName = cacheName, - windowSize = 10000L - )(statsPerStore) - } - } - -} diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/BUILD b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/BUILD deleted file mode 100644 index 8418563d5..000000000 --- a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-thrift-client", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/common", - "representation-manager/server/src/main/thrift:thrift-scala", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/BUILD.docx b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/BUILD.docx new file mode 100644 index 000000000..ac4787361 Binary files /dev/null and b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/BUILD.docx differ diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/ClientConfig.docx b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/ClientConfig.docx new file mode 100644 index 000000000..621c0b5f7 Binary files /dev/null and b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/ClientConfig.docx differ diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/ClientConfig.scala b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/ClientConfig.scala deleted file mode 100644 index 9ae0c49e7..000000000 --- a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/ClientConfig.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.representation_manager.config - -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion - -/* - * This is RMS client config class. - * We only support setting up in memory cache params for now, but we expect to enable other - * customisations in the near future e.g. request timeout - * - * -------------------------------------------- - * PLEASE NOTE: - * Having in-memory cache is not necessarily a free performance win, anyone considering it should - * investigate rather than blindly enabling it - * */ -class ClientConfig(inMemCacheParamsOverrides: Map[ - (EmbeddingType, ModelVersion), - InMemoryCacheParams -] = Map.empty) { - // In memory cache config per embedding - val inMemCacheParams = DefaultInMemoryCacheConfig.cacheParamsMap ++ inMemCacheParamsOverrides - val inMemoryCacheConfig = new InMemoryCacheConfig(inMemCacheParams) -} - -object DefaultClientConfig extends ClientConfig diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/InMemoryCacheConfig.docx b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/InMemoryCacheConfig.docx new file mode 100644 index 000000000..6a3723031 Binary files /dev/null and b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/InMemoryCacheConfig.docx differ diff --git a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/InMemoryCacheConfig.scala b/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/InMemoryCacheConfig.scala deleted file mode 100644 index eab569b51..000000000 --- a/representation-manager/client/src/main/scala/com/twitter/representation_manager/config/InMemoryCacheConfig.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.representation_manager.config - -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.util.Duration - -/* - * -------------------------------------------- - * PLEASE NOTE: - * Having in-memory cache is not necessarily a free performance win, anyone considering it should - * investigate rather than blindly enabling it - * -------------------------------------------- - * */ - -sealed trait InMemoryCacheParams - -/* - * This holds params that is required to set up a in-mem cache for a single embedding store - */ -case class EnabledInMemoryCacheParams( - ttl: Duration, - maxKeys: Int, - cacheName: String) - extends InMemoryCacheParams -object DisabledInMemoryCacheParams extends InMemoryCacheParams - -/* - * This is the class for the in-memory cache config. Client could pass in their own cacheParamsMap to - * create a new InMemoryCacheConfig instead of using the DefaultInMemoryCacheConfig object below - * */ -class InMemoryCacheConfig( - cacheParamsMap: Map[ - (EmbeddingType, ModelVersion), - InMemoryCacheParams - ] = Map.empty) { - - def getCacheSetup( - embeddingType: EmbeddingType, - modelVersion: ModelVersion - ): InMemoryCacheParams = { - // When requested embedding type doesn't exist, we return DisabledInMemoryCacheParams - cacheParamsMap.getOrElse((embeddingType, modelVersion), DisabledInMemoryCacheParams) - } -} - -/* - * Default config for the in-memory cache - * Clients can directly import and use this one if they don't want to set up a customised config - * */ -object DefaultInMemoryCacheConfig extends InMemoryCacheConfig { - // set default to no in-memory caching - val cacheParamsMap = Map.empty -} diff --git a/representation-manager/server/BUILD b/representation-manager/server/BUILD deleted file mode 100644 index 427fc1d3b..000000000 --- a/representation-manager/server/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -jvm_binary( - name = "bin", - basename = "representation-manager", - main = "com.twitter.representation_manager.RepresentationManagerFedServerMain", - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-logback/src/main/scala", - "loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", - "representation-manager/server/src/main/resources", - "representation-manager/server/src/main/scala/com/twitter/representation_manager", - "twitter-server/logback-classic/src/main/scala", - ], -) - -# Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app -jvm_app( - name = "representation-manager-app", - archive = "zip", - binary = ":bin", -) diff --git a/representation-manager/server/BUILD.docx b/representation-manager/server/BUILD.docx new file mode 100644 index 000000000..d7a2f787b Binary files /dev/null and b/representation-manager/server/BUILD.docx differ diff --git a/representation-manager/server/src/main/resources/BUILD b/representation-manager/server/src/main/resources/BUILD deleted file mode 100644 index b3a752276..000000000 --- a/representation-manager/server/src/main/resources/BUILD +++ /dev/null @@ -1,7 +0,0 @@ -resources( - sources = [ - "*.xml", - "config/*.yml", - ], - tags = ["bazel-compatible"], -) diff --git a/representation-manager/server/src/main/resources/BUILD.docx b/representation-manager/server/src/main/resources/BUILD.docx new file mode 100644 index 000000000..1561a4783 Binary files /dev/null and b/representation-manager/server/src/main/resources/BUILD.docx differ diff --git a/representation-manager/server/src/main/resources/config/decider.docx b/representation-manager/server/src/main/resources/config/decider.docx new file mode 100644 index 000000000..efa88823e Binary files /dev/null and b/representation-manager/server/src/main/resources/config/decider.docx differ diff --git a/representation-manager/server/src/main/resources/config/decider.yml b/representation-manager/server/src/main/resources/config/decider.yml deleted file mode 100644 index e75ebf89d..000000000 --- a/representation-manager/server/src/main/resources/config/decider.yml +++ /dev/null @@ -1,219 +0,0 @@ -# ---------- traffic percentage by embedding type and model version ---------- -# Decider strings are build dynamically following the rule in there -# i.e. s"enable_${embeddingType.name}_${modelVersion.name}" -# Hence this should be updated accordingly if usage is changed in the embedding stores - -# Tweet embeddings -"enable_LogFavBasedTweet_Model20m145k2020": - comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavBasedTweet - Model20m145k2020. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedTweet_Model20m145kUpdated": - comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavBasedTweet - Model20m145kUpdated. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavLongestL2EmbeddingTweet_Model20m145k2020": - comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavLongestL2EmbeddingTweet - Model20m145k2020. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavLongestL2EmbeddingTweet_Model20m145kUpdated": - comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavLongestL2EmbeddingTweet - Model20m145kUpdated. 0 means return EMPTY for all requests." - default_availability: 10000 - -# Topic embeddings -"enable_FavTfgTopic_Model20m145k2020": - comment: "Enable the read traffic to FavTfgTopic - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedKgoApeTopic_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedKgoApeTopic - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -# User embeddings - KnownFor -"enable_FavBasedProducer_Model20m145kUpdated": - comment: "Enable the read traffic to FavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FavBasedProducer_Model20m145k2020": - comment: "Enable the read traffic to FavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FollowBasedProducer_Model20m145k2020": - comment: "Enable the read traffic to FollowBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_AggregatableFavBasedProducer_Model20m145kUpdated": - comment: "Enable the read traffic to AggregatableFavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_AggregatableFavBasedProducer_Model20m145k2020": - comment: "Enable the read traffic to AggregatableFavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_AggregatableLogFavBasedProducer_Model20m145kUpdated": - comment: "Enable the read traffic to AggregatableLogFavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_AggregatableLogFavBasedProducer_Model20m145k2020": - comment: "Enable the read traffic to AggregatableLogFavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -enable_RelaxedAggregatableLogFavBasedProducer_Model20m145kUpdated: - comment: "Enable the read traffic to RelaxedAggregatableLogFavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -enable_RelaxedAggregatableLogFavBasedProducer_Model20m145k2020: - comment: "Enable the read traffic to RelaxedAggregatableLogFavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -# User embeddings - InterestedIn -"enable_LogFavBasedUserInterestedInFromAPE_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedInFromAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FollowBasedUserInterestedInFromAPE_Model20m145k2020": - comment: "Enable the read traffic to FollowBasedUserInterestedInFromAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FavBasedUserInterestedIn_Model20m145kUpdated": - comment: "Enable the read traffic to FavBasedUserInterestedIn - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FavBasedUserInterestedIn_Model20m145k2020": - comment: "Enable the read traffic to FavBasedUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FollowBasedUserInterestedIn_Model20m145k2020": - comment: "Enable the read traffic to FollowBasedUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedUserInterestedIn_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FavBasedUserInterestedInFromPE_Model20m145kUpdated": - comment: "Enable the read traffic to FavBasedUserInterestedInFromPE - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FilteredUserInterestedIn_Model20m145kUpdated": - comment: "Enable the read traffic to FilteredUserInterestedIn - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FilteredUserInterestedIn_Model20m145k2020": - comment: "Enable the read traffic to FilteredUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_FilteredUserInterestedInFromPE_Model20m145kUpdated": - comment: "Enable the read traffic to FilteredUserInterestedInFromPE - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_UnfilteredUserInterestedIn_Model20m145kUpdated": - comment: "Enable the read traffic to UnfilteredUserInterestedIn - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_UnfilteredUserInterestedIn_Model20m145k2020": - comment: "Enable the read traffic to UnfilteredUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_UserNextInterestedIn_Model20m145k2020": - comment: "Enable the read traffic to UserNextInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedUserInterestedAverageAddressBookFromIIAPE_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedAverageAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedAverageAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -"enable_LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE_Model20m145k2020": - comment: "Enable the read traffic to LogFavBasedUserInterestedAverageAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests." - default_availability: 10000 - -# ---------- load shedding by caller id ---------- -# To create a new decider, add here with the same format and caller's details : -# "representation-manager_load_shed_by_caller_id_twtr:{{role}}:{{name}}:{{environment}}:{{cluster}}" -# All the deciders below are generated by this script: -# ./strato/bin/fed deciders representation-manager --service-role=representation-manager --service-name=representation-manager -# If you need to run the script and paste the output, add ONLY the prod deciders here. -"representation-manager_load_shed_by_caller_id_all": - comment: "Reject all traffic from caller id: all" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:cr-mixer:cr-mixer:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:cr-mixer:cr-mixer:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:cr-mixer:cr-mixer:prod:pdxa": - comment: "Reject all traffic from caller id: twtr:svc:cr-mixer:cr-mixer:prod:pdxa" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-1:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-1:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-1:prod:pdxa": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-1:prod:pdxa" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-3:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-3:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-3:prod:pdxa": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-3:prod:pdxa" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-4:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-4:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-4:prod:pdxa": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-4:prod:pdxa" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:pdxa": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:pdxa" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann:prod:pdxa": - comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann:prod:pdxa" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:stratostore:stratoapi:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:stratostore:stratoapi:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:stratostore:stratoserver:prod:atla": - comment: "Reject all traffic from caller id: twtr:svc:stratostore:stratoserver:prod:atla" - default_availability: 0 - -"representation-manager_load_shed_by_caller_id_twtr:svc:stratostore:stratoserver:prod:pdxa": - comment: "Reject all traffic from caller id: twtr:svc:stratostore:stratoserver:prod:pdxa" - default_availability: 0 - -# ---------- Dark Traffic Proxy ---------- -representation-manager_forward_dark_traffic: - comment: "Defines the percentage of traffic to forward to diffy-proxy. Set to 0 to disable dark traffic forwarding" - default_availability: 0 diff --git a/representation-manager/server/src/main/resources/logback.docx b/representation-manager/server/src/main/resources/logback.docx new file mode 100644 index 000000000..506cc6ae9 Binary files /dev/null and b/representation-manager/server/src/main/resources/logback.docx differ diff --git a/representation-manager/server/src/main/resources/logback.xml b/representation-manager/server/src/main/resources/logback.xml deleted file mode 100644 index 47b3ed16d..000000000 --- a/representation-manager/server/src/main/resources/logback.xml +++ /dev/null @@ -1,165 +0,0 @@ - - - - - - - - - - - - - - - - - true - - - - - - - - - - - ${log.service.output} - - - ${log.service.output}.%d.gz - - 3GB - - 21 - true - - - %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n - - - - - - ${log.access.output} - - - ${log.access.output}.%d.gz - - 100MB - - 7 - true - - - ${DEFAULT_ACCESS_PATTERN}%n - - - - - - true - ${log.lens.category} - ${log.lens.index} - ${log.lens.tag}/service - - %msg - - - - - - true - ${log.lens.category} - ${log.lens.index} - ${log.lens.tag}/access - - %msg - - - - - - allow_listed_pipeline_executions.log - - - allow_listed_pipeline_executions.log.%d.gz - - 100MB - - 7 - true - - - %date %.-3level ${DEFAULT_SERVICE_PATTERN}%n - - - - - - - - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - ${async_queue_size} - ${async_max_flush_time} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/BUILD deleted file mode 100644 index d8ca301f6..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-thrift-client", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/BUILD.docx new file mode 100644 index 000000000..2bf2c038d Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/RepresentationManagerFedServer.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/RepresentationManagerFedServer.docx new file mode 100644 index 000000000..013e2c1d3 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/RepresentationManagerFedServer.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/RepresentationManagerFedServer.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/RepresentationManagerFedServer.scala deleted file mode 100644 index 5bc820bb4..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/RepresentationManagerFedServer.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.representation_manager - -import com.google.inject.Module -import com.twitter.inject.thrift.modules.ThriftClientIdModule -import com.twitter.representation_manager.columns.topic.LocaleEntityIdSimClustersEmbeddingCol -import com.twitter.representation_manager.columns.topic.TopicIdSimClustersEmbeddingCol -import com.twitter.representation_manager.columns.tweet.TweetSimClustersEmbeddingCol -import com.twitter.representation_manager.columns.user.UserSimClustersEmbeddingCol -import com.twitter.representation_manager.modules.CacheModule -import com.twitter.representation_manager.modules.InterestsThriftClientModule -import com.twitter.representation_manager.modules.LegacyRMSConfigModule -import com.twitter.representation_manager.modules.StoreModule -import com.twitter.representation_manager.modules.TimerModule -import com.twitter.representation_manager.modules.UttClientModule -import com.twitter.strato.fed._ -import com.twitter.strato.fed.server._ - -object RepresentationManagerFedServerMain extends RepresentationManagerFedServer - -trait RepresentationManagerFedServer extends StratoFedServer { - override def dest: String = "/s/representation-manager/representation-manager" - override val modules: Seq[Module] = - Seq( - CacheModule, - InterestsThriftClientModule, - LegacyRMSConfigModule, - StoreModule, - ThriftClientIdModule, - TimerModule, - UttClientModule - ) - - override def columns: Seq[Class[_ <: StratoFed.Column]] = - Seq( - classOf[TweetSimClustersEmbeddingCol], - classOf[UserSimClustersEmbeddingCol], - classOf[TopicIdSimClustersEmbeddingCol], - classOf[LocaleEntityIdSimClustersEmbeddingCol] - ) -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/BUILD deleted file mode 100644 index 6ebd77ef8..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/BUILD.docx new file mode 100644 index 000000000..ff46af5e4 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/ColumnConfigBase.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/ColumnConfigBase.docx new file mode 100644 index 000000000..391d21412 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/ColumnConfigBase.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/ColumnConfigBase.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/ColumnConfigBase.scala deleted file mode 100644 index 143ccdc4c..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/ColumnConfigBase.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.representation_manager.columns - -import com.twitter.strato.access.Access.LdapGroup -import com.twitter.strato.config.ContactInfo -import com.twitter.strato.config.FromColumns -import com.twitter.strato.config.Has -import com.twitter.strato.config.Prefix -import com.twitter.strato.config.ServiceIdentifierPattern - -object ColumnConfigBase { - - /****************** Internal permissions *******************/ - val recosPermissions: Seq[com.twitter.strato.config.Policy] = Seq() - - /****************** External permissions *******************/ - // This is used to grant limited access to members outside of RP team. - val externalPermissions: Seq[com.twitter.strato.config.Policy] = Seq() - - val contactInfo: ContactInfo = ContactInfo( - description = "Please contact Relevance Platform for more details", - contactEmail = "no-reply@twitter.com", - ldapGroup = "ldap", - jiraProject = "JIRA", - links = Seq("http://go/rms-runbook") - ) -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/BUILD deleted file mode 100644 index 26022ebe5..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/columns", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/modules", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/store", - "representation-manager/server/src/main/thrift:thrift-scala", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/BUILD.docx new file mode 100644 index 000000000..79349ec24 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/LocaleEntityIdSimClustersEmbeddingCol.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/LocaleEntityIdSimClustersEmbeddingCol.docx new file mode 100644 index 000000000..4e0c44448 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/LocaleEntityIdSimClustersEmbeddingCol.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/LocaleEntityIdSimClustersEmbeddingCol.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/LocaleEntityIdSimClustersEmbeddingCol.scala deleted file mode 100644 index 7b7952300..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/LocaleEntityIdSimClustersEmbeddingCol.scala +++ /dev/null @@ -1,77 +0,0 @@ -package com.twitter.representation_manager.columns.topic - -import com.twitter.representation_manager.columns.ColumnConfigBase -import com.twitter.representation_manager.store.TopicSimClustersEmbeddingStore -import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclusters_v2.thriftscala.LocaleEntityId -import com.twitter.stitch -import com.twitter.stitch.Stitch -import com.twitter.stitch.storehaus.StitchOfReadableStore -import com.twitter.strato.catalog.OpMetadata -import com.twitter.strato.config.AnyOf -import com.twitter.strato.config.ContactInfo -import com.twitter.strato.config.FromColumns -import com.twitter.strato.config.Policy -import com.twitter.strato.config.Prefix -import com.twitter.strato.data.Conv -import com.twitter.strato.data.Description.PlainText -import com.twitter.strato.data.Lifecycle -import com.twitter.strato.fed._ -import com.twitter.strato.thrift.ScroogeConv -import javax.inject.Inject - -class LocaleEntityIdSimClustersEmbeddingCol @Inject() ( - embeddingStore: TopicSimClustersEmbeddingStore) - extends StratoFed.Column( - "recommendations/representation_manager/simClustersEmbedding.LocaleEntityId") - with StratoFed.Fetch.Stitch { - - private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] = - StitchOfReadableStore(embeddingStore.topicSimClustersEmbeddingStore.mapValues(_.toThrift)) - - val colPermissions: Seq[com.twitter.strato.config.Policy] = - ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns( - Set( - Prefix("ml/featureStore/simClusters"), - )) - - override val policy: Policy = AnyOf({ - colPermissions - }) - - override type Key = LocaleEntityId - override type View = SimClustersEmbeddingView - override type Value = SimClustersEmbedding - - override val keyConv: Conv[Key] = ScroogeConv.fromStruct[LocaleEntityId] - override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView] - override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding] - - override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo - - override val metadata: OpMetadata = OpMetadata( - lifecycle = Some(Lifecycle.Production), - description = Some( - PlainText( - "The Topic SimClusters Embedding Endpoint in Representation Management Service with LocaleEntityId." + - " TDD: http://go/rms-tdd")) - ) - - override def fetch(key: Key, view: View): Stitch[Result[Value]] = { - val embeddingId = SimClustersEmbeddingId( - view.embeddingType, - view.modelVersion, - InternalId.LocaleEntityId(key) - ) - - storeStitch(embeddingId) - .map(embedding => found(embedding)) - .handle { - case stitch.NotFound => missing - } - } - -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/TopicIdSimClustersEmbeddingCol.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/TopicIdSimClustersEmbeddingCol.docx new file mode 100644 index 000000000..edc941a18 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/TopicIdSimClustersEmbeddingCol.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/TopicIdSimClustersEmbeddingCol.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/TopicIdSimClustersEmbeddingCol.scala deleted file mode 100644 index 4afddbb4c..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic/TopicIdSimClustersEmbeddingCol.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.representation_manager.columns.topic - -import com.twitter.representation_manager.columns.ColumnConfigBase -import com.twitter.representation_manager.store.TopicSimClustersEmbeddingStore -import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclusters_v2.thriftscala.TopicId -import com.twitter.stitch -import com.twitter.stitch.Stitch -import com.twitter.stitch.storehaus.StitchOfReadableStore -import com.twitter.strato.catalog.OpMetadata -import com.twitter.strato.config.AnyOf -import com.twitter.strato.config.ContactInfo -import com.twitter.strato.config.FromColumns -import com.twitter.strato.config.Policy -import com.twitter.strato.config.Prefix -import com.twitter.strato.data.Conv -import com.twitter.strato.data.Description.PlainText -import com.twitter.strato.data.Lifecycle -import com.twitter.strato.fed._ -import com.twitter.strato.thrift.ScroogeConv -import javax.inject.Inject - -class TopicIdSimClustersEmbeddingCol @Inject() (embeddingStore: TopicSimClustersEmbeddingStore) - extends StratoFed.Column("recommendations/representation_manager/simClustersEmbedding.TopicId") - with StratoFed.Fetch.Stitch { - - private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] = - StitchOfReadableStore(embeddingStore.topicSimClustersEmbeddingStore.mapValues(_.toThrift)) - - val colPermissions: Seq[com.twitter.strato.config.Policy] = - ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns( - Set( - Prefix("ml/featureStore/simClusters"), - )) - - override val policy: Policy = AnyOf({ - colPermissions - }) - - override type Key = TopicId - override type View = SimClustersEmbeddingView - override type Value = SimClustersEmbedding - - override val keyConv: Conv[Key] = ScroogeConv.fromStruct[TopicId] - override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView] - override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding] - - override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo - - override val metadata: OpMetadata = OpMetadata( - lifecycle = Some(Lifecycle.Production), - description = Some(PlainText( - "The Topic SimClusters Embedding Endpoint in Representation Management Service with TopicId." + - " TDD: http://go/rms-tdd")) - ) - - override def fetch(key: Key, view: View): Stitch[Result[Value]] = { - val embeddingId = SimClustersEmbeddingId( - view.embeddingType, - view.modelVersion, - InternalId.TopicId(key) - ) - - storeStitch(embeddingId) - .map(embedding => found(embedding)) - .handle { - case stitch.NotFound => missing - } - } - -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/BUILD deleted file mode 100644 index 26022ebe5..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/columns", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/modules", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/store", - "representation-manager/server/src/main/thrift:thrift-scala", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/BUILD.docx new file mode 100644 index 000000000..79349ec24 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/TweetSimClustersEmbeddingCol.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/TweetSimClustersEmbeddingCol.docx new file mode 100644 index 000000000..75317f336 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/TweetSimClustersEmbeddingCol.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/TweetSimClustersEmbeddingCol.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/TweetSimClustersEmbeddingCol.scala deleted file mode 100644 index 15cd4247c..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet/TweetSimClustersEmbeddingCol.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.representation_manager.columns.tweet - -import com.twitter.representation_manager.columns.ColumnConfigBase -import com.twitter.representation_manager.store.TweetSimClustersEmbeddingStore -import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.stitch -import com.twitter.stitch.Stitch -import com.twitter.stitch.storehaus.StitchOfReadableStore -import com.twitter.strato.catalog.OpMetadata -import com.twitter.strato.config.AnyOf -import com.twitter.strato.config.ContactInfo -import com.twitter.strato.config.FromColumns -import com.twitter.strato.config.Policy -import com.twitter.strato.config.Prefix -import com.twitter.strato.data.Conv -import com.twitter.strato.data.Description.PlainText -import com.twitter.strato.data.Lifecycle -import com.twitter.strato.fed._ -import com.twitter.strato.thrift.ScroogeConv -import javax.inject.Inject - -class TweetSimClustersEmbeddingCol @Inject() (embeddingStore: TweetSimClustersEmbeddingStore) - extends StratoFed.Column("recommendations/representation_manager/simClustersEmbedding.Tweet") - with StratoFed.Fetch.Stitch { - - private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] = - StitchOfReadableStore(embeddingStore.tweetSimClustersEmbeddingStore.mapValues(_.toThrift)) - - val colPermissions: Seq[com.twitter.strato.config.Policy] = - ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns( - Set( - Prefix("ml/featureStore/simClusters"), - )) - - override val policy: Policy = AnyOf({ - colPermissions - }) - - override type Key = Long // TweetId - override type View = SimClustersEmbeddingView - override type Value = SimClustersEmbedding - - override val keyConv: Conv[Key] = Conv.long - override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView] - override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding] - - override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo - - override val metadata: OpMetadata = OpMetadata( - lifecycle = Some(Lifecycle.Production), - description = Some( - PlainText("The Tweet SimClusters Embedding Endpoint in Representation Management Service." + - " TDD: http://go/rms-tdd")) - ) - - override def fetch(key: Key, view: View): Stitch[Result[Value]] = { - val embeddingId = SimClustersEmbeddingId( - view.embeddingType, - view.modelVersion, - InternalId.TweetId(key) - ) - - storeStitch(embeddingId) - .map(embedding => found(embedding)) - .handle { - case stitch.NotFound => missing - } - } - -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/BUILD deleted file mode 100644 index 26022ebe5..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/columns", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/modules", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/store", - "representation-manager/server/src/main/thrift:thrift-scala", - "strato/src/main/scala/com/twitter/strato/fed", - "strato/src/main/scala/com/twitter/strato/fed/server", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/BUILD.docx new file mode 100644 index 000000000..79349ec24 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/UserSimClustersEmbeddingCol.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/UserSimClustersEmbeddingCol.docx new file mode 100644 index 000000000..ea8a41494 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/UserSimClustersEmbeddingCol.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/UserSimClustersEmbeddingCol.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/UserSimClustersEmbeddingCol.scala deleted file mode 100644 index ebcf22a1d..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user/UserSimClustersEmbeddingCol.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.representation_manager.columns.user - -import com.twitter.representation_manager.columns.ColumnConfigBase -import com.twitter.representation_manager.store.UserSimClustersEmbeddingStore -import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.stitch -import com.twitter.stitch.Stitch -import com.twitter.stitch.storehaus.StitchOfReadableStore -import com.twitter.strato.catalog.OpMetadata -import com.twitter.strato.config.AnyOf -import com.twitter.strato.config.ContactInfo -import com.twitter.strato.config.FromColumns -import com.twitter.strato.config.Policy -import com.twitter.strato.config.Prefix -import com.twitter.strato.data.Conv -import com.twitter.strato.data.Description.PlainText -import com.twitter.strato.data.Lifecycle -import com.twitter.strato.fed._ -import com.twitter.strato.thrift.ScroogeConv -import javax.inject.Inject - -class UserSimClustersEmbeddingCol @Inject() (embeddingStore: UserSimClustersEmbeddingStore) - extends StratoFed.Column("recommendations/representation_manager/simClustersEmbedding.User") - with StratoFed.Fetch.Stitch { - - private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] = - StitchOfReadableStore(embeddingStore.userSimClustersEmbeddingStore.mapValues(_.toThrift)) - - val colPermissions: Seq[com.twitter.strato.config.Policy] = - ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns( - Set( - Prefix("ml/featureStore/simClusters"), - )) - - override val policy: Policy = AnyOf({ - colPermissions - }) - - override type Key = Long // UserId - override type View = SimClustersEmbeddingView - override type Value = SimClustersEmbedding - - override val keyConv: Conv[Key] = Conv.long - override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView] - override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding] - - override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo - - override val metadata: OpMetadata = OpMetadata( - lifecycle = Some(Lifecycle.Production), - description = Some( - PlainText("The User SimClusters Embedding Endpoint in Representation Management Service." + - " TDD: http://go/rms-tdd")) - ) - - override def fetch(key: Key, view: View): Stitch[Result[Value]] = { - val embeddingId = SimClustersEmbeddingId( - view.embeddingType, - view.modelVersion, - InternalId.UserId(key) - ) - - storeStitch(embeddingId) - .map(embedding => found(embedding)) - .handle { - case stitch.NotFound => missing - } - } - -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/BUILD deleted file mode 100644 index 62b8f5dd2..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "decider/src/main/scala", - "finagle/finagle-memcached", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", - "src/scala/com/twitter/simclusters_v2/common", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/BUILD.docx new file mode 100644 index 000000000..4881bd7d7 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/MemCacheConfig.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/MemCacheConfig.docx new file mode 100644 index 000000000..6a5d6ce03 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/MemCacheConfig.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/MemCacheConfig.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/MemCacheConfig.scala deleted file mode 100644 index 4741edb2d..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/MemCacheConfig.scala +++ /dev/null @@ -1,153 +0,0 @@ -package com.twitter.representation_manager.common - -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.common.SimClustersEmbeddingIdCacheKeyBuilder -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.EmbeddingType._ -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.simclusters_v2.thriftscala.ModelVersion._ -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} -import com.twitter.storehaus.ReadableStore -import com.twitter.util.Duration - -/* - * NOTE - ALL the cache configs here are just placeholders, NONE of them is used anyweher in RMS yet - * */ -sealed trait MemCacheParams -sealed trait MemCacheConfig - -/* - * This holds params that is required to set up a memcache cache for a single embedding store - * */ -case class EnabledMemCacheParams(ttl: Duration) extends MemCacheParams -object DisabledMemCacheParams extends MemCacheParams - -/* - * We use this MemcacheConfig as the single source to set up the memcache for all RMS use cases - * NO OVERRIDE FROM CLIENT - * */ -object MemCacheConfig { - val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - val hashKeyPrefix: String = "RMS" - val simclustersEmbeddingCacheKeyBuilder = - SimClustersEmbeddingIdCacheKeyBuilder(keyHasher.hashKey, hashKeyPrefix) - - val cacheParamsMap: Map[ - (EmbeddingType, ModelVersion), - MemCacheParams - ] = Map( - // Tweet Embeddings - (LogFavBasedTweet, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 10.minutes), - (LogFavBasedTweet, Model20m145k2020) -> EnabledMemCacheParams(ttl = 10.minutes), - (LogFavLongestL2EmbeddingTweet, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 10.minutes), - (LogFavLongestL2EmbeddingTweet, Model20m145k2020) -> EnabledMemCacheParams(ttl = 10.minutes), - // User - KnownFor Embeddings - (FavBasedProducer, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours), - (FavBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (FollowBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (AggregatableLogFavBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (RelaxedAggregatableLogFavBasedProducer, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = - 12.hours), - (RelaxedAggregatableLogFavBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl = - 12.hours), - // User - InterestedIn Embeddings - (LogFavBasedUserInterestedInFromAPE, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (FollowBasedUserInterestedInFromAPE, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (FavBasedUserInterestedIn, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours), - (FavBasedUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (FollowBasedUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (LogFavBasedUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (FavBasedUserInterestedInFromPE, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours), - (FilteredUserInterestedIn, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours), - (FilteredUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (FilteredUserInterestedInFromPE, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours), - (UnfilteredUserInterestedIn, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours), - (UnfilteredUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (UserNextInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = - 30.minutes), //embedding is updated every 2 hours, keeping it lower to avoid staleness - ( - LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE, - Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - ( - LogFavBasedUserInterestedAverageAddressBookFromIIAPE, - Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - ( - LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE, - Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - ( - LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE, - Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - ( - LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE, - Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - ( - LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE, - Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - // Topic Embeddings - (FavTfgTopic, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - (LogFavBasedKgoApeTopic, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours), - ) - - def getCacheSetup( - embeddingType: EmbeddingType, - modelVersion: ModelVersion - ): MemCacheParams = { - // When requested (embeddingType, modelVersion) doesn't exist, we return DisabledMemCacheParams - cacheParamsMap.getOrElse((embeddingType, modelVersion), DisabledMemCacheParams) - } - - def getCacheKeyPrefix(embeddingType: EmbeddingType, modelVersion: ModelVersion) = - s"${embeddingType.value}_${modelVersion.value}_" - - def getStatsName(embeddingType: EmbeddingType, modelVersion: ModelVersion) = - s"${embeddingType.name}_${modelVersion.name}_mem_cache" - - /** - * Build a ReadableStore based on MemCacheConfig. - * - * If memcache is disabled, it will return a normal readable store wrapper of the rawStore, - * with SimClustersEmbedding as value; - * If memcache is enabled, it will return a ObservedMemcachedReadableStore wrapper of the rawStore, - * with memcache set up according to the EnabledMemCacheParams - * */ - def buildMemCacheStoreForSimClustersEmbedding( - rawStore: ReadableStore[SimClustersEmbeddingId, ThriftSimClustersEmbedding], - cacheClient: Client, - embeddingType: EmbeddingType, - modelVersion: ModelVersion, - stats: StatsReceiver - ): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - val cacheParams = getCacheSetup(embeddingType, modelVersion) - val store = cacheParams match { - case DisabledMemCacheParams => rawStore - case EnabledMemCacheParams(ttl) => - val memCacheKeyPrefix = MemCacheConfig.getCacheKeyPrefix( - embeddingType, - modelVersion - ) - val statsName = MemCacheConfig.getStatsName( - embeddingType, - modelVersion - ) - ObservedMemcachedReadableStore.fromCacheClient( - backingStore = rawStore, - cacheClient = cacheClient, - ttl = ttl - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = stats.scope(statsName), - keyToString = { k => memCacheKeyPrefix + k.toString } - ) - } - store.mapValues(SimClustersEmbedding(_)) - } - -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/RepresentationManagerDecider.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/RepresentationManagerDecider.docx new file mode 100644 index 000000000..d265d98a1 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/RepresentationManagerDecider.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/RepresentationManagerDecider.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/RepresentationManagerDecider.scala deleted file mode 100644 index 97179e25f..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/common/RepresentationManagerDecider.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.representation_manager.common - -import com.twitter.decider.Decider -import com.twitter.decider.RandomRecipient -import com.twitter.decider.Recipient -import com.twitter.simclusters_v2.common.DeciderGateBuilderWithIdHashing -import javax.inject.Inject - -case class RepresentationManagerDecider @Inject() (decider: Decider) { - - val deciderGateBuilder = new DeciderGateBuilderWithIdHashing(decider) - - def isAvailable(feature: String, recipient: Option[Recipient]): Boolean = { - decider.isAvailable(feature, recipient) - } - - /** - * When useRandomRecipient is set to false, the decider is either completely on or off. - * When useRandomRecipient is set to true, the decider is on for the specified % of traffic. - */ - def isAvailable(feature: String, useRandomRecipient: Boolean = true): Boolean = { - if (useRandomRecipient) isAvailable(feature, Some(RandomRecipient)) - else isAvailable(feature, None) - } -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/BUILD deleted file mode 100644 index d8bf04fc0..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "content-recommender/server/src/main/scala/com/twitter/contentrecommender:representation-manager-deps", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection", - "relevance-platform/src/main/scala/com/twitter/relevance_platform/common/readablestore", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/common", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/store", - "src/scala/com/twitter/ml/api/embedding", - "src/scala/com/twitter/simclusters_v2/common", - "src/scala/com/twitter/simclusters_v2/score", - "src/scala/com/twitter/simclusters_v2/summingbird/stores", - "src/scala/com/twitter/storehaus_internal/manhattan", - "src/scala/com/twitter/storehaus_internal/util", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "storage/clients/manhattan/client/src/main/scala", - "tweetypie/src/scala/com/twitter/tweetypie/util", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/BUILD.docx new file mode 100644 index 000000000..d7a22b422 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/LegacyRMS.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/LegacyRMS.docx new file mode 100644 index 000000000..2be44430e Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/LegacyRMS.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/LegacyRMS.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/LegacyRMS.scala deleted file mode 100644 index 378f33594..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/migration/LegacyRMS.scala +++ /dev/null @@ -1,846 +0,0 @@ -package com.twitter.representation_manager.migration - -import com.twitter.bijection.Injection -import com.twitter.bijection.scrooge.BinaryScalaCodec -import com.twitter.contentrecommender.store.ApeEntityEmbeddingStore -import com.twitter.contentrecommender.store.InterestsOptOutStore -import com.twitter.contentrecommender.store.SemanticCoreTopicSeedStore -import com.twitter.contentrecommender.twistly -import com.twitter.conversions.DurationOps._ -import com.twitter.decider.Decider -import com.twitter.escherbird.util.uttclient.CacheConfigV2 -import com.twitter.escherbird.util.uttclient.CachedUttClientV2 -import com.twitter.escherbird.util.uttclient.UttClientCacheConfigsV2 -import com.twitter.escherbird.utt.strato.thriftscala.Environment -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.memcached.Client -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.frigate.common.store.strato.StratoFetchableStore -import com.twitter.frigate.common.util.SeqLongInjection -import com.twitter.hashing.KeyHasher -import com.twitter.hermit.store.common.DeciderableReadableStore -import com.twitter.hermit.store.common.ObservedCachedReadableStore -import com.twitter.hermit.store.common.ObservedMemcachedReadableStore -import com.twitter.hermit.store.common.ObservedReadableStore -import com.twitter.interests.thriftscala.InterestsThriftService -import com.twitter.relevance_platform.common.injection.LZ4Injection -import com.twitter.relevance_platform.common.readablestore.ReadableStoreWithTimeout -import com.twitter.representation_manager.common.RepresentationManagerDecider -import com.twitter.representation_manager.store.DeciderConstants -import com.twitter.representation_manager.store.DeciderKey -import com.twitter.simclusters_v2.common.ModelVersions -import com.twitter.simclusters_v2.common.SimClustersEmbedding -import com.twitter.simclusters_v2.common.SimClustersEmbeddingIdCacheKeyBuilder -import com.twitter.simclusters_v2.stores.SimClustersEmbeddingStore -import com.twitter.simclusters_v2.summingbird.stores.PersistentTweetEmbeddingStore -import com.twitter.simclusters_v2.summingbird.stores.ProducerClusterEmbeddingReadableStores -import com.twitter.simclusters_v2.summingbird.stores.UserInterestedInReadableStore -import com.twitter.simclusters_v2.thriftscala.ClustersUserIsInterestedIn -import com.twitter.simclusters_v2.thriftscala.EmbeddingType -import com.twitter.simclusters_v2.thriftscala.EmbeddingType._ -import com.twitter.simclusters_v2.thriftscala.InternalId -import com.twitter.simclusters_v2.thriftscala.ModelVersion -import com.twitter.simclusters_v2.thriftscala.ModelVersion.Model20m145k2020 -import com.twitter.simclusters_v2.thriftscala.ModelVersion.Model20m145kUpdated -import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId -import com.twitter.simclusters_v2.thriftscala.SimClustersMultiEmbedding -import com.twitter.simclusters_v2.thriftscala.SimClustersMultiEmbeddingId -import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding} -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams -import com.twitter.storehaus.ReadableStore -import com.twitter.storehaus_internal.manhattan.Athena -import com.twitter.storehaus_internal.manhattan.ManhattanRO -import com.twitter.storehaus_internal.manhattan.ManhattanROConfig -import com.twitter.storehaus_internal.util.ApplicationID -import com.twitter.storehaus_internal.util.DatasetName -import com.twitter.storehaus_internal.util.HDFSPath -import com.twitter.strato.client.Strato -import com.twitter.strato.client.{Client => StratoClient} -import com.twitter.strato.thrift.ScroogeConvImplicits._ -import com.twitter.tweetypie.util.UserId -import com.twitter.util.Duration -import com.twitter.util.Future -import com.twitter.util.Throw -import com.twitter.util.Timer -import javax.inject.Inject -import javax.inject.Named -import scala.reflect.ClassTag - -class LegacyRMS @Inject() ( - serviceIdentifier: ServiceIdentifier, - cacheClient: Client, - stats: StatsReceiver, - decider: Decider, - clientId: ClientId, - timer: Timer, - @Named("cacheHashKeyPrefix") val cacheHashKeyPrefix: String = "RMS", - @Named("useContentRecommenderConfiguration") val useContentRecommenderConfiguration: Boolean = - false) { - - private val mhMtlsParams: ManhattanKVClientMtlsParams = ManhattanKVClientMtlsParams( - serviceIdentifier) - private val rmsDecider = RepresentationManagerDecider(decider) - val keyHasher: KeyHasher = KeyHasher.FNV1A_64 - - private val embeddingCacheKeyBuilder = - SimClustersEmbeddingIdCacheKeyBuilder(keyHasher.hashKey, cacheHashKeyPrefix) - private val statsReceiver = stats.scope("representation_management") - - // Strato client, default timeout = 280ms - val stratoClient: StratoClient = - Strato.client - .withMutualTls(serviceIdentifier) - .build() - - // Builds ThriftMux client builder for Content-Recommender service - private def makeThriftClientBuilder( - requestTimeout: Duration - ): ThriftMux.Client = { - ThriftMux.client - .withClientId(clientId) - .withMutualTls(serviceIdentifier) - .withRequestTimeout(requestTimeout) - .withStatsReceiver(statsReceiver.scope("clnt")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } - } - - private def makeThriftClient[ThriftServiceType: ClassTag]( - dest: String, - label: String, - requestTimeout: Duration = 450.milliseconds - ): ThriftServiceType = { - makeThriftClientBuilder(requestTimeout) - .build[ThriftServiceType](dest, label) - } - - /** *** SimCluster Embedding Stores ******/ - implicit val simClustersEmbeddingIdInjection: Injection[SimClustersEmbeddingId, Array[Byte]] = - BinaryScalaCodec(SimClustersEmbeddingId) - implicit val simClustersEmbeddingInjection: Injection[ThriftSimClustersEmbedding, Array[Byte]] = - BinaryScalaCodec(ThriftSimClustersEmbedding) - implicit val simClustersMultiEmbeddingInjection: Injection[SimClustersMultiEmbedding, Array[ - Byte - ]] = - BinaryScalaCodec(SimClustersMultiEmbedding) - implicit val simClustersMultiEmbeddingIdInjection: Injection[SimClustersMultiEmbeddingId, Array[ - Byte - ]] = - BinaryScalaCodec(SimClustersMultiEmbeddingId) - - def getEmbeddingsDataset( - mhMtlsParams: ManhattanKVClientMtlsParams, - datasetName: String - ): ReadableStore[SimClustersEmbeddingId, ThriftSimClustersEmbedding] = { - ManhattanRO.getReadableStoreWithMtls[SimClustersEmbeddingId, ThriftSimClustersEmbedding]( - ManhattanROConfig( - HDFSPath(""), // not needed - ApplicationID("content_recommender_athena"), - DatasetName(datasetName), // this should be correct - Athena - ), - mhMtlsParams - ) - } - - lazy val logFavBasedLongestL2Tweet20M145K2020EmbeddingStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val rawStore = - PersistentTweetEmbeddingStore - .longestL2NormTweetEmbeddingStoreManhattan( - mhMtlsParams, - PersistentTweetEmbeddingStore.LogFavBased20m145k2020Dataset, - statsReceiver, - maxLength = 10, - ).mapValues(_.toThrift) - - val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = rawStore, - cacheClient = cacheClient, - ttl = 15.minutes - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = - statsReceiver.scope("log_fav_based_longest_l2_tweet_embedding_20m145k2020_mem_cache"), - keyToString = { k => - s"scez_l2:${LogFavBasedTweet}_${ModelVersions.Model20M145K2020}_$k" - } - ) - - val inMemoryCacheStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = - memcachedStore - .composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId( - LogFavLongestL2EmbeddingTweet, - Model20m145k2020, - InternalId.TweetId(tweetId)) => - tweetId - } - .mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - inMemoryCacheStore, - ttl = 12.minute, - maxKeys = 1048575, - cacheName = "log_fav_based_longest_l2_tweet_embedding_20m145k2020_cache", - windowSize = 10000L - )(statsReceiver.scope("log_fav_based_longest_l2_tweet_embedding_20m145k2020_store")) - } - - lazy val logFavBased20M145KUpdatedTweetEmbeddingStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val rawStore = - PersistentTweetEmbeddingStore - .mostRecentTweetEmbeddingStoreManhattan( - mhMtlsParams, - PersistentTweetEmbeddingStore.LogFavBased20m145kUpdatedDataset, - statsReceiver - ).mapValues(_.toThrift) - - val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = rawStore, - cacheClient = cacheClient, - ttl = 10.minutes - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("log_fav_based_tweet_embedding_mem_cache"), - keyToString = { k => - // SimClusters_embedding_LZ4/embeddingType_modelVersion_tweetId - s"scez:${LogFavBasedTweet}_${ModelVersions.Model20M145KUpdated}_$k" - } - ) - - val inMemoryCacheStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - memcachedStore - .composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId( - LogFavBasedTweet, - Model20m145kUpdated, - InternalId.TweetId(tweetId)) => - tweetId - } - .mapValues(SimClustersEmbedding(_)) - } - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - inMemoryCacheStore, - ttl = 5.minute, - maxKeys = 1048575, // 200MB - cacheName = "log_fav_based_tweet_embedding_cache", - windowSize = 10000L - )(statsReceiver.scope("log_fav_based_tweet_embedding_store")) - } - - lazy val logFavBased20M145K2020TweetEmbeddingStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val rawStore = - PersistentTweetEmbeddingStore - .mostRecentTweetEmbeddingStoreManhattan( - mhMtlsParams, - PersistentTweetEmbeddingStore.LogFavBased20m145k2020Dataset, - statsReceiver, - maxLength = 10, - ).mapValues(_.toThrift) - - val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = rawStore, - cacheClient = cacheClient, - ttl = 15.minutes - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("log_fav_based_tweet_embedding_20m145k2020_mem_cache"), - keyToString = { k => - // SimClusters_embedding_LZ4/embeddingType_modelVersion_tweetId - s"scez:${LogFavBasedTweet}_${ModelVersions.Model20M145K2020}_$k" - } - ) - - val inMemoryCacheStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = - memcachedStore - .composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId( - LogFavBasedTweet, - Model20m145k2020, - InternalId.TweetId(tweetId)) => - tweetId - } - .mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - inMemoryCacheStore, - ttl = 12.minute, - maxKeys = 16777215, - cacheName = "log_fav_based_tweet_embedding_20m145k2020_cache", - windowSize = 10000L - )(statsReceiver.scope("log_fav_based_tweet_embedding_20m145k2020_store")) - } - - lazy val favBasedTfgTopicEmbedding2020Store: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val stratoStore = - StratoFetchableStore - .withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding]( - stratoClient, - "recommendations/simclusters_v2/embeddings/favBasedTFGTopic20M145K2020") - - val truncatedStore = stratoStore.mapValues { embedding => - SimClustersEmbedding(embedding, truncate = 50) - } - - ObservedCachedReadableStore.from( - ObservedReadableStore(truncatedStore)( - statsReceiver.scope("fav_tfg_topic_embedding_2020_cache_backing_store")), - ttl = 12.hours, - maxKeys = 262143, // 200MB - cacheName = "fav_tfg_topic_embedding_2020_cache", - windowSize = 10000L - )(statsReceiver.scope("fav_tfg_topic_embedding_2020_cache")) - } - - lazy val logFavBasedApe20M145K2020EmbeddingStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - ObservedReadableStore( - StratoFetchableStore - .withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding]( - stratoClient, - "recommendations/simclusters_v2/embeddings/logFavBasedAPE20M145K2020") - .composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId( - AggregatableLogFavBasedProducer, - Model20m145k2020, - internalId) => - SimClustersEmbeddingId(AggregatableLogFavBasedProducer, Model20m145k2020, internalId) - } - .mapValues(embedding => SimClustersEmbedding(embedding, 50)) - )(statsReceiver.scope("aggregatable_producer_embeddings_by_logfav_score_2020")) - } - - val interestService: InterestsThriftService.MethodPerEndpoint = - makeThriftClient[InterestsThriftService.MethodPerEndpoint]( - "/s/interests-thrift-service/interests-thrift-service", - "interests_thrift_service" - ) - - val interestsOptOutStore: InterestsOptOutStore = InterestsOptOutStore(interestService) - - // Save 2 ^ 18 UTTs. Promising 100% cache rate - lazy val defaultCacheConfigV2: CacheConfigV2 = CacheConfigV2(262143) - lazy val uttClientCacheConfigsV2: UttClientCacheConfigsV2 = UttClientCacheConfigsV2( - getTaxonomyConfig = defaultCacheConfigV2, - getUttTaxonomyConfig = defaultCacheConfigV2, - getLeafIds = defaultCacheConfigV2, - getLeafUttEntities = defaultCacheConfigV2 - ) - - // CachedUttClient to use StratoClient - lazy val cachedUttClientV2: CachedUttClientV2 = new CachedUttClientV2( - stratoClient = stratoClient, - env = Environment.Prod, - cacheConfigs = uttClientCacheConfigsV2, - statsReceiver = statsReceiver.scope("cached_utt_client") - ) - - lazy val semanticCoreTopicSeedStore: ReadableStore[ - SemanticCoreTopicSeedStore.Key, - Seq[UserId] - ] = { - /* - Up to 1000 Long seeds per topic/language = 62.5kb per topic/language (worst case) - Assume ~10k active topic/languages ~= 650MB (worst case) - */ - val underlying = new SemanticCoreTopicSeedStore(cachedUttClientV2, interestsOptOutStore)( - statsReceiver.scope("semantic_core_topic_seed_store")) - - val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient( - backingStore = underlying, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = SeqLongInjection, - statsReceiver = statsReceiver.scope("topic_producer_seed_store_mem_cache"), - keyToString = { k => s"tpss:${k.entityId}_${k.languageCode}" } - ) - - ObservedCachedReadableStore.from[SemanticCoreTopicSeedStore.Key, Seq[UserId]]( - store = memcacheStore, - ttl = 6.hours, - maxKeys = 20e3.toInt, - cacheName = "topic_producer_seed_store_cache", - windowSize = 5000 - )(statsReceiver.scope("topic_producer_seed_store_cache")) - } - - lazy val logFavBasedApeEntity20M145K2020EmbeddingStore: ApeEntityEmbeddingStore = { - val apeStore = logFavBasedApe20M145K2020EmbeddingStore.composeKeyMapping[UserId]({ id => - SimClustersEmbeddingId( - AggregatableLogFavBasedProducer, - Model20m145k2020, - InternalId.UserId(id)) - }) - - new ApeEntityEmbeddingStore( - semanticCoreSeedStore = semanticCoreTopicSeedStore, - aggregatableProducerEmbeddingStore = apeStore, - statsReceiver = statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_store")) - } - - lazy val logFavBasedApeEntity20M145K2020EmbeddingCachedStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val truncatedStore = - logFavBasedApeEntity20M145K2020EmbeddingStore.mapValues(_.truncate(50).toThrift) - - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = truncatedStore, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_mem_cache"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - val inMemoryCachedStore = - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 6.hours, - maxKeys = 262143, - cacheName = "log_fav_based_ape_entity_2020_embedding_cache", - windowSize = 10000L - )(statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_cached_store")) - - DeciderableReadableStore( - inMemoryCachedStore, - rmsDecider.deciderGateBuilder.idGateWithHashing[SimClustersEmbeddingId]( - DeciderKey.enableLogFavBasedApeEntity20M145K2020EmbeddingCachedStore), - statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_deciderable_store") - ) - } - - lazy val relaxedLogFavBasedApe20M145K2020EmbeddingStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - ObservedReadableStore( - StratoFetchableStore - .withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding]( - stratoClient, - "recommendations/simclusters_v2/embeddings/logFavBasedAPERelaxedFavEngagementThreshold20M145K2020") - .composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId( - RelaxedAggregatableLogFavBasedProducer, - Model20m145k2020, - internalId) => - SimClustersEmbeddingId( - RelaxedAggregatableLogFavBasedProducer, - Model20m145k2020, - internalId) - } - .mapValues(embedding => SimClustersEmbedding(embedding).truncate(50)) - )(statsReceiver.scope( - "aggregatable_producer_embeddings_by_logfav_score_relaxed_fav_engagement_threshold_2020")) - } - - lazy val relaxedLogFavBasedApe20M145K2020EmbeddingCachedStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val truncatedStore = - relaxedLogFavBasedApe20M145K2020EmbeddingStore.mapValues(_.truncate(50).toThrift) - - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = truncatedStore, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = - statsReceiver.scope("relaxed_log_fav_based_ape_entity_2020_embedding_mem_cache"), - keyToString = { k: SimClustersEmbeddingId => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 6.hours, - maxKeys = 262143, - cacheName = "relaxed_log_fav_based_ape_entity_2020_embedding_cache", - windowSize = 10000L - )(statsReceiver.scope("relaxed_log_fav_based_ape_entity_2020_embedding_cache_store")) - } - - lazy val favBasedProducer20M145K2020EmbeddingStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val underlyingStore = ProducerClusterEmbeddingReadableStores - .getProducerTopKSimClusters2020EmbeddingsStore( - mhMtlsParams - ).composeKeyMapping[SimClustersEmbeddingId] { - case SimClustersEmbeddingId( - FavBasedProducer, - Model20m145k2020, - InternalId.UserId(userId)) => - userId - }.mapValues { topSimClustersWithScore => - ThriftSimClustersEmbedding(topSimClustersWithScore.topClusters.take(10)) - } - - // same memcache config as for favBasedUserInterestedIn20M145K2020Store - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = cacheClient, - ttl = 24.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("fav_based_producer_embedding_20M_145K_2020_mem_cache"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 12.hours, - maxKeys = 16777215, - cacheName = "fav_based_producer_embedding_20M_145K_2020_embedding_cache", - windowSize = 10000L - )(statsReceiver.scope("fav_based_producer_embedding_20M_145K_2020_embedding_store")) - } - - // Production - lazy val interestedIn20M145KUpdatedStore: ReadableStore[UserId, ClustersUserIsInterestedIn] = { - UserInterestedInReadableStore.defaultStoreWithMtls( - mhMtlsParams, - modelVersion = ModelVersions.Model20M145KUpdated - ) - } - - // Production - lazy val interestedIn20M145K2020Store: ReadableStore[UserId, ClustersUserIsInterestedIn] = { - UserInterestedInReadableStore.defaultStoreWithMtls( - mhMtlsParams, - modelVersion = ModelVersions.Model20M145K2020 - ) - } - - // Production - lazy val InterestedInFromPE20M145KUpdatedStore: ReadableStore[ - UserId, - ClustersUserIsInterestedIn - ] = { - UserInterestedInReadableStore.defaultIIPEStoreWithMtls( - mhMtlsParams, - modelVersion = ModelVersions.Model20M145KUpdated) - } - - lazy val simClustersInterestedInStore: ReadableStore[ - (UserId, ModelVersion), - ClustersUserIsInterestedIn - ] = { - new ReadableStore[(UserId, ModelVersion), ClustersUserIsInterestedIn] { - override def get(k: (UserId, ModelVersion)): Future[Option[ClustersUserIsInterestedIn]] = { - k match { - case (userId, Model20m145kUpdated) => - interestedIn20M145KUpdatedStore.get(userId) - case (userId, Model20m145k2020) => - interestedIn20M145K2020Store.get(userId) - case _ => - Future.None - } - } - } - } - - lazy val simClustersInterestedInFromProducerEmbeddingsStore: ReadableStore[ - (UserId, ModelVersion), - ClustersUserIsInterestedIn - ] = { - new ReadableStore[(UserId, ModelVersion), ClustersUserIsInterestedIn] { - override def get(k: (UserId, ModelVersion)): Future[Option[ClustersUserIsInterestedIn]] = { - k match { - case (userId, ModelVersion.Model20m145kUpdated) => - InterestedInFromPE20M145KUpdatedStore.get(userId) - case _ => - Future.None - } - } - } - } - - lazy val userInterestedInStore = - new twistly.interestedin.EmbeddingStore( - interestedInStore = simClustersInterestedInStore, - interestedInFromProducerEmbeddingStore = simClustersInterestedInFromProducerEmbeddingsStore, - statsReceiver = statsReceiver - ) - - // Production - lazy val favBasedUserInterestedIn20M145KUpdatedStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val underlyingStore = - UserInterestedInReadableStore - .defaultSimClustersEmbeddingStoreWithMtls( - mhMtlsParams, - EmbeddingType.FavBasedUserInterestedIn, - ModelVersion.Model20m145kUpdated) - .mapValues(_.toThrift) - - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("fav_based_user_interested_in_mem_cache"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 6.hours, - maxKeys = 262143, - cacheName = "fav_based_user_interested_in_cache", - windowSize = 10000L - )(statsReceiver.scope("fav_based_user_interested_in_store")) - } - - // Production - lazy val LogFavBasedInterestedInFromAPE20M145K2020Store: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val underlyingStore = - UserInterestedInReadableStore - .defaultIIAPESimClustersEmbeddingStoreWithMtls( - mhMtlsParams, - EmbeddingType.LogFavBasedUserInterestedInFromAPE, - ModelVersion.Model20m145k2020) - .mapValues(_.toThrift) - - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("log_fav_based_user_interested_in_from_ape_mem_cache"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 6.hours, - maxKeys = 262143, - cacheName = "log_fav_based_user_interested_in_from_ape_cache", - windowSize = 10000L - )(statsReceiver.scope("log_fav_based_user_interested_in_from_ape_store")) - } - - // Production - lazy val FollowBasedInterestedInFromAPE20M145K2020Store: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val underlyingStore = - UserInterestedInReadableStore - .defaultIIAPESimClustersEmbeddingStoreWithMtls( - mhMtlsParams, - EmbeddingType.FollowBasedUserInterestedInFromAPE, - ModelVersion.Model20m145k2020) - .mapValues(_.toThrift) - - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("follow_based_user_interested_in_from_ape_mem_cache"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 6.hours, - maxKeys = 262143, - cacheName = "follow_based_user_interested_in_from_ape_cache", - windowSize = 10000L - )(statsReceiver.scope("follow_based_user_interested_in_from_ape_store")) - } - - // production - lazy val favBasedUserInterestedIn20M145K2020Store: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val underlyingStore: ReadableStore[SimClustersEmbeddingId, ThriftSimClustersEmbedding] = - UserInterestedInReadableStore - .defaultSimClustersEmbeddingStoreWithMtls( - mhMtlsParams, - EmbeddingType.FavBasedUserInterestedIn, - ModelVersion.Model20m145k2020).mapValues(_.toThrift) - - ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("fav_based_user_interested_in_2020_mem_cache"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - } - - // Production - lazy val logFavBasedUserInterestedIn20M145K2020Store: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val underlyingStore = - UserInterestedInReadableStore - .defaultSimClustersEmbeddingStoreWithMtls( - mhMtlsParams, - EmbeddingType.LogFavBasedUserInterestedIn, - ModelVersion.Model20m145k2020) - - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore.mapValues(_.toThrift), - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("log_fav_based_user_interested_in_2020_store"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 6.hours, - maxKeys = 262143, - cacheName = "log_fav_based_user_interested_in_2020_cache", - windowSize = 10000L - )(statsReceiver.scope("log_fav_based_user_interested_in_2020_store")) - } - - // Production - lazy val favBasedUserInterestedInFromPE20M145KUpdatedStore: ReadableStore[ - SimClustersEmbeddingId, - SimClustersEmbedding - ] = { - val underlyingStore = - UserInterestedInReadableStore - .defaultIIPESimClustersEmbeddingStoreWithMtls( - mhMtlsParams, - EmbeddingType.FavBasedUserInterestedInFromPE, - ModelVersion.Model20m145kUpdated) - .mapValues(_.toThrift) - - val memcachedStore = ObservedMemcachedReadableStore - .fromCacheClient( - backingStore = underlyingStore, - cacheClient = cacheClient, - ttl = 12.hours - )( - valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)), - statsReceiver = statsReceiver.scope("fav_based_user_interested_in_from_pe_mem_cache"), - keyToString = { k => embeddingCacheKeyBuilder.apply(k) } - ).mapValues(SimClustersEmbedding(_)) - - ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding]( - memcachedStore, - ttl = 6.hours, - maxKeys = 262143, - cacheName = "fav_based_user_interested_in_from_pe_cache", - windowSize = 10000L - )(statsReceiver.scope("fav_based_user_interested_in_from_pe_cache")) - } - - private val underlyingStores: Map[ - (EmbeddingType, ModelVersion), - ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] - ] = Map( - // Tweet Embeddings - (LogFavBasedTweet, Model20m145kUpdated) -> logFavBased20M145KUpdatedTweetEmbeddingStore, - (LogFavBasedTweet, Model20m145k2020) -> logFavBased20M145K2020TweetEmbeddingStore, - ( - LogFavLongestL2EmbeddingTweet, - Model20m145k2020) -> logFavBasedLongestL2Tweet20M145K2020EmbeddingStore, - // Entity Embeddings - (FavTfgTopic, Model20m145k2020) -> favBasedTfgTopicEmbedding2020Store, - ( - LogFavBasedKgoApeTopic, - Model20m145k2020) -> logFavBasedApeEntity20M145K2020EmbeddingCachedStore, - // KnownFor Embeddings - (FavBasedProducer, Model20m145k2020) -> favBasedProducer20M145K2020EmbeddingStore, - ( - RelaxedAggregatableLogFavBasedProducer, - Model20m145k2020) -> relaxedLogFavBasedApe20M145K2020EmbeddingCachedStore, - // InterestedIn Embeddings - ( - LogFavBasedUserInterestedInFromAPE, - Model20m145k2020) -> LogFavBasedInterestedInFromAPE20M145K2020Store, - ( - FollowBasedUserInterestedInFromAPE, - Model20m145k2020) -> FollowBasedInterestedInFromAPE20M145K2020Store, - (FavBasedUserInterestedIn, Model20m145kUpdated) -> favBasedUserInterestedIn20M145KUpdatedStore, - (FavBasedUserInterestedIn, Model20m145k2020) -> favBasedUserInterestedIn20M145K2020Store, - (LogFavBasedUserInterestedIn, Model20m145k2020) -> logFavBasedUserInterestedIn20M145K2020Store, - ( - FavBasedUserInterestedInFromPE, - Model20m145kUpdated) -> favBasedUserInterestedInFromPE20M145KUpdatedStore, - (FilteredUserInterestedIn, Model20m145kUpdated) -> userInterestedInStore, - (FilteredUserInterestedIn, Model20m145k2020) -> userInterestedInStore, - (FilteredUserInterestedInFromPE, Model20m145kUpdated) -> userInterestedInStore, - (UnfilteredUserInterestedIn, Model20m145kUpdated) -> userInterestedInStore, - (UnfilteredUserInterestedIn, Model20m145k2020) -> userInterestedInStore, - ) - - val simClustersEmbeddingStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = { - val underlying: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = - SimClustersEmbeddingStore.buildWithDecider( - underlyingStores = underlyingStores, - decider = rmsDecider.decider, - statsReceiver = statsReceiver.scope("simClusters_embeddings_store_deciderable") - ) - - val underlyingWithTimeout: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = - new ReadableStoreWithTimeout( - rs = underlying, - decider = rmsDecider.decider, - enableTimeoutDeciderKey = DeciderConstants.enableSimClustersEmbeddingStoreTimeouts, - timeoutValueKey = DeciderConstants.simClustersEmbeddingStoreTimeoutValueMillis, - timer = timer, - statsReceiver = statsReceiver.scope("simClusters_embedding_store_timeouts") - ) - - ObservedReadableStore( - store = underlyingWithTimeout - )(statsReceiver.scope("simClusters_embeddings_store")) - } -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/BUILD deleted file mode 100644 index ab19a1dd7..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/BUILD +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication", - "finagle/finagle-stats", - "finatra/inject/inject-core/src/main/scala", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", - "interests-service/thrift/src/main/thrift:thrift-scala", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/common", - "servo/util", - "src/scala/com/twitter/storehaus_internal/manhattan", - "src/scala/com/twitter/storehaus_internal/memcache", - "src/scala/com/twitter/storehaus_internal/util", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/BUILD.docx new file mode 100644 index 000000000..6d0806dee Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/BUILD.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/CacheModule.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/CacheModule.docx new file mode 100644 index 000000000..9251fa53b Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/CacheModule.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/CacheModule.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/CacheModule.scala deleted file mode 100644 index a042225fa..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/CacheModule.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.representation_manager.modules - -import com.google.inject.Provides -import com.twitter.finagle.memcached.Client -import javax.inject.Singleton -import com.twitter.conversions.DurationOps._ -import com.twitter.inject.TwitterModule -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.storehaus_internal.memcache.MemcacheStore -import com.twitter.storehaus_internal.util.ClientName -import com.twitter.storehaus_internal.util.ZkEndPoint - -object CacheModule extends TwitterModule { - - private val cacheDest = flag[String]("cache_module.dest", "Path to memcache service") - private val timeout = flag[Int]("memcache.timeout", "Memcache client timeout") - private val retries = flag[Int]("memcache.retries", "Memcache timeout retries") - - @Singleton - @Provides - def providesCache( - serviceIdentifier: ServiceIdentifier, - stats: StatsReceiver - ): Client = - MemcacheStore.memcachedClient( - name = ClientName("memcache_representation_manager"), - dest = ZkEndPoint(cacheDest()), - timeout = timeout().milliseconds, - retries = retries(), - statsReceiver = stats.scope("cache_client"), - serviceIdentifier = serviceIdentifier - ) -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/InterestsThriftClientModule.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/InterestsThriftClientModule.docx new file mode 100644 index 000000000..3cfea5f25 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/InterestsThriftClientModule.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/InterestsThriftClientModule.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/InterestsThriftClientModule.scala deleted file mode 100644 index 82a5a5004..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/InterestsThriftClientModule.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.representation_manager.modules - -import com.google.inject.Provides -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.ThriftMux -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax -import com.twitter.finagle.mux.ClientDiscardedRequestException -import com.twitter.finagle.service.ReqRep -import com.twitter.finagle.service.ResponseClass -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.thrift.ClientId -import com.twitter.inject.TwitterModule -import com.twitter.interests.thriftscala.InterestsThriftService -import com.twitter.util.Throw -import javax.inject.Singleton - -object InterestsThriftClientModule extends TwitterModule { - - @Singleton - @Provides - def providesInterestsThriftClient( - clientId: ClientId, - serviceIdentifier: ServiceIdentifier, - statsReceiver: StatsReceiver - ): InterestsThriftService.MethodPerEndpoint = { - ThriftMux.client - .withClientId(clientId) - .withMutualTls(serviceIdentifier) - .withRequestTimeout(450.milliseconds) - .withStatsReceiver(statsReceiver.scope("InterestsThriftClient")) - .withResponseClassifier { - case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable - } - .build[InterestsThriftService.MethodPerEndpoint]( - dest = "/s/interests-thrift-service/interests-thrift-service", - label = "interests_thrift_service" - ) - } -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/LegacyRMSConfigModule.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/LegacyRMSConfigModule.docx new file mode 100644 index 000000000..1dd2d5e78 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/LegacyRMSConfigModule.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/LegacyRMSConfigModule.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/LegacyRMSConfigModule.scala deleted file mode 100644 index 0a06dffe6..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/LegacyRMSConfigModule.scala +++ /dev/null @@ -1,18 +0,0 @@ -package com.twitter.representation_manager.modules - -import com.google.inject.Provides -import com.twitter.inject.TwitterModule -import javax.inject.Named -import javax.inject.Singleton - -object LegacyRMSConfigModule extends TwitterModule { - @Singleton - @Provides - @Named("cacheHashKeyPrefix") - def providesCacheHashKeyPrefix: String = "RMS" - - @Singleton - @Provides - @Named("useContentRecommenderConfiguration") - def providesUseContentRecommenderConfiguration: Boolean = false -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/StoreModule.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/StoreModule.docx new file mode 100644 index 000000000..3b92bc6ce Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/StoreModule.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/StoreModule.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/StoreModule.scala deleted file mode 100644 index a2efe5925..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/StoreModule.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.representation_manager.modules - -import com.google.inject.Provides -import javax.inject.Singleton -import com.twitter.inject.TwitterModule -import com.twitter.decider.Decider -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.representation_manager.common.RepresentationManagerDecider -import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams - -object StoreModule extends TwitterModule { - @Singleton - @Provides - def providesMhMtlsParams( - serviceIdentifier: ServiceIdentifier - ): ManhattanKVClientMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier) - - @Singleton - @Provides - def providesRmsDecider( - decider: Decider - ): RepresentationManagerDecider = RepresentationManagerDecider(decider) - -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/TimerModule.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/TimerModule.docx new file mode 100644 index 000000000..7f41da40b Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/TimerModule.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/TimerModule.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/TimerModule.scala deleted file mode 100644 index fe7fddb45..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/TimerModule.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.representation_manager.modules - -import com.google.inject.Provides -import com.twitter.finagle.util.DefaultTimer -import com.twitter.inject.TwitterModule -import com.twitter.util.Timer -import javax.inject.Singleton - -object TimerModule extends TwitterModule { - @Singleton - @Provides - def providesTimer: Timer = DefaultTimer -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/UttClientModule.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/UttClientModule.docx new file mode 100644 index 000000000..ff9e3af11 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/UttClientModule.docx differ diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/UttClientModule.scala b/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/UttClientModule.scala deleted file mode 100644 index cc2100c1c..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/modules/UttClientModule.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.representation_manager.modules - -import com.google.inject.Provides -import com.twitter.escherbird.util.uttclient.CacheConfigV2 -import com.twitter.escherbird.util.uttclient.CachedUttClientV2 -import com.twitter.escherbird.util.uttclient.UttClientCacheConfigsV2 -import com.twitter.escherbird.utt.strato.thriftscala.Environment -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.inject.TwitterModule -import com.twitter.strato.client.{Client => StratoClient} -import javax.inject.Singleton - -object UttClientModule extends TwitterModule { - - @Singleton - @Provides - def providesUttClient( - stratoClient: StratoClient, - statsReceiver: StatsReceiver - ): CachedUttClientV2 = { - // Save 2 ^ 18 UTTs. Promising 100% cache rate - val defaultCacheConfigV2: CacheConfigV2 = CacheConfigV2(262143) - - val uttClientCacheConfigsV2: UttClientCacheConfigsV2 = UttClientCacheConfigsV2( - getTaxonomyConfig = defaultCacheConfigV2, - getUttTaxonomyConfig = defaultCacheConfigV2, - getLeafIds = defaultCacheConfigV2, - getLeafUttEntities = defaultCacheConfigV2 - ) - - // CachedUttClient to use StratoClient - new CachedUttClientV2( - stratoClient = stratoClient, - env = Environment.Prod, - cacheConfigs = uttClientCacheConfigsV2, - statsReceiver = statsReceiver.scope("cached_utt_client") - ) - } -} diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/store/BUILD b/representation-manager/server/src/main/scala/com/twitter/representation_manager/store/BUILD deleted file mode 100644 index 1731a2649..000000000 --- a/representation-manager/server/src/main/scala/com/twitter/representation_manager/store/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - "content-recommender/server/src/main/scala/com/twitter/contentrecommender:representation-manager-deps", - "frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util", - "hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common", - "representation-manager/server/src/main/scala/com/twitter/representation_manager/common", - "src/scala/com/twitter/simclusters_v2/stores", - "src/scala/com/twitter/simclusters_v2/summingbird/stores", - "src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala", - "storage/clients/manhattan/client/src/main/scala", - "tweetypie/src/scala/com/twitter/tweetypie/util", - ], -) diff --git a/representation-manager/server/src/main/scala/com/twitter/representation_manager/store/BUILD.docx b/representation-manager/server/src/main/scala/com/twitter/representation_manager/store/BUILD.docx new file mode 100644 index 000000000..347744b40 Binary files /dev/null and b/representation-manager/server/src/main/scala/com/twitter/representation_manager/store/BUILD.docx differ