mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-11-16 00:25:11 +01:00
[docx] split commit for file 3600
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
9ebf058832
commit
ce29360463
Binary file not shown.
@ -1,87 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.predicate.FatiguePredicate
|
||||
import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter
|
||||
import com.twitter.frigate.pushservice.predicate.{
|
||||
TargetNtabCaretClickFatiguePredicate => CommonNtabCaretClickFatiguePredicate
|
||||
}
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.params.PushParams
|
||||
import com.twitter.frigate.thriftscala.NotificationDisplayLocation
|
||||
import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT}
|
||||
import com.twitter.hermit.predicate.NamedPredicate
|
||||
import com.twitter.hermit.predicate.Predicate
|
||||
import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
|
||||
object RecTypeNtabCaretClickFatiguePredicate {
|
||||
val defaultName = "RecTypeNtabCaretClickFatiguePredicateForCandidate"
|
||||
|
||||
private def candidateFatiguePredicate(
|
||||
genericTypeCategories: Seq[String],
|
||||
crts: Set[CRT]
|
||||
)(
|
||||
implicit stats: StatsReceiver
|
||||
): NamedPredicate[
|
||||
PushCandidate
|
||||
] = {
|
||||
val name = "f1TriggeredCRTBasedFatiguePredciate"
|
||||
val scopedStats = stats.scope(s"predicate_$name")
|
||||
Predicate
|
||||
.fromAsync { candidate: PushCandidate =>
|
||||
if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) {
|
||||
if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) {
|
||||
NtabCaretClickContFnFatiguePredicate
|
||||
.ntabCaretClickContFnFatiguePredicates(
|
||||
filterHistory = FatiguePredicate.recTypesOnlyFilter(crts),
|
||||
filterCaretFeedbackHistory =
|
||||
CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories),
|
||||
filterInlineFeedbackHistory =
|
||||
NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(crts)
|
||||
).apply(Seq(candidate))
|
||||
.map(_.headOption.getOrElse(false))
|
||||
} else Future.True
|
||||
} else {
|
||||
Future.True
|
||||
}
|
||||
}.withStats(scopedStats)
|
||||
.withName(name)
|
||||
}
|
||||
|
||||
def apply(
|
||||
genericTypeCategories: Seq[String],
|
||||
crts: Set[CRT],
|
||||
calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration,
|
||||
useMostRecentDislikeTime: Boolean,
|
||||
name: String = defaultName
|
||||
)(
|
||||
implicit globalStats: StatsReceiver
|
||||
): NamedPredicate[PushCandidate] = {
|
||||
val scopedStats = globalStats.scope(name)
|
||||
val commonNtabCaretClickFatiguePredicate = CommonNtabCaretClickFatiguePredicate(
|
||||
filterCaretFeedbackHistory =
|
||||
CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories),
|
||||
filterHistory = FatiguePredicate.recTypesOnlyFilter(crts),
|
||||
calculateFatiguePeriod = calculateFatiguePeriod,
|
||||
useMostRecentDislikeTime = useMostRecentDislikeTime,
|
||||
name = name
|
||||
)(globalStats)
|
||||
|
||||
Predicate
|
||||
.fromAsync { candidate: PushCandidate =>
|
||||
if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) {
|
||||
if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) {
|
||||
commonNtabCaretClickFatiguePredicate
|
||||
.apply(Seq(candidate.target))
|
||||
.map(_.headOption.getOrElse(false))
|
||||
} else Future.True
|
||||
} else {
|
||||
Future.True
|
||||
}
|
||||
}.andThen(candidateFatiguePredicate(genericTypeCategories, crts))
|
||||
.withStats(scopedStats)
|
||||
.withName(name)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,44 +0,0 @@
|
||||
package com.twitter.frigate.pushservice
|
||||
|
||||
import com.twitter.frigate.common.base.Candidate
|
||||
import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap
|
||||
import com.twitter.frigate.common.base.TweetAuthor
|
||||
import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.hermit.predicate.Predicate
|
||||
|
||||
package object predicate {
|
||||
implicit class CandidatesWithAuthorFollowPredicates(
|
||||
predicate: Predicate[
|
||||
PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
||||
]) {
|
||||
def applyOnlyToAuthorBeingFollowPredicates: Predicate[Candidate] =
|
||||
predicate.optionalOn[Candidate](
|
||||
{
|
||||
case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
||||
if isInNetworkTweetType(candidate.commonRecType) =>
|
||||
Some(candidate)
|
||||
case _ =>
|
||||
None
|
||||
},
|
||||
missingResult = true
|
||||
)
|
||||
}
|
||||
|
||||
implicit class TweetCandidateWithTweetAuthor(
|
||||
predicate: Predicate[
|
||||
PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
||||
]) {
|
||||
def applyOnlyToBasicTweetPredicates: Predicate[Candidate] =
|
||||
predicate.optionalOn[Candidate](
|
||||
{
|
||||
case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
||||
if isInNetworkTweetType(candidate.commonRecType) =>
|
||||
Some(candidate)
|
||||
case _ =>
|
||||
None
|
||||
},
|
||||
missingResult = true
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,27 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.predicate.quality_model_predicate
|
||||
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
||||
import com.twitter.util.Future
|
||||
|
||||
object ExplicitOONCFilterPredicate extends QualityPredicateBase {
|
||||
override lazy val name = "open_or_ntab_click_explicit_threshold"
|
||||
|
||||
override lazy val thresholdExtractor = (t: Target) =>
|
||||
Future.value(t.params(PushFeatureSwitchParams.QualityPredicateExplicitThresholdParam))
|
||||
|
||||
override def scoreExtractor = (candidate: PushCandidate) =>
|
||||
candidate.mrWeightedOpenOrNtabClickRankingProbability
|
||||
}
|
||||
|
||||
object WeightedOpenOrNtabClickQualityPredicate extends QualityPredicateBase {
|
||||
override lazy val name = "weighted_open_or_ntab_click_model"
|
||||
|
||||
override lazy val thresholdExtractor = (t: Target) => {
|
||||
Future.value(0.0)
|
||||
}
|
||||
|
||||
override def scoreExtractor =
|
||||
(candidate: PushCandidate) => candidate.mrWeightedOpenOrNtabClickFilteringProbability
|
||||
}
|
||||
Binary file not shown.
@ -1,165 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.predicate.quality_model_predicate
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.pushservice.model.PushTypes
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.target.TargetScoringDetails
|
||||
import com.twitter.hermit.predicate.NamedPredicate
|
||||
import com.twitter.hermit.predicate.Predicate
|
||||
import com.twitter.util.Future
|
||||
|
||||
object PDauCohort extends Enumeration {
|
||||
type PDauCohort = Value
|
||||
|
||||
val cohort1 = Value
|
||||
val cohort2 = Value
|
||||
val cohort3 = Value
|
||||
val cohort4 = Value
|
||||
val cohort5 = Value
|
||||
val cohort6 = Value
|
||||
}
|
||||
|
||||
object PDauCohortUtil {
|
||||
|
||||
case class DauThreshold(
|
||||
threshold1: Double,
|
||||
threshold2: Double,
|
||||
threshold3: Double,
|
||||
threshold4: Double,
|
||||
threshold5: Double)
|
||||
|
||||
val defaultDAUProb = 0.0
|
||||
|
||||
val dauProbThresholds = DauThreshold(
|
||||
threshold1 = 0.05,
|
||||
threshold2 = 0.14,
|
||||
threshold3 = 0.33,
|
||||
threshold4 = 0.7,
|
||||
threshold5 = 0.959
|
||||
)
|
||||
|
||||
val finerThresholdMap =
|
||||
Map(
|
||||
PDauCohort.cohort2 -> List(0.05, 0.0539, 0.0563, 0.0600, 0.0681, 0.0733, 0.0800, 0.0849,
|
||||
0.0912, 0.0975, 0.1032, 0.1092, 0.1134, 0.1191, 0.1252, 0.1324, 0.14),
|
||||
PDauCohort.cohort3 -> List(0.14, 0.1489, 0.1544, 0.1625, 0.1704, 0.1797, 0.1905, 0.2001,
|
||||
0.2120, 0.2248, 0.2363, 0.2500, 0.2650, 0.2801, 0.2958, 0.3119, 0.33),
|
||||
PDauCohort.cohort4 -> List(0.33, 0.3484, 0.3686, 0.3893, 0.4126, 0.4350, 0.4603, 0.4856,
|
||||
0.5092, 0.5348, 0.5602, 0.5850, 0.6087, 0.6319, 0.6548, 0.6779, 0.7),
|
||||
PDauCohort.cohort5 -> List(0.7, 0.7295, 0.7581, 0.7831, 0.8049, 0.8251, 0.8444, 0.8612,
|
||||
0.8786, 0.8936, 0.9043, 0.9175, 0.9290, 0.9383, 0.9498, 0.9587, 0.959)
|
||||
)
|
||||
|
||||
def getBucket(targetUser: PushTypes.Target, doImpression: Boolean) = {
|
||||
implicit val stats = targetUser.stats.scope("PDauCohortUtil")
|
||||
if (doImpression) targetUser.getBucket _ else targetUser.getBucketWithoutImpression _
|
||||
}
|
||||
|
||||
def threshold1(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold1
|
||||
|
||||
def threshold2(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold2
|
||||
|
||||
def threshold3(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold3
|
||||
|
||||
def threshold4(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold4
|
||||
|
||||
def threshold5(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold5
|
||||
|
||||
def thresholdForCohort(targetUser: PushTypes.Target, dauCohort: Int): Double = {
|
||||
if (dauCohort == 0) 0.0
|
||||
else if (dauCohort == 1) threshold1(targetUser)
|
||||
else if (dauCohort == 2) threshold2(targetUser)
|
||||
else if (dauCohort == 3) threshold3(targetUser)
|
||||
else if (dauCohort == 4) threshold4(targetUser)
|
||||
else if (dauCohort == 5) threshold5(targetUser)
|
||||
else 1.0
|
||||
}
|
||||
|
||||
def getPDauCohort(dauProbability: Double, thresholds: DauThreshold): PDauCohort.Value = {
|
||||
dauProbability match {
|
||||
case dauProb if dauProb >= 0.0 && dauProb < thresholds.threshold1 => PDauCohort.cohort1
|
||||
case dauProb if dauProb >= thresholds.threshold1 && dauProb < thresholds.threshold2 =>
|
||||
PDauCohort.cohort2
|
||||
case dauProb if dauProb >= thresholds.threshold2 && dauProb < thresholds.threshold3 =>
|
||||
PDauCohort.cohort3
|
||||
case dauProb if dauProb >= thresholds.threshold3 && dauProb < thresholds.threshold4 =>
|
||||
PDauCohort.cohort4
|
||||
case dauProb if dauProb >= thresholds.threshold4 && dauProb < thresholds.threshold5 =>
|
||||
PDauCohort.cohort5
|
||||
case dauProb if dauProb >= thresholds.threshold5 && dauProb <= 1.0 => PDauCohort.cohort6
|
||||
}
|
||||
}
|
||||
|
||||
def getDauProb(target: TargetScoringDetails): Future[Double] = {
|
||||
target.dauProbability.map { dauProb =>
|
||||
dauProb.map(_.probability).getOrElse(defaultDAUProb)
|
||||
}
|
||||
}
|
||||
|
||||
def getPDauCohort(target: TargetScoringDetails): Future[PDauCohort.Value] = {
|
||||
getDauProb(target).map { getPDauCohort(_, dauProbThresholds) }
|
||||
}
|
||||
|
||||
def getPDauCohortWithPDau(target: TargetScoringDetails): Future[(PDauCohort.Value, Double)] = {
|
||||
getDauProb(target).map { prob =>
|
||||
(getPDauCohort(prob, dauProbThresholds), prob)
|
||||
}
|
||||
}
|
||||
|
||||
def updateStats(
|
||||
target: PushTypes.Target,
|
||||
modelName: String,
|
||||
predicateResult: Boolean
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver
|
||||
): Unit = {
|
||||
val dauCohortOp = getPDauCohort(target)
|
||||
dauCohortOp.map { dauCohort =>
|
||||
val cohortStats = statsReceiver.scope(modelName).scope(dauCohort.toString)
|
||||
cohortStats.counter(s"filter_$predicateResult").incr()
|
||||
}
|
||||
if (target.isNewSignup) {
|
||||
val newUserModelStats = statsReceiver.scope(modelName)
|
||||
newUserModelStats.counter(s"new_user_filter_$predicateResult").incr()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait QualityPredicateBase {
|
||||
def name: String
|
||||
def thresholdExtractor: Target => Future[Double]
|
||||
def scoreExtractor: PushCandidate => Future[Option[Double]]
|
||||
def isPredicateEnabled: PushCandidate => Future[Boolean] = _ => Future.True
|
||||
def comparator: (Double, Double) => Boolean =
|
||||
(score: Double, threshold: Double) => score >= threshold
|
||||
def updateCustomStats(
|
||||
candidate: PushCandidate,
|
||||
score: Double,
|
||||
threshold: Double,
|
||||
result: Boolean
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver
|
||||
): Unit = {}
|
||||
|
||||
def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = {
|
||||
Predicate
|
||||
.fromAsync { candidate: PushCandidate =>
|
||||
isPredicateEnabled(candidate).flatMap {
|
||||
case true =>
|
||||
scoreExtractor(candidate).flatMap { scoreOpt =>
|
||||
thresholdExtractor(candidate.target).map { threshold =>
|
||||
val score = scoreOpt.getOrElse(0.0)
|
||||
val result = comparator(score, threshold)
|
||||
PDauCohortUtil.updateStats(candidate.target, name, result)
|
||||
updateCustomStats(candidate, score, threshold, result)
|
||||
result
|
||||
}
|
||||
}
|
||||
case _ => Future.True
|
||||
}
|
||||
}
|
||||
.withStats(statsReceiver.scope(s"predicate_$name"))
|
||||
.withName(name)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,21 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.predicate.quality_model_predicate
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.params.QualityPredicateEnum
|
||||
import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate
|
||||
import com.twitter.hermit.predicate.NamedPredicate
|
||||
|
||||
object QualityPredicateMap {
|
||||
|
||||
def apply(
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver
|
||||
): Map[QualityPredicateEnum.Value, NamedPredicate[PushCandidate]] = {
|
||||
Map(
|
||||
QualityPredicateEnum.WeightedOpenOrNtabClick -> WeightedOpenOrNtabClickQualityPredicate(),
|
||||
QualityPredicateEnum.ExplicitOpenOrNtabClickFilter -> ExplicitOONCFilterPredicate(),
|
||||
QualityPredicateEnum.AlwaysTrue -> PredicatesForCandidate.alwaysTruePushCandidatePredicate,
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,54 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
|
||||
/**
|
||||
* This Ranker re-ranks MR candidates, boosting input CRTs.
|
||||
* Relative ranking between input CRTs and rest of the candidates doesn't change
|
||||
*
|
||||
* Ex: T: Tweet candidate, F: input CRT candidatess
|
||||
*
|
||||
* T3, F2, T1, T2, F1 => F2, F1, T3, T1, T2
|
||||
*/
|
||||
case class CRTBoostRanker(statsReceiver: StatsReceiver) {
|
||||
|
||||
private val recsToBoostStat = statsReceiver.stat("recs_to_boost")
|
||||
private val otherRecsStat = statsReceiver.stat("other_recs")
|
||||
|
||||
private def boostCrtToTop(
|
||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
crtToBoost: CommonRecommendationType
|
||||
): Seq[CandidateDetails[PushCandidate]] = {
|
||||
val (upRankedCandidates, otherCandidates) =
|
||||
inputCandidates.partition(_.candidate.commonRecType == crtToBoost)
|
||||
recsToBoostStat.add(upRankedCandidates.size)
|
||||
otherRecsStat.add(otherCandidates.size)
|
||||
upRankedCandidates ++ otherCandidates
|
||||
}
|
||||
|
||||
final def boostCrtsToTop(
|
||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
crtsToBoost: Seq[CommonRecommendationType]
|
||||
): Seq[CandidateDetails[PushCandidate]] = {
|
||||
crtsToBoost.headOption match {
|
||||
case Some(crt) =>
|
||||
val upRankedCandidates = boostCrtToTop(inputCandidates, crt)
|
||||
boostCrtsToTop(upRankedCandidates, crtsToBoost.tail)
|
||||
case None => inputCandidates
|
||||
}
|
||||
}
|
||||
|
||||
final def boostCrtsToTopStableOrder(
|
||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
crtsToBoost: Seq[CommonRecommendationType]
|
||||
): Seq[CandidateDetails[PushCandidate]] = {
|
||||
val crtsToBoostSet = crtsToBoost.toSet
|
||||
val (upRankedCandidates, otherCandidates) = inputCandidates.partition(candidateDetail =>
|
||||
crtsToBoostSet.contains(candidateDetail.candidate.commonRecType))
|
||||
|
||||
upRankedCandidates ++ otherCandidates
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,45 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
|
||||
/**
|
||||
* This Ranker re-ranks MR candidates, down ranks input CRTs.
|
||||
* Relative ranking between input CRTs and rest of the candidates doesn't change
|
||||
*
|
||||
* Ex: T: Tweet candidate, F: input CRT candidates
|
||||
*
|
||||
* T3, F2, T1, T2, F1 => T3, T1, T2, F2, F1
|
||||
*/
|
||||
case class CRTDownRanker(statsReceiver: StatsReceiver) {
|
||||
|
||||
private val recsToDownRankStat = statsReceiver.stat("recs_to_down_rank")
|
||||
private val otherRecsStat = statsReceiver.stat("other_recs")
|
||||
private val downRankerRequests = statsReceiver.counter("down_ranker_requests")
|
||||
|
||||
private def downRank(
|
||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
crtToDownRank: CommonRecommendationType
|
||||
): Seq[CandidateDetails[PushCandidate]] = {
|
||||
downRankerRequests.incr()
|
||||
val (downRankedCandidates, otherCandidates) =
|
||||
inputCandidates.partition(_.candidate.commonRecType == crtToDownRank)
|
||||
recsToDownRankStat.add(downRankedCandidates.size)
|
||||
otherRecsStat.add(otherCandidates.size)
|
||||
otherCandidates ++ downRankedCandidates
|
||||
}
|
||||
|
||||
final def downRank(
|
||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
crtsToDownRank: Seq[CommonRecommendationType]
|
||||
): Seq[CandidateDetails[PushCandidate]] = {
|
||||
crtsToDownRank.headOption match {
|
||||
case Some(crt) =>
|
||||
val downRankedCandidates = downRank(inputCandidates, crt)
|
||||
downRank(downRankedCandidates, crtsToDownRank.tail)
|
||||
case None => inputCandidates
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,45 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.TweetCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
class LoggedOutRanker(tweetyPieStore: ReadableStore[Long, TweetyPieResult], stats: StatsReceiver) {
|
||||
private val statsReceiver = stats.scope(this.getClass.getSimpleName)
|
||||
private val rankedCandidates = statsReceiver.counter("ranked_candidates_count")
|
||||
|
||||
def rank(
|
||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
val tweetIds = candidates.map { cand => cand.candidate.asInstanceOf[TweetCandidate].tweetId }
|
||||
val results = tweetyPieStore.multiGet(tweetIds.toSet).values.toSeq
|
||||
val futureOfResults = Future.traverseSequentially(results)(r => r)
|
||||
val tweetsFut = futureOfResults.map { tweetyPieResults =>
|
||||
tweetyPieResults.map(_.map(_.tweet))
|
||||
}
|
||||
val sortedTweetsFuture = tweetsFut.map { tweets =>
|
||||
tweets
|
||||
.map { tweet =>
|
||||
if (tweet.isDefined && tweet.get.counts.isDefined) {
|
||||
tweet.get.id -> tweet.get.counts.get.favoriteCount.getOrElse(0L)
|
||||
} else {
|
||||
0 -> 0L
|
||||
}
|
||||
}.sortBy(_._2)(Ordering[Long].reverse)
|
||||
}
|
||||
val finalCandidates = sortedTweetsFuture.map { sortedTweets =>
|
||||
sortedTweets
|
||||
.map { tweet =>
|
||||
candidates.find(_.candidate.asInstanceOf[TweetCandidate].tweetId == tweet._1).orNull
|
||||
}.filter { cand => cand != null }
|
||||
}
|
||||
finalCandidates.map { fc =>
|
||||
rankedCandidates.incr(fc.size)
|
||||
}
|
||||
finalCandidates
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,204 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.util.Future
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum
|
||||
import com.twitter.frigate.common.base.TweetCandidate
|
||||
import com.twitter.frigate.common.rec_types.RecTypes
|
||||
import com.twitter.frigate.pushservice.params.PushConstants.OoncQualityCombinedScore
|
||||
|
||||
object ModelBasedRanker {
|
||||
|
||||
def rankBySpecifiedScore(
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
scoreExtractor: PushCandidate => Future[Option[Double]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
val scoredCandidatesFutures = candidatesDetails.map { cand =>
|
||||
scoreExtractor(cand.candidate).map { scoreOp => (cand, scoreOp.getOrElse(0.0)) }
|
||||
}
|
||||
|
||||
Future.collect(scoredCandidatesFutures).map { scores =>
|
||||
val sorted = scores.sortBy { candidateDetails => -1 * candidateDetails._2 }
|
||||
sorted.map(_._1)
|
||||
}
|
||||
}
|
||||
|
||||
def populatePredictionScoreStats(
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
scoreExtractor: PushCandidate => Future[Option[Double]],
|
||||
predictionScoreStats: StatsReceiver
|
||||
): Unit = {
|
||||
val scoreScaleFactorForStat = 10000
|
||||
val statName = "prediction_scores"
|
||||
candidatesDetails.map {
|
||||
case CandidateDetails(candidate, source) =>
|
||||
val crt = candidate.commonRecType
|
||||
scoreExtractor(candidate).map { scoreOp =>
|
||||
val scaledScore = (scoreOp.getOrElse(0.0) * scoreScaleFactorForStat).toFloat
|
||||
predictionScoreStats.scope("all_candidates").stat(statName).add(scaledScore)
|
||||
predictionScoreStats.scope(crt.toString()).stat(statName).add(scaledScore)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def populateMrWeightedOpenOrNtabClickScoreStats(
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
predictionScoreStats: StatsReceiver
|
||||
): Unit = {
|
||||
populatePredictionScoreStats(
|
||||
candidatesDetails,
|
||||
candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability,
|
||||
predictionScoreStats
|
||||
)
|
||||
}
|
||||
|
||||
def populateMrQualityUprankingScoreStats(
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
predictionScoreStats: StatsReceiver
|
||||
): Unit = {
|
||||
populatePredictionScoreStats(
|
||||
candidatesDetails,
|
||||
candidate => candidate.mrQualityUprankingProbability,
|
||||
predictionScoreStats
|
||||
)
|
||||
}
|
||||
|
||||
def rankByMrWeightedOpenOrNtabClickScore(
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
rankBySpecifiedScore(
|
||||
candidatesDetails,
|
||||
candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability
|
||||
)
|
||||
}
|
||||
|
||||
def transformSigmoid(
|
||||
score: Double,
|
||||
weight: Double = 1.0,
|
||||
bias: Double = 0.0
|
||||
): Double = {
|
||||
val base = -1.0 * (weight * score + bias)
|
||||
val cappedBase = math.max(math.min(base, 100.0), -100.0)
|
||||
1.0 / (1.0 + math.exp(cappedBase))
|
||||
}
|
||||
|
||||
def transformLinear(
|
||||
score: Double,
|
||||
bar: Double = 1.0
|
||||
): Double = {
|
||||
val positiveBar = math.abs(bar)
|
||||
val cappedScore = math.max(math.min(score, positiveBar), -1.0 * positiveBar)
|
||||
cappedScore / positiveBar
|
||||
}
|
||||
|
||||
def transformIdentity(
|
||||
score: Double
|
||||
): Double = score
|
||||
|
||||
def rankByQualityOoncCombinedScore(
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
qualityScoreTransform: Double => Double,
|
||||
qualityScoreBoost: Double = 1.0
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
rankBySpecifiedScore(
|
||||
candidatesDetails,
|
||||
candidate => {
|
||||
val ooncScoreFutOpt: Future[Option[Double]] =
|
||||
candidate.mrWeightedOpenOrNtabClickRankingProbability
|
||||
val qualityScoreFutOpt: Future[Option[Double]] =
|
||||
candidate.mrQualityUprankingProbability
|
||||
Future
|
||||
.join(
|
||||
ooncScoreFutOpt,
|
||||
qualityScoreFutOpt
|
||||
).map {
|
||||
case (Some(ooncScore), Some(qualityScore)) =>
|
||||
val transformedQualityScore = qualityScoreTransform(qualityScore)
|
||||
val combinedScore = ooncScore * (1.0 + qualityScoreBoost * transformedQualityScore)
|
||||
candidate
|
||||
.cacheExternalScore(OoncQualityCombinedScore, Future.value(Some(combinedScore)))
|
||||
Some(combinedScore)
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
def rerankByProducerQualityOoncCombinedScore(
|
||||
candidateDetails: Seq[CandidateDetails[PushCandidate]]
|
||||
)(
|
||||
implicit stat: StatsReceiver
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
val scopedStat = stat.scope("producer_quality_reranking")
|
||||
val oonCandidates = candidateDetails.filter {
|
||||
case CandidateDetails(pushCandidate: PushCandidate, _) =>
|
||||
tweetCandidateSelector(pushCandidate, MrQualityUprankingPartialTypeEnum.Oon)
|
||||
}
|
||||
|
||||
val rankedOonCandidatesFut = rankBySpecifiedScore(
|
||||
oonCandidates,
|
||||
candidate => {
|
||||
val baseScoreFutureOpt: Future[Option[Double]] = {
|
||||
val qualityCombinedScoreFutureOpt =
|
||||
candidate.getExternalCachedScoreByName(OoncQualityCombinedScore)
|
||||
val ooncScoreFutureOpt = candidate.mrWeightedOpenOrNtabClickRankingProbability
|
||||
Future.join(qualityCombinedScoreFutureOpt, ooncScoreFutureOpt).map {
|
||||
case (Some(qualityCombinedScore), _) =>
|
||||
scopedStat.counter("quality_combined_score").incr()
|
||||
Some(qualityCombinedScore)
|
||||
case (_, ooncScoreOpt) =>
|
||||
scopedStat.counter("oonc_score").incr()
|
||||
ooncScoreOpt
|
||||
}
|
||||
}
|
||||
baseScoreFutureOpt.map {
|
||||
case Some(baseScore) =>
|
||||
val boostRatio = candidate.mrProducerQualityUprankingBoost.getOrElse(1.0)
|
||||
if (boostRatio > 1.0) scopedStat.counter("author_uprank").incr()
|
||||
else if (boostRatio < 1.0) scopedStat.counter("author_downrank").incr()
|
||||
else scopedStat.counter("author_noboost").incr()
|
||||
Some(baseScore * boostRatio)
|
||||
case _ =>
|
||||
scopedStat.counter("empty_score").incr()
|
||||
None
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
rankedOonCandidatesFut.map { rankedOonCandidates =>
|
||||
val sortedOonCandidateIterator = rankedOonCandidates.toIterator
|
||||
candidateDetails.map { ooncRankedCandidate =>
|
||||
val isOon = tweetCandidateSelector(
|
||||
ooncRankedCandidate.candidate,
|
||||
MrQualityUprankingPartialTypeEnum.Oon)
|
||||
|
||||
if (sortedOonCandidateIterator.hasNext && isOon)
|
||||
sortedOonCandidateIterator.next()
|
||||
else ooncRankedCandidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def tweetCandidateSelector(
|
||||
pushCandidate: PushCandidate,
|
||||
selectedCandidateType: MrQualityUprankingPartialTypeEnum.Value
|
||||
): Boolean = {
|
||||
pushCandidate match {
|
||||
case candidate: PushCandidate with TweetCandidate =>
|
||||
selectedCandidateType match {
|
||||
case MrQualityUprankingPartialTypeEnum.Oon =>
|
||||
val crt = candidate.commonRecType
|
||||
RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes
|
||||
.contains(crt)
|
||||
case _ => true
|
||||
}
|
||||
case _ => false
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,31 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.Ranker
|
||||
import com.twitter.util.Future
|
||||
|
||||
trait PushserviceRanker[T, C] extends Ranker[T, C] {
|
||||
|
||||
/**
|
||||
* Initial Ranking of input candidates
|
||||
*/
|
||||
def initialRank(target: T, candidates: Seq[CandidateDetails[C]]): Future[Seq[CandidateDetails[C]]]
|
||||
|
||||
/**
|
||||
* Re-ranks input ranked candidates. Useful when a subset of candidates are ranked
|
||||
* by a different logic, while preserving the initial ranking for the rest
|
||||
*/
|
||||
def reRank(
|
||||
target: T,
|
||||
rankedCandidates: Seq[CandidateDetails[C]]
|
||||
): Future[Seq[CandidateDetails[C]]]
|
||||
|
||||
/**
|
||||
* Final ranking that does Initial + Rerank
|
||||
*/
|
||||
override final def rank(target: T, candidates: Seq[CandidateDetails[C]]): (
|
||||
Future[Seq[CandidateDetails[C]]]
|
||||
) = {
|
||||
initialRank(target, candidates).flatMap { rankedCandidates => reRank(target, rankedCandidates) }
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,139 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
import com.twitter.contentrecommender.thriftscala.LightRankingCandidate
|
||||
import com.twitter.contentrecommender.thriftscala.LightRankingFeatureHydrationContext
|
||||
import com.twitter.contentrecommender.thriftscala.MagicRecsFeatureHydrationContext
|
||||
import com.twitter.finagle.stats.Counter
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.RandomRanker
|
||||
import com.twitter.frigate.common.base.Ranker
|
||||
import com.twitter.frigate.common.base.TweetAuthor
|
||||
import com.twitter.frigate.common.base.TweetCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.params.PushConstants
|
||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
||||
import com.twitter.frigate.pushservice.params.PushParams
|
||||
import com.twitter.ml.featurestore.lib.UserId
|
||||
import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker
|
||||
import com.twitter.util.Future
|
||||
|
||||
class RFPHLightRanker(
|
||||
lightRanker: MagicRecsServeDataRecordLightRanker,
|
||||
stats: StatsReceiver)
|
||||
extends Ranker[Target, PushCandidate] {
|
||||
|
||||
private val statsReceiver = stats.scope(this.getClass.getSimpleName)
|
||||
|
||||
private val lightRankerCandidateCounter = statsReceiver.counter("light_ranker_candidate_count")
|
||||
private val lightRankerRequestCounter = statsReceiver.counter("light_ranker_request_count")
|
||||
private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking")
|
||||
private val restrictLightRankingCounter: Counter =
|
||||
lightRankingStats.counter("restrict_light_ranking")
|
||||
private val selectedLightRankerScribedTargetCandidateCountStats: Stat =
|
||||
lightRankingStats.stat("selected_light_ranker_scribed_target_candidate_count")
|
||||
private val selectedLightRankerScribedCandidatesStats: Stat =
|
||||
lightRankingStats.stat("selected_light_ranker_scribed_candidates")
|
||||
private val lightRankingRandomBaselineStats: StatsReceiver =
|
||||
statsReceiver.scope("light_ranking_random_baseline")
|
||||
|
||||
override def rank(
|
||||
target: Target,
|
||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
val enableLightRanker = target.params(PushFeatureSwitchParams.EnableLightRankingParam)
|
||||
val restrictLightRanker = target.params(PushParams.RestrictLightRankingParam)
|
||||
val lightRankerSelectionThreshold =
|
||||
target.params(PushFeatureSwitchParams.LightRankingNumberOfCandidatesParam)
|
||||
val randomRanker = RandomRanker[Target, PushCandidate]()(lightRankingRandomBaselineStats)
|
||||
|
||||
if (enableLightRanker && candidates.length > lightRankerSelectionThreshold && !target.scribeFeatureForRequestScribe) {
|
||||
val (tweetCandidates, nonTweetCandidates) =
|
||||
candidates.partition {
|
||||
case CandidateDetails(pushCandidate: PushCandidate with TweetCandidate, source) => true
|
||||
case _ => false
|
||||
}
|
||||
val lightRankerSelectedTweetCandidatesFut = {
|
||||
if (restrictLightRanker) {
|
||||
restrictLightRankingCounter.incr()
|
||||
lightRankThenTake(
|
||||
target,
|
||||
tweetCandidates
|
||||
.asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]],
|
||||
PushConstants.RestrictLightRankingCandidatesThreshold
|
||||
)
|
||||
} else if (target.params(PushFeatureSwitchParams.EnableRandomBaselineLightRankingParam)) {
|
||||
randomRanker.rank(target, tweetCandidates).map { randomLightRankerCands =>
|
||||
randomLightRankerCands.take(lightRankerSelectionThreshold)
|
||||
}
|
||||
} else {
|
||||
lightRankThenTake(
|
||||
target,
|
||||
tweetCandidates
|
||||
.asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]],
|
||||
lightRankerSelectionThreshold
|
||||
)
|
||||
}
|
||||
}
|
||||
lightRankerSelectedTweetCandidatesFut.map { returnedTweetCandidates =>
|
||||
nonTweetCandidates ++ returnedTweetCandidates
|
||||
}
|
||||
} else if (target.scribeFeatureForRequestScribe) {
|
||||
val downSampleRate: Double =
|
||||
if (target.params(PushParams.DownSampleLightRankingScribeCandidatesParam))
|
||||
PushConstants.DownSampleLightRankingScribeCandidatesRate
|
||||
else target.params(PushFeatureSwitchParams.LightRankingScribeCandidatesDownSamplingParam)
|
||||
val selectedCandidateCounter: Int = math.ceil(candidates.size * downSampleRate).toInt
|
||||
selectedLightRankerScribedTargetCandidateCountStats.add(selectedCandidateCounter.toFloat)
|
||||
|
||||
randomRanker.rank(target, candidates).map { randomLightRankerCands =>
|
||||
val selectedCandidates = randomLightRankerCands.take(selectedCandidateCounter)
|
||||
selectedLightRankerScribedCandidatesStats.add(selectedCandidates.size.toFloat)
|
||||
selectedCandidates
|
||||
}
|
||||
} else Future.value(candidates)
|
||||
}
|
||||
|
||||
private def lightRankThenTake(
|
||||
target: Target,
|
||||
candidates: Seq[CandidateDetails[PushCandidate with TweetCandidate]],
|
||||
numOfCandidates: Int
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
lightRankerCandidateCounter.incr(candidates.length)
|
||||
lightRankerRequestCounter.incr()
|
||||
val lightRankerCandidates: Seq[LightRankingCandidate] = candidates.map {
|
||||
case CandidateDetails(tweetCandidate, _) =>
|
||||
val tweetAuthor = tweetCandidate match {
|
||||
case t: TweetCandidate with TweetAuthor => t.authorId
|
||||
case _ => None
|
||||
}
|
||||
val hydrationContext: LightRankingFeatureHydrationContext =
|
||||
LightRankingFeatureHydrationContext.MagicRecsHydrationContext(
|
||||
MagicRecsFeatureHydrationContext(
|
||||
tweetAuthor = tweetAuthor,
|
||||
pushString = tweetCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString))
|
||||
)
|
||||
LightRankingCandidate(
|
||||
tweetId = tweetCandidate.tweetId,
|
||||
hydrationContext = Some(hydrationContext)
|
||||
)
|
||||
}
|
||||
val modelName = target.params(PushFeatureSwitchParams.LightRankingModelTypeParam)
|
||||
val lightRankedCandidatesFut = {
|
||||
lightRanker
|
||||
.rank(UserId(target.targetId), lightRankerCandidates, modelName)
|
||||
}
|
||||
|
||||
lightRankedCandidatesFut.map { lightRankedCandidates =>
|
||||
val lrScoreMap = lightRankedCandidates.map { lrCand =>
|
||||
lrCand.tweetId -> lrCand.score
|
||||
}.toMap
|
||||
val candScoreMap: Seq[Option[Double]] = candidates.map { candidateDetails =>
|
||||
lrScoreMap.get(candidateDetails.candidate.tweetId)
|
||||
}
|
||||
sortCandidatesByScore(candidates, candScoreMap)
|
||||
.take(numOfCandidates)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,297 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.Ranker
|
||||
import com.twitter.frigate.common.rec_types.RecTypes
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.ml.HealthFeatureGetter
|
||||
import com.twitter.frigate.pushservice.ml.PushMLModelScorer
|
||||
import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum
|
||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
||||
import com.twitter.frigate.pushservice.params.PushMLModel
|
||||
import com.twitter.frigate.pushservice.params.PushModelName
|
||||
import com.twitter.frigate.pushservice.params.PushParams
|
||||
import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.updateMediaCategoryStats
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.frigate.pushservice.params.MrQualityUprankingTransformTypeEnum
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.frigate.thriftscala.UserMediaRepresentation
|
||||
import com.twitter.hss.api.thriftscala.UserHealthSignalResponse
|
||||
|
||||
class RFPHRanker(
|
||||
randomRanker: Ranker[Target, PushCandidate],
|
||||
weightedOpenOrNtabClickModelScorer: PushMLModelScorer,
|
||||
subscriptionCreatorRanker: SubscriptionCreatorRanker,
|
||||
userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse],
|
||||
producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation],
|
||||
stats: StatsReceiver)
|
||||
extends PushserviceRanker[Target, PushCandidate] {
|
||||
|
||||
private val statsReceiver = stats.scope(this.getClass.getSimpleName)
|
||||
|
||||
private val boostCRTsRanker = CRTBoostRanker(statsReceiver.scope("boost_desired_crts"))
|
||||
private val crtDownRanker = CRTDownRanker(statsReceiver.scope("down_rank_desired_crts"))
|
||||
|
||||
private val crtsToDownRank = statsReceiver.stat("crts_to_downrank")
|
||||
private val crtsToUprank = statsReceiver.stat("crts_to_uprank")
|
||||
|
||||
private val randomRankingCounter = stats.counter("randomRanking")
|
||||
private val mlRankingCounter = stats.counter("mlRanking")
|
||||
private val disableAllRelevanceCounter = stats.counter("disableAllRelevance")
|
||||
private val disableHeavyRankingCounter = stats.counter("disableHeavyRanking")
|
||||
|
||||
private val heavyRankerCandidateCounter = stats.counter("heavy_ranker_candidate_count")
|
||||
private val heavyRankerScoreStats = statsReceiver.scope("heavy_ranker_prediction_scores")
|
||||
|
||||
private val producerUprankingCounter = statsReceiver.counter("producer_quality_upranking")
|
||||
private val producerBoostedCounter = statsReceiver.counter("producer_boosted_candidates")
|
||||
private val producerDownboostedCounter = statsReceiver.counter("producer_downboosted_candidates")
|
||||
|
||||
override def initialRank(
|
||||
target: Target,
|
||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
heavyRankerCandidateCounter.incr(candidates.size)
|
||||
|
||||
updateMediaCategoryStats(candidates)(stats)
|
||||
target.targetUserState
|
||||
.flatMap { targetUserState =>
|
||||
val useRandomRanking = target.skipMlRanker || target.params(
|
||||
PushParams.UseRandomRankingParam
|
||||
)
|
||||
|
||||
if (useRandomRanking) {
|
||||
randomRankingCounter.incr()
|
||||
randomRanker.rank(target, candidates)
|
||||
} else if (target.params(PushParams.DisableAllRelevanceParam)) {
|
||||
disableAllRelevanceCounter.incr()
|
||||
Future.value(candidates)
|
||||
} else if (target.params(PushParams.DisableHeavyRankingParam) || target.params(
|
||||
PushFeatureSwitchParams.DisableHeavyRankingModelFSParam)) {
|
||||
disableHeavyRankingCounter.incr()
|
||||
Future.value(candidates)
|
||||
} else {
|
||||
mlRankingCounter.incr()
|
||||
|
||||
val scoredCandidatesFut = scoring(target, candidates)
|
||||
|
||||
target.rankingModelParam.map { rankingModelParam =>
|
||||
val modelName = PushModelName(
|
||||
PushMLModel.WeightedOpenOrNtabClickProbability,
|
||||
target.params(rankingModelParam)).toString
|
||||
ModelBasedRanker.populateMrWeightedOpenOrNtabClickScoreStats(
|
||||
candidates,
|
||||
heavyRankerScoreStats.scope(modelName)
|
||||
)
|
||||
}
|
||||
|
||||
if (target.params(
|
||||
PushFeatureSwitchParams.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam)) {
|
||||
val modelName = PushModelName(
|
||||
PushMLModel.FilteringProbability,
|
||||
target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam)
|
||||
).toString
|
||||
ModelBasedRanker.populateMrQualityUprankingScoreStats(
|
||||
candidates,
|
||||
heavyRankerScoreStats.scope(modelName)
|
||||
)
|
||||
}
|
||||
|
||||
val ooncRankedCandidatesFut =
|
||||
scoredCandidatesFut.flatMap(ModelBasedRanker.rankByMrWeightedOpenOrNtabClickScore)
|
||||
|
||||
val qualityUprankedCandidatesFut =
|
||||
if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) {
|
||||
ooncRankedCandidatesFut.flatMap { ooncRankedCandidates =>
|
||||
val transformFunc: Double => Double =
|
||||
target.params(PushFeatureSwitchParams.QualityUprankingTransformTypeParam) match {
|
||||
case MrQualityUprankingTransformTypeEnum.Linear =>
|
||||
ModelBasedRanker.transformLinear(
|
||||
_,
|
||||
bar = target.params(
|
||||
PushFeatureSwitchParams.QualityUprankingLinearBarForHeavyRankingParam))
|
||||
case MrQualityUprankingTransformTypeEnum.Sigmoid =>
|
||||
ModelBasedRanker.transformSigmoid(
|
||||
_,
|
||||
weight = target.params(
|
||||
PushFeatureSwitchParams.QualityUprankingSigmoidWeightForHeavyRankingParam),
|
||||
bias = target.params(
|
||||
PushFeatureSwitchParams.QualityUprankingSigmoidBiasForHeavyRankingParam)
|
||||
)
|
||||
case _ => ModelBasedRanker.transformIdentity
|
||||
}
|
||||
|
||||
ModelBasedRanker.rankByQualityOoncCombinedScore(
|
||||
ooncRankedCandidates,
|
||||
transformFunc,
|
||||
target.params(PushFeatureSwitchParams.QualityUprankingBoostForHeavyRankingParam)
|
||||
)
|
||||
}
|
||||
} else ooncRankedCandidatesFut
|
||||
|
||||
if (target.params(
|
||||
PushFeatureSwitchParams.EnableProducersQualityBoostingForHeavyRankingParam)) {
|
||||
producerUprankingCounter.incr()
|
||||
qualityUprankedCandidatesFut.flatMap(cands =>
|
||||
ModelBasedRanker.rerankByProducerQualityOoncCombinedScore(cands)(statsReceiver))
|
||||
} else qualityUprankedCandidatesFut
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private def scoring(
|
||||
target: Target,
|
||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
val ooncScoredCandidatesFut = target.rankingModelParam.map { rankingModelParam =>
|
||||
weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion(
|
||||
target,
|
||||
candidates,
|
||||
rankingModelParam
|
||||
)
|
||||
}
|
||||
|
||||
val scoredCandidatesFut = {
|
||||
if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) {
|
||||
ooncScoredCandidatesFut.map { candidates =>
|
||||
weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion(
|
||||
target = target,
|
||||
candidatesDetails = candidates,
|
||||
modelVersionParam = PushFeatureSwitchParams.QualityUprankingModelTypeParam,
|
||||
overridePushMLModelOpt = Some(PushMLModel.FilteringProbability)
|
||||
)
|
||||
}
|
||||
} else ooncScoredCandidatesFut
|
||||
}
|
||||
|
||||
scoredCandidatesFut.foreach { candidates =>
|
||||
val oonCandidates = candidates.filter {
|
||||
case CandidateDetails(pushCandidate: PushCandidate, _) =>
|
||||
ModelBasedRanker.tweetCandidateSelector(
|
||||
pushCandidate,
|
||||
MrQualityUprankingPartialTypeEnum.Oon)
|
||||
}
|
||||
setProducerQuality(
|
||||
target,
|
||||
oonCandidates,
|
||||
userHealthSignalStore,
|
||||
producerMediaRepresentationStore)
|
||||
}
|
||||
}
|
||||
|
||||
private def setProducerQuality(
|
||||
target: Target,
|
||||
candidates: Seq[CandidateDetails[PushCandidate]],
|
||||
userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse],
|
||||
producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation]
|
||||
): Unit = {
|
||||
lazy val boostRatio =
|
||||
target.params(PushFeatureSwitchParams.QualityUprankingBoostForHighQualityProducersParam)
|
||||
lazy val downboostRatio =
|
||||
target.params(PushFeatureSwitchParams.QualityUprankingDownboostForLowQualityProducersParam)
|
||||
candidates.foreach {
|
||||
case CandidateDetails(pushCandidate, _) =>
|
||||
HealthFeatureGetter
|
||||
.getFeatures(pushCandidate, producerMediaRepresentationStore, userHealthSignalStore).map {
|
||||
featureMap =>
|
||||
val agathaNsfwScore = featureMap.numericFeatures.getOrElse("agathaNsfwScore", 0.5)
|
||||
val textNsfwScore = featureMap.numericFeatures.getOrElse("textNsfwScore", 0.15)
|
||||
val nudityRate = featureMap.numericFeatures.getOrElse("nudityRate", 0.0)
|
||||
val activeFollowers = featureMap.numericFeatures.getOrElse("activeFollowers", 0.0)
|
||||
val favorsRcvd28Days = featureMap.numericFeatures.getOrElse("favorsRcvd28Days", 0.0)
|
||||
val tweets28Days = featureMap.numericFeatures.getOrElse("tweets28Days", 0.0)
|
||||
val authorDislikeCount = featureMap.numericFeatures
|
||||
.getOrElse("authorDislikeCount", 0.0)
|
||||
val authorDislikeRate = featureMap.numericFeatures.getOrElse("authorDislikeRate", 0.0)
|
||||
val authorReportRate = featureMap.numericFeatures.getOrElse("authorReportRate", 0.0)
|
||||
val abuseStrikeTop2Percent =
|
||||
featureMap.booleanFeatures.getOrElse("abuseStrikeTop2Percent", false)
|
||||
val abuseStrikeTop1Percent =
|
||||
featureMap.booleanFeatures.getOrElse("abuseStrikeTop1Percent", false)
|
||||
val hasNsfwToken = featureMap.booleanFeatures.getOrElse("hasNsfwToken", false)
|
||||
|
||||
if ((activeFollowers > 3000000) ||
|
||||
(activeFollowers > 1000000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent) ||
|
||||
(activeFollowers > 100000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent &&
|
||||
tweets28Days > 0 && favorsRcvd28Days / tweets28Days > 3000 && authorReportRate < 0.000001 && authorDislikeRate < 0.0005)) {
|
||||
producerBoostedCounter.incr()
|
||||
pushCandidate.setProducerQualityUprankingBoost(boostRatio)
|
||||
} else if (activeFollowers < 5 || agathaNsfwScore > 0.9 || nudityRate > 0.03 || hasNsfwToken || abuseStrikeTop1Percent ||
|
||||
textNsfwScore > 0.4 || (authorDislikeRate > 0.005 && authorDislikeCount > 5) ||
|
||||
(tweets28Days > 56 && favorsRcvd28Days / tweets28Days < 100)) {
|
||||
producerDownboostedCounter.incr()
|
||||
pushCandidate.setProducerQualityUprankingBoost(downboostRatio)
|
||||
} else pushCandidate.setProducerQualityUprankingBoost(1.0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private def rerankBySubscriptionCreatorRanker(
|
||||
target: Target,
|
||||
rankedCandidates: Future[Seq[CandidateDetails[PushCandidate]]],
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
if (target.params(PushFeatureSwitchParams.SoftRankCandidatesFromSubscriptionCreators)) {
|
||||
val factor = target.params(PushFeatureSwitchParams.SoftRankFactorForSubscriptionCreators)
|
||||
subscriptionCreatorRanker.boostByScoreFactor(rankedCandidates, factor)
|
||||
} else
|
||||
subscriptionCreatorRanker.boostSubscriptionCreator(rankedCandidates)
|
||||
}
|
||||
|
||||
override def reRank(
|
||||
target: Target,
|
||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
val numberOfF1Candidates =
|
||||
rankedCandidates.count(candidateDetails =>
|
||||
RecTypes.isF1Type(candidateDetails.candidate.commonRecType))
|
||||
lazy val threshold =
|
||||
target.params(PushFeatureSwitchParams.NumberOfF1CandidatesThresholdForOONBackfill)
|
||||
lazy val enableOONBackfillBasedOnF1 =
|
||||
target.params(PushFeatureSwitchParams.EnableOONBackfillBasedOnF1Candidates)
|
||||
|
||||
val f1BoostedCandidates =
|
||||
if (enableOONBackfillBasedOnF1 && numberOfF1Candidates > threshold) {
|
||||
boostCRTsRanker.boostCrtsToTopStableOrder(
|
||||
rankedCandidates,
|
||||
RecTypes.f1FirstDegreeTypes.toSeq)
|
||||
} else rankedCandidates
|
||||
|
||||
val topTweetsByGeoDownRankedCandidates =
|
||||
if (target.params(PushFeatureSwitchParams.BackfillRankTopTweetsByGeoCandidates)) {
|
||||
crtDownRanker.downRank(
|
||||
f1BoostedCandidates,
|
||||
Seq(CommonRecommendationType.GeoPopTweet)
|
||||
)
|
||||
} else f1BoostedCandidates
|
||||
|
||||
val reRankedCandidatesWithBoostedCrts = {
|
||||
val listOfCrtsToUpRank = target
|
||||
.params(PushFeatureSwitchParams.ListOfCrtsToUpRank)
|
||||
.flatMap(CommonRecommendationType.valueOf)
|
||||
crtsToUprank.add(listOfCrtsToUpRank.size)
|
||||
boostCRTsRanker.boostCrtsToTop(topTweetsByGeoDownRankedCandidates, listOfCrtsToUpRank)
|
||||
}
|
||||
|
||||
val reRankedCandidatesWithDownRankedCrts = {
|
||||
val listOfCrtsToDownRank = target
|
||||
.params(PushFeatureSwitchParams.ListOfCrtsToDownRank)
|
||||
.flatMap(CommonRecommendationType.valueOf)
|
||||
crtsToDownRank.add(listOfCrtsToDownRank.size)
|
||||
crtDownRanker.downRank(reRankedCandidatesWithBoostedCrts, listOfCrtsToDownRank)
|
||||
}
|
||||
|
||||
val rerankBySubscriptionCreatorFut = {
|
||||
if (target.params(PushFeatureSwitchParams.BoostCandidatesFromSubscriptionCreators)) {
|
||||
rerankBySubscriptionCreatorRanker(
|
||||
target,
|
||||
Future.value(reRankedCandidatesWithDownRankedCrts))
|
||||
} else Future.value(reRankedCandidatesWithDownRankedCrts)
|
||||
}
|
||||
|
||||
rerankBySubscriptionCreatorFut
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,110 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.rank
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.TweetAuthor
|
||||
import com.twitter.frigate.common.base.TweetCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.storehaus.FutureOps
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
class SubscriptionCreatorRanker(
|
||||
superFollowEligibilityUserStore: ReadableStore[Long, Boolean],
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
private val scopedStats = statsReceiver.scope("SubscriptionCreatorRanker")
|
||||
private val boostStats = scopedStats.scope("boostSubscriptionCreator")
|
||||
private val softUprankStats = scopedStats.scope("boostByScoreFactor")
|
||||
private val boostTotalCandidates = boostStats.stat("total_input_candidates")
|
||||
private val softRankTotalCandidates = softUprankStats.stat("total_input_candidates")
|
||||
private val softRankNumCandidatesCreators = softUprankStats.counter("candidates_from_creators")
|
||||
private val softRankNumCandidatesNonCreators =
|
||||
softUprankStats.counter("candidates_not_from_creators")
|
||||
private val boostNumCandidatesCreators = boostStats.counter("candidates_from_creators")
|
||||
private val boostNumCandidatesNonCreators =
|
||||
boostStats.counter("candidates_not_from_creators")
|
||||
|
||||
def boostSubscriptionCreator(
|
||||
inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
inputCandidatesFut.flatMap { inputCandidates =>
|
||||
boostTotalCandidates.add(inputCandidates.size)
|
||||
val tweetAuthorIds = inputCandidates.flatMap {
|
||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
||||
candidate.authorId
|
||||
case _ => None
|
||||
}.toSet
|
||||
|
||||
FutureOps
|
||||
.mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds))
|
||||
.map { creatorAuthorMap =>
|
||||
val (upRankedCandidates, otherCandidates) = inputCandidates.partition {
|
||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
||||
candidate.authorId match {
|
||||
case Some(authorId) =>
|
||||
creatorAuthorMap(authorId).getOrElse(false)
|
||||
case _ => false
|
||||
}
|
||||
case _ => false
|
||||
}
|
||||
boostNumCandidatesCreators.incr(upRankedCandidates.size)
|
||||
boostNumCandidatesNonCreators.incr(otherCandidates.size)
|
||||
upRankedCandidates ++ otherCandidates
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def boostByScoreFactor(
|
||||
inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]],
|
||||
factor: Double = 1.0,
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
inputCandidatesFut.flatMap { inputCandidates =>
|
||||
softRankTotalCandidates.add(inputCandidates.size)
|
||||
val tweetAuthorIds = inputCandidates.flatMap {
|
||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
||||
candidate.authorId
|
||||
case _ => None
|
||||
}.toSet
|
||||
|
||||
FutureOps
|
||||
.mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds))
|
||||
.flatMap { creatorAuthorMap =>
|
||||
val (upRankedCandidates, otherCandidates) = inputCandidates.partition {
|
||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
||||
candidate.authorId match {
|
||||
case Some(authorId) =>
|
||||
creatorAuthorMap(authorId).getOrElse(false)
|
||||
case _ => false
|
||||
}
|
||||
case _ => false
|
||||
}
|
||||
softRankNumCandidatesCreators.incr(upRankedCandidates.size)
|
||||
softRankNumCandidatesNonCreators.incr(otherCandidates.size)
|
||||
|
||||
ModelBasedRanker.rankBySpecifiedScore(
|
||||
inputCandidates,
|
||||
candidate => {
|
||||
val isFromCreator = candidate match {
|
||||
case candidate: TweetCandidate with TweetAuthor =>
|
||||
candidate.authorId match {
|
||||
case Some(authorId) =>
|
||||
creatorAuthorMap(authorId).getOrElse(false)
|
||||
case _ => false
|
||||
}
|
||||
case _ => false
|
||||
}
|
||||
candidate.mrWeightedOpenOrNtabClickRankingProbability.map {
|
||||
case Some(score) =>
|
||||
if (isFromCreator) Some(score * factor)
|
||||
else Some(score)
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,259 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.finagle.stats.Counter
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.CandidateResult
|
||||
import com.twitter.frigate.common.base.CandidateSource
|
||||
import com.twitter.frigate.common.base.FetchRankFlowWithHydratedCandidates
|
||||
import com.twitter.frigate.common.base.Invalid
|
||||
import com.twitter.frigate.common.base.OK
|
||||
import com.twitter.frigate.common.base.Response
|
||||
import com.twitter.frigate.common.base.Result
|
||||
import com.twitter.frigate.common.base.Stats.track
|
||||
import com.twitter.frigate.common.base.Stats.trackSeq
|
||||
import com.twitter.frigate.common.logger.MRLogger
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator
|
||||
import com.twitter.frigate.pushservice.predicate.LoggedOutPreRankingPredicates
|
||||
import com.twitter.frigate.pushservice.predicate.LoggedOutTargetPredicates
|
||||
import com.twitter.frigate.pushservice.rank.LoggedOutRanker
|
||||
import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier
|
||||
import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler
|
||||
import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder
|
||||
import com.twitter.frigate.pushservice.thriftscala.LoggedOutRequest
|
||||
import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse
|
||||
import com.twitter.frigate.pushservice.thriftscala.PushContext
|
||||
import com.twitter.hermit.predicate.NamedPredicate
|
||||
import com.twitter.hermit.predicate.Predicate
|
||||
import com.twitter.hermit.predicate.SequentialPredicate
|
||||
import com.twitter.util.Future
|
||||
|
||||
class LoggedOutRefreshForPushHandler(
|
||||
val loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder,
|
||||
val loPushCandidateSourceGenerator: LoggedOutPushCandidateSourceGenerator,
|
||||
candidateHydrator: PushCandidateHydrator,
|
||||
val loRanker: LoggedOutRanker,
|
||||
val loRfphNotifier: LoggedOutRefreshForPushNotifier,
|
||||
loMrRequestScriberNode: String
|
||||
)(
|
||||
globalStats: StatsReceiver)
|
||||
extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] {
|
||||
|
||||
val log = MRLogger("LORefreshForPushHandler")
|
||||
implicit val statsReceiver: StatsReceiver =
|
||||
globalStats.scope("LORefreshForPushHandler")
|
||||
private val loggedOutBuildStats = statsReceiver.scope("logged_out_build_target")
|
||||
private val loggedOutProcessStats = statsReceiver.scope("logged_out_process")
|
||||
private val loggedOutNotifyStats = statsReceiver.scope("logged_out_notify")
|
||||
private val loCandidateHydrationStats: StatsReceiver =
|
||||
statsReceiver.scope("logged_out_candidate_hydration")
|
||||
val mrLORequestCandidateScribeStats =
|
||||
statsReceiver.scope("mr_logged_out_request_scribe_candidates")
|
||||
|
||||
val mrRequestScribeHandler =
|
||||
new MrRequestScribeHandler(loMrRequestScriberNode, statsReceiver.scope("lo_mr_request_scribe"))
|
||||
val loMrRequestTargetScribeStats = statsReceiver.scope("lo_mr_request_scribe_target")
|
||||
|
||||
lazy val loCandSourceEligibleCounter: Counter =
|
||||
loCandidateStats.counter("logged_out_cand_source_eligible")
|
||||
lazy val loCandSourceNotEligibleCounter: Counter =
|
||||
loCandidateStats.counter("logged_out_cand_source_not_eligible")
|
||||
lazy val allCandidatesCounter: Counter = statsReceiver.counter("all_logged_out_candidates")
|
||||
val allCandidatesFilteredPreRank = filterStats.counter("all_logged_out_candidates_filtered")
|
||||
|
||||
override def targetPredicates(target: Target): List[Predicate[Target]] = List(
|
||||
LoggedOutTargetPredicates.targetFatiguePredicate(),
|
||||
LoggedOutTargetPredicates.loggedOutRecsHoldbackPredicate()
|
||||
)
|
||||
|
||||
override def isTargetValid(target: Target): Future[Result] = {
|
||||
val resultFut =
|
||||
if (target.skipFilters) {
|
||||
Future.value(OK)
|
||||
} else {
|
||||
predicateSeq(target).track(Seq(target)).map { resultArr =>
|
||||
trackTargetPredStats(resultArr(0))
|
||||
}
|
||||
}
|
||||
track(targetStats)(resultFut)
|
||||
}
|
||||
|
||||
override def rank(
|
||||
target: Target,
|
||||
candidateDetails: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
loRanker.rank(candidateDetails)
|
||||
}
|
||||
|
||||
override def validCandidates(
|
||||
target: Target,
|
||||
candidates: Seq[PushCandidate]
|
||||
): Future[Seq[Result]] = {
|
||||
Future.value(candidates.map { c => OK })
|
||||
}
|
||||
|
||||
override def desiredCandidateCount(target: Target): Int = 1
|
||||
|
||||
private val loggedOutPreRankingPredicates =
|
||||
LoggedOutPreRankingPredicates(filterStats.scope("logged_out_predicates"))
|
||||
|
||||
private val loggedOutPreRankingPredicateChain =
|
||||
new SequentialPredicate[PushCandidate](loggedOutPreRankingPredicates)
|
||||
|
||||
override def filter(
|
||||
target: Target,
|
||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[
|
||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
||||
] = {
|
||||
val predicateChain = loggedOutPreRankingPredicateChain
|
||||
predicateChain
|
||||
.track(candidates.map(_.candidate))
|
||||
.map { results =>
|
||||
val resultForPreRankingFiltering =
|
||||
results
|
||||
.zip(candidates)
|
||||
.foldLeft(
|
||||
(
|
||||
Seq.empty[CandidateDetails[PushCandidate]],
|
||||
Seq.empty[CandidateResult[PushCandidate, Result]]
|
||||
)
|
||||
) {
|
||||
case ((goodCandidates, filteredCandidates), (result, candidateDetails)) =>
|
||||
result match {
|
||||
case None =>
|
||||
(goodCandidates :+ candidateDetails, filteredCandidates)
|
||||
|
||||
case Some(pred: NamedPredicate[_]) =>
|
||||
val r = Invalid(Some(pred.name))
|
||||
(
|
||||
goodCandidates,
|
||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
||||
candidateDetails.candidate,
|
||||
candidateDetails.source,
|
||||
r
|
||||
)
|
||||
)
|
||||
case Some(_) =>
|
||||
val r = Invalid(Some("Filtered by un-named predicate"))
|
||||
(
|
||||
goodCandidates,
|
||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
||||
candidateDetails.candidate,
|
||||
candidateDetails.source,
|
||||
r
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
resultForPreRankingFiltering match {
|
||||
case (validCandidates, _) if validCandidates.isEmpty && candidates.nonEmpty =>
|
||||
allCandidatesFilteredPreRank.incr()
|
||||
case _ => ()
|
||||
|
||||
}
|
||||
resultForPreRankingFiltering
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
override def candidateSources(
|
||||
target: Target
|
||||
): Future[Seq[CandidateSource[Target, RawCandidate]]] = {
|
||||
Future
|
||||
.collect(loPushCandidateSourceGenerator.sources.map { cs =>
|
||||
cs.isCandidateSourceAvailable(target).map { isEligible =>
|
||||
if (isEligible) {
|
||||
loCandSourceEligibleCounter.incr()
|
||||
Some(cs)
|
||||
} else {
|
||||
loCandSourceNotEligibleCounter.incr()
|
||||
None
|
||||
}
|
||||
}
|
||||
}).map(_.flatten)
|
||||
}
|
||||
|
||||
override def process(
|
||||
target: Target,
|
||||
externalCandidates: Seq[RawCandidate] = Nil
|
||||
): Future[Response[PushCandidate, Result]] = {
|
||||
isTargetValid(target).flatMap {
|
||||
case OK =>
|
||||
for {
|
||||
candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target))
|
||||
externalCandidateDetails = externalCandidates.map(
|
||||
CandidateDetails(_, "logged_out_refresh_for_push_handler_external_candidates"))
|
||||
allCandidates = candidatesFromSources ++ externalCandidateDetails
|
||||
hydratedCandidatesWithCopy <-
|
||||
trackSeq(loCandidateHydrationStats)(hydrateCandidates(allCandidates))
|
||||
(candidates, preRankingFilteredCandidates) <-
|
||||
track(filterStats)(filter(target, hydratedCandidatesWithCopy))
|
||||
rankedCandidates <- trackSeq(rankingStats)(rank(target, candidates))
|
||||
allTakeCandidateResults <- track(takeStats)(
|
||||
take(target, rankedCandidates, desiredCandidateCount(target))
|
||||
)
|
||||
_ <- track(mrLORequestCandidateScribeStats)(
|
||||
mrRequestScribeHandler.scribeForCandidateFiltering(
|
||||
target,
|
||||
hydratedCandidatesWithCopy,
|
||||
preRankingFilteredCandidates,
|
||||
rankedCandidates,
|
||||
rankedCandidates,
|
||||
rankedCandidates,
|
||||
allTakeCandidateResults
|
||||
))
|
||||
|
||||
} yield {
|
||||
val takeCandidateResults = allTakeCandidateResults.filterNot { candResult =>
|
||||
candResult.result == MoreThanDesiredCandidates
|
||||
}
|
||||
val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates
|
||||
allCandidatesCounter.incr(allCandidateResults.size)
|
||||
Response(OK, allCandidateResults)
|
||||
}
|
||||
|
||||
case result: Result =>
|
||||
for (_ <- track(loMrRequestTargetScribeStats)(
|
||||
mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield {
|
||||
Response(result, Nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def buildTarget(
|
||||
guestId: Long,
|
||||
inputPushContext: Option[PushContext]
|
||||
): Future[Target] =
|
||||
loPushTargetUserBuilder.buildTarget(guestId, inputPushContext)
|
||||
|
||||
/**
|
||||
* Hydrate candidate by querying downstream services
|
||||
*
|
||||
* @param candidates - candidates
|
||||
*
|
||||
* @return - hydrated candidates
|
||||
*/
|
||||
override def hydrateCandidates(
|
||||
candidates: Seq[CandidateDetails[RawCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates)
|
||||
|
||||
override def batchForCandidatesCheck(target: Target): Int = 1
|
||||
|
||||
def refreshAndSend(request: LoggedOutRequest): Future[LoggedOutResponse] = {
|
||||
for {
|
||||
target <- track(loggedOutBuildStats)(
|
||||
loPushTargetUserBuilder.buildTarget(request.guestId, request.context))
|
||||
response <- track(loggedOutProcessStats)(process(target, externalCandidates = Seq.empty))
|
||||
loggedOutRefreshResponse <-
|
||||
track(loggedOutNotifyStats)(loRfphNotifier.checkResponseAndNotify(response))
|
||||
} yield {
|
||||
loggedOutRefreshResponse
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
@ -1,239 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.channels.common.thriftscala.ApiList
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base._
|
||||
import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.TrendTweetPushCandidate
|
||||
import com.twitter.frigate.pushservice.ml.PushMLModelScorer
|
||||
import com.twitter.frigate.pushservice.model.candidate.CopyIds
|
||||
import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion
|
||||
import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._
|
||||
import com.twitter.frigate.pushservice.util.MrUserStateUtil
|
||||
import com.twitter.frigate.pushservice.util.RelationshipUtil
|
||||
import com.twitter.gizmoduck.thriftscala.User
|
||||
import com.twitter.hermit.predicate.socialgraph.RelationEdge
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class PushCandidateHydrator(
|
||||
socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean],
|
||||
safeUserStore: ReadableStore[Long, User],
|
||||
apiListStore: ReadableStore[Long, ApiList],
|
||||
candidateCopyCross: CandidateCopyExpansion
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver,
|
||||
implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) {
|
||||
|
||||
lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num")
|
||||
lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates")
|
||||
lazy val mrUserStateStat = statsReceiver.scope("mr_user_state")
|
||||
|
||||
lazy val queryStep = statsReceiver.scope("query_step")
|
||||
lazy val relationEdgeWithoutDuplicateInQueryStep =
|
||||
queryStep.counter("number_of_relationEdge_without_duplicate_in_query_step")
|
||||
lazy val relationEdgeWithoutDuplicateInQueryStepDistribution =
|
||||
queryStep.stat("number_of_relationEdge_without_duplicate_in_query_step_distribution")
|
||||
|
||||
case class Entities(
|
||||
users: Set[Long] = Set.empty[Long],
|
||||
relationshipEdges: Set[RelationEdge] = Set.empty[RelationEdge]) {
|
||||
def merge(otherEntities: Entities): Entities = {
|
||||
this.copy(
|
||||
users = this.users ++ otherEntities.users,
|
||||
relationshipEdges =
|
||||
this.relationshipEdges ++ otherEntities.relationshipEdges
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
case class EntitiesMap(
|
||||
userMap: Map[Long, User] = Map.empty[Long, User],
|
||||
relationshipMap: Map[RelationEdge, Boolean] = Map.empty[RelationEdge, Boolean])
|
||||
|
||||
private def updateCandidateAndCrtStats(
|
||||
candidate: RawCandidate,
|
||||
candidateType: String,
|
||||
numEntities: Int = 1
|
||||
): Unit = {
|
||||
statsReceiver
|
||||
.scope(candidateType).scope(candidate.commonRecType.name).stat(
|
||||
"totalEntitiesPerCandidateTypePerCrt").add(numEntities)
|
||||
statsReceiver.scope(candidateType).stat("totalEntitiesPerCandidateType").add(numEntities)
|
||||
}
|
||||
|
||||
private def collectEntities(
|
||||
candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]]
|
||||
): Entities = {
|
||||
candidateDetailsSeq
|
||||
.map { candidateDetails =>
|
||||
val pushCandidate = candidateDetails.candidate
|
||||
|
||||
val userEntities = pushCandidate match {
|
||||
case tweetWithSocialContext: RawCandidate with TweetWithSocialContextTraits =>
|
||||
val authorIdOpt = getAuthorIdFromTweetCandidate(tweetWithSocialContext)
|
||||
val scUserIds = tweetWithSocialContext.socialContextUserIds.toSet
|
||||
updateCandidateAndCrtStats(pushCandidate, "tweetWithSocialContext", scUserIds.size + 1)
|
||||
Entities(users = scUserIds ++ authorIdOpt.toSet)
|
||||
|
||||
case _ => Entities()
|
||||
}
|
||||
|
||||
val relationEntities = {
|
||||
if (isInNetworkTweetType(pushCandidate.commonRecType)) {
|
||||
Entities(
|
||||
relationshipEdges =
|
||||
RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(pushCandidate).toSet
|
||||
)
|
||||
} else Entities()
|
||||
}
|
||||
|
||||
userEntities.merge(relationEntities)
|
||||
}
|
||||
.foldLeft(Entities()) { (e1, e2) => e1.merge(e2) }
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* This method calls Gizmoduck and Social Graph Service, keep the results in EntitiesMap
|
||||
* and passed onto the update candidate phase in the hydration step
|
||||
*
|
||||
* @param entities contains all userIds and relationEdges for all candidates
|
||||
* @return EntitiesMap contains userMap and relationshipMap
|
||||
*/
|
||||
private def queryEntities(entities: Entities): Future[EntitiesMap] = {
|
||||
|
||||
relationEdgeWithoutDuplicateInQueryStep.incr(entities.relationshipEdges.size)
|
||||
relationEdgeWithoutDuplicateInQueryStepDistribution.add(entities.relationshipEdges.size)
|
||||
|
||||
val relationshipMapFuture = Future
|
||||
.collect(socialGraphServiceProcessStore.multiGet(entities.relationshipEdges))
|
||||
.map { resultMap =>
|
||||
resultMap.collect {
|
||||
case (relationshipEdge, Some(res)) => relationshipEdge -> res
|
||||
case (relationshipEdge, None) => relationshipEdge -> false
|
||||
}
|
||||
}
|
||||
|
||||
val userMapFuture = Future
|
||||
.collect(safeUserStore.multiGet(entities.users))
|
||||
.map { userMap =>
|
||||
userMap.collect {
|
||||
case (userId, Some(user)) =>
|
||||
userId -> user
|
||||
}
|
||||
}
|
||||
|
||||
Future.join(userMapFuture, relationshipMapFuture).map {
|
||||
case (uMap, rMap) => EntitiesMap(userMap = uMap, relationshipMap = rMap)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param candidateDetails: recommendation candidates for a user
|
||||
* @return sequence of candidates tagged with push and ntab copy id
|
||||
*/
|
||||
private def expandCandidatesWithCopy(
|
||||
candidateDetails: Seq[CandidateDetails[RawCandidate]]
|
||||
): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = {
|
||||
candidateCopyCross.expandCandidatesWithCopyId(candidateDetails)
|
||||
}
|
||||
|
||||
def updateCandidates(
|
||||
candidateDetailsWithCopies: Seq[(CandidateDetails[RawCandidate], CopyIds)],
|
||||
entitiesMaps: EntitiesMap
|
||||
): Seq[CandidateDetails[PushCandidate]] = {
|
||||
candidateDetailsWithCopies.map {
|
||||
case (candidateDetail, copyIds) =>
|
||||
val pushCandidate = candidateDetail.candidate
|
||||
val userMap = entitiesMaps.userMap
|
||||
val relationshipMap = entitiesMaps.relationshipMap
|
||||
|
||||
val hydratedCandidate = pushCandidate match {
|
||||
|
||||
case f1TweetCandidate: F1FirstDegree =>
|
||||
getHydratedCandidateForF1FirstDegreeTweet(
|
||||
f1TweetCandidate,
|
||||
userMap,
|
||||
relationshipMap,
|
||||
copyIds)
|
||||
|
||||
case tweetRetweet: TweetRetweetCandidate =>
|
||||
getHydratedCandidateForTweetRetweet(tweetRetweet, userMap, copyIds)
|
||||
|
||||
case tweetFavorite: TweetFavoriteCandidate =>
|
||||
getHydratedCandidateForTweetFavorite(tweetFavorite, userMap, copyIds)
|
||||
|
||||
case tripTweetCandidate: OutOfNetworkTweetCandidate with TripCandidate =>
|
||||
getHydratedCandidateForTripTweetCandidate(tripTweetCandidate, userMap, copyIds)
|
||||
|
||||
case outOfNetworkTweetCandidate: OutOfNetworkTweetCandidate with TopicCandidate =>
|
||||
getHydratedCandidateForOutOfNetworkTweetCandidate(
|
||||
outOfNetworkTweetCandidate,
|
||||
userMap,
|
||||
copyIds)
|
||||
|
||||
case topicProofTweetCandidate: TopicProofTweetCandidate =>
|
||||
getHydratedTopicProofTweetCandidate(topicProofTweetCandidate, userMap, copyIds)
|
||||
|
||||
case subscribedSearchTweetCandidate: SubscribedSearchTweetCandidate =>
|
||||
getHydratedSubscribedSearchTweetCandidate(
|
||||
subscribedSearchTweetCandidate,
|
||||
userMap,
|
||||
copyIds)
|
||||
|
||||
case listRecommendation: ListPushCandidate =>
|
||||
getHydratedListCandidate(apiListStore, listRecommendation, copyIds)
|
||||
|
||||
case discoverTwitterCandidate: DiscoverTwitterCandidate =>
|
||||
getHydratedCandidateForDiscoverTwitterCandidate(discoverTwitterCandidate, copyIds)
|
||||
|
||||
case topTweetImpressionsCandidate: TopTweetImpressionsCandidate =>
|
||||
getHydratedCandidateForTopTweetImpressionsCandidate(
|
||||
topTweetImpressionsCandidate,
|
||||
copyIds)
|
||||
|
||||
case trendTweetCandidate: TrendTweetCandidate =>
|
||||
new TrendTweetPushCandidate(
|
||||
trendTweetCandidate,
|
||||
trendTweetCandidate.authorId.flatMap(userMap.get),
|
||||
copyIds)
|
||||
|
||||
case unknownCandidate =>
|
||||
throw new IllegalArgumentException(
|
||||
s"Incorrect candidate for hydration: ${unknownCandidate.commonRecType}")
|
||||
}
|
||||
|
||||
CandidateDetails(
|
||||
hydratedCandidate,
|
||||
source = candidateDetail.source
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
def apply(
|
||||
candidateDetails: Seq[CandidateDetails[RawCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
val isLoggedOutRequest =
|
||||
candidateDetails.headOption.exists(_.candidate.target.isLoggedOutUser)
|
||||
if (!isLoggedOutRequest) {
|
||||
candidateDetails.headOption.map { cd =>
|
||||
MrUserStateUtil.updateMrUserStateStats(cd.candidate.target)(mrUserStateStat)
|
||||
}
|
||||
}
|
||||
|
||||
expandCandidatesWithCopy(candidateDetails).flatMap { candidateDetailsWithCopy =>
|
||||
candidateWithCopyNumStat.add(candidateDetailsWithCopy.size)
|
||||
val entities = collectEntities(candidateDetailsWithCopy.map(_._1))
|
||||
queryEntities(entities).flatMap { entitiesMap =>
|
||||
val updatedCandidates = updateCandidates(candidateDetailsWithCopy, entitiesMap)
|
||||
updatedCandidates.foreach { cand =>
|
||||
hydratedCandidateStat.counter(cand.candidate.commonRecType.name).incr()
|
||||
}
|
||||
Future.value(updatedCandidates)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,69 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.FeatureMap
|
||||
import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.ml.HydrationContextBuilder
|
||||
import com.twitter.frigate.pushservice.params.PushParams
|
||||
import com.twitter.frigate.pushservice.util.MrUserStateUtil
|
||||
import com.twitter.nrel.heavyranker.FeatureHydrator
|
||||
import com.twitter.util.Future
|
||||
|
||||
class RFPHFeatureHydrator(
|
||||
featureHydrator: FeatureHydrator
|
||||
)(
|
||||
implicit globalStats: StatsReceiver) {
|
||||
|
||||
implicit val statsReceiver: StatsReceiver =
|
||||
globalStats.scope("RefreshForPushHandler")
|
||||
|
||||
//stat for feature hydration
|
||||
private val featureHydrationEnabledCounter = statsReceiver.counter("featureHydrationEnabled")
|
||||
private val mrUserStateStat = statsReceiver.scope("mr_user_state")
|
||||
|
||||
private def hydrateFromRelevanceHydrator(
|
||||
candidateDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
mrRequestContextForFeatureStore: MrRequestContextForFeatureStore
|
||||
): Future[Unit] = {
|
||||
val pushCandidates = candidateDetails.map(_.candidate)
|
||||
val candidatesAndContextsFut = Future.collect(pushCandidates.map { pc =>
|
||||
val contextFut = HydrationContextBuilder.build(pc)
|
||||
contextFut.map { ctx => (pc, ctx) }
|
||||
})
|
||||
candidatesAndContextsFut.flatMap { candidatesAndContexts =>
|
||||
val contexts = candidatesAndContexts.map(_._2)
|
||||
val resultsFut = featureHydrator.hydrateCandidate(contexts, mrRequestContextForFeatureStore)
|
||||
resultsFut.map { hydrationResult =>
|
||||
candidatesAndContexts.foreach {
|
||||
case (pushCandidate, context) =>
|
||||
val resultFeatures = hydrationResult.getOrElse(context, FeatureMap())
|
||||
pushCandidate.mergeFeatures(resultFeatures)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def candidateFeatureHydration(
|
||||
candidateDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
mrRequestContextForFeatureStore: MrRequestContextForFeatureStore
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
candidateDetails.headOption match {
|
||||
case Some(cand) =>
|
||||
val target = cand.candidate.target
|
||||
MrUserStateUtil.updateMrUserStateStats(target)(mrUserStateStat)
|
||||
if (target.params(PushParams.DisableAllRelevanceParam)) {
|
||||
Future.value(candidateDetails)
|
||||
} else {
|
||||
featureHydrationEnabledCounter.incr()
|
||||
for {
|
||||
_ <- hydrateFromRelevanceHydrator(candidateDetails, mrRequestContextForFeatureStore)
|
||||
} yield {
|
||||
candidateDetails
|
||||
}
|
||||
}
|
||||
case _ => Future.Nil
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,104 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.finagle.stats.Counter
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base._
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.predicate.PreRankingPredicates
|
||||
import com.twitter.hermit.predicate.NamedPredicate
|
||||
import com.twitter.hermit.predicate.SequentialPredicate
|
||||
import com.twitter.util._
|
||||
|
||||
class RFPHPrerankFilter(
|
||||
)(
|
||||
globalStats: StatsReceiver) {
|
||||
def filter(
|
||||
target: Target,
|
||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[
|
||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
||||
] = {
|
||||
lazy val filterStats: StatsReceiver = globalStats.scope("RefreshForPushHandler/filter")
|
||||
lazy val okFilterCounter: Counter = filterStats.counter("ok")
|
||||
lazy val invalidFilterCounter: Counter = filterStats.counter("invalid")
|
||||
lazy val invalidFilterStat: StatsReceiver = filterStats.scope("invalid")
|
||||
lazy val invalidFilterReasonStat: StatsReceiver = invalidFilterStat.scope("reason")
|
||||
val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered")
|
||||
|
||||
lazy val preRankingPredicates = PreRankingPredicates(
|
||||
filterStats.scope("predicates")
|
||||
)
|
||||
|
||||
lazy val preRankingPredicateChain =
|
||||
new SequentialPredicate[PushCandidate](preRankingPredicates)
|
||||
|
||||
val predicateChain = if (target.pushContext.exists(_.predicatesToEnable.exists(_.nonEmpty))) {
|
||||
val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil)
|
||||
new SequentialPredicate[PushCandidate](preRankingPredicates.filter { pred =>
|
||||
predicatesToEnable.contains(pred.name)
|
||||
})
|
||||
} else preRankingPredicateChain
|
||||
|
||||
predicateChain
|
||||
.track(hydratedCandidates.map(_.candidate))
|
||||
.map { results =>
|
||||
val resultForPreRankFiltering = results
|
||||
.zip(hydratedCandidates)
|
||||
.foldLeft(
|
||||
(
|
||||
Seq.empty[CandidateDetails[PushCandidate]],
|
||||
Seq.empty[CandidateResult[PushCandidate, Result]]
|
||||
)
|
||||
) {
|
||||
case ((goodCandidates, filteredCandidates), (result, candidateDetails)) =>
|
||||
result match {
|
||||
case None =>
|
||||
okFilterCounter.incr()
|
||||
(goodCandidates :+ candidateDetails, filteredCandidates)
|
||||
|
||||
case Some(pred: NamedPredicate[_]) =>
|
||||
invalidFilterCounter.incr()
|
||||
invalidFilterReasonStat.counter(pred.name).incr()
|
||||
invalidFilterReasonStat
|
||||
.scope(candidateDetails.candidate.commonRecType.toString).counter(
|
||||
pred.name).incr()
|
||||
|
||||
val r = Invalid(Some(pred.name))
|
||||
(
|
||||
goodCandidates,
|
||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
||||
candidateDetails.candidate,
|
||||
candidateDetails.source,
|
||||
r
|
||||
)
|
||||
)
|
||||
case Some(_) =>
|
||||
invalidFilterCounter.incr()
|
||||
invalidFilterReasonStat.counter("unknown").incr()
|
||||
invalidFilterReasonStat
|
||||
.scope(candidateDetails.candidate.commonRecType.toString).counter(
|
||||
"unknown").incr()
|
||||
|
||||
val r = Invalid(Some("Filtered by un-named predicate"))
|
||||
(
|
||||
goodCandidates,
|
||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
||||
candidateDetails.candidate,
|
||||
candidateDetails.source,
|
||||
r
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
resultForPreRankFiltering match {
|
||||
case (validCandidates, _) if validCandidates.isEmpty && hydratedCandidates.nonEmpty =>
|
||||
allCandidatesFilteredPreRank.incr()
|
||||
case _ => ()
|
||||
}
|
||||
|
||||
resultForPreRankFiltering
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,34 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.TargetUser
|
||||
import com.twitter.frigate.common.candidate.TargetABDecider
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
||||
import com.twitter.frigate.pushservice.target.TargetScoringDetails
|
||||
|
||||
class RFPHRestrictStep()(implicit stats: StatsReceiver) {
|
||||
|
||||
private val statsReceiver: StatsReceiver = stats.scope("RefreshForPushHandler")
|
||||
private val restrictStepStats: StatsReceiver = statsReceiver.scope("restrict")
|
||||
private val restrictStepNumCandidatesDroppedStat: Stat =
|
||||
restrictStepStats.stat("candidates_dropped")
|
||||
|
||||
/**
|
||||
* Limit the number of candidates that enter the Take step
|
||||
*/
|
||||
def restrict(
|
||||
target: TargetUser with TargetABDecider with TargetScoringDetails,
|
||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): (Seq[CandidateDetails[PushCandidate]], Seq[CandidateDetails[PushCandidate]]) = {
|
||||
if (target.params(PushFeatureSwitchParams.EnableRestrictStep)) {
|
||||
val restrictSizeParam = PushFeatureSwitchParams.RestrictStepSize
|
||||
val (newCandidates, filteredCandidates) = candidates.splitAt(target.params(restrictSizeParam))
|
||||
val numDropped = candidates.length - newCandidates.length
|
||||
restrictStepNumCandidatesDroppedStat.add(numDropped)
|
||||
(newCandidates, filteredCandidates)
|
||||
} else (candidates, Seq.empty)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,77 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
|
||||
class RFPHStatsRecorder(implicit statsReceiver: StatsReceiver) {
|
||||
|
||||
private val selectedCandidateScoreStats: StatsReceiver =
|
||||
statsReceiver.scope("score_of_sent_candidate_times_10000")
|
||||
|
||||
private val emptyScoreStats: StatsReceiver =
|
||||
statsReceiver.scope("score_of_sent_candidate_empty")
|
||||
|
||||
def trackPredictionScoreStats(candidate: PushCandidate): Unit = {
|
||||
candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach {
|
||||
case Some(s) =>
|
||||
selectedCandidateScoreStats
|
||||
.stat("weighted_open_or_ntab_click_ranking")
|
||||
.add((s * 10000).toFloat)
|
||||
case None =>
|
||||
emptyScoreStats.counter("weighted_open_or_ntab_click_ranking").incr()
|
||||
}
|
||||
candidate.mrWeightedOpenOrNtabClickFilteringProbability.foreach {
|
||||
case Some(s) =>
|
||||
selectedCandidateScoreStats
|
||||
.stat("weighted_open_or_ntab_click_filtering")
|
||||
.add((s * 10000).toFloat)
|
||||
case None =>
|
||||
emptyScoreStats.counter("weighted_open_or_ntab_click_filtering").incr()
|
||||
}
|
||||
candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach {
|
||||
case Some(s) =>
|
||||
selectedCandidateScoreStats
|
||||
.scope(candidate.commonRecType.toString)
|
||||
.stat("weighted_open_or_ntab_click_ranking")
|
||||
.add((s * 10000).toFloat)
|
||||
case None =>
|
||||
emptyScoreStats
|
||||
.scope(candidate.commonRecType.toString)
|
||||
.counter("weighted_open_or_ntab_click_ranking")
|
||||
.incr()
|
||||
}
|
||||
}
|
||||
|
||||
def refreshRequestExceptionStats(
|
||||
exception: Throwable,
|
||||
bStats: StatsReceiver
|
||||
): Unit = {
|
||||
bStats.counter("failures").incr()
|
||||
bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr()
|
||||
}
|
||||
|
||||
def loggedOutRequestExceptionStats(
|
||||
exception: Throwable,
|
||||
bStats: StatsReceiver
|
||||
): Unit = {
|
||||
bStats.counter("logged_out_failures").incr()
|
||||
bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr()
|
||||
}
|
||||
|
||||
def rankDistributionStats(
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
numRecsPerTypeStat: (CommonRecommendationType => Stat)
|
||||
): Unit = {
|
||||
candidatesDetails
|
||||
.groupBy { c =>
|
||||
c.candidate.commonRecType
|
||||
}
|
||||
.mapValues { s =>
|
||||
s.size
|
||||
}
|
||||
.foreach { case (crt, numRecs) => numRecsPerTypeStat(crt).add(numRecs) }
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,292 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.finagle.stats.Counter
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.Stats.track
|
||||
import com.twitter.frigate.common.base.Stats.trackSeq
|
||||
import com.twitter.frigate.common.base._
|
||||
import com.twitter.frigate.common.logger.MRLogger
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.adaptor._
|
||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
||||
import com.twitter.frigate.pushservice.rank.RFPHLightRanker
|
||||
import com.twitter.frigate.pushservice.rank.RFPHRanker
|
||||
import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler
|
||||
import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator
|
||||
import com.twitter.frigate.pushservice.target.PushTargetUserBuilder
|
||||
import com.twitter.frigate.pushservice.target.RFPHTargetPredicates
|
||||
import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil
|
||||
import com.twitter.frigate.pushservice.util.AdhocStatsUtil
|
||||
import com.twitter.frigate.pushservice.thriftscala.PushContext
|
||||
import com.twitter.frigate.pushservice.thriftscala.RefreshRequest
|
||||
import com.twitter.frigate.pushservice.thriftscala.RefreshResponse
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.hermit.predicate.Predicate
|
||||
import com.twitter.timelines.configapi.FeatureValue
|
||||
import com.twitter.util._
|
||||
|
||||
case class ResultWithDebugInfo(result: Result, predicateResults: Seq[PredicateWithResult])
|
||||
|
||||
class RefreshForPushHandler(
|
||||
val pushTargetUserBuilder: PushTargetUserBuilder,
|
||||
val candSourceGenerator: PushCandidateSourceGenerator,
|
||||
rfphRanker: RFPHRanker,
|
||||
candidateHydrator: PushCandidateHydrator,
|
||||
candidateValidator: RFPHCandidateValidator,
|
||||
rfphTakeStepUtil: RFPHTakeStepUtil,
|
||||
rfphRestrictStep: RFPHRestrictStep,
|
||||
val rfphNotifier: RefreshForPushNotifier,
|
||||
rfphStatsRecorder: RFPHStatsRecorder,
|
||||
mrRequestScriberNode: String,
|
||||
rfphFeatureHydrator: RFPHFeatureHydrator,
|
||||
rfphPrerankFilter: RFPHPrerankFilter,
|
||||
rfphLightRanker: RFPHLightRanker
|
||||
)(
|
||||
globalStats: StatsReceiver)
|
||||
extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] {
|
||||
|
||||
val log = MRLogger("RefreshForPushHandler")
|
||||
|
||||
implicit val statsReceiver: StatsReceiver =
|
||||
globalStats.scope("RefreshForPushHandler")
|
||||
private val maxCandidatesToBatchInTakeStat: Stat =
|
||||
statsReceiver.stat("max_cands_to_batch_in_take")
|
||||
|
||||
private val rfphRequestCounter = statsReceiver.counter("requests")
|
||||
|
||||
private val buildTargetStats = statsReceiver.scope("build_target")
|
||||
private val processStats = statsReceiver.scope("process")
|
||||
private val notifyStats = statsReceiver.scope("notify")
|
||||
|
||||
private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking")
|
||||
private val reRankingStats: StatsReceiver = statsReceiver.scope("rerank")
|
||||
private val featureHydrationLatency: StatsReceiver =
|
||||
statsReceiver.scope("featureHydrationLatency")
|
||||
private val candidateHydrationStats: StatsReceiver = statsReceiver.scope("candidate_hydration")
|
||||
|
||||
lazy val candSourceEligibleCounter: Counter =
|
||||
candidateStats.counter("cand_source_eligible")
|
||||
lazy val candSourceNotEligibleCounter: Counter =
|
||||
candidateStats.counter("cand_source_not_eligible")
|
||||
|
||||
//pre-ranking stats
|
||||
val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered")
|
||||
|
||||
// total invalid candidates
|
||||
val totalStats: StatsReceiver = statsReceiver.scope("total")
|
||||
val totalInvalidCandidatesStat: Stat = totalStats.stat("candidates_invalid")
|
||||
|
||||
val mrRequestScribeBuiltStats: Counter = statsReceiver.counter("mr_request_scribe_built")
|
||||
|
||||
val mrRequestCandidateScribeStats = statsReceiver.scope("mr_request_scribe_candidates")
|
||||
val mrRequestTargetScribeStats = statsReceiver.scope("mr_request_scribe_target")
|
||||
|
||||
val mrRequestScribeHandler =
|
||||
new MrRequestScribeHandler(mrRequestScriberNode, statsReceiver.scope("mr_request_scribe"))
|
||||
|
||||
val adhocStatsUtil = new AdhocStatsUtil(statsReceiver.scope("adhoc_stats"))
|
||||
|
||||
private def numRecsPerTypeStat(crt: CommonRecommendationType) =
|
||||
fetchStats.scope(crt.toString).stat("dist")
|
||||
|
||||
// static list of target predicates
|
||||
private val targetPredicates = RFPHTargetPredicates(targetStats.scope("predicates"))
|
||||
|
||||
def buildTarget(
|
||||
userId: Long,
|
||||
inputPushContext: Option[PushContext],
|
||||
forcedFeatureValues: Option[Map[String, FeatureValue]] = None
|
||||
): Future[Target] =
|
||||
pushTargetUserBuilder.buildTarget(userId, inputPushContext, forcedFeatureValues)
|
||||
|
||||
override def targetPredicates(target: Target): List[Predicate[Target]] = targetPredicates
|
||||
|
||||
override def isTargetValid(target: Target): Future[Result] = {
|
||||
val resultFut = if (target.skipFilters) {
|
||||
Future.value(trackTargetPredStats(None))
|
||||
} else {
|
||||
predicateSeq(target).track(Seq(target)).map { resultArr =>
|
||||
trackTargetPredStats(resultArr(0))
|
||||
}
|
||||
}
|
||||
track(targetStats)(resultFut)
|
||||
}
|
||||
|
||||
override def candidateSources(
|
||||
target: Target
|
||||
): Future[Seq[CandidateSource[Target, RawCandidate]]] = {
|
||||
Future
|
||||
.collect(candSourceGenerator.sources.map { cs =>
|
||||
cs.isCandidateSourceAvailable(target).map { isEligible =>
|
||||
if (isEligible) {
|
||||
candSourceEligibleCounter.incr()
|
||||
Some(cs)
|
||||
} else {
|
||||
candSourceNotEligibleCounter.incr()
|
||||
None
|
||||
}
|
||||
}
|
||||
}).map(_.flatten)
|
||||
}
|
||||
|
||||
override def updateCandidateCounter(
|
||||
candidateResults: Seq[CandidateResult[PushCandidate, Result]]
|
||||
): Unit = {
|
||||
candidateResults.foreach {
|
||||
case candidateResult if candidateResult.result == OK =>
|
||||
okCandidateCounter.incr()
|
||||
case candidateResult if candidateResult.result.isInstanceOf[Invalid] =>
|
||||
invalidCandidateCounter.incr()
|
||||
case _ =>
|
||||
}
|
||||
}
|
||||
|
||||
override def hydrateCandidates(
|
||||
candidates: Seq[CandidateDetails[RawCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates)
|
||||
|
||||
override def filter(
|
||||
target: Target,
|
||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[
|
||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
||||
] = rfphPrerankFilter.filter(target, hydratedCandidates)
|
||||
|
||||
def lightRankAndTake(
|
||||
target: Target,
|
||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
rfphLightRanker.rank(target, candidates)
|
||||
}
|
||||
|
||||
override def rank(
|
||||
target: Target,
|
||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
val featureHydratedCandidatesFut = trackSeq(featureHydrationLatency)(
|
||||
rfphFeatureHydrator
|
||||
.candidateFeatureHydration(candidatesDetails, target.mrRequestContextForFeatureStore)
|
||||
)
|
||||
featureHydratedCandidatesFut.flatMap { featureHydratedCandidates =>
|
||||
rfphStatsRecorder.rankDistributionStats(featureHydratedCandidates, numRecsPerTypeStat)
|
||||
rfphRanker.initialRank(target, candidatesDetails)
|
||||
}
|
||||
}
|
||||
|
||||
def reRank(
|
||||
target: Target,
|
||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
rfphRanker.reRank(target, rankedCandidates)
|
||||
}
|
||||
|
||||
override def validCandidates(
|
||||
target: Target,
|
||||
candidates: Seq[PushCandidate]
|
||||
): Future[Seq[Result]] = {
|
||||
Future.collect(candidates.map { candidate =>
|
||||
rfphTakeStepUtil.isCandidateValid(candidate, candidateValidator).map(res => res.result)
|
||||
})
|
||||
}
|
||||
|
||||
override def desiredCandidateCount(target: Target): Int = target.desiredCandidateCount
|
||||
|
||||
override def batchForCandidatesCheck(target: Target): Int = {
|
||||
val fsParam = PushFeatureSwitchParams.NumberOfMaxCandidatesToBatchInRFPHTakeStep
|
||||
val maxToBatch = target.params(fsParam)
|
||||
maxCandidatesToBatchInTakeStat.add(maxToBatch)
|
||||
maxToBatch
|
||||
}
|
||||
|
||||
override def process(
|
||||
target: Target,
|
||||
externalCandidates: Seq[RawCandidate] = Nil
|
||||
): Future[Response[PushCandidate, Result]] = {
|
||||
isTargetValid(target).flatMap {
|
||||
case OK =>
|
||||
for {
|
||||
candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target))
|
||||
externalCandidateDetails = externalCandidates.map(
|
||||
CandidateDetails(_, "refresh_for_push_handler_external_candidate"))
|
||||
allCandidates = candidatesFromSources ++ externalCandidateDetails
|
||||
hydratedCandidatesWithCopy <-
|
||||
trackSeq(candidateHydrationStats)(hydrateCandidates(allCandidates))
|
||||
_ = adhocStatsUtil.getCandidateSourceStats(hydratedCandidatesWithCopy)
|
||||
(candidates, preRankingFilteredCandidates) <-
|
||||
track(filterStats)(filter(target, hydratedCandidatesWithCopy))
|
||||
_ = adhocStatsUtil.getPreRankingFilterStats(preRankingFilteredCandidates)
|
||||
lightRankerFilteredCandidates <-
|
||||
trackSeq(lightRankingStats)(lightRankAndTake(target, candidates))
|
||||
_ = adhocStatsUtil.getLightRankingStats(lightRankerFilteredCandidates)
|
||||
rankedCandidates <- trackSeq(rankingStats)(rank(target, lightRankerFilteredCandidates))
|
||||
_ = adhocStatsUtil.getRankingStats(rankedCandidates)
|
||||
rerankedCandidates <- trackSeq(reRankingStats)(reRank(target, rankedCandidates))
|
||||
_ = adhocStatsUtil.getReRankingStats(rerankedCandidates)
|
||||
(restrictedCandidates, restrictFilteredCandidates) =
|
||||
rfphRestrictStep.restrict(target, rerankedCandidates)
|
||||
allTakeCandidateResults <- track(takeStats)(
|
||||
take(target, restrictedCandidates, desiredCandidateCount(target))
|
||||
)
|
||||
_ = adhocStatsUtil.getTakeCandidateResultStats(allTakeCandidateResults)
|
||||
_ <- track(mrRequestCandidateScribeStats)(
|
||||
mrRequestScribeHandler.scribeForCandidateFiltering(
|
||||
target,
|
||||
hydratedCandidatesWithCopy,
|
||||
preRankingFilteredCandidates,
|
||||
rankedCandidates,
|
||||
rerankedCandidates,
|
||||
restrictFilteredCandidates,
|
||||
allTakeCandidateResults
|
||||
))
|
||||
} yield {
|
||||
|
||||
/**
|
||||
* Take processes post restrict step candidates and returns both:
|
||||
* 1. valid + invalid candidates
|
||||
* 2. Candidates that are not processed (more than desired) + restricted candidates
|
||||
* We need #2 only for importance sampling
|
||||
*/
|
||||
val takeCandidateResults =
|
||||
allTakeCandidateResults.filterNot { candResult =>
|
||||
candResult.result == MoreThanDesiredCandidates
|
||||
}
|
||||
|
||||
val totalInvalidCandidates = {
|
||||
preRankingFilteredCandidates.size + //pre-ranking filtered candidates
|
||||
(rerankedCandidates.length - restrictedCandidates.length) + //candidates reject in restrict step
|
||||
takeCandidateResults.count(_.result != OK) //candidates reject in take step
|
||||
}
|
||||
takeInvalidCandidateDist.add(
|
||||
takeCandidateResults
|
||||
.count(_.result != OK)
|
||||
) // take step invalid candidates
|
||||
totalInvalidCandidatesStat.add(totalInvalidCandidates)
|
||||
val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates
|
||||
Response(OK, allCandidateResults)
|
||||
}
|
||||
|
||||
case result: Result =>
|
||||
for (_ <- track(mrRequestTargetScribeStats)(
|
||||
mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield {
|
||||
mrRequestScribeBuiltStats.incr()
|
||||
Response(result, Nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def refreshAndSend(request: RefreshRequest): Future[RefreshResponse] = {
|
||||
rfphRequestCounter.incr()
|
||||
for {
|
||||
target <- track(buildTargetStats)(
|
||||
pushTargetUserBuilder
|
||||
.buildTarget(request.userId, request.context))
|
||||
response <- track(processStats)(process(target, externalCandidates = Seq.empty))
|
||||
refreshResponse <- track(notifyStats)(rfphNotifier.checkResponseAndNotify(response, target))
|
||||
} yield {
|
||||
refreshResponse
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,128 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler
|
||||
|
||||
import com.twitter.finagle.stats.BroadcastStatsReceiver
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.Stats.track
|
||||
import com.twitter.frigate.common.base._
|
||||
import com.twitter.frigate.common.config.CommonConstants
|
||||
import com.twitter.frigate.common.util.PushServiceUtil.FilteredRefreshResponseFut
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.take.CandidateNotifier
|
||||
import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest
|
||||
import com.twitter.frigate.pushservice.thriftscala.PushStatus
|
||||
import com.twitter.frigate.pushservice.thriftscala.RefreshResponse
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.JavaTimer
|
||||
import com.twitter.util.Timer
|
||||
|
||||
class RefreshForPushNotifier(
|
||||
rfphStatsRecorder: RFPHStatsRecorder,
|
||||
candidateNotifier: CandidateNotifier
|
||||
)(
|
||||
globalStats: StatsReceiver) {
|
||||
|
||||
private implicit val statsReceiver: StatsReceiver =
|
||||
globalStats.scope("RefreshForPushHandler")
|
||||
|
||||
private val pushStats: StatsReceiver = statsReceiver.scope("push")
|
||||
private val sendLatency: StatsReceiver = statsReceiver.scope("send_handler")
|
||||
implicit private val timer: Timer = new JavaTimer(true)
|
||||
|
||||
private def notify(
|
||||
candidatesResult: CandidateResult[PushCandidate, Result],
|
||||
target: Target,
|
||||
receivers: Seq[StatsReceiver]
|
||||
): Future[RefreshResponse] = {
|
||||
|
||||
val candidate = candidatesResult.candidate
|
||||
|
||||
val predsResult = candidatesResult.result
|
||||
|
||||
if (predsResult != OK) {
|
||||
val invalidResult = predsResult
|
||||
invalidResult match {
|
||||
case Invalid(Some(reason)) =>
|
||||
Future.value(RefreshResponse(PushStatus.Filtered, Some(reason)))
|
||||
case _ =>
|
||||
Future.value(RefreshResponse(PushStatus.Filtered, None))
|
||||
}
|
||||
} else {
|
||||
rfphStatsRecorder.trackPredictionScoreStats(candidate)
|
||||
|
||||
val isQualityUprankingCandidate = candidate.mrQualityUprankingBoost.isDefined
|
||||
val commonRecTypeStats = Seq(
|
||||
statsReceiver.scope(candidate.commonRecType.toString),
|
||||
globalStats.scope(candidate.commonRecType.toString)
|
||||
)
|
||||
val qualityUprankingStats = Seq(
|
||||
statsReceiver.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString),
|
||||
globalStats.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString)
|
||||
)
|
||||
|
||||
val receiversWithRecTypeStats = {
|
||||
if (isQualityUprankingCandidate) {
|
||||
receivers ++ commonRecTypeStats ++ qualityUprankingStats
|
||||
} else {
|
||||
receivers ++ commonRecTypeStats
|
||||
}
|
||||
}
|
||||
track(sendLatency)(candidateNotifier.notify(candidate).map { res =>
|
||||
trackStatsForResponseToRequest(
|
||||
candidate.commonRecType,
|
||||
candidate.target,
|
||||
res,
|
||||
receiversWithRecTypeStats
|
||||
)(globalStats)
|
||||
RefreshResponse(res.status)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
def checkResponseAndNotify(
|
||||
response: Response[PushCandidate, Result],
|
||||
targetUserContext: Target
|
||||
): Future[RefreshResponse] = {
|
||||
val receivers = Seq(statsReceiver)
|
||||
val refreshResponse = response match {
|
||||
case Response(OK, processedCandidates) =>
|
||||
// valid rec candidates
|
||||
val validCandidates = processedCandidates.filter(_.result == OK)
|
||||
|
||||
// top rec candidate
|
||||
validCandidates.headOption match {
|
||||
case Some(candidatesResult) =>
|
||||
candidatesResult.result match {
|
||||
case OK =>
|
||||
notify(candidatesResult, targetUserContext, receivers)
|
||||
.onSuccess { nr =>
|
||||
pushStats.scope("result").counter(nr.status.name).incr()
|
||||
}
|
||||
case _ =>
|
||||
targetUserContext.isTeamMember.flatMap { isTeamMember =>
|
||||
FilteredRefreshResponseFut
|
||||
}
|
||||
}
|
||||
case _ =>
|
||||
FilteredRefreshResponseFut
|
||||
}
|
||||
case Response(Invalid(reason), _) =>
|
||||
// invalid target with known reason
|
||||
FilteredRefreshResponseFut.map(_.copy(targetFilteredBy = reason))
|
||||
case _ =>
|
||||
// invalid target
|
||||
FilteredRefreshResponseFut
|
||||
}
|
||||
|
||||
val bStats = BroadcastStatsReceiver(receivers)
|
||||
Stat
|
||||
.timeFuture(bStats.stat("latency"))(
|
||||
refreshResponse
|
||||
.raiseWithin(CommonConstants.maxPushRequestDuration)
|
||||
)
|
||||
.onFailure { exception =>
|
||||
rfphStatsRecorder.refreshRequestExceptionStats(exception, bStats)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,79 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.util.MRNtabCopy
|
||||
import com.twitter.frigate.common.util.MRPushCopy
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.util.Future
|
||||
|
||||
abstract class BaseCopyFramework(statsReceiver: StatsReceiver) {
|
||||
|
||||
private val NoAvailableCopyStat = statsReceiver.scope("no_copy_for_crt")
|
||||
private val NoAvailableNtabCopyStat = statsReceiver.scope("no_ntab_copy")
|
||||
|
||||
/**
|
||||
* Instantiate push copy filters
|
||||
*/
|
||||
protected final val copyFilters = new CopyFilters(statsReceiver.scope("filters"))
|
||||
|
||||
/**
|
||||
*
|
||||
* The following method fetches all the push copies for a [[com.twitter.frigate.thriftscala.CommonRecommendationType]]
|
||||
* associated with a candidate and then filters the eligible copies based on
|
||||
* [[PushTypes.PushCandidate]] features. These filters are defined in
|
||||
* [[CopyFilters]]
|
||||
*
|
||||
* @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate
|
||||
*
|
||||
* @return - set of eligible push copies for a given candidate
|
||||
*/
|
||||
protected[cross] final def getEligiblePushCopiesFromCandidate(
|
||||
rawCandidate: RawCandidate
|
||||
): Future[Seq[MRPushCopy]] = {
|
||||
val pushCopiesFromRectype = CandidateToCopy.getPushCopiesFromRectype(rawCandidate.commonRecType)
|
||||
|
||||
if (pushCopiesFromRectype.isEmpty) {
|
||||
NoAvailableCopyStat.counter(rawCandidate.commonRecType.name).incr()
|
||||
throw new IllegalStateException(s"No Copy defined for CRT: " + rawCandidate.commonRecType)
|
||||
}
|
||||
pushCopiesFromRectype
|
||||
.map(pushCopySet => copyFilters.execute(rawCandidate, pushCopySet.toSeq))
|
||||
.getOrElse(Future.value(Seq.empty))
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* This method essentially forms the base for cross-step for the MagicRecs Copy Framework. Given
|
||||
* a recommendation type this returns a set of tuples wherein each tuple is a pair of push and
|
||||
* ntab copy eligible for the said recommendation type
|
||||
*
|
||||
* @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate
|
||||
* @return - Set of eligible [[MRPushCopy]], Option[[MRNtabCopy]] for a given recommendation type
|
||||
*/
|
||||
protected[cross] final def getEligiblePushAndNtabCopiesFromCandidate(
|
||||
rawCandidate: RawCandidate
|
||||
): Future[Seq[(MRPushCopy, Option[MRNtabCopy])]] = {
|
||||
|
||||
val eligiblePushCopies = getEligiblePushCopiesFromCandidate(rawCandidate)
|
||||
|
||||
eligiblePushCopies.map { pushCopies =>
|
||||
val setBuilder = Set.newBuilder[(MRPushCopy, Option[MRNtabCopy])]
|
||||
pushCopies.foreach { pushCopy =>
|
||||
val ntabCopies = CandidateToCopy.getNtabcopiesFromPushcopy(pushCopy)
|
||||
val pushNtabCopyPairs = ntabCopies match {
|
||||
case Some(ntabCopySet) =>
|
||||
if (ntabCopySet.isEmpty) {
|
||||
NoAvailableNtabCopyStat.counter(s"copy_id: ${pushCopy.copyId}").incr()
|
||||
Set(pushCopy -> None)
|
||||
} // push copy only
|
||||
else ntabCopySet.map(pushCopy -> Some(_))
|
||||
|
||||
case None =>
|
||||
Set.empty[(MRPushCopy, Option[MRNtabCopy])] // no push or ntab copy
|
||||
}
|
||||
setBuilder ++= pushNtabCopyPairs
|
||||
}
|
||||
setBuilder.result().toSeq
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,56 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.util.MRNtabCopy
|
||||
import com.twitter.frigate.common.util.MRPushCopy
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.candidate.CopyIds
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* @param statsReceiver - stats receiver object
|
||||
*/
|
||||
class CandidateCopyExpansion(statsReceiver: StatsReceiver)
|
||||
extends BaseCopyFramework(statsReceiver) {
|
||||
|
||||
/**
|
||||
*
|
||||
* Given a [[CandidateDetails]] object representing a push recommendation candidate this method
|
||||
* expands it to multiple candidates, each tagged with a push copy id and ntab copy id to
|
||||
* represent the eligible copies for the given recommendation candidate
|
||||
*
|
||||
* @param candidateDetails - [[CandidateDetails]] objects containing a recommendation candidate
|
||||
*
|
||||
* @return - list of tuples of [[PushTypes.RawCandidate]] and [[CopyIds]]
|
||||
*/
|
||||
private final def crossCandidateDetailsWithCopyId(
|
||||
candidateDetails: CandidateDetails[RawCandidate]
|
||||
): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = {
|
||||
val eligibleCopyPairs = getEligiblePushAndNtabCopiesFromCandidate(candidateDetails.candidate)
|
||||
val copyPairs = eligibleCopyPairs.map(_.map {
|
||||
case (pushCopy: MRPushCopy, ntabCopy: Option[MRNtabCopy]) =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(pushCopy.copyId),
|
||||
ntabCopyId = ntabCopy.map(_.copyId)
|
||||
)
|
||||
})
|
||||
|
||||
copyPairs.map(_.map((candidateDetails, _)))
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* This method takes as input a list of [[CandidateDetails]] objects which contain the push
|
||||
* recommendation candidates for a given target user. It expands each input candidate into
|
||||
* multiple candidates, each tagged with a push copy id and ntab copy id to represent the eligible
|
||||
* copies for the given recommendation candidate
|
||||
*
|
||||
* @param candidateDetailsSeq - list of fetched candidates for push recommendation
|
||||
* @return - list of tuples of [[RawCandidate]] and [[CopyIds]]
|
||||
*/
|
||||
final def expandCandidatesWithCopyId(
|
||||
candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]]
|
||||
): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] =
|
||||
Future.collect(candidateDetailsSeq.map(crossCandidateDetailsWithCopyId)).map(_.flatten)
|
||||
}
|
||||
Binary file not shown.
@ -1,11 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
||||
|
||||
import com.twitter.frigate.common.util.MRPushCopy
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
|
||||
/**
|
||||
*
|
||||
* @param candidate: [[RawCandidate]] is a recommendation candidate
|
||||
* @param pushCopy: [[MRPushCopy]] eligible for candidate
|
||||
*/
|
||||
case class CandidateCopyPair(candidate: RawCandidate, pushCopy: MRPushCopy)
|
||||
Binary file not shown.
@ -1,263 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
||||
|
||||
import com.twitter.frigate.common.util.MrNtabCopyObjects
|
||||
import com.twitter.frigate.common.util.MrPushCopyObjects
|
||||
import com.twitter.frigate.common.util._
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType._
|
||||
|
||||
object CandidateToCopy {
|
||||
|
||||
// Static map from a CommonRecommendationType to set of eligible push notification copies
|
||||
private[cross] val rectypeToPushCopy: Map[CommonRecommendationType, Set[
|
||||
MRPushCopy
|
||||
]] =
|
||||
Map[CommonRecommendationType, Set[MRPushCopy]](
|
||||
F1FirstdegreeTweet -> Set(
|
||||
MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle
|
||||
),
|
||||
F1FirstdegreePhoto -> Set(
|
||||
MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle
|
||||
),
|
||||
F1FirstdegreeVideo -> Set(
|
||||
MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle
|
||||
),
|
||||
TweetRetweet -> Set(
|
||||
MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText,
|
||||
MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText,
|
||||
MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText
|
||||
),
|
||||
TweetRetweetPhoto -> Set(
|
||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText,
|
||||
MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText,
|
||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText
|
||||
),
|
||||
TweetRetweetVideo -> Set(
|
||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText,
|
||||
MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText,
|
||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText
|
||||
),
|
||||
TweetFavorite -> Set(
|
||||
MrPushCopyObjects.TweetLikeOneSocialContextWithText,
|
||||
MrPushCopyObjects.TweetLikeTwoSocialContextWithText,
|
||||
MrPushCopyObjects.TweetLikeMultipleSocialContextWithText
|
||||
),
|
||||
TweetFavoritePhoto -> Set(
|
||||
MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText,
|
||||
MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText,
|
||||
MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText
|
||||
),
|
||||
TweetFavoriteVideo -> Set(
|
||||
MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText,
|
||||
MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText,
|
||||
MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText
|
||||
),
|
||||
UnreadBadgeCount -> Set(MrPushCopyObjects.UnreadBadgeCount),
|
||||
InterestBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
InterestBasedPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
||||
InterestBasedVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
||||
UserFollow -> Set(
|
||||
MrPushCopyObjects.UserFollowWithOneSocialContext,
|
||||
MrPushCopyObjects.UserFollowWithTwoSocialContext,
|
||||
MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext
|
||||
),
|
||||
HermitUser -> Set(
|
||||
MrPushCopyObjects.HermitUserWithOneSocialContext,
|
||||
MrPushCopyObjects.HermitUserWithTwoSocialContext,
|
||||
MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
TriangularLoopUser -> Set(
|
||||
MrPushCopyObjects.TriangularLoopUserWithOneSocialContext,
|
||||
MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts,
|
||||
MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext
|
||||
),
|
||||
ForwardAddressbookUserFollow -> Set(MrPushCopyObjects.ForwardAddressBookUserFollow),
|
||||
NewsArticleNewsLanding -> Set(MrPushCopyObjects.NewsArticleNewsLandingCopy),
|
||||
TopicProofTweet -> Set(MrPushCopyObjects.TopicProofTweet),
|
||||
UserInterestinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
UserInterestinPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
||||
UserInterestinVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
||||
TwistlyTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
TwistlyPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
||||
TwistlyVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
||||
ElasticTimelineTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
ElasticTimelinePhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
||||
ElasticTimelineVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
||||
ExploreVideoTweet -> Set(MrPushCopyObjects.ExploreVideoTweet),
|
||||
List -> Set(MrPushCopyObjects.ListRecommendation),
|
||||
InterestBasedUserFollow -> Set(MrPushCopyObjects.UserFollowInterestBasedCopy),
|
||||
PastEmailEngagementTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
PastEmailEngagementPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
||||
PastEmailEngagementVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
||||
ExplorePush -> Set(MrPushCopyObjects.ExplorePush),
|
||||
ConnectTabPush -> Set(MrPushCopyObjects.ConnectTabPush),
|
||||
ConnectTabWithUserPush -> Set(MrPushCopyObjects.ConnectTabWithUserPush),
|
||||
AddressBookUploadPush -> Set(MrPushCopyObjects.AddressBookPush),
|
||||
InterestPickerPush -> Set(MrPushCopyObjects.InterestPickerPush),
|
||||
CompleteOnboardingPush -> Set(MrPushCopyObjects.CompleteOnboardingPush),
|
||||
GeoPopTweet -> Set(MrPushCopyObjects.GeoPopPushCopy),
|
||||
TagSpaceTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
FrsTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
TwhinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
MrModelingBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
DetopicTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
TweetImpressions -> Set(MrPushCopyObjects.TopTweetImpressions),
|
||||
TrendTweet -> Set(MrPushCopyObjects.TrendTweet),
|
||||
ReverseAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
ForwardAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
SpaceInNetwork -> Set(MrPushCopyObjects.SpaceHost),
|
||||
SpaceOutOfNetwork -> Set(MrPushCopyObjects.SpaceHost),
|
||||
SubscribedSearch -> Set(MrPushCopyObjects.SubscribedSearchTweet),
|
||||
TripGeoTweet -> Set(MrPushCopyObjects.TripGeoTweetPushCopy),
|
||||
CrowdSearchTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
||||
Digest -> Set(MrPushCopyObjects.Digest),
|
||||
TripHqTweet -> Set(MrPushCopyObjects.TripHqTweetPushCopy)
|
||||
)
|
||||
|
||||
// Static map from a push copy to set of eligible ntab copies
|
||||
private[cross] val pushcopyToNtabcopy: Map[MRPushCopy, Set[MRNtabCopy]] =
|
||||
Map[MRPushCopy, Set[MRNtabCopy]](
|
||||
MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle -> Set(
|
||||
MrNtabCopyObjects.FirstDegreeTweetRecent),
|
||||
MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle -> Set(
|
||||
MrNtabCopyObjects.FirstDegreeTweetRecent
|
||||
),
|
||||
MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle -> Set(
|
||||
MrNtabCopyObjects.FirstDegreeTweetRecent
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetVideoWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText -> Set(
|
||||
MrNtabCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetLikeOneSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikeWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.TweetLikeTwoSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikeWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetLikeMultipleSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikeWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikePhotoWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikePhotoWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikePhotoWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikeVideoWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikeVideoWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText -> Set(
|
||||
MrNtabCopyObjects.TweetLikeVideoWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.UnreadBadgeCount -> Set.empty[MRNtabCopy],
|
||||
MrPushCopyObjects.RecommendedForYouTweet -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
||||
MrPushCopyObjects.RecommendedForYouPhoto -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
||||
MrPushCopyObjects.RecommendedForYouVideo -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
||||
MrPushCopyObjects.GeoPopPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
||||
MrPushCopyObjects.UserFollowWithOneSocialContext -> Set(
|
||||
MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.UserFollowWithTwoSocialContext -> Set(
|
||||
MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext -> Set(
|
||||
MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.HermitUserWithOneSocialContext -> Set(
|
||||
MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext
|
||||
),
|
||||
MrPushCopyObjects.HermitUserWithTwoSocialContext -> Set(
|
||||
MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts
|
||||
),
|
||||
MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts -> Set(
|
||||
MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TriangularLoopUserWithOneSocialContext -> Set(
|
||||
MrNtabCopyObjects.TriangularLoopUserWithOneSocialContext
|
||||
),
|
||||
MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts -> Set(
|
||||
MrNtabCopyObjects.TriangularLoopUserWithTwoSocialContexts
|
||||
),
|
||||
MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext -> Set(
|
||||
MrNtabCopyObjects.TriangularLoopUserOneDisplayAndKOtherSocialContext
|
||||
),
|
||||
MrPushCopyObjects.NewsArticleNewsLandingCopy -> Set(
|
||||
MrNtabCopyObjects.NewsArticleNewsLandingCopy
|
||||
),
|
||||
MrPushCopyObjects.UserFollowInterestBasedCopy -> Set(
|
||||
MrNtabCopyObjects.UserFollowInterestBasedCopy
|
||||
),
|
||||
MrPushCopyObjects.ForwardAddressBookUserFollow -> Set(
|
||||
MrNtabCopyObjects.ForwardAddressBookUserFollow),
|
||||
MrPushCopyObjects.ConnectTabPush -> Set(
|
||||
MrNtabCopyObjects.ConnectTabPush
|
||||
),
|
||||
MrPushCopyObjects.ExplorePush -> Set.empty[MRNtabCopy],
|
||||
MrPushCopyObjects.ConnectTabWithUserPush -> Set(
|
||||
MrNtabCopyObjects.UserFollowInterestBasedCopy),
|
||||
MrPushCopyObjects.AddressBookPush -> Set(MrNtabCopyObjects.AddressBook),
|
||||
MrPushCopyObjects.InterestPickerPush -> Set(MrNtabCopyObjects.InterestPicker),
|
||||
MrPushCopyObjects.CompleteOnboardingPush -> Set(MrNtabCopyObjects.CompleteOnboarding),
|
||||
MrPushCopyObjects.TopicProofTweet -> Set(MrNtabCopyObjects.TopicProofTweet),
|
||||
MrPushCopyObjects.TopTweetImpressions -> Set(MrNtabCopyObjects.TopTweetImpressions),
|
||||
MrPushCopyObjects.TrendTweet -> Set(MrNtabCopyObjects.TrendTweet),
|
||||
MrPushCopyObjects.SpaceHost -> Set(MrNtabCopyObjects.SpaceHost),
|
||||
MrPushCopyObjects.SubscribedSearchTweet -> Set(MrNtabCopyObjects.SubscribedSearchTweet),
|
||||
MrPushCopyObjects.TripGeoTweetPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
||||
MrPushCopyObjects.Digest -> Set(MrNtabCopyObjects.Digest),
|
||||
MrPushCopyObjects.TripHqTweetPushCopy -> Set(MrNtabCopyObjects.HighQualityTweet),
|
||||
MrPushCopyObjects.ExploreVideoTweet -> Set(MrNtabCopyObjects.ExploreVideoTweet),
|
||||
MrPushCopyObjects.ListRecommendation -> Set(MrNtabCopyObjects.ListRecommendation),
|
||||
MrPushCopyObjects.MagicFanoutCreatorSubscription -> Set(
|
||||
MrNtabCopyObjects.MagicFanoutCreatorSubscription),
|
||||
MrPushCopyObjects.MagicFanoutNewCreator -> Set(MrNtabCopyObjects.MagicFanoutNewCreator)
|
||||
)
|
||||
|
||||
/**
|
||||
*
|
||||
* @param crt - [[CommonRecommendationType]] used for a frigate push notification
|
||||
*
|
||||
* @return - Set of [[MRPushCopy]] objects representing push copies eligibile for a
|
||||
* [[CommonRecommendationType]]
|
||||
*/
|
||||
def getPushCopiesFromRectype(crt: CommonRecommendationType): Option[Set[MRPushCopy]] =
|
||||
rectypeToPushCopy.get(crt)
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pushcopy - [[MRPushCopy]] object representing a push notification copy
|
||||
* @return - Set of [[MRNtabCopy]] objects that can be paired with a given [[MRPushCopy]]
|
||||
*/
|
||||
def getNtabcopiesFromPushcopy(pushcopy: MRPushCopy): Option[Set[MRNtabCopy]] =
|
||||
pushcopyToNtabcopy.get(pushcopy)
|
||||
}
|
||||
Binary file not shown.
@ -1,41 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base._
|
||||
import com.twitter.frigate.common.util.MRPushCopy
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.hermit.predicate.Predicate
|
||||
import com.twitter.util.Future
|
||||
|
||||
private[cross] class CopyFilters(statsReceiver: StatsReceiver) {
|
||||
|
||||
private val copyPredicates = new CopyPredicates(statsReceiver.scope("copy_predicate"))
|
||||
|
||||
def execute(rawCandidate: RawCandidate, pushCopies: Seq[MRPushCopy]): Future[Seq[MRPushCopy]] = {
|
||||
val candidateCopyPairs: Seq[CandidateCopyPair] =
|
||||
pushCopies.map(CandidateCopyPair(rawCandidate, _))
|
||||
|
||||
val compositePredicate: Predicate[CandidateCopyPair] = rawCandidate match {
|
||||
case _: F1FirstDegree | _: OutOfNetworkTweetCandidate | _: EventCandidate |
|
||||
_: TopicProofTweetCandidate | _: ListPushCandidate | _: HermitInterestBasedUserFollow |
|
||||
_: UserFollowWithoutSocialContextCandidate | _: DiscoverTwitterCandidate |
|
||||
_: TopTweetImpressionsCandidate | _: TrendTweetCandidate |
|
||||
_: SubscribedSearchTweetCandidate | _: DigestCandidate =>
|
||||
copyPredicates.alwaysTruePredicate
|
||||
|
||||
case _: SocialContextActions => copyPredicates.displaySocialContextPredicate
|
||||
|
||||
case _ => copyPredicates.unrecognizedCandidatePredicate // block unrecognised candidates
|
||||
}
|
||||
|
||||
// apply predicate to all [[MRPushCopy]] objects
|
||||
val filterResults: Future[Seq[Boolean]] = compositePredicate(candidateCopyPairs)
|
||||
filterResults.map { results: Seq[Boolean] =>
|
||||
val seqBuilder = Seq.newBuilder[MRPushCopy]
|
||||
results.zip(pushCopies).foreach {
|
||||
case (result, pushCopy) => if (result) seqBuilder += pushCopy
|
||||
}
|
||||
seqBuilder.result()
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,36 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.SocialContextActions
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.hermit.predicate.Predicate
|
||||
|
||||
class CopyPredicates(statsReceiver: StatsReceiver) {
|
||||
val alwaysTruePredicate = Predicate
|
||||
.from { _: CandidateCopyPair =>
|
||||
true
|
||||
}.withStats(statsReceiver.scope("always_true_copy_predicate"))
|
||||
|
||||
val unrecognizedCandidatePredicate = alwaysTruePredicate.flip
|
||||
.withStats(statsReceiver.scope("unrecognized_candidate"))
|
||||
|
||||
val displaySocialContextPredicate = Predicate
|
||||
.from { candidateCopyPair: CandidateCopyPair =>
|
||||
candidateCopyPair.candidate match {
|
||||
case candidateWithScActions: RawCandidate with SocialContextActions =>
|
||||
val socialContextUserIds = candidateWithScActions.socialContextActions.map(_.userId)
|
||||
val countSocialContext = socialContextUserIds.size
|
||||
val pushCopy = candidateCopyPair.pushCopy
|
||||
|
||||
countSocialContext match {
|
||||
case 1 => pushCopy.hasOneDisplaySocialContext && !pushCopy.hasOtherSocialContext
|
||||
case 2 => pushCopy.hasTwoDisplayContext && !pushCopy.hasOtherSocialContext
|
||||
case c if c > 2 =>
|
||||
pushCopy.hasOneDisplaySocialContext && pushCopy.hasOtherSocialContext
|
||||
case _ => false
|
||||
}
|
||||
|
||||
case _ => false
|
||||
}
|
||||
}.withStats(statsReceiver.scope("display_social_context_predicate"))
|
||||
}
|
||||
Binary file not shown.
@ -1,388 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.scriber
|
||||
|
||||
import com.twitter.bijection.Base64String
|
||||
import com.twitter.bijection.Injection
|
||||
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
||||
import com.twitter.core_workflows.user_model.thriftscala.{UserState => ThriftUserState}
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.tracing.Trace
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.CandidateResult
|
||||
import com.twitter.frigate.common.base.Invalid
|
||||
import com.twitter.frigate.common.base.OK
|
||||
import com.twitter.frigate.common.base.Result
|
||||
import com.twitter.frigate.common.rec_types.RecTypes
|
||||
import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
||||
import com.twitter.frigate.pushservice.params.PushParams
|
||||
import com.twitter.frigate.scribe.thriftscala.CandidateFilteredOutStep
|
||||
import com.twitter.frigate.scribe.thriftscala.CandidateRequestInfo
|
||||
import com.twitter.frigate.scribe.thriftscala.MrRequestScribe
|
||||
import com.twitter.frigate.scribe.thriftscala.TargetUserInfo
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.frigate.thriftscala.TweetNotification
|
||||
import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction}
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.ml.api.Feature
|
||||
import com.twitter.ml.api.FeatureType
|
||||
import com.twitter.ml.api.util.SRichDataRecord
|
||||
import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions
|
||||
import com.twitter.nrel.heavyranker.PushPredictionHelper
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Time
|
||||
import java.util.UUID
|
||||
import scala.collection.mutable
|
||||
|
||||
class MrRequestScribeHandler(mrRequestScriberNode: String, stats: StatsReceiver) {
|
||||
|
||||
private val mrRequestScribeLogger = Logger(mrRequestScriberNode)
|
||||
|
||||
private val mrRequestScribeTargetFilteringStats =
|
||||
stats.counter("MrRequestScribeHandler_target_filtering")
|
||||
private val mrRequestScribeCandidateFilteringStats =
|
||||
stats.counter("MrRequestScribeHandler_candidate_filtering")
|
||||
private val mrRequestScribeInvalidStats =
|
||||
stats.counter("MrRequestScribeHandler_invalid_filtering")
|
||||
private val mrRequestScribeUnsupportedFeatureTypeStats =
|
||||
stats.counter("MrRequestScribeHandler_unsupported_feature_type")
|
||||
private val mrRequestScribeNotIncludedFeatureStats =
|
||||
stats.counter("MrRequestScribeHandler_not_included_features")
|
||||
|
||||
private final val MrRequestScribeInjection: Injection[MrRequestScribe, String] = BinaryScalaCodec(
|
||||
MrRequestScribe
|
||||
) andThen Injection.connect[Array[Byte], Base64String, String]
|
||||
|
||||
/**
|
||||
*
|
||||
* @param target : Target user id
|
||||
* @param result : Result for target filtering
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
def scribeForTargetFiltering(target: Target, result: Result): Future[Option[MrRequestScribe]] = {
|
||||
if (target.isLoggedOutUser || !enableTargetFilteringScribing(target)) {
|
||||
Future.None
|
||||
} else {
|
||||
val predicate = result match {
|
||||
case Invalid(reason) => reason
|
||||
case _ =>
|
||||
mrRequestScribeInvalidStats.incr()
|
||||
throw new IllegalStateException("Invalid reason for Target Filtering " + result)
|
||||
}
|
||||
buildScribeThrift(target, predicate, None).map { targetFilteredScribe =>
|
||||
writeAtTargetFilteringStep(target, targetFilteredScribe)
|
||||
Some(targetFilteredScribe)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param target : Target user id
|
||||
* @param hydratedCandidates : Candidates hydrated with details: impressionId, frigateNotification and source
|
||||
* @param preRankingFilteredCandidates : Candidates result filtered out at preRanking filtering step
|
||||
* @param rankedCandidates : Sorted candidates details ranked by ranking step
|
||||
* @param rerankedCandidates : Sorted candidates details ranked by reranking step
|
||||
* @param restrictFilteredCandidates : Candidates details filtered out at restrict step
|
||||
* @param allTakeCandidateResults : Candidates results at take step, include the candidates we take and the candidates filtered out at take step [with different result]
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
def scribeForCandidateFiltering(
|
||||
target: Target,
|
||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]],
|
||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
rerankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]]
|
||||
): Future[Seq[MrRequestScribe]] = {
|
||||
if (target.isLoggedOutUser || target.isEmailUser) {
|
||||
Future.Nil
|
||||
} else if (enableCandidateFilteringScribing(target)) {
|
||||
val hydrateFeature =
|
||||
target.params(PushFeatureSwitchParams.EnableMrRequestScribingWithFeatureHydrating) ||
|
||||
target.scribeFeatureForRequestScribe
|
||||
|
||||
val candidateRequestInfoSeq = generateCandidatesScribeInfo(
|
||||
hydratedCandidates,
|
||||
preRankingFilteredCandidates,
|
||||
rankedCandidates,
|
||||
rerankedCandidates,
|
||||
restrictFilteredCandidates,
|
||||
allTakeCandidateResults,
|
||||
isFeatureHydratingEnabled = hydrateFeature
|
||||
)
|
||||
val flattenStructure =
|
||||
target.params(PushFeatureSwitchParams.EnableFlattenMrRequestScribing) || hydrateFeature
|
||||
candidateRequestInfoSeq.flatMap { candidateRequestInfos =>
|
||||
if (flattenStructure) {
|
||||
Future.collect {
|
||||
candidateRequestInfos.map { candidateRequestInfo =>
|
||||
buildScribeThrift(target, None, Some(Seq(candidateRequestInfo)))
|
||||
.map { mrRequestScribe =>
|
||||
writeAtCandidateFilteringStep(target, mrRequestScribe)
|
||||
mrRequestScribe
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
buildScribeThrift(target, None, Some(candidateRequestInfos))
|
||||
.map { mrRequestScribe =>
|
||||
writeAtCandidateFilteringStep(target, mrRequestScribe)
|
||||
Seq(mrRequestScribe)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else Future.Nil
|
||||
|
||||
}
|
||||
|
||||
private def buildScribeThrift(
|
||||
target: Target,
|
||||
targetFilteredOutPredicate: Option[String],
|
||||
candidatesRequestInfo: Option[Seq[CandidateRequestInfo]]
|
||||
): Future[MrRequestScribe] = {
|
||||
Future
|
||||
.join(
|
||||
target.targetUserState,
|
||||
generateTargetFeatureScribeInfo(target),
|
||||
target.targetUser).map {
|
||||
case (userStateOption, targetFeatureOption, gizmoduckUserOpt) =>
|
||||
val userState = userStateOption.map(userState => ThriftUserState(userState.id))
|
||||
val targetFeatures =
|
||||
targetFeatureOption.map(ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord)
|
||||
val traceId = Trace.id.traceId.toLong
|
||||
|
||||
MrRequestScribe(
|
||||
requestId = UUID.randomUUID.toString.replaceAll("-", ""),
|
||||
scribedTimeMs = Time.now.inMilliseconds,
|
||||
targetUserId = target.targetId,
|
||||
targetUserInfo = Some(
|
||||
TargetUserInfo(
|
||||
userState,
|
||||
features = targetFeatures,
|
||||
userType = gizmoduckUserOpt.map(_.userType))
|
||||
),
|
||||
targetFilteredOutPredicate = targetFilteredOutPredicate,
|
||||
candidates = candidatesRequestInfo,
|
||||
traceId = Some(traceId)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private def generateTargetFeatureScribeInfo(
|
||||
target: Target
|
||||
): Future[Option[DataRecord]] = {
|
||||
val featureList =
|
||||
target.params(PushFeatureSwitchParams.TargetLevelFeatureListForMrRequestScribing)
|
||||
if (featureList.nonEmpty) {
|
||||
PushPredictionHelper
|
||||
.getDataRecordFromTargetFeatureMap(
|
||||
target.targetId,
|
||||
target.featureMap,
|
||||
stats
|
||||
).map { dataRecord =>
|
||||
val richRecord =
|
||||
new SRichDataRecord(dataRecord, PushQualityModelFeatureContext.featureContext)
|
||||
|
||||
val selectedRecord =
|
||||
SRichDataRecord(new DataRecord(), PushQualityModelFeatureContext.featureContext)
|
||||
featureList.map { featureName =>
|
||||
val feature: Feature[_] = {
|
||||
try {
|
||||
PushQualityModelFeatureContext.featureContext.getFeature(featureName)
|
||||
} catch {
|
||||
case _: Exception =>
|
||||
mrRequestScribeNotIncludedFeatureStats.incr()
|
||||
throw new IllegalStateException(
|
||||
"Scribing features not included in FeatureContext: " + featureName)
|
||||
}
|
||||
}
|
||||
|
||||
richRecord.getFeatureValueOpt(feature).foreach { featureVal =>
|
||||
feature.getFeatureType() match {
|
||||
case FeatureType.BINARY =>
|
||||
selectedRecord.setFeatureValue(
|
||||
feature.asInstanceOf[Feature[Boolean]],
|
||||
featureVal.asInstanceOf[Boolean])
|
||||
case FeatureType.CONTINUOUS =>
|
||||
selectedRecord.setFeatureValue(
|
||||
feature.asInstanceOf[Feature[Double]],
|
||||
featureVal.asInstanceOf[Double])
|
||||
case FeatureType.STRING =>
|
||||
selectedRecord.setFeatureValue(
|
||||
feature.asInstanceOf[Feature[String]],
|
||||
featureVal.asInstanceOf[String])
|
||||
case FeatureType.DISCRETE =>
|
||||
selectedRecord.setFeatureValue(
|
||||
feature.asInstanceOf[Feature[Long]],
|
||||
featureVal.asInstanceOf[Long])
|
||||
case _ =>
|
||||
mrRequestScribeUnsupportedFeatureTypeStats.incr()
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(selectedRecord.getRecord)
|
||||
}
|
||||
} else Future.None
|
||||
}
|
||||
|
||||
private def generateCandidatesScribeInfo(
|
||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]],
|
||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
rerankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]],
|
||||
allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]],
|
||||
isFeatureHydratingEnabled: Boolean
|
||||
): Future[Seq[CandidateRequestInfo]] = {
|
||||
val candidatesMap = new mutable.HashMap[String, CandidateRequestInfo]
|
||||
|
||||
hydratedCandidates.foreach { hydratedCandidate =>
|
||||
val frgNotif = hydratedCandidate.candidate.frigateNotification
|
||||
val simplifiedTweetNotificationOpt = frgNotif.tweetNotification.map { tweetNotification =>
|
||||
TweetNotification(
|
||||
tweetNotification.tweetId,
|
||||
Seq.empty[TSocialContextAction],
|
||||
tweetNotification.tweetAuthorId)
|
||||
}
|
||||
val simplifiedFrigateNotification = FrigateNotification(
|
||||
frgNotif.commonRecommendationType,
|
||||
frgNotif.notificationDisplayLocation,
|
||||
tweetNotification = simplifiedTweetNotificationOpt
|
||||
)
|
||||
candidatesMap(hydratedCandidate.candidate.impressionId) = CandidateRequestInfo(
|
||||
candidateId = "",
|
||||
candidateSource = hydratedCandidate.source.substring(
|
||||
0,
|
||||
Math.min(6, hydratedCandidate.source.length)
|
||||
),
|
||||
frigateNotification = Some(simplifiedFrigateNotification),
|
||||
modelScore = None,
|
||||
rankPosition = None,
|
||||
rerankPosition = None,
|
||||
features = None,
|
||||
isSent = Some(false)
|
||||
)
|
||||
}
|
||||
|
||||
preRankingFilteredCandidates.foreach { preRankingFilteredCandidateResult =>
|
||||
candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) =
|
||||
candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId)
|
||||
.copy(
|
||||
candidateFilteredOutPredicate = preRankingFilteredCandidateResult.result match {
|
||||
case Invalid(reason) => reason
|
||||
case _ => {
|
||||
mrRequestScribeInvalidStats.incr()
|
||||
throw new IllegalStateException(
|
||||
"Invalid reason for Candidate Filtering " + preRankingFilteredCandidateResult.result)
|
||||
}
|
||||
},
|
||||
candidateFilteredOutStep = Some(CandidateFilteredOutStep.PreRankFiltering)
|
||||
)
|
||||
}
|
||||
|
||||
for {
|
||||
_ <- Future.collectToTry {
|
||||
rankedCandidates.zipWithIndex.map {
|
||||
case (rankedCandidateDetail, index) =>
|
||||
val modelScoresFut = {
|
||||
val crt = rankedCandidateDetail.candidate.commonRecType
|
||||
if (RecTypes.notEligibleForModelScoreTracking.contains(crt)) Future.None
|
||||
else rankedCandidateDetail.candidate.modelScores.map(Some(_))
|
||||
}
|
||||
|
||||
modelScoresFut.map { modelScores =>
|
||||
candidatesMap(rankedCandidateDetail.candidate.impressionId) =
|
||||
candidatesMap(rankedCandidateDetail.candidate.impressionId).copy(
|
||||
rankPosition = Some(index),
|
||||
modelScore = modelScores
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_ = rerankedCandidates.zipWithIndex.foreach {
|
||||
case (rerankedCandidateDetail, index) => {
|
||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId) =
|
||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy(
|
||||
rerankPosition = Some(index)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
_ <- Future.collectToTry {
|
||||
rerankedCandidates.map { rerankedCandidateDetail =>
|
||||
if (isFeatureHydratingEnabled) {
|
||||
PushPredictionHelper
|
||||
.getDataRecord(
|
||||
rerankedCandidateDetail.candidate.target.targetHydrationContext,
|
||||
rerankedCandidateDetail.candidate.target.featureMap,
|
||||
rerankedCandidateDetail.candidate.candidateHydrationContext,
|
||||
rerankedCandidateDetail.candidate.candidateFeatureMap(),
|
||||
stats
|
||||
).map { features =>
|
||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId) =
|
||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy(
|
||||
features = Some(
|
||||
ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(features))
|
||||
)
|
||||
}
|
||||
} else Future.Unit
|
||||
}
|
||||
}
|
||||
|
||||
_ = restrictFilteredCandidates.foreach { restrictFilteredCandidateDetatil =>
|
||||
candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) =
|
||||
candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId)
|
||||
.copy(candidateFilteredOutStep = Some(CandidateFilteredOutStep.Restrict))
|
||||
}
|
||||
|
||||
_ = allTakeCandidateResults.foreach { allTakeCandidateResult =>
|
||||
allTakeCandidateResult.result match {
|
||||
case OK =>
|
||||
candidatesMap(allTakeCandidateResult.candidate.impressionId) =
|
||||
candidatesMap(allTakeCandidateResult.candidate.impressionId).copy(isSent = Some(true))
|
||||
case Invalid(reason) =>
|
||||
candidatesMap(allTakeCandidateResult.candidate.impressionId) =
|
||||
candidatesMap(allTakeCandidateResult.candidate.impressionId).copy(
|
||||
candidateFilteredOutPredicate = reason,
|
||||
candidateFilteredOutStep = Some(CandidateFilteredOutStep.PostRankFiltering))
|
||||
case _ =>
|
||||
mrRequestScribeInvalidStats.incr()
|
||||
throw new IllegalStateException(
|
||||
"Invalid reason for Candidate Filtering " + allTakeCandidateResult.result)
|
||||
}
|
||||
}
|
||||
} yield candidatesMap.values.toSeq
|
||||
}
|
||||
|
||||
private def enableTargetFilteringScribing(target: Target): Boolean = {
|
||||
target.params(PushParams.EnableMrRequestScribing) && target.params(
|
||||
PushFeatureSwitchParams.EnableMrRequestScribingForTargetFiltering)
|
||||
}
|
||||
|
||||
private def enableCandidateFilteringScribing(target: Target): Boolean = {
|
||||
target.params(PushParams.EnableMrRequestScribing) && target.params(
|
||||
PushFeatureSwitchParams.EnableMrRequestScribingForCandidateFiltering)
|
||||
}
|
||||
|
||||
private def writeAtTargetFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = {
|
||||
logToScribe(mrRequestScribe)
|
||||
mrRequestScribeTargetFilteringStats.incr()
|
||||
}
|
||||
|
||||
private def writeAtCandidateFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = {
|
||||
logToScribe(mrRequestScribe)
|
||||
mrRequestScribeCandidateFilteringStats.incr()
|
||||
}
|
||||
|
||||
private def logToScribe(mrRequestScribe: MrRequestScribe): Unit = {
|
||||
val logEntry: String = MrRequestScribeInjection(mrRequestScribe)
|
||||
mrRequestScribeLogger.info(logEntry)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,250 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler
|
||||
|
||||
import com.twitter.finagle.stats.BroadcastStatsReceiver
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base.CandidateDetails
|
||||
import com.twitter.frigate.common.base.CandidateFilteringOnlyFlow
|
||||
import com.twitter.frigate.common.base.CandidateResult
|
||||
import com.twitter.frigate.common.base.FeatureMap
|
||||
import com.twitter.frigate.common.base.OK
|
||||
import com.twitter.frigate.common.base.Response
|
||||
import com.twitter.frigate.common.base.Result
|
||||
import com.twitter.frigate.common.base.Stats.track
|
||||
import com.twitter.frigate.common.config.CommonConstants
|
||||
import com.twitter.frigate.common.logger.MRLogger
|
||||
import com.twitter.frigate.common.rec_types.RecTypes
|
||||
import com.twitter.frigate.common.util.InvalidRequestException
|
||||
import com.twitter.frigate.common.util.MrNtabCopyObjects
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.config.Config
|
||||
import com.twitter.frigate.pushservice.ml.HydrationContextBuilder
|
||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableMagicFanoutNewsForYouNtabCopy
|
||||
import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler
|
||||
import com.twitter.frigate.pushservice.send_handler.generator.PushRequestToCandidate
|
||||
import com.twitter.frigate.pushservice.take.SendHandlerNotifier
|
||||
import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator
|
||||
import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator
|
||||
import com.twitter.frigate.pushservice.target.PushTargetUserBuilder
|
||||
import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest
|
||||
import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil
|
||||
import com.twitter.frigate.pushservice.thriftscala.PushRequest
|
||||
import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe
|
||||
import com.twitter.frigate.pushservice.thriftscala.PushResponse
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.nrel.heavyranker.FeatureHydrator
|
||||
import com.twitter.util._
|
||||
|
||||
/**
|
||||
* A handler for sending PushRequests
|
||||
*/
|
||||
class SendHandler(
|
||||
pushTargetUserBuilder: PushTargetUserBuilder,
|
||||
preCandidateValidator: SendHandlerPreCandidateValidator,
|
||||
postCandidateValidator: SendHandlerPostCandidateValidator,
|
||||
sendHandlerNotifier: SendHandlerNotifier,
|
||||
candidateHydrator: SendHandlerPushCandidateHydrator,
|
||||
featureHydrator: FeatureHydrator,
|
||||
sendHandlerPredicateUtil: SendHandlerPredicateUtil,
|
||||
mrRequestScriberNode: String
|
||||
)(
|
||||
implicit val statsReceiver: StatsReceiver,
|
||||
implicit val config: Config)
|
||||
extends CandidateFilteringOnlyFlow[Target, RawCandidate, PushCandidate] {
|
||||
|
||||
implicit private val timer: Timer = new JavaTimer(true)
|
||||
val stats = statsReceiver.scope("SendHandler")
|
||||
val log = MRLogger("SendHandler")
|
||||
|
||||
private val buildTargetStats = stats.scope("build_target")
|
||||
|
||||
private val candidateHydrationLatency: Stat =
|
||||
stats.stat("candidateHydrationLatency")
|
||||
|
||||
private val candidatePreValidatorLatency: Stat =
|
||||
stats.stat("candidatePreValidatorLatency")
|
||||
|
||||
private val candidatePostValidatorLatency: Stat =
|
||||
stats.stat("candidatePostValidatorLatency")
|
||||
|
||||
private val featureHydrationLatency: StatsReceiver =
|
||||
stats.scope("featureHydrationLatency")
|
||||
|
||||
private val mrRequestScribeHandler =
|
||||
new MrRequestScribeHandler(mrRequestScriberNode, stats.scope("mr_request_scribe"))
|
||||
|
||||
def apply(request: PushRequest): Future[PushResponse] = {
|
||||
val receivers = Seq(
|
||||
stats,
|
||||
stats.scope(request.notification.commonRecommendationType.toString)
|
||||
)
|
||||
val bStats = BroadcastStatsReceiver(receivers)
|
||||
bStats.counter("requests").incr()
|
||||
Stat
|
||||
.timeFuture(bStats.stat("latency"))(
|
||||
process(request).raiseWithin(CommonConstants.maxPushRequestDuration))
|
||||
.onSuccess {
|
||||
case (pushResp, rawCandidate) =>
|
||||
trackStatsForResponseToRequest(
|
||||
rawCandidate.commonRecType,
|
||||
rawCandidate.target,
|
||||
pushResp,
|
||||
receivers)(statsReceiver)
|
||||
if (!request.context.exists(_.darkWrite.contains(true))) {
|
||||
config.requestScribe(PushRequestScribe(request, pushResp))
|
||||
}
|
||||
}
|
||||
.onFailure { ex =>
|
||||
bStats.counter("failures").incr()
|
||||
bStats.scope("failures").counter(ex.getClass.getCanonicalName).incr()
|
||||
}
|
||||
.map {
|
||||
case (pushResp, _) => pushResp
|
||||
}
|
||||
}
|
||||
|
||||
private def process(request: PushRequest): Future[(PushResponse, RawCandidate)] = {
|
||||
val recType = request.notification.commonRecommendationType
|
||||
|
||||
track(buildTargetStats)(
|
||||
pushTargetUserBuilder
|
||||
.buildTarget(
|
||||
request.userId,
|
||||
request.context
|
||||
)
|
||||
).flatMap { targetUser =>
|
||||
val responseWithScribedInfo = request.context.exists { context =>
|
||||
context.responseWithScribedInfo.contains(true)
|
||||
}
|
||||
val newRequest =
|
||||
if (request.notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent &&
|
||||
targetUser.params(EnableMagicFanoutNewsForYouNtabCopy)) {
|
||||
val newNotification = request.notification.copy(ntabCopyId =
|
||||
Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId))
|
||||
request.copy(notification = newNotification)
|
||||
} else request
|
||||
|
||||
if (RecTypes.isSendHandlerType(recType) || newRequest.context.exists(
|
||||
_.allowCRT.contains(true))) {
|
||||
|
||||
val rawCandidateFut = PushRequestToCandidate.generatePushCandidate(
|
||||
newRequest.notification,
|
||||
targetUser
|
||||
)
|
||||
|
||||
rawCandidateFut.flatMap { rawCandidate =>
|
||||
val pushResponse = process(targetUser, Seq(rawCandidate)).flatMap {
|
||||
sendHandlerNotifier.checkResponseAndNotify(_, responseWithScribedInfo)
|
||||
}
|
||||
|
||||
pushResponse.map { pushResponse =>
|
||||
(pushResponse, rawCandidate)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Future.exception(InvalidRequestException(s"${recType.name} not supported in SendHandler"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private def hydrateFeatures(
|
||||
candidateDetails: Seq[CandidateDetails[PushCandidate]],
|
||||
target: Target,
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
candidateDetails.headOption match {
|
||||
case Some(candidateDetail)
|
||||
if RecTypes.notEligibleForModelScoreTracking(candidateDetail.candidate.commonRecType) =>
|
||||
Future.value(candidateDetails)
|
||||
|
||||
case Some(candidateDetail) =>
|
||||
val hydrationContextFut = HydrationContextBuilder.build(candidateDetail.candidate)
|
||||
hydrationContextFut.flatMap { hc =>
|
||||
featureHydrator
|
||||
.hydrateCandidate(Seq(hc), target.mrRequestContextForFeatureStore)
|
||||
.map { hydrationResult =>
|
||||
val features = hydrationResult.getOrElse(hc, FeatureMap())
|
||||
candidateDetail.candidate.mergeFeatures(features)
|
||||
candidateDetails
|
||||
}
|
||||
}
|
||||
case _ => Future.Nil
|
||||
}
|
||||
}
|
||||
|
||||
override def process(
|
||||
target: Target,
|
||||
externalCandidates: Seq[RawCandidate]
|
||||
): Future[Response[PushCandidate, Result]] = {
|
||||
val candidate = externalCandidates.map(CandidateDetails(_, "realtime"))
|
||||
|
||||
for {
|
||||
hydratedCandidatesWithCopy <- hydrateCandidates(candidate)
|
||||
|
||||
(candidates, preHydrationFilteredCandidates) <- track(filterStats)(
|
||||
filter(target, hydratedCandidatesWithCopy)
|
||||
)
|
||||
|
||||
featureHydratedCandidates <-
|
||||
track(featureHydrationLatency)(hydrateFeatures(candidates, target))
|
||||
|
||||
allTakeCandidateResults <- track(takeStats)(
|
||||
take(target, featureHydratedCandidates, desiredCandidateCount(target))
|
||||
)
|
||||
|
||||
_ <- mrRequestScribeHandler.scribeForCandidateFiltering(
|
||||
target = target,
|
||||
hydratedCandidates = hydratedCandidatesWithCopy,
|
||||
preRankingFilteredCandidates = preHydrationFilteredCandidates,
|
||||
rankedCandidates = featureHydratedCandidates,
|
||||
rerankedCandidates = Seq.empty,
|
||||
restrictFilteredCandidates = Seq.empty, // no restrict step
|
||||
allTakeCandidateResults = allTakeCandidateResults
|
||||
)
|
||||
} yield {
|
||||
|
||||
/**
|
||||
* We combine the results for all filtering steps and pass on in sequence to next step
|
||||
*
|
||||
* This is done to ensure the filtering reason for the candidate from multiple levels of
|
||||
* filtering is carried all the way until [[PushResponse]] is built and returned from
|
||||
* frigate-pushservice-send
|
||||
*/
|
||||
Response(OK, allTakeCandidateResults ++ preHydrationFilteredCandidates)
|
||||
}
|
||||
}
|
||||
|
||||
override def hydrateCandidates(
|
||||
candidates: Seq[CandidateDetails[RawCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
Stat.timeFuture(candidateHydrationLatency)(candidateHydrator(candidates))
|
||||
}
|
||||
|
||||
// Filter Step - pre-predicates and app specific predicates
|
||||
override def filter(
|
||||
target: Target,
|
||||
hydratedCandidatesDetails: Seq[CandidateDetails[PushCandidate]]
|
||||
): Future[
|
||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
||||
] = {
|
||||
Stat.timeFuture(candidatePreValidatorLatency)(
|
||||
sendHandlerPredicateUtil.preValidationForCandidate(
|
||||
hydratedCandidatesDetails,
|
||||
preCandidateValidator
|
||||
))
|
||||
}
|
||||
|
||||
// Post Validation - Take step
|
||||
override def validCandidates(
|
||||
target: Target,
|
||||
candidates: Seq[PushCandidate]
|
||||
): Future[Seq[Result]] = {
|
||||
Stat.timeFuture(candidatePostValidatorLatency)(Future.collect(candidates.map { candidate =>
|
||||
sendHandlerPredicateUtil
|
||||
.postValidationForCandidate(candidate, postCandidateValidator)
|
||||
.map(res => res.result)
|
||||
}))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,184 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler
|
||||
|
||||
import com.twitter.escherbird.metadata.thriftscala.EntityMegadata
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.base._
|
||||
import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext
|
||||
import com.twitter.frigate.common.util.MrNtabCopyObjects
|
||||
import com.twitter.frigate.common.util.MrPushCopyObjects
|
||||
import com.twitter.frigate.magic_events.thriftscala.FanoutEvent
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.ml.PushMLModelScorer
|
||||
import com.twitter.frigate.pushservice.model.candidate.CopyIds
|
||||
import com.twitter.frigate.pushservice.store.EventRequest
|
||||
import com.twitter.frigate.pushservice.store.UttEntityHydrationStore
|
||||
import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.gizmoduck.thriftscala.User
|
||||
import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery
|
||||
import com.twitter.interests.thriftscala.UserInterests
|
||||
import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent}
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.strato.client.UserId
|
||||
import com.twitter.ubs.thriftscala.AudioSpace
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class SendHandlerPushCandidateHydrator(
|
||||
lexServiceStore: ReadableStore[EventRequest, LiveEvent],
|
||||
fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent],
|
||||
semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata],
|
||||
safeUserStore: ReadableStore[Long, User],
|
||||
simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities],
|
||||
audioSpaceStore: ReadableStore[String, AudioSpace],
|
||||
interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests],
|
||||
uttEntityHydrationStore: UttEntityHydrationStore,
|
||||
superFollowCreatorTweetCountStore: ReadableStore[UserId, Int]
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver,
|
||||
implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) {
|
||||
|
||||
lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num")
|
||||
lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates")
|
||||
|
||||
def updateCandidates(
|
||||
candidateDetails: Seq[CandidateDetails[RawCandidate]],
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
|
||||
Future.collect {
|
||||
candidateDetails.map { candidateDetail =>
|
||||
val pushCandidate = candidateDetail.candidate
|
||||
|
||||
val copyIds = getCopyIdsByCRT(pushCandidate.commonRecType)
|
||||
|
||||
val hydratedCandidateFut = pushCandidate match {
|
||||
case magicFanoutNewsEventCandidate: MagicFanoutNewsEventCandidate =>
|
||||
getHydratedCandidateForMagicFanoutNewsEvent(
|
||||
magicFanoutNewsEventCandidate,
|
||||
copyIds,
|
||||
lexServiceStore,
|
||||
fanoutMetadataStore,
|
||||
semanticCoreMegadataStore,
|
||||
simClusterToEntityStore,
|
||||
interestsLookupStore,
|
||||
uttEntityHydrationStore
|
||||
)
|
||||
|
||||
case scheduledSpaceSubscriberCandidate: ScheduledSpaceSubscriberCandidate =>
|
||||
getHydratedCandidateForScheduledSpaceSubscriber(
|
||||
scheduledSpaceSubscriberCandidate,
|
||||
safeUserStore,
|
||||
copyIds,
|
||||
audioSpaceStore
|
||||
)
|
||||
case scheduledSpaceSpeakerCandidate: ScheduledSpaceSpeakerCandidate =>
|
||||
getHydratedCandidateForScheduledSpaceSpeaker(
|
||||
scheduledSpaceSpeakerCandidate,
|
||||
safeUserStore,
|
||||
copyIds,
|
||||
audioSpaceStore
|
||||
)
|
||||
case magicFanoutSportsEventCandidate: MagicFanoutSportsEventCandidate with MagicFanoutSportsScoreInformation =>
|
||||
getHydratedCandidateForMagicFanoutSportsEvent(
|
||||
magicFanoutSportsEventCandidate,
|
||||
copyIds,
|
||||
lexServiceStore,
|
||||
fanoutMetadataStore,
|
||||
semanticCoreMegadataStore,
|
||||
interestsLookupStore,
|
||||
uttEntityHydrationStore
|
||||
)
|
||||
case magicFanoutProductLaunchCandidate: MagicFanoutProductLaunchCandidate =>
|
||||
getHydratedCandidateForMagicFanoutProductLaunch(
|
||||
magicFanoutProductLaunchCandidate,
|
||||
copyIds)
|
||||
case creatorEventCandidate: MagicFanoutCreatorEventCandidate =>
|
||||
getHydratedCandidateForMagicFanoutCreatorEvent(
|
||||
creatorEventCandidate,
|
||||
safeUserStore,
|
||||
copyIds,
|
||||
superFollowCreatorTweetCountStore)
|
||||
case _ =>
|
||||
throw new IllegalArgumentException("Incorrect candidate type when update candidates")
|
||||
}
|
||||
|
||||
hydratedCandidateFut.map { hydratedCandidate =>
|
||||
hydratedCandidateStat.counter(hydratedCandidate.commonRecType.name).incr()
|
||||
CandidateDetails(
|
||||
hydratedCandidate,
|
||||
source = candidateDetail.source
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private def getCopyIdsByCRT(crt: CommonRecommendationType): CopyIds = {
|
||||
crt match {
|
||||
case CommonRecommendationType.MagicFanoutNewsEvent =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewsPushCopy.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
|
||||
case CommonRecommendationType.ScheduledSpaceSubscriber =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSubscriber.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSubscriber.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case CommonRecommendationType.ScheduledSpaceSpeaker =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSpeaker.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSpeakerNow.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case CommonRecommendationType.SpaceSpeaker =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.SpaceSpeaker.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.SpaceSpeaker.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case CommonRecommendationType.SpaceHost =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.SpaceHost.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.SpaceHost.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case CommonRecommendationType.MagicFanoutSportsEvent =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutSportsPushCopy.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutSportsCopy.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case CommonRecommendationType.MagicFanoutProductLaunch =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutProductLaunch.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.ProductLaunch.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case CommonRecommendationType.CreatorSubscriber =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutCreatorSubscription.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutCreatorSubscription.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case CommonRecommendationType.NewCreator =>
|
||||
CopyIds(
|
||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewCreator.copyId),
|
||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewCreator.copyId),
|
||||
aggregationId = None
|
||||
)
|
||||
case _ =>
|
||||
throw new IllegalArgumentException("Incorrect candidate type when fetch copy ids")
|
||||
}
|
||||
}
|
||||
|
||||
def apply(
|
||||
candidateDetails: Seq[CandidateDetails[RawCandidate]]
|
||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
||||
updateCandidates(candidateDetails)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,17 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.util.Future
|
||||
|
||||
trait CandidateGenerator {
|
||||
|
||||
/**
|
||||
* Build RawCandidate from FrigateNotification
|
||||
* @param target
|
||||
* @param frigateNotification
|
||||
* @return RawCandidate
|
||||
*/
|
||||
def getCandidate(target: Target, frigateNotification: FrigateNotification): Future[RawCandidate]
|
||||
}
|
||||
Binary file not shown.
@ -1,70 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate
|
||||
import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType
|
||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.model.PushTypes
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.util.Future
|
||||
|
||||
object MagicFanoutCreatorEventCandidateGenerator extends CandidateGenerator {
|
||||
override def getCandidate(
|
||||
targetUser: PushTypes.Target,
|
||||
notification: FrigateNotification
|
||||
): Future[PushTypes.RawCandidate] = {
|
||||
|
||||
require(
|
||||
notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber || notification.commonRecommendationType == CommonRecommendationType.NewCreator,
|
||||
"MagicFanoutCreatorEvent: unexpected CRT " + notification.commonRecommendationType
|
||||
)
|
||||
require(
|
||||
notification.creatorSubscriptionNotification.isDefined,
|
||||
"MagicFanoutCreatorEvent: creatorSubscriptionNotification is not defined")
|
||||
require(
|
||||
notification.creatorSubscriptionNotification.exists(_.magicFanoutPushId.isDefined),
|
||||
"MagicFanoutCreatorEvent: magicFanoutPushId is not defined")
|
||||
require(
|
||||
notification.creatorSubscriptionNotification.exists(_.fanoutReasons.isDefined),
|
||||
"MagicFanoutCreatorEvent: fanoutReasons is not defined")
|
||||
require(
|
||||
notification.creatorSubscriptionNotification.exists(_.creatorId.isDefined),
|
||||
"MagicFanoutCreatorEvent: creatorId is not defined")
|
||||
if (notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber) {
|
||||
require(
|
||||
notification.creatorSubscriptionNotification
|
||||
.exists(_.subscriberId.isDefined),
|
||||
"MagicFanoutCreatorEvent: subscriber id is not defined"
|
||||
)
|
||||
}
|
||||
|
||||
val creatorSubscriptionNotification = notification.creatorSubscriptionNotification.get
|
||||
|
||||
val candidate = new RawCandidate with MagicFanoutCreatorEventCandidate {
|
||||
|
||||
override val target: Target = targetUser
|
||||
|
||||
override val pushId: Long =
|
||||
creatorSubscriptionNotification.magicFanoutPushId.get
|
||||
|
||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
||||
creatorSubscriptionNotification.fanoutReasons.get
|
||||
|
||||
override val creatorFanoutType: CreatorFanoutType =
|
||||
creatorSubscriptionNotification.creatorFanoutType
|
||||
|
||||
override val commonRecType: CommonRecommendationType =
|
||||
notification.commonRecommendationType
|
||||
|
||||
override val frigateNotification: FrigateNotification = notification
|
||||
|
||||
override val subscriberId: Option[Long] = creatorSubscriptionNotification.subscriberId
|
||||
|
||||
override val creatorId: Long = creatorSubscriptionNotification.creatorId.get
|
||||
}
|
||||
|
||||
Future.value(candidate)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,57 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate
|
||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails
|
||||
import com.twitter.util.Future
|
||||
|
||||
object MagicFanoutNewsEventCandidateGenerator extends CandidateGenerator {
|
||||
|
||||
override def getCandidate(
|
||||
targetUser: Target,
|
||||
notification: FrigateNotification
|
||||
): Future[RawCandidate] = {
|
||||
|
||||
/**
|
||||
* frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutNewsEvent]]
|
||||
* AND pushId field should be set
|
||||
**/
|
||||
require(
|
||||
notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent,
|
||||
"MagicFanoutNewsEvent: unexpected CRT " + notification.commonRecommendationType
|
||||
)
|
||||
|
||||
require(
|
||||
notification.magicFanoutEventNotification.exists(_.pushId.isDefined),
|
||||
"MagicFanoutNewsEvent: pushId is not defined")
|
||||
|
||||
val magicFanoutEventNotification = notification.magicFanoutEventNotification.get
|
||||
|
||||
val candidate = new RawCandidate with MagicFanoutNewsEventCandidate {
|
||||
|
||||
override val target: Target = targetUser
|
||||
|
||||
override val eventId: Long = magicFanoutEventNotification.eventId
|
||||
|
||||
override val pushId: Long = magicFanoutEventNotification.pushId.get
|
||||
|
||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
||||
magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty)
|
||||
|
||||
override val momentId: Option[Long] = magicFanoutEventNotification.momentId
|
||||
|
||||
override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage
|
||||
|
||||
override val details: Option[MagicFanoutEventNotificationDetails] =
|
||||
magicFanoutEventNotification.details
|
||||
|
||||
override val frigateNotification: FrigateNotification = notification
|
||||
}
|
||||
|
||||
Future.value(candidate)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,54 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate
|
||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
||||
import com.twitter.frigate.magic_events.thriftscala.ProductType
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.model.PushTypes
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.util.Future
|
||||
|
||||
object MagicFanoutProductLaunchCandidateGenerator extends CandidateGenerator {
|
||||
|
||||
override def getCandidate(
|
||||
targetUser: PushTypes.Target,
|
||||
notification: FrigateNotification
|
||||
): Future[PushTypes.RawCandidate] = {
|
||||
|
||||
require(
|
||||
notification.commonRecommendationType == CommonRecommendationType.MagicFanoutProductLaunch,
|
||||
"MagicFanoutProductLaunch: unexpected CRT " + notification.commonRecommendationType
|
||||
)
|
||||
require(
|
||||
notification.magicFanoutProductLaunchNotification.isDefined,
|
||||
"MagicFanoutProductLaunch: magicFanoutProductLaunchNotification is not defined")
|
||||
require(
|
||||
notification.magicFanoutProductLaunchNotification.exists(_.magicFanoutPushId.isDefined),
|
||||
"MagicFanoutProductLaunch: magicFanoutPushId is not defined")
|
||||
require(
|
||||
notification.magicFanoutProductLaunchNotification.exists(_.fanoutReasons.isDefined),
|
||||
"MagicFanoutProductLaunch: fanoutReasons is not defined")
|
||||
|
||||
val magicFanoutProductLaunchNotification = notification.magicFanoutProductLaunchNotification.get
|
||||
|
||||
val candidate = new RawCandidate with MagicFanoutProductLaunchCandidate {
|
||||
|
||||
override val target: Target = targetUser
|
||||
|
||||
override val pushId: Long =
|
||||
magicFanoutProductLaunchNotification.magicFanoutPushId.get
|
||||
|
||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
||||
magicFanoutProductLaunchNotification.fanoutReasons.get
|
||||
|
||||
override val productLaunchType: ProductType =
|
||||
magicFanoutProductLaunchNotification.productLaunchType
|
||||
|
||||
override val frigateNotification: FrigateNotification = notification
|
||||
}
|
||||
|
||||
Future.value(candidate)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,153 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate
|
||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate
|
||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate
|
||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate
|
||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate
|
||||
import com.twitter.escherbird.common.thriftscala.Domains
|
||||
import com.twitter.escherbird.common.thriftscala.QualifiedId
|
||||
import com.twitter.escherbird.metadata.thriftscala.EntityMegadata
|
||||
import com.twitter.frigate.common.base.BaseGameScore
|
||||
import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate
|
||||
import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation
|
||||
import com.twitter.frigate.common.base.TeamInfo
|
||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
||||
import com.twitter.frigate.pushservice.exception.InvalidSportDomainException
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.params.PushConstants
|
||||
import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails
|
||||
import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
object MagicFanoutSportsEventCandidateGenerator {
|
||||
|
||||
final def getCandidate(
|
||||
targetUser: Target,
|
||||
notification: FrigateNotification,
|
||||
basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate],
|
||||
baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate],
|
||||
cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate],
|
||||
soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate],
|
||||
nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate],
|
||||
semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata],
|
||||
): Future[RawCandidate] = {
|
||||
|
||||
/**
|
||||
* frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutSportsEvent]]
|
||||
* AND pushId field should be set
|
||||
*
|
||||
* */
|
||||
require(
|
||||
notification.commonRecommendationType == CommonRecommendationType.MagicFanoutSportsEvent,
|
||||
"MagicFanoutSports: unexpected CRT " + notification.commonRecommendationType
|
||||
)
|
||||
|
||||
require(
|
||||
notification.magicFanoutEventNotification.exists(_.pushId.isDefined),
|
||||
"MagicFanoutSportsEvent: pushId is not defined")
|
||||
|
||||
val magicFanoutEventNotification = notification.magicFanoutEventNotification.get
|
||||
val eventId = magicFanoutEventNotification.eventId
|
||||
val _isScoreUpdate = magicFanoutEventNotification.isScoreUpdate.getOrElse(false)
|
||||
|
||||
val gameScoresFut: Future[Option[BaseGameScore]] = {
|
||||
if (_isScoreUpdate) {
|
||||
semanticCoreMegadataStore
|
||||
.get(SemanticEntityForQuery(PushConstants.SportsEventDomainId, eventId))
|
||||
.flatMap {
|
||||
case Some(megadata) =>
|
||||
if (megadata.domains.contains(Domains.BasketballGame)) {
|
||||
basketballGameScoreStore
|
||||
.get(QualifiedId(Domains.BasketballGame.value, eventId)).map {
|
||||
case Some(game) if game.status.isDefined =>
|
||||
val status = game.status.get
|
||||
MagicFanoutSportsUtil.transformToGameScore(game.score, status)
|
||||
case _ => None
|
||||
}
|
||||
} else if (megadata.domains.contains(Domains.BaseballGame)) {
|
||||
baseballGameScoreStore
|
||||
.get(QualifiedId(Domains.BaseballGame.value, eventId)).map {
|
||||
case Some(game) if game.status.isDefined =>
|
||||
val status = game.status.get
|
||||
MagicFanoutSportsUtil.transformToGameScore(game.runs, status)
|
||||
case _ => None
|
||||
}
|
||||
} else if (megadata.domains.contains(Domains.NflFootballGame)) {
|
||||
nflGameScoreStore
|
||||
.get(QualifiedId(Domains.NflFootballGame.value, eventId)).map {
|
||||
case Some(game) if game.status.isDefined =>
|
||||
val nflScore = MagicFanoutSportsUtil.transformNFLGameScore(game)
|
||||
nflScore
|
||||
case _ => None
|
||||
}
|
||||
} else if (megadata.domains.contains(Domains.SoccerMatch)) {
|
||||
soccerMatchScoreStore
|
||||
.get(QualifiedId(Domains.SoccerMatch.value, eventId)).map {
|
||||
case Some(game) if game.status.isDefined =>
|
||||
val soccerScore = MagicFanoutSportsUtil.transformSoccerGameScore(game)
|
||||
soccerScore
|
||||
case _ => None
|
||||
}
|
||||
} else {
|
||||
// The domains are not in our list of supported sports
|
||||
throw new InvalidSportDomainException(
|
||||
s"Domain for entity ${eventId} is not supported")
|
||||
}
|
||||
case _ => Future.None
|
||||
}
|
||||
} else Future.None
|
||||
}
|
||||
|
||||
val homeTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap {
|
||||
case Some(gameScore) =>
|
||||
MagicFanoutSportsUtil.getTeamInfo(gameScore.home, semanticCoreMegadataStore)
|
||||
case _ => Future.None
|
||||
}
|
||||
|
||||
val awayTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap {
|
||||
case Some(gameScore) =>
|
||||
MagicFanoutSportsUtil.getTeamInfo(gameScore.away, semanticCoreMegadataStore)
|
||||
case _ => Future.None
|
||||
}
|
||||
|
||||
val candidate = new RawCandidate
|
||||
with MagicFanoutSportsEventCandidate
|
||||
with MagicFanoutSportsScoreInformation {
|
||||
|
||||
override val target: Target = targetUser
|
||||
|
||||
override val eventId: Long = magicFanoutEventNotification.eventId
|
||||
|
||||
override val pushId: Long = magicFanoutEventNotification.pushId.get
|
||||
|
||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
||||
magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty)
|
||||
|
||||
override val momentId: Option[Long] = magicFanoutEventNotification.momentId
|
||||
|
||||
override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage
|
||||
|
||||
override val details: Option[MagicFanoutEventNotificationDetails] =
|
||||
magicFanoutEventNotification.details
|
||||
|
||||
override val frigateNotification: FrigateNotification = notification
|
||||
|
||||
override val homeTeamInfo: Future[Option[TeamInfo]] = homeTeamInfoFut
|
||||
|
||||
override val awayTeamInfo: Future[Option[TeamInfo]] = awayTeamInfoFut
|
||||
|
||||
override val gameScores: Future[Option[BaseGameScore]] = gameScoresFut
|
||||
|
||||
override val isScoreUpdate: Boolean = _isScoreUpdate
|
||||
}
|
||||
|
||||
Future.value(candidate)
|
||||
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,49 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.pushservice.config.Config
|
||||
import com.twitter.frigate.pushservice.exception.UnsupportedCrtException
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT}
|
||||
import com.twitter.util.Future
|
||||
|
||||
object PushRequestToCandidate {
|
||||
final def generatePushCandidate(
|
||||
frigateNotification: FrigateNotification,
|
||||
target: Target
|
||||
)(
|
||||
implicit config: Config
|
||||
): Future[RawCandidate] = {
|
||||
|
||||
val candidateGenerator: (Target, FrigateNotification) => Future[RawCandidate] = {
|
||||
frigateNotification.commonRecommendationType match {
|
||||
case CRT.MagicFanoutNewsEvent => MagicFanoutNewsEventCandidateGenerator.getCandidate
|
||||
case CRT.ScheduledSpaceSubscriber => ScheduledSpaceSubscriberCandidateGenerator.getCandidate
|
||||
case CRT.ScheduledSpaceSpeaker => ScheduledSpaceSpeakerCandidateGenerator.getCandidate
|
||||
case CRT.MagicFanoutSportsEvent =>
|
||||
MagicFanoutSportsEventCandidateGenerator.getCandidate(
|
||||
_,
|
||||
_,
|
||||
config.basketballGameScoreStore,
|
||||
config.baseballGameScoreStore,
|
||||
config.cricketMatchScoreStore,
|
||||
config.soccerMatchScoreStore,
|
||||
config.nflGameScoreStore,
|
||||
config.semanticCoreMegadataStore
|
||||
)
|
||||
case CRT.MagicFanoutProductLaunch =>
|
||||
MagicFanoutProductLaunchCandidateGenerator.getCandidate
|
||||
case CRT.NewCreator =>
|
||||
MagicFanoutCreatorEventCandidateGenerator.getCandidate
|
||||
case CRT.CreatorSubscriber =>
|
||||
MagicFanoutCreatorEventCandidateGenerator.getCandidate
|
||||
case _ =>
|
||||
throw new UnsupportedCrtException(
|
||||
"UnsupportedCrtException for SendHandler: " + frigateNotification.commonRecommendationType)
|
||||
}
|
||||
}
|
||||
|
||||
candidateGenerator(target, frigateNotification)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,55 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.util.Future
|
||||
|
||||
object ScheduledSpaceSpeakerCandidateGenerator extends CandidateGenerator {
|
||||
|
||||
override def getCandidate(
|
||||
targetUser: Target,
|
||||
notification: FrigateNotification
|
||||
): Future[RawCandidate] = {
|
||||
|
||||
/**
|
||||
* frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSpeaker]]
|
||||
*
|
||||
**/
|
||||
require(
|
||||
notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSpeaker,
|
||||
"ScheduledSpaceSpeaker: unexpected CRT " + notification.commonRecommendationType
|
||||
)
|
||||
|
||||
val spaceNotification = notification.spaceNotification.getOrElse(
|
||||
throw new IllegalStateException("ScheduledSpaceSpeaker notification object not defined"))
|
||||
|
||||
require(
|
||||
spaceNotification.hostUserId.isDefined,
|
||||
"ScheduledSpaceSpeaker notification - hostUserId not defined"
|
||||
)
|
||||
|
||||
val spaceHostId = spaceNotification.hostUserId
|
||||
|
||||
require(
|
||||
spaceNotification.scheduledStartTime.isDefined,
|
||||
"ScheduledSpaceSpeaker notification - scheduledStartTime not defined"
|
||||
)
|
||||
|
||||
val scheduledStartTime = spaceNotification.scheduledStartTime.get
|
||||
|
||||
val candidate = new RawCandidate with ScheduledSpaceSpeakerCandidate {
|
||||
override val target: Target = targetUser
|
||||
override val frigateNotification: FrigateNotification = notification
|
||||
override val spaceId: String = spaceNotification.broadcastId
|
||||
override val hostId: Option[Long] = spaceHostId
|
||||
override val startTime: Long = scheduledStartTime
|
||||
override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers
|
||||
override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners
|
||||
}
|
||||
|
||||
Future.value(candidate)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,55 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.send_handler.generator
|
||||
|
||||
import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.util.Future
|
||||
|
||||
object ScheduledSpaceSubscriberCandidateGenerator extends CandidateGenerator {
|
||||
|
||||
override def getCandidate(
|
||||
targetUser: Target,
|
||||
notification: FrigateNotification
|
||||
): Future[RawCandidate] = {
|
||||
|
||||
/**
|
||||
* frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSubscriber]]
|
||||
*
|
||||
**/
|
||||
require(
|
||||
notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSubscriber,
|
||||
"ScheduledSpaceSubscriber: unexpected CRT " + notification.commonRecommendationType
|
||||
)
|
||||
|
||||
val spaceNotification = notification.spaceNotification.getOrElse(
|
||||
throw new IllegalStateException("ScheduledSpaceSubscriber notification object not defined"))
|
||||
|
||||
require(
|
||||
spaceNotification.hostUserId.isDefined,
|
||||
"ScheduledSpaceSubscriber notification - hostUserId not defined"
|
||||
)
|
||||
|
||||
val spaceHostId = spaceNotification.hostUserId
|
||||
|
||||
require(
|
||||
spaceNotification.scheduledStartTime.isDefined,
|
||||
"ScheduledSpaceSubscriber notification - scheduledStartTime not defined"
|
||||
)
|
||||
|
||||
val scheduledStartTime = spaceNotification.scheduledStartTime.get
|
||||
|
||||
val candidate = new RawCandidate with ScheduledSpaceSubscriberCandidate {
|
||||
override val target: Target = targetUser
|
||||
override val frigateNotification: FrigateNotification = notification
|
||||
override val spaceId: String = spaceNotification.broadcastId
|
||||
override val hostId: Option[Long] = spaceHostId
|
||||
override val startTime: Long = scheduledStartTime
|
||||
override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers
|
||||
override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners
|
||||
}
|
||||
|
||||
Future.value(candidate)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,17 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.content_mixer.thriftscala.ContentMixer
|
||||
import com.twitter.content_mixer.thriftscala.ContentMixerRequest
|
||||
import com.twitter.content_mixer.thriftscala.ContentMixerResponse
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class ContentMixerStore(contentMixer: ContentMixer.MethodPerEndpoint)
|
||||
extends ReadableStore[ContentMixerRequest, ContentMixerResponse] {
|
||||
|
||||
override def get(request: ContentMixerRequest): Future[Option[ContentMixerResponse]] = {
|
||||
contentMixer.getCandidates(request).map { response =>
|
||||
Some(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,15 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.copyselectionservice.thriftscala._
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
class CopySelectionServiceStore(copySelectionServiceClient: CopySelectionService.FinagledClient)
|
||||
extends ReadableStore[CopySelectionRequestV1, Copy] {
|
||||
override def get(k: CopySelectionRequestV1): Future[Option[Copy]] =
|
||||
copySelectionServiceClient.getSelectedCopy(CopySelectionRequest.V1(k)).map {
|
||||
case CopySelectionResponse.V1(response) =>
|
||||
Some(response.selectedCopy)
|
||||
case _ => throw CopyServiceException(CopyServiceErrorCode.VersionNotFound)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,58 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.cr_mixer.thriftscala.CrMixer
|
||||
import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest
|
||||
import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse
|
||||
import com.twitter.cr_mixer.thriftscala.FrsTweetRequest
|
||||
import com.twitter.cr_mixer.thriftscala.FrsTweetResponse
|
||||
import com.twitter.finagle.stats.NullStatsReceiver
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Store to get content recs from content recommender.
|
||||
*/
|
||||
case class CrMixerTweetStore(
|
||||
crMixer: CrMixer.MethodPerEndpoint
|
||||
)(
|
||||
implicit statsReceiver: StatsReceiver = NullStatsReceiver) {
|
||||
|
||||
private val requestsCounter = statsReceiver.counter("requests")
|
||||
private val successCounter = statsReceiver.counter("success")
|
||||
private val failuresCounter = statsReceiver.counter("failures")
|
||||
private val nonEmptyCounter = statsReceiver.counter("non_empty")
|
||||
private val emptyCounter = statsReceiver.counter("empty")
|
||||
private val failuresScope = statsReceiver.scope("failures")
|
||||
private val latencyStat = statsReceiver.stat("latency")
|
||||
|
||||
private def updateStats[T](f: => Future[Option[T]]): Future[Option[T]] = {
|
||||
requestsCounter.incr()
|
||||
Stat
|
||||
.timeFuture(latencyStat)(f)
|
||||
.onSuccess { r =>
|
||||
if (r.isDefined) nonEmptyCounter.incr() else emptyCounter.incr()
|
||||
successCounter.incr()
|
||||
}
|
||||
.onFailure { e =>
|
||||
{
|
||||
failuresCounter.incr()
|
||||
failuresScope.counter(e.getClass.getName).incr()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getTweetRecommendations(
|
||||
request: CrMixerTweetRequest
|
||||
): Future[Option[CrMixerTweetResponse]] = {
|
||||
updateStats(crMixer.getTweetRecommendations(request).map { response =>
|
||||
Some(response)
|
||||
})
|
||||
}
|
||||
|
||||
def getFRSTweetCandidates(request: FrsTweetRequest): Future[Option[FrsTweetResponse]] = {
|
||||
updateStats(crMixer.getFrsBasedTweetRecommendations(request).map { response =>
|
||||
Some(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,28 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.explore_ranker.thriftscala.ExploreRanker
|
||||
import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse
|
||||
import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
/** A Store for Video Tweet Recommendations from Explore
|
||||
*
|
||||
* @param exploreRankerService
|
||||
*/
|
||||
case class ExploreRankerStore(exploreRankerService: ExploreRanker.MethodPerEndpoint)
|
||||
extends ReadableStore[ExploreRankerRequest, ExploreRankerResponse] {
|
||||
|
||||
/** Method to get video recommendations
|
||||
*
|
||||
* @param request explore ranker request object
|
||||
* @return
|
||||
*/
|
||||
override def get(
|
||||
request: ExploreRankerRequest
|
||||
): Future[Option[ExploreRankerResponse]] = {
|
||||
exploreRankerService.getRankedResults(request).map { response =>
|
||||
Some(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,46 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService
|
||||
import com.twitter.follow_recommendations.thriftscala.Recommendation
|
||||
import com.twitter.follow_recommendations.thriftscala.RecommendationRequest
|
||||
import com.twitter.follow_recommendations.thriftscala.RecommendationResponse
|
||||
import com.twitter.follow_recommendations.thriftscala.UserRecommendation
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class FollowRecommendationsStore(
|
||||
frsClient: FollowRecommendationsThriftService.MethodPerEndpoint,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends ReadableStore[RecommendationRequest, RecommendationResponse]
|
||||
with Logging {
|
||||
|
||||
private val scopedStats = statsReceiver.scope(getClass.getSimpleName)
|
||||
private val requests = scopedStats.counter("requests")
|
||||
private val valid = scopedStats.counter("valid")
|
||||
private val invalid = scopedStats.counter("invalid")
|
||||
private val numTotalResults = scopedStats.stat("total_results")
|
||||
private val numValidResults = scopedStats.stat("valid_results")
|
||||
|
||||
override def get(request: RecommendationRequest): Future[Option[RecommendationResponse]] = {
|
||||
requests.incr()
|
||||
frsClient.getRecommendations(request).map { response =>
|
||||
numTotalResults.add(response.recommendations.size)
|
||||
val validRecs = response.recommendations.filter {
|
||||
case Recommendation.User(_: UserRecommendation) =>
|
||||
valid.incr()
|
||||
true
|
||||
case _ =>
|
||||
invalid.incr()
|
||||
false
|
||||
}
|
||||
|
||||
numValidResults.add(validRecs.size)
|
||||
Some(
|
||||
RecommendationResponse(
|
||||
recommendations = validRecs
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,190 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.finagle.stats.BroadcastStatsReceiver
|
||||
import com.twitter.finagle.stats.NullStatsReceiver
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.logger.MRLogger
|
||||
import com.twitter.frigate.common.store
|
||||
import com.twitter.frigate.common.store.Fail
|
||||
import com.twitter.frigate.common.store.IbisRequestInfo
|
||||
import com.twitter.frigate.common.store.IbisResponse
|
||||
import com.twitter.frigate.common.store.Sent
|
||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
||||
import com.twitter.ibis2.service.thriftscala.Flags
|
||||
import com.twitter.ibis2.service.thriftscala.FlowControl
|
||||
import com.twitter.ibis2.service.thriftscala.Ibis2Request
|
||||
import com.twitter.ibis2.service.thriftscala.Ibis2Response
|
||||
import com.twitter.ibis2.service.thriftscala.Ibis2ResponseStatus
|
||||
import com.twitter.ibis2.service.thriftscala.Ibis2Service
|
||||
import com.twitter.ibis2.service.thriftscala.NotificationNotSentCode
|
||||
import com.twitter.ibis2.service.thriftscala.TargetFanoutResult.NotSentReason
|
||||
import com.twitter.util.Future
|
||||
|
||||
trait Ibis2Store extends store.Ibis2Store {
|
||||
def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse]
|
||||
}
|
||||
|
||||
case class PushIbis2Store(
|
||||
ibisClient: Ibis2Service.MethodPerEndpoint
|
||||
)(
|
||||
implicit val statsReceiver: StatsReceiver = NullStatsReceiver)
|
||||
extends Ibis2Store {
|
||||
private val log = MRLogger(this.getClass.getSimpleName)
|
||||
private val stats = statsReceiver.scope("ibis_v2_store")
|
||||
private val statsByCrt = stats.scope("byCrt")
|
||||
private val requestsByCrt = statsByCrt.scope("requests")
|
||||
private val failuresByCrt = statsByCrt.scope("failures")
|
||||
private val successByCrt = statsByCrt.scope("success")
|
||||
|
||||
private val statsByIbisModel = stats.scope("byIbisModel")
|
||||
private val requestsByIbisModel = statsByIbisModel.scope("requests")
|
||||
private val failuresByIbisModel = statsByIbisModel.scope("failures")
|
||||
private val successByIbisModel = statsByIbisModel.scope("success")
|
||||
|
||||
private[this] def ibisSend(
|
||||
ibis2Request: Ibis2Request,
|
||||
commonRecommendationType: CommonRecommendationType
|
||||
): Future[IbisResponse] = {
|
||||
val ibisModel = ibis2Request.modelName
|
||||
|
||||
val bStats = if (ibis2Request.flags.getOrElse(Flags()).darkWrite.contains(true)) {
|
||||
BroadcastStatsReceiver(
|
||||
Seq(
|
||||
stats,
|
||||
stats.scope("dark_write")
|
||||
)
|
||||
)
|
||||
} else BroadcastStatsReceiver(Seq(stats))
|
||||
|
||||
bStats.counter("requests").incr()
|
||||
requestsByCrt.counter(commonRecommendationType.name).incr()
|
||||
requestsByIbisModel.counter(ibisModel).incr()
|
||||
|
||||
retry(ibisClient, ibis2Request, 3, bStats)
|
||||
.map { response =>
|
||||
bStats.counter(response.status.status.name).incr()
|
||||
successByCrt.counter(response.status.status.name, commonRecommendationType.name).incr()
|
||||
successByIbisModel.counter(response.status.status.name, ibisModel).incr()
|
||||
response.status.status match {
|
||||
case Ibis2ResponseStatus.SuccessWithDeliveries |
|
||||
Ibis2ResponseStatus.SuccessNoDeliveries =>
|
||||
IbisResponse(Sent, Some(response))
|
||||
case _ =>
|
||||
IbisResponse(Fail, Some(response))
|
||||
}
|
||||
}
|
||||
.onFailure { ex =>
|
||||
bStats.counter("failures").incr()
|
||||
val exceptionName = ex.getClass.getCanonicalName
|
||||
bStats.scope("failures").counter(exceptionName).incr()
|
||||
failuresByCrt.counter(exceptionName, commonRecommendationType.name).incr()
|
||||
failuresByIbisModel.counter(exceptionName, ibisModel).incr()
|
||||
}
|
||||
}
|
||||
|
||||
private def getNotifNotSentReason(
|
||||
ibis2Response: Ibis2Response
|
||||
): Option[NotificationNotSentCode] = {
|
||||
ibis2Response.status.fanoutResults match {
|
||||
case Some(fanoutResult) =>
|
||||
fanoutResult.pushResult.flatMap { pushResult =>
|
||||
pushResult.results.headOption match {
|
||||
case Some(NotSentReason(notSentInfo)) => Some(notSentInfo.notSentCode)
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
|
||||
def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = {
|
||||
val requestWithIID = if (ibis2Request.flowControl.exists(_.externalIid.isDefined)) {
|
||||
ibis2Request
|
||||
} else {
|
||||
ibis2Request.copy(
|
||||
flowControl = Some(
|
||||
ibis2Request.flowControl
|
||||
.getOrElse(FlowControl())
|
||||
.copy(externalIid = Some(candidate.impressionId))
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
val commonRecommendationType = candidate.frigateNotification.commonRecommendationType
|
||||
|
||||
ibisSend(requestWithIID, commonRecommendationType)
|
||||
.onSuccess { response =>
|
||||
response.ibis2Response.foreach { ibis2Response =>
|
||||
getNotifNotSentReason(ibis2Response).foreach { notifNotSentCode =>
|
||||
stats.scope(ibis2Response.status.status.name).counter(s"$notifNotSentCode").incr()
|
||||
}
|
||||
if (ibis2Response.status.status != Ibis2ResponseStatus.SuccessWithDeliveries) {
|
||||
log.warning(
|
||||
s"Request dropped on ibis for ${ibis2Request.recipientSelector.recipientId}: $ibis2Response")
|
||||
}
|
||||
}
|
||||
}
|
||||
.onFailure { ex =>
|
||||
log.warning(
|
||||
s"Ibis Request failure: ${ex.getClass.getCanonicalName} \n For IbisRequest: $ibis2Request")
|
||||
log.error(ex, ex.getMessage)
|
||||
}
|
||||
}
|
||||
|
||||
// retry request when Ibis2ResponseStatus is PreFanoutError
|
||||
def retry(
|
||||
ibisClient: Ibis2Service.MethodPerEndpoint,
|
||||
request: Ibis2Request,
|
||||
retryCount: Int,
|
||||
bStats: StatsReceiver
|
||||
): Future[Ibis2Response] = {
|
||||
ibisClient.sendNotification(request).flatMap { response =>
|
||||
response.status.status match {
|
||||
case Ibis2ResponseStatus.PreFanoutError if retryCount > 0 =>
|
||||
bStats.scope("requests").counter("retry").incr()
|
||||
bStats.counter(response.status.status.name).incr()
|
||||
retry(ibisClient, request, retryCount - 1, bStats)
|
||||
case _ =>
|
||||
Future.value(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override def send(
|
||||
ibis2Request: Ibis2Request,
|
||||
requestInfo: IbisRequestInfo
|
||||
): Future[IbisResponse] = {
|
||||
ibisSend(ibis2Request, requestInfo.commonRecommendationType)
|
||||
}
|
||||
}
|
||||
|
||||
case class StagingIbis2Store(remoteIbis2Store: PushIbis2Store) extends Ibis2Store {
|
||||
|
||||
final def addDarkWriteFlagIbis2Request(
|
||||
isTeamMember: Boolean,
|
||||
ibis2Request: Ibis2Request
|
||||
): Ibis2Request = {
|
||||
val flags =
|
||||
ibis2Request.flags.getOrElse(Flags())
|
||||
val darkWrite: Boolean = !isTeamMember || flags.darkWrite.getOrElse(false)
|
||||
ibis2Request.copy(flags = Some(flags.copy(darkWrite = Some(darkWrite))))
|
||||
}
|
||||
|
||||
override def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = {
|
||||
candidate.target.isTeamMember.flatMap { isTeamMember =>
|
||||
val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request)
|
||||
remoteIbis2Store.send(ibis2Req, candidate)
|
||||
}
|
||||
}
|
||||
|
||||
override def send(
|
||||
ibis2Request: Ibis2Request,
|
||||
requestInfo: IbisRequestInfo
|
||||
): Future[IbisResponse] = {
|
||||
requestInfo.isTeamMember.flatMap { isTeamMember =>
|
||||
val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request)
|
||||
remoteIbis2Store.send(ibis2Req, requestInfo)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,16 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService
|
||||
import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest
|
||||
import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class InterestDiscoveryStore(
|
||||
client: InterestsDiscoveryService.MethodPerEndpoint)
|
||||
extends ReadableStore[RecommendedListsRequest, RecommendedListsResponse] {
|
||||
|
||||
override def get(request: RecommendedListsRequest): Future[Option[RecommendedListsResponse]] = {
|
||||
client.getListRecos(request).map(Some(_))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,156 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.candidate.TargetDecider
|
||||
import com.twitter.frigate.common.history.History
|
||||
import com.twitter.frigate.common.history.HistoryStoreKeyContext
|
||||
import com.twitter.frigate.common.history.PushServiceHistoryStore
|
||||
import com.twitter.frigate.data_pipeline.thriftscala._
|
||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
||||
import com.twitter.hermit.store.labeled_push_recs.LabeledPushRecsJoinedWithNotificationHistoryStore
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Time
|
||||
|
||||
case class LabeledPushRecsVerifyingStoreKey(
|
||||
historyStoreKey: HistoryStoreKeyContext,
|
||||
useHydratedDataset: Boolean,
|
||||
verifyHydratedDatasetResults: Boolean) {
|
||||
def userId: Long = historyStoreKey.targetUserId
|
||||
}
|
||||
|
||||
case class LabeledPushRecsVerifyingStoreResponse(
|
||||
userHistory: UserHistoryValue,
|
||||
unequalNotificationsUnhydratedToHydrated: Option[
|
||||
Map[(Time, FrigateNotification), FrigateNotification]
|
||||
],
|
||||
missingFromHydrated: Option[Map[Time, FrigateNotification]])
|
||||
|
||||
case class LabeledPushRecsVerifyingStore(
|
||||
labeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue],
|
||||
historyStore: PushServiceHistoryStore
|
||||
)(
|
||||
implicit stats: StatsReceiver)
|
||||
extends ReadableStore[LabeledPushRecsVerifyingStoreKey, LabeledPushRecsVerifyingStoreResponse] {
|
||||
|
||||
private def getByJoiningWithRealHistory(
|
||||
key: HistoryStoreKeyContext
|
||||
): Future[Option[UserHistoryValue]] = {
|
||||
val historyFut = historyStore.get(key, Some(365.days))
|
||||
val toJoinWithRealHistoryFut = labeledPushRecsStore.get(UserHistoryKey.UserId(key.targetUserId))
|
||||
Future.join(historyFut, toJoinWithRealHistoryFut).map {
|
||||
case (_, None) => None
|
||||
case (History(realtimeHistoryMap), Some(uhValue)) =>
|
||||
Some(
|
||||
LabeledPushRecsJoinedWithNotificationHistoryStore
|
||||
.joinLabeledPushRecsSentWithNotificationHistory(uhValue, realtimeHistoryMap, stats)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private def processUserHistoryValue(uhValue: UserHistoryValue): Map[Time, FrigateNotification] = {
|
||||
uhValue.events
|
||||
.getOrElse(Nil)
|
||||
.collect {
|
||||
case Event(
|
||||
EventType.LabeledPushRecSend,
|
||||
Some(tsMillis),
|
||||
Some(EventUnion.LabeledPushRecSendEvent(lprs: LabeledPushRecSendEvent))
|
||||
) if lprs.pushRecSendEvent.frigateNotification.isDefined =>
|
||||
Time.fromMilliseconds(tsMillis) -> lprs.pushRecSendEvent.frigateNotification.get
|
||||
}
|
||||
.toMap
|
||||
}
|
||||
|
||||
override def get(
|
||||
key: LabeledPushRecsVerifyingStoreKey
|
||||
): Future[Option[LabeledPushRecsVerifyingStoreResponse]] = {
|
||||
val uhKey = UserHistoryKey.UserId(key.userId)
|
||||
if (!key.useHydratedDataset) {
|
||||
getByJoiningWithRealHistory(key.historyStoreKey).map { uhValueOpt =>
|
||||
uhValueOpt.map { uhValue => LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) }
|
||||
}
|
||||
} else {
|
||||
labeledPushRecsStore.get(uhKey).flatMap { hydratedValueOpt: Option[UserHistoryValue] =>
|
||||
if (!key.verifyHydratedDatasetResults) {
|
||||
Future.value(hydratedValueOpt.map { uhValue =>
|
||||
LabeledPushRecsVerifyingStoreResponse(uhValue, None, None)
|
||||
})
|
||||
} else {
|
||||
getByJoiningWithRealHistory(key.historyStoreKey).map {
|
||||
joinedWithRealHistoryOpt: Option[UserHistoryValue] =>
|
||||
val joinedWithRealHistoryMap =
|
||||
joinedWithRealHistoryOpt.map(processUserHistoryValue).getOrElse(Map.empty)
|
||||
val hydratedMap = hydratedValueOpt.map(processUserHistoryValue).getOrElse(Map.empty)
|
||||
val unequal = joinedWithRealHistoryMap.flatMap {
|
||||
case (time, frigateNotif) =>
|
||||
hydratedMap.get(time).collect {
|
||||
case n if n != frigateNotif => ((time, frigateNotif), n)
|
||||
}
|
||||
}
|
||||
val missing = joinedWithRealHistoryMap.filter {
|
||||
case (time, frigateNotif) => !hydratedMap.contains(time)
|
||||
}
|
||||
hydratedValueOpt.map { hydratedValue =>
|
||||
LabeledPushRecsVerifyingStoreResponse(hydratedValue, Some(unequal), Some(missing))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case class LabeledPushRecsStoreKey(target: TargetDecider, historyStoreKey: HistoryStoreKeyContext) {
|
||||
def userId: Long = historyStoreKey.targetUserId
|
||||
}
|
||||
|
||||
case class LabeledPushRecsDecideredStore(
|
||||
verifyingStore: ReadableStore[
|
||||
LabeledPushRecsVerifyingStoreKey,
|
||||
LabeledPushRecsVerifyingStoreResponse
|
||||
],
|
||||
useHydratedLabeledSendsDatasetDeciderKey: String,
|
||||
verifyHydratedLabeledSendsForHistoryDeciderKey: String
|
||||
)(
|
||||
implicit globalStats: StatsReceiver)
|
||||
extends ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] {
|
||||
private val log = Logger()
|
||||
private val stats = globalStats.scope("LabeledPushRecsDecideredStore")
|
||||
private val numComparisons = stats.counter("num_comparisons")
|
||||
private val numMissingStat = stats.stat("num_missing")
|
||||
private val numUnequalStat = stats.stat("num_unequal")
|
||||
|
||||
override def get(key: LabeledPushRecsStoreKey): Future[Option[UserHistoryValue]] = {
|
||||
val useHydrated = key.target.isDeciderEnabled(
|
||||
useHydratedLabeledSendsDatasetDeciderKey,
|
||||
stats,
|
||||
useRandomRecipient = true
|
||||
)
|
||||
|
||||
val verifyHydrated = if (useHydrated) {
|
||||
key.target.isDeciderEnabled(
|
||||
verifyHydratedLabeledSendsForHistoryDeciderKey,
|
||||
stats,
|
||||
useRandomRecipient = true
|
||||
)
|
||||
} else false
|
||||
|
||||
val newKey = LabeledPushRecsVerifyingStoreKey(key.historyStoreKey, useHydrated, verifyHydrated)
|
||||
verifyingStore.get(newKey).map {
|
||||
case None => None
|
||||
case Some(LabeledPushRecsVerifyingStoreResponse(uhValue, unequalOpt, missingOpt)) =>
|
||||
(unequalOpt, missingOpt) match {
|
||||
case (Some(unequal), Some(missing)) =>
|
||||
numComparisons.incr()
|
||||
numMissingStat.add(missing.size)
|
||||
numUnequalStat.add(unequal.size)
|
||||
case _ => //no-op
|
||||
}
|
||||
Some(uhValue)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
@ -1,26 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.livevideo.common.ids.EventId
|
||||
import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient
|
||||
import com.twitter.livevideo.timeline.domain.v2.Event
|
||||
import com.twitter.livevideo.timeline.domain.v2.LookupContext
|
||||
import com.twitter.stitch.storehaus.ReadableStoreOfStitch
|
||||
import com.twitter.stitch.NotFound
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
|
||||
case class EventRequest(eventId: Long, lookupContext: LookupContext = LookupContext.default)
|
||||
|
||||
object LexServiceStore {
|
||||
def apply(
|
||||
liveVideoTimelineClient: LiveVideoTimelineClient
|
||||
): ReadableStore[EventRequest, Event] = {
|
||||
ReadableStoreOfStitch { eventRequest =>
|
||||
liveVideoTimelineClient.getEvent(
|
||||
EventId(eventRequest.eventId),
|
||||
eventRequest.lookupContext) rescue {
|
||||
case NotFound => Stitch.NotFound
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,45 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.hermit.store.common.ReadableWritableStore
|
||||
import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection
|
||||
import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection
|
||||
import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint
|
||||
import com.twitter.storage.client.manhattan.kv.impl.Component
|
||||
import com.twitter.storage.client.manhattan.kv.impl.DescriptorP1L1
|
||||
import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor
|
||||
import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class NTabHistoryStore(mhEndpoint: ManhattanKVEndpoint, dataset: String)
|
||||
extends ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] {
|
||||
|
||||
private val keyDesc: DescriptorP1L1.EmptyKey[Long, String] =
|
||||
KeyDescriptor(Component(LongInjection), Component(StringInjection))
|
||||
|
||||
private val genericNotifKeyValDesc: ValueDescriptor.EmptyValue[GenericNotificationOverrideKey] =
|
||||
ValueDescriptor[GenericNotificationOverrideKey](
|
||||
BinaryCompactScalaInjection(GenericNotificationOverrideKey)
|
||||
)
|
||||
|
||||
override def get(key: (Long, String)): Future[Option[GenericNotificationOverrideKey]] = {
|
||||
val (userId, impressionId) = key
|
||||
val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId)
|
||||
|
||||
Stitch
|
||||
.run(mhEndpoint.get(mhKey, genericNotifKeyValDesc))
|
||||
.map { optionMhValue =>
|
||||
optionMhValue.map(_.contents)
|
||||
}
|
||||
}
|
||||
|
||||
override def put(keyValue: ((Long, String), GenericNotificationOverrideKey)): Future[Unit] = {
|
||||
val ((userId, impressionId), genericNotifOverrideKey) = keyValue
|
||||
val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId)
|
||||
val mhVal = genericNotifKeyValDesc.withValue(genericNotifOverrideKey)
|
||||
Stitch.run(mhEndpoint.insert(mhKey, mhVal))
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
@ -1,73 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection
|
||||
import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection
|
||||
import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection
|
||||
import com.twitter.storage.client.manhattan.kv.impl.Component
|
||||
import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor
|
||||
import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClient
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder
|
||||
import com.twitter.storage.client.manhattan.kv.NoMtlsParams
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.storehaus_internal.manhattan.Omega
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Time
|
||||
|
||||
case class OCFHistoryStoreKey(userId: Long, fatigueDuration: Duration, fatigueGroup: String)
|
||||
|
||||
class OCFPromptHistoryStore(
|
||||
manhattanAppId: String,
|
||||
dataset: String,
|
||||
mtlsParams: ManhattanKVClientMtlsParams = NoMtlsParams
|
||||
)(
|
||||
implicit stats: StatsReceiver)
|
||||
extends ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] {
|
||||
|
||||
import ManhattanInjections._
|
||||
|
||||
private val client = ManhattanKVClient(
|
||||
appId = manhattanAppId,
|
||||
dest = Omega.wilyName,
|
||||
mtlsParams = mtlsParams,
|
||||
label = "ocf_history_store"
|
||||
)
|
||||
private val endpoint = ManhattanKVEndpointBuilder(client, defaultMaxTimeout = 5.seconds)
|
||||
.statsReceiver(stats.scope("ocf_history_store"))
|
||||
.build()
|
||||
|
||||
private val limitResultsTo = 1
|
||||
|
||||
private val datasetKey = keyDesc.withDataset(dataset)
|
||||
|
||||
override def get(storeKey: OCFHistoryStoreKey): Future[Option[FatigueFlowEnrollment]] = {
|
||||
val userId = storeKey.userId
|
||||
val fatigueGroup = storeKey.fatigueGroup
|
||||
val fatigueLength = storeKey.fatigueDuration.inMilliseconds
|
||||
val currentTime = Time.now.inMilliseconds
|
||||
val fullKey = datasetKey
|
||||
.withPkey(userId)
|
||||
.from(fatigueGroup)
|
||||
.to(fatigueGroup, fatigueLength - currentTime)
|
||||
|
||||
Stitch
|
||||
.run(endpoint.slice(fullKey, valDesc, limit = Some(limitResultsTo)))
|
||||
.map { results =>
|
||||
if (results.nonEmpty) {
|
||||
val (_, mhValue) = results.head
|
||||
Some(mhValue.contents)
|
||||
} else None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object ManhattanInjections {
|
||||
val keyDesc = KeyDescriptor(Component(LongInjection), Component(StringInjection, LongInjection))
|
||||
val valDesc = ValueDescriptor(BinaryScalaInjection(FatigueFlowEnrollment))
|
||||
}
|
||||
Binary file not shown.
@ -1,81 +0,0 @@
|
||||
package com.twitter.frigate.pushservice.store
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.frigate.common.history.History
|
||||
import com.twitter.frigate.common.store.RealTimeClientEventStore
|
||||
import com.twitter.frigate.data_pipeline.common.HistoryJoin
|
||||
import com.twitter.frigate.data_pipeline.thriftscala.Event
|
||||
import com.twitter.frigate.data_pipeline.thriftscala.EventUnion
|
||||
import com.twitter.frigate.data_pipeline.thriftscala.PushRecSendEvent
|
||||
import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Time
|
||||
|
||||
case class OnlineUserHistoryKey(
|
||||
userId: Long,
|
||||
offlineUserHistory: Option[UserHistoryValue],
|
||||
history: Option[History])
|
||||
|
||||
case class OnlineUserHistoryStore(
|
||||
realTimeClientEventStore: RealTimeClientEventStore,
|
||||
duration: Duration = 3.days)
|
||||
extends ReadableStore[OnlineUserHistoryKey, UserHistoryValue] {
|
||||
|
||||
override def get(key: OnlineUserHistoryKey): Future[Option[UserHistoryValue]] = {
|
||||
val now = Time.now
|
||||
|
||||
val pushRecSends = key.history
|
||||
.getOrElse(History(Nil.toMap))
|
||||
.sortedPushDmHistory
|
||||
.filter(_._1 > now - (duration + 1.day))
|
||||
.map {
|
||||
case (time, frigateNotification) =>
|
||||
val pushRecSendEvent = PushRecSendEvent(
|
||||
frigateNotification = Some(frigateNotification),
|
||||
impressionId = frigateNotification.impressionId
|
||||
)
|
||||
pushRecSendEvent -> time
|
||||
}
|
||||
|
||||
realTimeClientEventStore
|
||||
.get(key.userId, now - duration, now)
|
||||
.map { attributedEventHistory =>
|
||||
val attributedClientEvents = attributedEventHistory.sortedHistory.flatMap {
|
||||
case (time, event) =>
|
||||
event.eventUnion match {
|
||||
case Some(eventUnion: EventUnion.AttributedPushRecClientEvent) =>
|
||||
Some((eventUnion.attributedPushRecClientEvent, event.eventType, time))
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
|
||||
val realtimeLabeledSends: Seq[Event] = HistoryJoin.getLabeledPushRecSends(
|
||||
pushRecSends,
|
||||
attributedClientEvents,
|
||||
Seq(),
|
||||
Seq(),
|
||||
Seq(),
|
||||
now
|
||||
)
|
||||
|
||||
key.offlineUserHistory.map { offlineUserHistory =>
|
||||
val combinedEvents = offlineUserHistory.events.map { offlineEvents =>
|
||||
(offlineEvents ++ realtimeLabeledSends)
|
||||
.map { event =>
|
||||
event.timestampMillis -> event
|
||||
}
|
||||
.toMap
|
||||
.values
|
||||
.toSeq
|
||||
.sortBy { event =>
|
||||
-1 * event.timestampMillis.getOrElse(0L)
|
||||
}
|
||||
}
|
||||
|
||||
offlineUserHistory.copy(events = combinedEvents)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user