mirror of
https://github.com/twitter/the-algorithm.git
synced 2025-03-09 18:01:56 +01:00
[docx] split commit for file 3600
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
9ebf058832
commit
ce29360463
Binary file not shown.
@ -1,87 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.predicate.ntab_caret_fatigue
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.predicate.FatiguePredicate
|
|
||||||
import com.twitter.frigate.pushservice.predicate.CaretFeedbackHistoryFilter
|
|
||||||
import com.twitter.frigate.pushservice.predicate.{
|
|
||||||
TargetNtabCaretClickFatiguePredicate => CommonNtabCaretClickFatiguePredicate
|
|
||||||
}
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.params.PushParams
|
|
||||||
import com.twitter.frigate.thriftscala.NotificationDisplayLocation
|
|
||||||
import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT}
|
|
||||||
import com.twitter.hermit.predicate.NamedPredicate
|
|
||||||
import com.twitter.hermit.predicate.Predicate
|
|
||||||
import com.twitter.notificationservice.thriftscala.CaretFeedbackDetails
|
|
||||||
import com.twitter.util.Duration
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object RecTypeNtabCaretClickFatiguePredicate {
|
|
||||||
val defaultName = "RecTypeNtabCaretClickFatiguePredicateForCandidate"
|
|
||||||
|
|
||||||
private def candidateFatiguePredicate(
|
|
||||||
genericTypeCategories: Seq[String],
|
|
||||||
crts: Set[CRT]
|
|
||||||
)(
|
|
||||||
implicit stats: StatsReceiver
|
|
||||||
): NamedPredicate[
|
|
||||||
PushCandidate
|
|
||||||
] = {
|
|
||||||
val name = "f1TriggeredCRTBasedFatiguePredciate"
|
|
||||||
val scopedStats = stats.scope(s"predicate_$name")
|
|
||||||
Predicate
|
|
||||||
.fromAsync { candidate: PushCandidate =>
|
|
||||||
if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) {
|
|
||||||
if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) {
|
|
||||||
NtabCaretClickContFnFatiguePredicate
|
|
||||||
.ntabCaretClickContFnFatiguePredicates(
|
|
||||||
filterHistory = FatiguePredicate.recTypesOnlyFilter(crts),
|
|
||||||
filterCaretFeedbackHistory =
|
|
||||||
CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories),
|
|
||||||
filterInlineFeedbackHistory =
|
|
||||||
NtabCaretClickFatigueUtils.feedbackModelFilterByCRT(crts)
|
|
||||||
).apply(Seq(candidate))
|
|
||||||
.map(_.headOption.getOrElse(false))
|
|
||||||
} else Future.True
|
|
||||||
} else {
|
|
||||||
Future.True
|
|
||||||
}
|
|
||||||
}.withStats(scopedStats)
|
|
||||||
.withName(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
def apply(
|
|
||||||
genericTypeCategories: Seq[String],
|
|
||||||
crts: Set[CRT],
|
|
||||||
calculateFatiguePeriod: Seq[CaretFeedbackDetails] => Duration,
|
|
||||||
useMostRecentDislikeTime: Boolean,
|
|
||||||
name: String = defaultName
|
|
||||||
)(
|
|
||||||
implicit globalStats: StatsReceiver
|
|
||||||
): NamedPredicate[PushCandidate] = {
|
|
||||||
val scopedStats = globalStats.scope(name)
|
|
||||||
val commonNtabCaretClickFatiguePredicate = CommonNtabCaretClickFatiguePredicate(
|
|
||||||
filterCaretFeedbackHistory =
|
|
||||||
CaretFeedbackHistoryFilter.caretFeedbackHistoryFilter(genericTypeCategories),
|
|
||||||
filterHistory = FatiguePredicate.recTypesOnlyFilter(crts),
|
|
||||||
calculateFatiguePeriod = calculateFatiguePeriod,
|
|
||||||
useMostRecentDislikeTime = useMostRecentDislikeTime,
|
|
||||||
name = name
|
|
||||||
)(globalStats)
|
|
||||||
|
|
||||||
Predicate
|
|
||||||
.fromAsync { candidate: PushCandidate =>
|
|
||||||
if (candidate.frigateNotification.notificationDisplayLocation == NotificationDisplayLocation.PushToMobileDevice) {
|
|
||||||
if (candidate.target.params(PushParams.EnableFatigueNtabCaretClickingParam)) {
|
|
||||||
commonNtabCaretClickFatiguePredicate
|
|
||||||
.apply(Seq(candidate.target))
|
|
||||||
.map(_.headOption.getOrElse(false))
|
|
||||||
} else Future.True
|
|
||||||
} else {
|
|
||||||
Future.True
|
|
||||||
}
|
|
||||||
}.andThen(candidateFatiguePredicate(genericTypeCategories, crts))
|
|
||||||
.withStats(scopedStats)
|
|
||||||
.withName(name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,44 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.Candidate
|
|
||||||
import com.twitter.frigate.common.base.SocialGraphServiceRelationshipMap
|
|
||||||
import com.twitter.frigate.common.base.TweetAuthor
|
|
||||||
import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.hermit.predicate.Predicate
|
|
||||||
|
|
||||||
package object predicate {
|
|
||||||
implicit class CandidatesWithAuthorFollowPredicates(
|
|
||||||
predicate: Predicate[
|
|
||||||
PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
|
||||||
]) {
|
|
||||||
def applyOnlyToAuthorBeingFollowPredicates: Predicate[Candidate] =
|
|
||||||
predicate.optionalOn[Candidate](
|
|
||||||
{
|
|
||||||
case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
|
||||||
if isInNetworkTweetType(candidate.commonRecType) =>
|
|
||||||
Some(candidate)
|
|
||||||
case _ =>
|
|
||||||
None
|
|
||||||
},
|
|
||||||
missingResult = true
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
implicit class TweetCandidateWithTweetAuthor(
|
|
||||||
predicate: Predicate[
|
|
||||||
PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
|
||||||
]) {
|
|
||||||
def applyOnlyToBasicTweetPredicates: Predicate[Candidate] =
|
|
||||||
predicate.optionalOn[Candidate](
|
|
||||||
{
|
|
||||||
case candidate: PushCandidate with TweetAuthor with SocialGraphServiceRelationshipMap
|
|
||||||
if isInNetworkTweetType(candidate.commonRecType) =>
|
|
||||||
Some(candidate)
|
|
||||||
case _ =>
|
|
||||||
None
|
|
||||||
},
|
|
||||||
missingResult = true
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,27 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.predicate.quality_model_predicate
|
|
||||||
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object ExplicitOONCFilterPredicate extends QualityPredicateBase {
|
|
||||||
override lazy val name = "open_or_ntab_click_explicit_threshold"
|
|
||||||
|
|
||||||
override lazy val thresholdExtractor = (t: Target) =>
|
|
||||||
Future.value(t.params(PushFeatureSwitchParams.QualityPredicateExplicitThresholdParam))
|
|
||||||
|
|
||||||
override def scoreExtractor = (candidate: PushCandidate) =>
|
|
||||||
candidate.mrWeightedOpenOrNtabClickRankingProbability
|
|
||||||
}
|
|
||||||
|
|
||||||
object WeightedOpenOrNtabClickQualityPredicate extends QualityPredicateBase {
|
|
||||||
override lazy val name = "weighted_open_or_ntab_click_model"
|
|
||||||
|
|
||||||
override lazy val thresholdExtractor = (t: Target) => {
|
|
||||||
Future.value(0.0)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def scoreExtractor =
|
|
||||||
(candidate: PushCandidate) => candidate.mrWeightedOpenOrNtabClickFilteringProbability
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,165 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.predicate.quality_model_predicate
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.target.TargetScoringDetails
|
|
||||||
import com.twitter.hermit.predicate.NamedPredicate
|
|
||||||
import com.twitter.hermit.predicate.Predicate
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object PDauCohort extends Enumeration {
|
|
||||||
type PDauCohort = Value
|
|
||||||
|
|
||||||
val cohort1 = Value
|
|
||||||
val cohort2 = Value
|
|
||||||
val cohort3 = Value
|
|
||||||
val cohort4 = Value
|
|
||||||
val cohort5 = Value
|
|
||||||
val cohort6 = Value
|
|
||||||
}
|
|
||||||
|
|
||||||
object PDauCohortUtil {
|
|
||||||
|
|
||||||
case class DauThreshold(
|
|
||||||
threshold1: Double,
|
|
||||||
threshold2: Double,
|
|
||||||
threshold3: Double,
|
|
||||||
threshold4: Double,
|
|
||||||
threshold5: Double)
|
|
||||||
|
|
||||||
val defaultDAUProb = 0.0
|
|
||||||
|
|
||||||
val dauProbThresholds = DauThreshold(
|
|
||||||
threshold1 = 0.05,
|
|
||||||
threshold2 = 0.14,
|
|
||||||
threshold3 = 0.33,
|
|
||||||
threshold4 = 0.7,
|
|
||||||
threshold5 = 0.959
|
|
||||||
)
|
|
||||||
|
|
||||||
val finerThresholdMap =
|
|
||||||
Map(
|
|
||||||
PDauCohort.cohort2 -> List(0.05, 0.0539, 0.0563, 0.0600, 0.0681, 0.0733, 0.0800, 0.0849,
|
|
||||||
0.0912, 0.0975, 0.1032, 0.1092, 0.1134, 0.1191, 0.1252, 0.1324, 0.14),
|
|
||||||
PDauCohort.cohort3 -> List(0.14, 0.1489, 0.1544, 0.1625, 0.1704, 0.1797, 0.1905, 0.2001,
|
|
||||||
0.2120, 0.2248, 0.2363, 0.2500, 0.2650, 0.2801, 0.2958, 0.3119, 0.33),
|
|
||||||
PDauCohort.cohort4 -> List(0.33, 0.3484, 0.3686, 0.3893, 0.4126, 0.4350, 0.4603, 0.4856,
|
|
||||||
0.5092, 0.5348, 0.5602, 0.5850, 0.6087, 0.6319, 0.6548, 0.6779, 0.7),
|
|
||||||
PDauCohort.cohort5 -> List(0.7, 0.7295, 0.7581, 0.7831, 0.8049, 0.8251, 0.8444, 0.8612,
|
|
||||||
0.8786, 0.8936, 0.9043, 0.9175, 0.9290, 0.9383, 0.9498, 0.9587, 0.959)
|
|
||||||
)
|
|
||||||
|
|
||||||
def getBucket(targetUser: PushTypes.Target, doImpression: Boolean) = {
|
|
||||||
implicit val stats = targetUser.stats.scope("PDauCohortUtil")
|
|
||||||
if (doImpression) targetUser.getBucket _ else targetUser.getBucketWithoutImpression _
|
|
||||||
}
|
|
||||||
|
|
||||||
def threshold1(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold1
|
|
||||||
|
|
||||||
def threshold2(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold2
|
|
||||||
|
|
||||||
def threshold3(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold3
|
|
||||||
|
|
||||||
def threshold4(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold4
|
|
||||||
|
|
||||||
def threshold5(targetUser: PushTypes.Target): Double = dauProbThresholds.threshold5
|
|
||||||
|
|
||||||
def thresholdForCohort(targetUser: PushTypes.Target, dauCohort: Int): Double = {
|
|
||||||
if (dauCohort == 0) 0.0
|
|
||||||
else if (dauCohort == 1) threshold1(targetUser)
|
|
||||||
else if (dauCohort == 2) threshold2(targetUser)
|
|
||||||
else if (dauCohort == 3) threshold3(targetUser)
|
|
||||||
else if (dauCohort == 4) threshold4(targetUser)
|
|
||||||
else if (dauCohort == 5) threshold5(targetUser)
|
|
||||||
else 1.0
|
|
||||||
}
|
|
||||||
|
|
||||||
def getPDauCohort(dauProbability: Double, thresholds: DauThreshold): PDauCohort.Value = {
|
|
||||||
dauProbability match {
|
|
||||||
case dauProb if dauProb >= 0.0 && dauProb < thresholds.threshold1 => PDauCohort.cohort1
|
|
||||||
case dauProb if dauProb >= thresholds.threshold1 && dauProb < thresholds.threshold2 =>
|
|
||||||
PDauCohort.cohort2
|
|
||||||
case dauProb if dauProb >= thresholds.threshold2 && dauProb < thresholds.threshold3 =>
|
|
||||||
PDauCohort.cohort3
|
|
||||||
case dauProb if dauProb >= thresholds.threshold3 && dauProb < thresholds.threshold4 =>
|
|
||||||
PDauCohort.cohort4
|
|
||||||
case dauProb if dauProb >= thresholds.threshold4 && dauProb < thresholds.threshold5 =>
|
|
||||||
PDauCohort.cohort5
|
|
||||||
case dauProb if dauProb >= thresholds.threshold5 && dauProb <= 1.0 => PDauCohort.cohort6
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def getDauProb(target: TargetScoringDetails): Future[Double] = {
|
|
||||||
target.dauProbability.map { dauProb =>
|
|
||||||
dauProb.map(_.probability).getOrElse(defaultDAUProb)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def getPDauCohort(target: TargetScoringDetails): Future[PDauCohort.Value] = {
|
|
||||||
getDauProb(target).map { getPDauCohort(_, dauProbThresholds) }
|
|
||||||
}
|
|
||||||
|
|
||||||
def getPDauCohortWithPDau(target: TargetScoringDetails): Future[(PDauCohort.Value, Double)] = {
|
|
||||||
getDauProb(target).map { prob =>
|
|
||||||
(getPDauCohort(prob, dauProbThresholds), prob)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def updateStats(
|
|
||||||
target: PushTypes.Target,
|
|
||||||
modelName: String,
|
|
||||||
predicateResult: Boolean
|
|
||||||
)(
|
|
||||||
implicit statsReceiver: StatsReceiver
|
|
||||||
): Unit = {
|
|
||||||
val dauCohortOp = getPDauCohort(target)
|
|
||||||
dauCohortOp.map { dauCohort =>
|
|
||||||
val cohortStats = statsReceiver.scope(modelName).scope(dauCohort.toString)
|
|
||||||
cohortStats.counter(s"filter_$predicateResult").incr()
|
|
||||||
}
|
|
||||||
if (target.isNewSignup) {
|
|
||||||
val newUserModelStats = statsReceiver.scope(modelName)
|
|
||||||
newUserModelStats.counter(s"new_user_filter_$predicateResult").incr()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
trait QualityPredicateBase {
|
|
||||||
def name: String
|
|
||||||
def thresholdExtractor: Target => Future[Double]
|
|
||||||
def scoreExtractor: PushCandidate => Future[Option[Double]]
|
|
||||||
def isPredicateEnabled: PushCandidate => Future[Boolean] = _ => Future.True
|
|
||||||
def comparator: (Double, Double) => Boolean =
|
|
||||||
(score: Double, threshold: Double) => score >= threshold
|
|
||||||
def updateCustomStats(
|
|
||||||
candidate: PushCandidate,
|
|
||||||
score: Double,
|
|
||||||
threshold: Double,
|
|
||||||
result: Boolean
|
|
||||||
)(
|
|
||||||
implicit statsReceiver: StatsReceiver
|
|
||||||
): Unit = {}
|
|
||||||
|
|
||||||
def apply()(implicit statsReceiver: StatsReceiver): NamedPredicate[PushCandidate] = {
|
|
||||||
Predicate
|
|
||||||
.fromAsync { candidate: PushCandidate =>
|
|
||||||
isPredicateEnabled(candidate).flatMap {
|
|
||||||
case true =>
|
|
||||||
scoreExtractor(candidate).flatMap { scoreOpt =>
|
|
||||||
thresholdExtractor(candidate.target).map { threshold =>
|
|
||||||
val score = scoreOpt.getOrElse(0.0)
|
|
||||||
val result = comparator(score, threshold)
|
|
||||||
PDauCohortUtil.updateStats(candidate.target, name, result)
|
|
||||||
updateCustomStats(candidate, score, threshold, result)
|
|
||||||
result
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case _ => Future.True
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.withStats(statsReceiver.scope(s"predicate_$name"))
|
|
||||||
.withName(name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,21 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.predicate.quality_model_predicate
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.params.QualityPredicateEnum
|
|
||||||
import com.twitter.frigate.pushservice.predicate.PredicatesForCandidate
|
|
||||||
import com.twitter.hermit.predicate.NamedPredicate
|
|
||||||
|
|
||||||
object QualityPredicateMap {
|
|
||||||
|
|
||||||
def apply(
|
|
||||||
)(
|
|
||||||
implicit statsReceiver: StatsReceiver
|
|
||||||
): Map[QualityPredicateEnum.Value, NamedPredicate[PushCandidate]] = {
|
|
||||||
Map(
|
|
||||||
QualityPredicateEnum.WeightedOpenOrNtabClick -> WeightedOpenOrNtabClickQualityPredicate(),
|
|
||||||
QualityPredicateEnum.ExplicitOpenOrNtabClickFilter -> ExplicitOONCFilterPredicate(),
|
|
||||||
QualityPredicateEnum.AlwaysTrue -> PredicatesForCandidate.alwaysTruePushCandidatePredicate,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,54 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This Ranker re-ranks MR candidates, boosting input CRTs.
|
|
||||||
* Relative ranking between input CRTs and rest of the candidates doesn't change
|
|
||||||
*
|
|
||||||
* Ex: T: Tweet candidate, F: input CRT candidatess
|
|
||||||
*
|
|
||||||
* T3, F2, T1, T2, F1 => F2, F1, T3, T1, T2
|
|
||||||
*/
|
|
||||||
case class CRTBoostRanker(statsReceiver: StatsReceiver) {
|
|
||||||
|
|
||||||
private val recsToBoostStat = statsReceiver.stat("recs_to_boost")
|
|
||||||
private val otherRecsStat = statsReceiver.stat("other_recs")
|
|
||||||
|
|
||||||
private def boostCrtToTop(
|
|
||||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
crtToBoost: CommonRecommendationType
|
|
||||||
): Seq[CandidateDetails[PushCandidate]] = {
|
|
||||||
val (upRankedCandidates, otherCandidates) =
|
|
||||||
inputCandidates.partition(_.candidate.commonRecType == crtToBoost)
|
|
||||||
recsToBoostStat.add(upRankedCandidates.size)
|
|
||||||
otherRecsStat.add(otherCandidates.size)
|
|
||||||
upRankedCandidates ++ otherCandidates
|
|
||||||
}
|
|
||||||
|
|
||||||
final def boostCrtsToTop(
|
|
||||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
crtsToBoost: Seq[CommonRecommendationType]
|
|
||||||
): Seq[CandidateDetails[PushCandidate]] = {
|
|
||||||
crtsToBoost.headOption match {
|
|
||||||
case Some(crt) =>
|
|
||||||
val upRankedCandidates = boostCrtToTop(inputCandidates, crt)
|
|
||||||
boostCrtsToTop(upRankedCandidates, crtsToBoost.tail)
|
|
||||||
case None => inputCandidates
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
final def boostCrtsToTopStableOrder(
|
|
||||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
crtsToBoost: Seq[CommonRecommendationType]
|
|
||||||
): Seq[CandidateDetails[PushCandidate]] = {
|
|
||||||
val crtsToBoostSet = crtsToBoost.toSet
|
|
||||||
val (upRankedCandidates, otherCandidates) = inputCandidates.partition(candidateDetail =>
|
|
||||||
crtsToBoostSet.contains(candidateDetail.candidate.commonRecType))
|
|
||||||
|
|
||||||
upRankedCandidates ++ otherCandidates
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,45 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This Ranker re-ranks MR candidates, down ranks input CRTs.
|
|
||||||
* Relative ranking between input CRTs and rest of the candidates doesn't change
|
|
||||||
*
|
|
||||||
* Ex: T: Tweet candidate, F: input CRT candidates
|
|
||||||
*
|
|
||||||
* T3, F2, T1, T2, F1 => T3, T1, T2, F2, F1
|
|
||||||
*/
|
|
||||||
case class CRTDownRanker(statsReceiver: StatsReceiver) {
|
|
||||||
|
|
||||||
private val recsToDownRankStat = statsReceiver.stat("recs_to_down_rank")
|
|
||||||
private val otherRecsStat = statsReceiver.stat("other_recs")
|
|
||||||
private val downRankerRequests = statsReceiver.counter("down_ranker_requests")
|
|
||||||
|
|
||||||
private def downRank(
|
|
||||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
crtToDownRank: CommonRecommendationType
|
|
||||||
): Seq[CandidateDetails[PushCandidate]] = {
|
|
||||||
downRankerRequests.incr()
|
|
||||||
val (downRankedCandidates, otherCandidates) =
|
|
||||||
inputCandidates.partition(_.candidate.commonRecType == crtToDownRank)
|
|
||||||
recsToDownRankStat.add(downRankedCandidates.size)
|
|
||||||
otherRecsStat.add(otherCandidates.size)
|
|
||||||
otherCandidates ++ downRankedCandidates
|
|
||||||
}
|
|
||||||
|
|
||||||
final def downRank(
|
|
||||||
inputCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
crtsToDownRank: Seq[CommonRecommendationType]
|
|
||||||
): Seq[CandidateDetails[PushCandidate]] = {
|
|
||||||
crtsToDownRank.headOption match {
|
|
||||||
case Some(crt) =>
|
|
||||||
val downRankedCandidates = downRank(inputCandidates, crt)
|
|
||||||
downRank(downRankedCandidates, crtsToDownRank.tail)
|
|
||||||
case None => inputCandidates
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,45 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.TweetCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.stitch.tweetypie.TweetyPie.TweetyPieResult
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
class LoggedOutRanker(tweetyPieStore: ReadableStore[Long, TweetyPieResult], stats: StatsReceiver) {
|
|
||||||
private val statsReceiver = stats.scope(this.getClass.getSimpleName)
|
|
||||||
private val rankedCandidates = statsReceiver.counter("ranked_candidates_count")
|
|
||||||
|
|
||||||
def rank(
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
val tweetIds = candidates.map { cand => cand.candidate.asInstanceOf[TweetCandidate].tweetId }
|
|
||||||
val results = tweetyPieStore.multiGet(tweetIds.toSet).values.toSeq
|
|
||||||
val futureOfResults = Future.traverseSequentially(results)(r => r)
|
|
||||||
val tweetsFut = futureOfResults.map { tweetyPieResults =>
|
|
||||||
tweetyPieResults.map(_.map(_.tweet))
|
|
||||||
}
|
|
||||||
val sortedTweetsFuture = tweetsFut.map { tweets =>
|
|
||||||
tweets
|
|
||||||
.map { tweet =>
|
|
||||||
if (tweet.isDefined && tweet.get.counts.isDefined) {
|
|
||||||
tweet.get.id -> tweet.get.counts.get.favoriteCount.getOrElse(0L)
|
|
||||||
} else {
|
|
||||||
0 -> 0L
|
|
||||||
}
|
|
||||||
}.sortBy(_._2)(Ordering[Long].reverse)
|
|
||||||
}
|
|
||||||
val finalCandidates = sortedTweetsFuture.map { sortedTweets =>
|
|
||||||
sortedTweets
|
|
||||||
.map { tweet =>
|
|
||||||
candidates.find(_.candidate.asInstanceOf[TweetCandidate].tweetId == tweet._1).orNull
|
|
||||||
}.filter { cand => cand != null }
|
|
||||||
}
|
|
||||||
finalCandidates.map { fc =>
|
|
||||||
rankedCandidates.incr(fc.size)
|
|
||||||
}
|
|
||||||
finalCandidates
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,204 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum
|
|
||||||
import com.twitter.frigate.common.base.TweetCandidate
|
|
||||||
import com.twitter.frigate.common.rec_types.RecTypes
|
|
||||||
import com.twitter.frigate.pushservice.params.PushConstants.OoncQualityCombinedScore
|
|
||||||
|
|
||||||
object ModelBasedRanker {
|
|
||||||
|
|
||||||
def rankBySpecifiedScore(
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
scoreExtractor: PushCandidate => Future[Option[Double]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
val scoredCandidatesFutures = candidatesDetails.map { cand =>
|
|
||||||
scoreExtractor(cand.candidate).map { scoreOp => (cand, scoreOp.getOrElse(0.0)) }
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.collect(scoredCandidatesFutures).map { scores =>
|
|
||||||
val sorted = scores.sortBy { candidateDetails => -1 * candidateDetails._2 }
|
|
||||||
sorted.map(_._1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def populatePredictionScoreStats(
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
scoreExtractor: PushCandidate => Future[Option[Double]],
|
|
||||||
predictionScoreStats: StatsReceiver
|
|
||||||
): Unit = {
|
|
||||||
val scoreScaleFactorForStat = 10000
|
|
||||||
val statName = "prediction_scores"
|
|
||||||
candidatesDetails.map {
|
|
||||||
case CandidateDetails(candidate, source) =>
|
|
||||||
val crt = candidate.commonRecType
|
|
||||||
scoreExtractor(candidate).map { scoreOp =>
|
|
||||||
val scaledScore = (scoreOp.getOrElse(0.0) * scoreScaleFactorForStat).toFloat
|
|
||||||
predictionScoreStats.scope("all_candidates").stat(statName).add(scaledScore)
|
|
||||||
predictionScoreStats.scope(crt.toString()).stat(statName).add(scaledScore)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def populateMrWeightedOpenOrNtabClickScoreStats(
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
predictionScoreStats: StatsReceiver
|
|
||||||
): Unit = {
|
|
||||||
populatePredictionScoreStats(
|
|
||||||
candidatesDetails,
|
|
||||||
candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability,
|
|
||||||
predictionScoreStats
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
def populateMrQualityUprankingScoreStats(
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
predictionScoreStats: StatsReceiver
|
|
||||||
): Unit = {
|
|
||||||
populatePredictionScoreStats(
|
|
||||||
candidatesDetails,
|
|
||||||
candidate => candidate.mrQualityUprankingProbability,
|
|
||||||
predictionScoreStats
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
def rankByMrWeightedOpenOrNtabClickScore(
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
rankBySpecifiedScore(
|
|
||||||
candidatesDetails,
|
|
||||||
candidate => candidate.mrWeightedOpenOrNtabClickRankingProbability
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
def transformSigmoid(
|
|
||||||
score: Double,
|
|
||||||
weight: Double = 1.0,
|
|
||||||
bias: Double = 0.0
|
|
||||||
): Double = {
|
|
||||||
val base = -1.0 * (weight * score + bias)
|
|
||||||
val cappedBase = math.max(math.min(base, 100.0), -100.0)
|
|
||||||
1.0 / (1.0 + math.exp(cappedBase))
|
|
||||||
}
|
|
||||||
|
|
||||||
def transformLinear(
|
|
||||||
score: Double,
|
|
||||||
bar: Double = 1.0
|
|
||||||
): Double = {
|
|
||||||
val positiveBar = math.abs(bar)
|
|
||||||
val cappedScore = math.max(math.min(score, positiveBar), -1.0 * positiveBar)
|
|
||||||
cappedScore / positiveBar
|
|
||||||
}
|
|
||||||
|
|
||||||
def transformIdentity(
|
|
||||||
score: Double
|
|
||||||
): Double = score
|
|
||||||
|
|
||||||
def rankByQualityOoncCombinedScore(
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
qualityScoreTransform: Double => Double,
|
|
||||||
qualityScoreBoost: Double = 1.0
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
rankBySpecifiedScore(
|
|
||||||
candidatesDetails,
|
|
||||||
candidate => {
|
|
||||||
val ooncScoreFutOpt: Future[Option[Double]] =
|
|
||||||
candidate.mrWeightedOpenOrNtabClickRankingProbability
|
|
||||||
val qualityScoreFutOpt: Future[Option[Double]] =
|
|
||||||
candidate.mrQualityUprankingProbability
|
|
||||||
Future
|
|
||||||
.join(
|
|
||||||
ooncScoreFutOpt,
|
|
||||||
qualityScoreFutOpt
|
|
||||||
).map {
|
|
||||||
case (Some(ooncScore), Some(qualityScore)) =>
|
|
||||||
val transformedQualityScore = qualityScoreTransform(qualityScore)
|
|
||||||
val combinedScore = ooncScore * (1.0 + qualityScoreBoost * transformedQualityScore)
|
|
||||||
candidate
|
|
||||||
.cacheExternalScore(OoncQualityCombinedScore, Future.value(Some(combinedScore)))
|
|
||||||
Some(combinedScore)
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
def rerankByProducerQualityOoncCombinedScore(
|
|
||||||
candidateDetails: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
)(
|
|
||||||
implicit stat: StatsReceiver
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
val scopedStat = stat.scope("producer_quality_reranking")
|
|
||||||
val oonCandidates = candidateDetails.filter {
|
|
||||||
case CandidateDetails(pushCandidate: PushCandidate, _) =>
|
|
||||||
tweetCandidateSelector(pushCandidate, MrQualityUprankingPartialTypeEnum.Oon)
|
|
||||||
}
|
|
||||||
|
|
||||||
val rankedOonCandidatesFut = rankBySpecifiedScore(
|
|
||||||
oonCandidates,
|
|
||||||
candidate => {
|
|
||||||
val baseScoreFutureOpt: Future[Option[Double]] = {
|
|
||||||
val qualityCombinedScoreFutureOpt =
|
|
||||||
candidate.getExternalCachedScoreByName(OoncQualityCombinedScore)
|
|
||||||
val ooncScoreFutureOpt = candidate.mrWeightedOpenOrNtabClickRankingProbability
|
|
||||||
Future.join(qualityCombinedScoreFutureOpt, ooncScoreFutureOpt).map {
|
|
||||||
case (Some(qualityCombinedScore), _) =>
|
|
||||||
scopedStat.counter("quality_combined_score").incr()
|
|
||||||
Some(qualityCombinedScore)
|
|
||||||
case (_, ooncScoreOpt) =>
|
|
||||||
scopedStat.counter("oonc_score").incr()
|
|
||||||
ooncScoreOpt
|
|
||||||
}
|
|
||||||
}
|
|
||||||
baseScoreFutureOpt.map {
|
|
||||||
case Some(baseScore) =>
|
|
||||||
val boostRatio = candidate.mrProducerQualityUprankingBoost.getOrElse(1.0)
|
|
||||||
if (boostRatio > 1.0) scopedStat.counter("author_uprank").incr()
|
|
||||||
else if (boostRatio < 1.0) scopedStat.counter("author_downrank").incr()
|
|
||||||
else scopedStat.counter("author_noboost").incr()
|
|
||||||
Some(baseScore * boostRatio)
|
|
||||||
case _ =>
|
|
||||||
scopedStat.counter("empty_score").incr()
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
rankedOonCandidatesFut.map { rankedOonCandidates =>
|
|
||||||
val sortedOonCandidateIterator = rankedOonCandidates.toIterator
|
|
||||||
candidateDetails.map { ooncRankedCandidate =>
|
|
||||||
val isOon = tweetCandidateSelector(
|
|
||||||
ooncRankedCandidate.candidate,
|
|
||||||
MrQualityUprankingPartialTypeEnum.Oon)
|
|
||||||
|
|
||||||
if (sortedOonCandidateIterator.hasNext && isOon)
|
|
||||||
sortedOonCandidateIterator.next()
|
|
||||||
else ooncRankedCandidate
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def tweetCandidateSelector(
|
|
||||||
pushCandidate: PushCandidate,
|
|
||||||
selectedCandidateType: MrQualityUprankingPartialTypeEnum.Value
|
|
||||||
): Boolean = {
|
|
||||||
pushCandidate match {
|
|
||||||
case candidate: PushCandidate with TweetCandidate =>
|
|
||||||
selectedCandidateType match {
|
|
||||||
case MrQualityUprankingPartialTypeEnum.Oon =>
|
|
||||||
val crt = candidate.commonRecType
|
|
||||||
RecTypes.isOutOfNetworkTweetRecType(crt) || RecTypes.outOfNetworkTopicTweetTypes
|
|
||||||
.contains(crt)
|
|
||||||
case _ => true
|
|
||||||
}
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,31 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.Ranker
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
trait PushserviceRanker[T, C] extends Ranker[T, C] {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initial Ranking of input candidates
|
|
||||||
*/
|
|
||||||
def initialRank(target: T, candidates: Seq[CandidateDetails[C]]): Future[Seq[CandidateDetails[C]]]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Re-ranks input ranked candidates. Useful when a subset of candidates are ranked
|
|
||||||
* by a different logic, while preserving the initial ranking for the rest
|
|
||||||
*/
|
|
||||||
def reRank(
|
|
||||||
target: T,
|
|
||||||
rankedCandidates: Seq[CandidateDetails[C]]
|
|
||||||
): Future[Seq[CandidateDetails[C]]]
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Final ranking that does Initial + Rerank
|
|
||||||
*/
|
|
||||||
override final def rank(target: T, candidates: Seq[CandidateDetails[C]]): (
|
|
||||||
Future[Seq[CandidateDetails[C]]]
|
|
||||||
) = {
|
|
||||||
initialRank(target, candidates).flatMap { rankedCandidates => reRank(target, rankedCandidates) }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,139 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
import com.twitter.contentrecommender.thriftscala.LightRankingCandidate
|
|
||||||
import com.twitter.contentrecommender.thriftscala.LightRankingFeatureHydrationContext
|
|
||||||
import com.twitter.contentrecommender.thriftscala.MagicRecsFeatureHydrationContext
|
|
||||||
import com.twitter.finagle.stats.Counter
|
|
||||||
import com.twitter.finagle.stats.Stat
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.RandomRanker
|
|
||||||
import com.twitter.frigate.common.base.Ranker
|
|
||||||
import com.twitter.frigate.common.base.TweetAuthor
|
|
||||||
import com.twitter.frigate.common.base.TweetCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.params.PushConstants
|
|
||||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
|
||||||
import com.twitter.frigate.pushservice.params.PushParams
|
|
||||||
import com.twitter.ml.featurestore.lib.UserId
|
|
||||||
import com.twitter.nrel.lightranker.MagicRecsServeDataRecordLightRanker
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
class RFPHLightRanker(
|
|
||||||
lightRanker: MagicRecsServeDataRecordLightRanker,
|
|
||||||
stats: StatsReceiver)
|
|
||||||
extends Ranker[Target, PushCandidate] {
|
|
||||||
|
|
||||||
private val statsReceiver = stats.scope(this.getClass.getSimpleName)
|
|
||||||
|
|
||||||
private val lightRankerCandidateCounter = statsReceiver.counter("light_ranker_candidate_count")
|
|
||||||
private val lightRankerRequestCounter = statsReceiver.counter("light_ranker_request_count")
|
|
||||||
private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking")
|
|
||||||
private val restrictLightRankingCounter: Counter =
|
|
||||||
lightRankingStats.counter("restrict_light_ranking")
|
|
||||||
private val selectedLightRankerScribedTargetCandidateCountStats: Stat =
|
|
||||||
lightRankingStats.stat("selected_light_ranker_scribed_target_candidate_count")
|
|
||||||
private val selectedLightRankerScribedCandidatesStats: Stat =
|
|
||||||
lightRankingStats.stat("selected_light_ranker_scribed_candidates")
|
|
||||||
private val lightRankingRandomBaselineStats: StatsReceiver =
|
|
||||||
statsReceiver.scope("light_ranking_random_baseline")
|
|
||||||
|
|
||||||
override def rank(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
val enableLightRanker = target.params(PushFeatureSwitchParams.EnableLightRankingParam)
|
|
||||||
val restrictLightRanker = target.params(PushParams.RestrictLightRankingParam)
|
|
||||||
val lightRankerSelectionThreshold =
|
|
||||||
target.params(PushFeatureSwitchParams.LightRankingNumberOfCandidatesParam)
|
|
||||||
val randomRanker = RandomRanker[Target, PushCandidate]()(lightRankingRandomBaselineStats)
|
|
||||||
|
|
||||||
if (enableLightRanker && candidates.length > lightRankerSelectionThreshold && !target.scribeFeatureForRequestScribe) {
|
|
||||||
val (tweetCandidates, nonTweetCandidates) =
|
|
||||||
candidates.partition {
|
|
||||||
case CandidateDetails(pushCandidate: PushCandidate with TweetCandidate, source) => true
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
val lightRankerSelectedTweetCandidatesFut = {
|
|
||||||
if (restrictLightRanker) {
|
|
||||||
restrictLightRankingCounter.incr()
|
|
||||||
lightRankThenTake(
|
|
||||||
target,
|
|
||||||
tweetCandidates
|
|
||||||
.asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]],
|
|
||||||
PushConstants.RestrictLightRankingCandidatesThreshold
|
|
||||||
)
|
|
||||||
} else if (target.params(PushFeatureSwitchParams.EnableRandomBaselineLightRankingParam)) {
|
|
||||||
randomRanker.rank(target, tweetCandidates).map { randomLightRankerCands =>
|
|
||||||
randomLightRankerCands.take(lightRankerSelectionThreshold)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
lightRankThenTake(
|
|
||||||
target,
|
|
||||||
tweetCandidates
|
|
||||||
.asInstanceOf[Seq[CandidateDetails[PushCandidate with TweetCandidate]]],
|
|
||||||
lightRankerSelectionThreshold
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
lightRankerSelectedTweetCandidatesFut.map { returnedTweetCandidates =>
|
|
||||||
nonTweetCandidates ++ returnedTweetCandidates
|
|
||||||
}
|
|
||||||
} else if (target.scribeFeatureForRequestScribe) {
|
|
||||||
val downSampleRate: Double =
|
|
||||||
if (target.params(PushParams.DownSampleLightRankingScribeCandidatesParam))
|
|
||||||
PushConstants.DownSampleLightRankingScribeCandidatesRate
|
|
||||||
else target.params(PushFeatureSwitchParams.LightRankingScribeCandidatesDownSamplingParam)
|
|
||||||
val selectedCandidateCounter: Int = math.ceil(candidates.size * downSampleRate).toInt
|
|
||||||
selectedLightRankerScribedTargetCandidateCountStats.add(selectedCandidateCounter.toFloat)
|
|
||||||
|
|
||||||
randomRanker.rank(target, candidates).map { randomLightRankerCands =>
|
|
||||||
val selectedCandidates = randomLightRankerCands.take(selectedCandidateCounter)
|
|
||||||
selectedLightRankerScribedCandidatesStats.add(selectedCandidates.size.toFloat)
|
|
||||||
selectedCandidates
|
|
||||||
}
|
|
||||||
} else Future.value(candidates)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def lightRankThenTake(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate with TweetCandidate]],
|
|
||||||
numOfCandidates: Int
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
lightRankerCandidateCounter.incr(candidates.length)
|
|
||||||
lightRankerRequestCounter.incr()
|
|
||||||
val lightRankerCandidates: Seq[LightRankingCandidate] = candidates.map {
|
|
||||||
case CandidateDetails(tweetCandidate, _) =>
|
|
||||||
val tweetAuthor = tweetCandidate match {
|
|
||||||
case t: TweetCandidate with TweetAuthor => t.authorId
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
val hydrationContext: LightRankingFeatureHydrationContext =
|
|
||||||
LightRankingFeatureHydrationContext.MagicRecsHydrationContext(
|
|
||||||
MagicRecsFeatureHydrationContext(
|
|
||||||
tweetAuthor = tweetAuthor,
|
|
||||||
pushString = tweetCandidate.getPushCopy.flatMap(_.pushStringGroup).map(_.toString))
|
|
||||||
)
|
|
||||||
LightRankingCandidate(
|
|
||||||
tweetId = tweetCandidate.tweetId,
|
|
||||||
hydrationContext = Some(hydrationContext)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
val modelName = target.params(PushFeatureSwitchParams.LightRankingModelTypeParam)
|
|
||||||
val lightRankedCandidatesFut = {
|
|
||||||
lightRanker
|
|
||||||
.rank(UserId(target.targetId), lightRankerCandidates, modelName)
|
|
||||||
}
|
|
||||||
|
|
||||||
lightRankedCandidatesFut.map { lightRankedCandidates =>
|
|
||||||
val lrScoreMap = lightRankedCandidates.map { lrCand =>
|
|
||||||
lrCand.tweetId -> lrCand.score
|
|
||||||
}.toMap
|
|
||||||
val candScoreMap: Seq[Option[Double]] = candidates.map { candidateDetails =>
|
|
||||||
lrScoreMap.get(candidateDetails.candidate.tweetId)
|
|
||||||
}
|
|
||||||
sortCandidatesByScore(candidates, candScoreMap)
|
|
||||||
.take(numOfCandidates)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,297 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.Ranker
|
|
||||||
import com.twitter.frigate.common.rec_types.RecTypes
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.ml.HealthFeatureGetter
|
|
||||||
import com.twitter.frigate.pushservice.ml.PushMLModelScorer
|
|
||||||
import com.twitter.frigate.pushservice.params.MrQualityUprankingPartialTypeEnum
|
|
||||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
|
||||||
import com.twitter.frigate.pushservice.params.PushMLModel
|
|
||||||
import com.twitter.frigate.pushservice.params.PushModelName
|
|
||||||
import com.twitter.frigate.pushservice.params.PushParams
|
|
||||||
import com.twitter.frigate.pushservice.util.MediaAnnotationsUtil.updateMediaCategoryStats
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.util.Future
|
|
||||||
import com.twitter.frigate.pushservice.params.MrQualityUprankingTransformTypeEnum
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.frigate.thriftscala.UserMediaRepresentation
|
|
||||||
import com.twitter.hss.api.thriftscala.UserHealthSignalResponse
|
|
||||||
|
|
||||||
class RFPHRanker(
|
|
||||||
randomRanker: Ranker[Target, PushCandidate],
|
|
||||||
weightedOpenOrNtabClickModelScorer: PushMLModelScorer,
|
|
||||||
subscriptionCreatorRanker: SubscriptionCreatorRanker,
|
|
||||||
userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse],
|
|
||||||
producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation],
|
|
||||||
stats: StatsReceiver)
|
|
||||||
extends PushserviceRanker[Target, PushCandidate] {
|
|
||||||
|
|
||||||
private val statsReceiver = stats.scope(this.getClass.getSimpleName)
|
|
||||||
|
|
||||||
private val boostCRTsRanker = CRTBoostRanker(statsReceiver.scope("boost_desired_crts"))
|
|
||||||
private val crtDownRanker = CRTDownRanker(statsReceiver.scope("down_rank_desired_crts"))
|
|
||||||
|
|
||||||
private val crtsToDownRank = statsReceiver.stat("crts_to_downrank")
|
|
||||||
private val crtsToUprank = statsReceiver.stat("crts_to_uprank")
|
|
||||||
|
|
||||||
private val randomRankingCounter = stats.counter("randomRanking")
|
|
||||||
private val mlRankingCounter = stats.counter("mlRanking")
|
|
||||||
private val disableAllRelevanceCounter = stats.counter("disableAllRelevance")
|
|
||||||
private val disableHeavyRankingCounter = stats.counter("disableHeavyRanking")
|
|
||||||
|
|
||||||
private val heavyRankerCandidateCounter = stats.counter("heavy_ranker_candidate_count")
|
|
||||||
private val heavyRankerScoreStats = statsReceiver.scope("heavy_ranker_prediction_scores")
|
|
||||||
|
|
||||||
private val producerUprankingCounter = statsReceiver.counter("producer_quality_upranking")
|
|
||||||
private val producerBoostedCounter = statsReceiver.counter("producer_boosted_candidates")
|
|
||||||
private val producerDownboostedCounter = statsReceiver.counter("producer_downboosted_candidates")
|
|
||||||
|
|
||||||
override def initialRank(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
heavyRankerCandidateCounter.incr(candidates.size)
|
|
||||||
|
|
||||||
updateMediaCategoryStats(candidates)(stats)
|
|
||||||
target.targetUserState
|
|
||||||
.flatMap { targetUserState =>
|
|
||||||
val useRandomRanking = target.skipMlRanker || target.params(
|
|
||||||
PushParams.UseRandomRankingParam
|
|
||||||
)
|
|
||||||
|
|
||||||
if (useRandomRanking) {
|
|
||||||
randomRankingCounter.incr()
|
|
||||||
randomRanker.rank(target, candidates)
|
|
||||||
} else if (target.params(PushParams.DisableAllRelevanceParam)) {
|
|
||||||
disableAllRelevanceCounter.incr()
|
|
||||||
Future.value(candidates)
|
|
||||||
} else if (target.params(PushParams.DisableHeavyRankingParam) || target.params(
|
|
||||||
PushFeatureSwitchParams.DisableHeavyRankingModelFSParam)) {
|
|
||||||
disableHeavyRankingCounter.incr()
|
|
||||||
Future.value(candidates)
|
|
||||||
} else {
|
|
||||||
mlRankingCounter.incr()
|
|
||||||
|
|
||||||
val scoredCandidatesFut = scoring(target, candidates)
|
|
||||||
|
|
||||||
target.rankingModelParam.map { rankingModelParam =>
|
|
||||||
val modelName = PushModelName(
|
|
||||||
PushMLModel.WeightedOpenOrNtabClickProbability,
|
|
||||||
target.params(rankingModelParam)).toString
|
|
||||||
ModelBasedRanker.populateMrWeightedOpenOrNtabClickScoreStats(
|
|
||||||
candidates,
|
|
||||||
heavyRankerScoreStats.scope(modelName)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (target.params(
|
|
||||||
PushFeatureSwitchParams.EnableQualityUprankingCrtScoreStatsForHeavyRankingParam)) {
|
|
||||||
val modelName = PushModelName(
|
|
||||||
PushMLModel.FilteringProbability,
|
|
||||||
target.params(PushFeatureSwitchParams.QualityUprankingModelTypeParam)
|
|
||||||
).toString
|
|
||||||
ModelBasedRanker.populateMrQualityUprankingScoreStats(
|
|
||||||
candidates,
|
|
||||||
heavyRankerScoreStats.scope(modelName)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
val ooncRankedCandidatesFut =
|
|
||||||
scoredCandidatesFut.flatMap(ModelBasedRanker.rankByMrWeightedOpenOrNtabClickScore)
|
|
||||||
|
|
||||||
val qualityUprankedCandidatesFut =
|
|
||||||
if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) {
|
|
||||||
ooncRankedCandidatesFut.flatMap { ooncRankedCandidates =>
|
|
||||||
val transformFunc: Double => Double =
|
|
||||||
target.params(PushFeatureSwitchParams.QualityUprankingTransformTypeParam) match {
|
|
||||||
case MrQualityUprankingTransformTypeEnum.Linear =>
|
|
||||||
ModelBasedRanker.transformLinear(
|
|
||||||
_,
|
|
||||||
bar = target.params(
|
|
||||||
PushFeatureSwitchParams.QualityUprankingLinearBarForHeavyRankingParam))
|
|
||||||
case MrQualityUprankingTransformTypeEnum.Sigmoid =>
|
|
||||||
ModelBasedRanker.transformSigmoid(
|
|
||||||
_,
|
|
||||||
weight = target.params(
|
|
||||||
PushFeatureSwitchParams.QualityUprankingSigmoidWeightForHeavyRankingParam),
|
|
||||||
bias = target.params(
|
|
||||||
PushFeatureSwitchParams.QualityUprankingSigmoidBiasForHeavyRankingParam)
|
|
||||||
)
|
|
||||||
case _ => ModelBasedRanker.transformIdentity
|
|
||||||
}
|
|
||||||
|
|
||||||
ModelBasedRanker.rankByQualityOoncCombinedScore(
|
|
||||||
ooncRankedCandidates,
|
|
||||||
transformFunc,
|
|
||||||
target.params(PushFeatureSwitchParams.QualityUprankingBoostForHeavyRankingParam)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} else ooncRankedCandidatesFut
|
|
||||||
|
|
||||||
if (target.params(
|
|
||||||
PushFeatureSwitchParams.EnableProducersQualityBoostingForHeavyRankingParam)) {
|
|
||||||
producerUprankingCounter.incr()
|
|
||||||
qualityUprankedCandidatesFut.flatMap(cands =>
|
|
||||||
ModelBasedRanker.rerankByProducerQualityOoncCombinedScore(cands)(statsReceiver))
|
|
||||||
} else qualityUprankedCandidatesFut
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def scoring(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
val ooncScoredCandidatesFut = target.rankingModelParam.map { rankingModelParam =>
|
|
||||||
weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion(
|
|
||||||
target,
|
|
||||||
candidates,
|
|
||||||
rankingModelParam
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
val scoredCandidatesFut = {
|
|
||||||
if (target.params(PushFeatureSwitchParams.EnableQualityUprankingForHeavyRankingParam)) {
|
|
||||||
ooncScoredCandidatesFut.map { candidates =>
|
|
||||||
weightedOpenOrNtabClickModelScorer.scoreByBatchPredictionForModelVersion(
|
|
||||||
target = target,
|
|
||||||
candidatesDetails = candidates,
|
|
||||||
modelVersionParam = PushFeatureSwitchParams.QualityUprankingModelTypeParam,
|
|
||||||
overridePushMLModelOpt = Some(PushMLModel.FilteringProbability)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} else ooncScoredCandidatesFut
|
|
||||||
}
|
|
||||||
|
|
||||||
scoredCandidatesFut.foreach { candidates =>
|
|
||||||
val oonCandidates = candidates.filter {
|
|
||||||
case CandidateDetails(pushCandidate: PushCandidate, _) =>
|
|
||||||
ModelBasedRanker.tweetCandidateSelector(
|
|
||||||
pushCandidate,
|
|
||||||
MrQualityUprankingPartialTypeEnum.Oon)
|
|
||||||
}
|
|
||||||
setProducerQuality(
|
|
||||||
target,
|
|
||||||
oonCandidates,
|
|
||||||
userHealthSignalStore,
|
|
||||||
producerMediaRepresentationStore)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def setProducerQuality(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
userHealthSignalStore: ReadableStore[Long, UserHealthSignalResponse],
|
|
||||||
producerMediaRepresentationStore: ReadableStore[Long, UserMediaRepresentation]
|
|
||||||
): Unit = {
|
|
||||||
lazy val boostRatio =
|
|
||||||
target.params(PushFeatureSwitchParams.QualityUprankingBoostForHighQualityProducersParam)
|
|
||||||
lazy val downboostRatio =
|
|
||||||
target.params(PushFeatureSwitchParams.QualityUprankingDownboostForLowQualityProducersParam)
|
|
||||||
candidates.foreach {
|
|
||||||
case CandidateDetails(pushCandidate, _) =>
|
|
||||||
HealthFeatureGetter
|
|
||||||
.getFeatures(pushCandidate, producerMediaRepresentationStore, userHealthSignalStore).map {
|
|
||||||
featureMap =>
|
|
||||||
val agathaNsfwScore = featureMap.numericFeatures.getOrElse("agathaNsfwScore", 0.5)
|
|
||||||
val textNsfwScore = featureMap.numericFeatures.getOrElse("textNsfwScore", 0.15)
|
|
||||||
val nudityRate = featureMap.numericFeatures.getOrElse("nudityRate", 0.0)
|
|
||||||
val activeFollowers = featureMap.numericFeatures.getOrElse("activeFollowers", 0.0)
|
|
||||||
val favorsRcvd28Days = featureMap.numericFeatures.getOrElse("favorsRcvd28Days", 0.0)
|
|
||||||
val tweets28Days = featureMap.numericFeatures.getOrElse("tweets28Days", 0.0)
|
|
||||||
val authorDislikeCount = featureMap.numericFeatures
|
|
||||||
.getOrElse("authorDislikeCount", 0.0)
|
|
||||||
val authorDislikeRate = featureMap.numericFeatures.getOrElse("authorDislikeRate", 0.0)
|
|
||||||
val authorReportRate = featureMap.numericFeatures.getOrElse("authorReportRate", 0.0)
|
|
||||||
val abuseStrikeTop2Percent =
|
|
||||||
featureMap.booleanFeatures.getOrElse("abuseStrikeTop2Percent", false)
|
|
||||||
val abuseStrikeTop1Percent =
|
|
||||||
featureMap.booleanFeatures.getOrElse("abuseStrikeTop1Percent", false)
|
|
||||||
val hasNsfwToken = featureMap.booleanFeatures.getOrElse("hasNsfwToken", false)
|
|
||||||
|
|
||||||
if ((activeFollowers > 3000000) ||
|
|
||||||
(activeFollowers > 1000000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent) ||
|
|
||||||
(activeFollowers > 100000 && agathaNsfwScore < 0.7 && nudityRate < 0.01 && !hasNsfwToken && !abuseStrikeTop2Percent &&
|
|
||||||
tweets28Days > 0 && favorsRcvd28Days / tweets28Days > 3000 && authorReportRate < 0.000001 && authorDislikeRate < 0.0005)) {
|
|
||||||
producerBoostedCounter.incr()
|
|
||||||
pushCandidate.setProducerQualityUprankingBoost(boostRatio)
|
|
||||||
} else if (activeFollowers < 5 || agathaNsfwScore > 0.9 || nudityRate > 0.03 || hasNsfwToken || abuseStrikeTop1Percent ||
|
|
||||||
textNsfwScore > 0.4 || (authorDislikeRate > 0.005 && authorDislikeCount > 5) ||
|
|
||||||
(tweets28Days > 56 && favorsRcvd28Days / tweets28Days < 100)) {
|
|
||||||
producerDownboostedCounter.incr()
|
|
||||||
pushCandidate.setProducerQualityUprankingBoost(downboostRatio)
|
|
||||||
} else pushCandidate.setProducerQualityUprankingBoost(1.0)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def rerankBySubscriptionCreatorRanker(
|
|
||||||
target: Target,
|
|
||||||
rankedCandidates: Future[Seq[CandidateDetails[PushCandidate]]],
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
if (target.params(PushFeatureSwitchParams.SoftRankCandidatesFromSubscriptionCreators)) {
|
|
||||||
val factor = target.params(PushFeatureSwitchParams.SoftRankFactorForSubscriptionCreators)
|
|
||||||
subscriptionCreatorRanker.boostByScoreFactor(rankedCandidates, factor)
|
|
||||||
} else
|
|
||||||
subscriptionCreatorRanker.boostSubscriptionCreator(rankedCandidates)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def reRank(
|
|
||||||
target: Target,
|
|
||||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
val numberOfF1Candidates =
|
|
||||||
rankedCandidates.count(candidateDetails =>
|
|
||||||
RecTypes.isF1Type(candidateDetails.candidate.commonRecType))
|
|
||||||
lazy val threshold =
|
|
||||||
target.params(PushFeatureSwitchParams.NumberOfF1CandidatesThresholdForOONBackfill)
|
|
||||||
lazy val enableOONBackfillBasedOnF1 =
|
|
||||||
target.params(PushFeatureSwitchParams.EnableOONBackfillBasedOnF1Candidates)
|
|
||||||
|
|
||||||
val f1BoostedCandidates =
|
|
||||||
if (enableOONBackfillBasedOnF1 && numberOfF1Candidates > threshold) {
|
|
||||||
boostCRTsRanker.boostCrtsToTopStableOrder(
|
|
||||||
rankedCandidates,
|
|
||||||
RecTypes.f1FirstDegreeTypes.toSeq)
|
|
||||||
} else rankedCandidates
|
|
||||||
|
|
||||||
val topTweetsByGeoDownRankedCandidates =
|
|
||||||
if (target.params(PushFeatureSwitchParams.BackfillRankTopTweetsByGeoCandidates)) {
|
|
||||||
crtDownRanker.downRank(
|
|
||||||
f1BoostedCandidates,
|
|
||||||
Seq(CommonRecommendationType.GeoPopTweet)
|
|
||||||
)
|
|
||||||
} else f1BoostedCandidates
|
|
||||||
|
|
||||||
val reRankedCandidatesWithBoostedCrts = {
|
|
||||||
val listOfCrtsToUpRank = target
|
|
||||||
.params(PushFeatureSwitchParams.ListOfCrtsToUpRank)
|
|
||||||
.flatMap(CommonRecommendationType.valueOf)
|
|
||||||
crtsToUprank.add(listOfCrtsToUpRank.size)
|
|
||||||
boostCRTsRanker.boostCrtsToTop(topTweetsByGeoDownRankedCandidates, listOfCrtsToUpRank)
|
|
||||||
}
|
|
||||||
|
|
||||||
val reRankedCandidatesWithDownRankedCrts = {
|
|
||||||
val listOfCrtsToDownRank = target
|
|
||||||
.params(PushFeatureSwitchParams.ListOfCrtsToDownRank)
|
|
||||||
.flatMap(CommonRecommendationType.valueOf)
|
|
||||||
crtsToDownRank.add(listOfCrtsToDownRank.size)
|
|
||||||
crtDownRanker.downRank(reRankedCandidatesWithBoostedCrts, listOfCrtsToDownRank)
|
|
||||||
}
|
|
||||||
|
|
||||||
val rerankBySubscriptionCreatorFut = {
|
|
||||||
if (target.params(PushFeatureSwitchParams.BoostCandidatesFromSubscriptionCreators)) {
|
|
||||||
rerankBySubscriptionCreatorRanker(
|
|
||||||
target,
|
|
||||||
Future.value(reRankedCandidatesWithDownRankedCrts))
|
|
||||||
} else Future.value(reRankedCandidatesWithDownRankedCrts)
|
|
||||||
}
|
|
||||||
|
|
||||||
rerankBySubscriptionCreatorFut
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,110 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.rank
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.TweetAuthor
|
|
||||||
import com.twitter.frigate.common.base.TweetCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.storehaus.FutureOps
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
class SubscriptionCreatorRanker(
|
|
||||||
superFollowEligibilityUserStore: ReadableStore[Long, Boolean],
|
|
||||||
statsReceiver: StatsReceiver) {
|
|
||||||
|
|
||||||
private val scopedStats = statsReceiver.scope("SubscriptionCreatorRanker")
|
|
||||||
private val boostStats = scopedStats.scope("boostSubscriptionCreator")
|
|
||||||
private val softUprankStats = scopedStats.scope("boostByScoreFactor")
|
|
||||||
private val boostTotalCandidates = boostStats.stat("total_input_candidates")
|
|
||||||
private val softRankTotalCandidates = softUprankStats.stat("total_input_candidates")
|
|
||||||
private val softRankNumCandidatesCreators = softUprankStats.counter("candidates_from_creators")
|
|
||||||
private val softRankNumCandidatesNonCreators =
|
|
||||||
softUprankStats.counter("candidates_not_from_creators")
|
|
||||||
private val boostNumCandidatesCreators = boostStats.counter("candidates_from_creators")
|
|
||||||
private val boostNumCandidatesNonCreators =
|
|
||||||
boostStats.counter("candidates_not_from_creators")
|
|
||||||
|
|
||||||
def boostSubscriptionCreator(
|
|
||||||
inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
inputCandidatesFut.flatMap { inputCandidates =>
|
|
||||||
boostTotalCandidates.add(inputCandidates.size)
|
|
||||||
val tweetAuthorIds = inputCandidates.flatMap {
|
|
||||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
|
||||||
candidate.authorId
|
|
||||||
case _ => None
|
|
||||||
}.toSet
|
|
||||||
|
|
||||||
FutureOps
|
|
||||||
.mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds))
|
|
||||||
.map { creatorAuthorMap =>
|
|
||||||
val (upRankedCandidates, otherCandidates) = inputCandidates.partition {
|
|
||||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
|
||||||
candidate.authorId match {
|
|
||||||
case Some(authorId) =>
|
|
||||||
creatorAuthorMap(authorId).getOrElse(false)
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
boostNumCandidatesCreators.incr(upRankedCandidates.size)
|
|
||||||
boostNumCandidatesNonCreators.incr(otherCandidates.size)
|
|
||||||
upRankedCandidates ++ otherCandidates
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def boostByScoreFactor(
|
|
||||||
inputCandidatesFut: Future[Seq[CandidateDetails[PushCandidate]]],
|
|
||||||
factor: Double = 1.0,
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
inputCandidatesFut.flatMap { inputCandidates =>
|
|
||||||
softRankTotalCandidates.add(inputCandidates.size)
|
|
||||||
val tweetAuthorIds = inputCandidates.flatMap {
|
|
||||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
|
||||||
candidate.authorId
|
|
||||||
case _ => None
|
|
||||||
}.toSet
|
|
||||||
|
|
||||||
FutureOps
|
|
||||||
.mapCollect(superFollowEligibilityUserStore.multiGet(tweetAuthorIds))
|
|
||||||
.flatMap { creatorAuthorMap =>
|
|
||||||
val (upRankedCandidates, otherCandidates) = inputCandidates.partition {
|
|
||||||
case CandidateDetails(candidate: TweetCandidate with TweetAuthor, s) =>
|
|
||||||
candidate.authorId match {
|
|
||||||
case Some(authorId) =>
|
|
||||||
creatorAuthorMap(authorId).getOrElse(false)
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
softRankNumCandidatesCreators.incr(upRankedCandidates.size)
|
|
||||||
softRankNumCandidatesNonCreators.incr(otherCandidates.size)
|
|
||||||
|
|
||||||
ModelBasedRanker.rankBySpecifiedScore(
|
|
||||||
inputCandidates,
|
|
||||||
candidate => {
|
|
||||||
val isFromCreator = candidate match {
|
|
||||||
case candidate: TweetCandidate with TweetAuthor =>
|
|
||||||
candidate.authorId match {
|
|
||||||
case Some(authorId) =>
|
|
||||||
creatorAuthorMap(authorId).getOrElse(false)
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
candidate.mrWeightedOpenOrNtabClickRankingProbability.map {
|
|
||||||
case Some(score) =>
|
|
||||||
if (isFromCreator) Some(score * factor)
|
|
||||||
else Some(score)
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,259 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.Counter
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.CandidateResult
|
|
||||||
import com.twitter.frigate.common.base.CandidateSource
|
|
||||||
import com.twitter.frigate.common.base.FetchRankFlowWithHydratedCandidates
|
|
||||||
import com.twitter.frigate.common.base.Invalid
|
|
||||||
import com.twitter.frigate.common.base.OK
|
|
||||||
import com.twitter.frigate.common.base.Response
|
|
||||||
import com.twitter.frigate.common.base.Result
|
|
||||||
import com.twitter.frigate.common.base.Stats.track
|
|
||||||
import com.twitter.frigate.common.base.Stats.trackSeq
|
|
||||||
import com.twitter.frigate.common.logger.MRLogger
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.adaptor.LoggedOutPushCandidateSourceGenerator
|
|
||||||
import com.twitter.frigate.pushservice.predicate.LoggedOutPreRankingPredicates
|
|
||||||
import com.twitter.frigate.pushservice.predicate.LoggedOutTargetPredicates
|
|
||||||
import com.twitter.frigate.pushservice.rank.LoggedOutRanker
|
|
||||||
import com.twitter.frigate.pushservice.take.LoggedOutRefreshForPushNotifier
|
|
||||||
import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler
|
|
||||||
import com.twitter.frigate.pushservice.target.LoggedOutPushTargetUserBuilder
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.LoggedOutRequest
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.LoggedOutResponse
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.PushContext
|
|
||||||
import com.twitter.hermit.predicate.NamedPredicate
|
|
||||||
import com.twitter.hermit.predicate.Predicate
|
|
||||||
import com.twitter.hermit.predicate.SequentialPredicate
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
class LoggedOutRefreshForPushHandler(
|
|
||||||
val loPushTargetUserBuilder: LoggedOutPushTargetUserBuilder,
|
|
||||||
val loPushCandidateSourceGenerator: LoggedOutPushCandidateSourceGenerator,
|
|
||||||
candidateHydrator: PushCandidateHydrator,
|
|
||||||
val loRanker: LoggedOutRanker,
|
|
||||||
val loRfphNotifier: LoggedOutRefreshForPushNotifier,
|
|
||||||
loMrRequestScriberNode: String
|
|
||||||
)(
|
|
||||||
globalStats: StatsReceiver)
|
|
||||||
extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] {
|
|
||||||
|
|
||||||
val log = MRLogger("LORefreshForPushHandler")
|
|
||||||
implicit val statsReceiver: StatsReceiver =
|
|
||||||
globalStats.scope("LORefreshForPushHandler")
|
|
||||||
private val loggedOutBuildStats = statsReceiver.scope("logged_out_build_target")
|
|
||||||
private val loggedOutProcessStats = statsReceiver.scope("logged_out_process")
|
|
||||||
private val loggedOutNotifyStats = statsReceiver.scope("logged_out_notify")
|
|
||||||
private val loCandidateHydrationStats: StatsReceiver =
|
|
||||||
statsReceiver.scope("logged_out_candidate_hydration")
|
|
||||||
val mrLORequestCandidateScribeStats =
|
|
||||||
statsReceiver.scope("mr_logged_out_request_scribe_candidates")
|
|
||||||
|
|
||||||
val mrRequestScribeHandler =
|
|
||||||
new MrRequestScribeHandler(loMrRequestScriberNode, statsReceiver.scope("lo_mr_request_scribe"))
|
|
||||||
val loMrRequestTargetScribeStats = statsReceiver.scope("lo_mr_request_scribe_target")
|
|
||||||
|
|
||||||
lazy val loCandSourceEligibleCounter: Counter =
|
|
||||||
loCandidateStats.counter("logged_out_cand_source_eligible")
|
|
||||||
lazy val loCandSourceNotEligibleCounter: Counter =
|
|
||||||
loCandidateStats.counter("logged_out_cand_source_not_eligible")
|
|
||||||
lazy val allCandidatesCounter: Counter = statsReceiver.counter("all_logged_out_candidates")
|
|
||||||
val allCandidatesFilteredPreRank = filterStats.counter("all_logged_out_candidates_filtered")
|
|
||||||
|
|
||||||
override def targetPredicates(target: Target): List[Predicate[Target]] = List(
|
|
||||||
LoggedOutTargetPredicates.targetFatiguePredicate(),
|
|
||||||
LoggedOutTargetPredicates.loggedOutRecsHoldbackPredicate()
|
|
||||||
)
|
|
||||||
|
|
||||||
override def isTargetValid(target: Target): Future[Result] = {
|
|
||||||
val resultFut =
|
|
||||||
if (target.skipFilters) {
|
|
||||||
Future.value(OK)
|
|
||||||
} else {
|
|
||||||
predicateSeq(target).track(Seq(target)).map { resultArr =>
|
|
||||||
trackTargetPredStats(resultArr(0))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
track(targetStats)(resultFut)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def rank(
|
|
||||||
target: Target,
|
|
||||||
candidateDetails: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
loRanker.rank(candidateDetails)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def validCandidates(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[PushCandidate]
|
|
||||||
): Future[Seq[Result]] = {
|
|
||||||
Future.value(candidates.map { c => OK })
|
|
||||||
}
|
|
||||||
|
|
||||||
override def desiredCandidateCount(target: Target): Int = 1
|
|
||||||
|
|
||||||
private val loggedOutPreRankingPredicates =
|
|
||||||
LoggedOutPreRankingPredicates(filterStats.scope("logged_out_predicates"))
|
|
||||||
|
|
||||||
private val loggedOutPreRankingPredicateChain =
|
|
||||||
new SequentialPredicate[PushCandidate](loggedOutPreRankingPredicates)
|
|
||||||
|
|
||||||
override def filter(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[
|
|
||||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
|
||||||
] = {
|
|
||||||
val predicateChain = loggedOutPreRankingPredicateChain
|
|
||||||
predicateChain
|
|
||||||
.track(candidates.map(_.candidate))
|
|
||||||
.map { results =>
|
|
||||||
val resultForPreRankingFiltering =
|
|
||||||
results
|
|
||||||
.zip(candidates)
|
|
||||||
.foldLeft(
|
|
||||||
(
|
|
||||||
Seq.empty[CandidateDetails[PushCandidate]],
|
|
||||||
Seq.empty[CandidateResult[PushCandidate, Result]]
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
case ((goodCandidates, filteredCandidates), (result, candidateDetails)) =>
|
|
||||||
result match {
|
|
||||||
case None =>
|
|
||||||
(goodCandidates :+ candidateDetails, filteredCandidates)
|
|
||||||
|
|
||||||
case Some(pred: NamedPredicate[_]) =>
|
|
||||||
val r = Invalid(Some(pred.name))
|
|
||||||
(
|
|
||||||
goodCandidates,
|
|
||||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
|
||||||
candidateDetails.candidate,
|
|
||||||
candidateDetails.source,
|
|
||||||
r
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case Some(_) =>
|
|
||||||
val r = Invalid(Some("Filtered by un-named predicate"))
|
|
||||||
(
|
|
||||||
goodCandidates,
|
|
||||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
|
||||||
candidateDetails.candidate,
|
|
||||||
candidateDetails.source,
|
|
||||||
r
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resultForPreRankingFiltering match {
|
|
||||||
case (validCandidates, _) if validCandidates.isEmpty && candidates.nonEmpty =>
|
|
||||||
allCandidatesFilteredPreRank.incr()
|
|
||||||
case _ => ()
|
|
||||||
|
|
||||||
}
|
|
||||||
resultForPreRankingFiltering
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
override def candidateSources(
|
|
||||||
target: Target
|
|
||||||
): Future[Seq[CandidateSource[Target, RawCandidate]]] = {
|
|
||||||
Future
|
|
||||||
.collect(loPushCandidateSourceGenerator.sources.map { cs =>
|
|
||||||
cs.isCandidateSourceAvailable(target).map { isEligible =>
|
|
||||||
if (isEligible) {
|
|
||||||
loCandSourceEligibleCounter.incr()
|
|
||||||
Some(cs)
|
|
||||||
} else {
|
|
||||||
loCandSourceNotEligibleCounter.incr()
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}).map(_.flatten)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def process(
|
|
||||||
target: Target,
|
|
||||||
externalCandidates: Seq[RawCandidate] = Nil
|
|
||||||
): Future[Response[PushCandidate, Result]] = {
|
|
||||||
isTargetValid(target).flatMap {
|
|
||||||
case OK =>
|
|
||||||
for {
|
|
||||||
candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target))
|
|
||||||
externalCandidateDetails = externalCandidates.map(
|
|
||||||
CandidateDetails(_, "logged_out_refresh_for_push_handler_external_candidates"))
|
|
||||||
allCandidates = candidatesFromSources ++ externalCandidateDetails
|
|
||||||
hydratedCandidatesWithCopy <-
|
|
||||||
trackSeq(loCandidateHydrationStats)(hydrateCandidates(allCandidates))
|
|
||||||
(candidates, preRankingFilteredCandidates) <-
|
|
||||||
track(filterStats)(filter(target, hydratedCandidatesWithCopy))
|
|
||||||
rankedCandidates <- trackSeq(rankingStats)(rank(target, candidates))
|
|
||||||
allTakeCandidateResults <- track(takeStats)(
|
|
||||||
take(target, rankedCandidates, desiredCandidateCount(target))
|
|
||||||
)
|
|
||||||
_ <- track(mrLORequestCandidateScribeStats)(
|
|
||||||
mrRequestScribeHandler.scribeForCandidateFiltering(
|
|
||||||
target,
|
|
||||||
hydratedCandidatesWithCopy,
|
|
||||||
preRankingFilteredCandidates,
|
|
||||||
rankedCandidates,
|
|
||||||
rankedCandidates,
|
|
||||||
rankedCandidates,
|
|
||||||
allTakeCandidateResults
|
|
||||||
))
|
|
||||||
|
|
||||||
} yield {
|
|
||||||
val takeCandidateResults = allTakeCandidateResults.filterNot { candResult =>
|
|
||||||
candResult.result == MoreThanDesiredCandidates
|
|
||||||
}
|
|
||||||
val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates
|
|
||||||
allCandidatesCounter.incr(allCandidateResults.size)
|
|
||||||
Response(OK, allCandidateResults)
|
|
||||||
}
|
|
||||||
|
|
||||||
case result: Result =>
|
|
||||||
for (_ <- track(loMrRequestTargetScribeStats)(
|
|
||||||
mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield {
|
|
||||||
Response(result, Nil)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def buildTarget(
|
|
||||||
guestId: Long,
|
|
||||||
inputPushContext: Option[PushContext]
|
|
||||||
): Future[Target] =
|
|
||||||
loPushTargetUserBuilder.buildTarget(guestId, inputPushContext)
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Hydrate candidate by querying downstream services
|
|
||||||
*
|
|
||||||
* @param candidates - candidates
|
|
||||||
*
|
|
||||||
* @return - hydrated candidates
|
|
||||||
*/
|
|
||||||
override def hydrateCandidates(
|
|
||||||
candidates: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates)
|
|
||||||
|
|
||||||
override def batchForCandidatesCheck(target: Target): Int = 1
|
|
||||||
|
|
||||||
def refreshAndSend(request: LoggedOutRequest): Future[LoggedOutResponse] = {
|
|
||||||
for {
|
|
||||||
target <- track(loggedOutBuildStats)(
|
|
||||||
loPushTargetUserBuilder.buildTarget(request.guestId, request.context))
|
|
||||||
response <- track(loggedOutProcessStats)(process(target, externalCandidates = Seq.empty))
|
|
||||||
loggedOutRefreshResponse <-
|
|
||||||
track(loggedOutNotifyStats)(loRfphNotifier.checkResponseAndNotify(response))
|
|
||||||
} yield {
|
|
||||||
loggedOutRefreshResponse
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,239 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.channels.common.thriftscala.ApiList
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base._
|
|
||||||
import com.twitter.frigate.common.rec_types.RecTypes.isInNetworkTweetType
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.TrendTweetPushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.ml.PushMLModelScorer
|
|
||||||
import com.twitter.frigate.pushservice.model.candidate.CopyIds
|
|
||||||
import com.twitter.frigate.pushservice.refresh_handler.cross.CandidateCopyExpansion
|
|
||||||
import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._
|
|
||||||
import com.twitter.frigate.pushservice.util.MrUserStateUtil
|
|
||||||
import com.twitter.frigate.pushservice.util.RelationshipUtil
|
|
||||||
import com.twitter.gizmoduck.thriftscala.User
|
|
||||||
import com.twitter.hermit.predicate.socialgraph.RelationEdge
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
case class PushCandidateHydrator(
|
|
||||||
socialGraphServiceProcessStore: ReadableStore[RelationEdge, Boolean],
|
|
||||||
safeUserStore: ReadableStore[Long, User],
|
|
||||||
apiListStore: ReadableStore[Long, ApiList],
|
|
||||||
candidateCopyCross: CandidateCopyExpansion
|
|
||||||
)(
|
|
||||||
implicit statsReceiver: StatsReceiver,
|
|
||||||
implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) {
|
|
||||||
|
|
||||||
lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num")
|
|
||||||
lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates")
|
|
||||||
lazy val mrUserStateStat = statsReceiver.scope("mr_user_state")
|
|
||||||
|
|
||||||
lazy val queryStep = statsReceiver.scope("query_step")
|
|
||||||
lazy val relationEdgeWithoutDuplicateInQueryStep =
|
|
||||||
queryStep.counter("number_of_relationEdge_without_duplicate_in_query_step")
|
|
||||||
lazy val relationEdgeWithoutDuplicateInQueryStepDistribution =
|
|
||||||
queryStep.stat("number_of_relationEdge_without_duplicate_in_query_step_distribution")
|
|
||||||
|
|
||||||
case class Entities(
|
|
||||||
users: Set[Long] = Set.empty[Long],
|
|
||||||
relationshipEdges: Set[RelationEdge] = Set.empty[RelationEdge]) {
|
|
||||||
def merge(otherEntities: Entities): Entities = {
|
|
||||||
this.copy(
|
|
||||||
users = this.users ++ otherEntities.users,
|
|
||||||
relationshipEdges =
|
|
||||||
this.relationshipEdges ++ otherEntities.relationshipEdges
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case class EntitiesMap(
|
|
||||||
userMap: Map[Long, User] = Map.empty[Long, User],
|
|
||||||
relationshipMap: Map[RelationEdge, Boolean] = Map.empty[RelationEdge, Boolean])
|
|
||||||
|
|
||||||
private def updateCandidateAndCrtStats(
|
|
||||||
candidate: RawCandidate,
|
|
||||||
candidateType: String,
|
|
||||||
numEntities: Int = 1
|
|
||||||
): Unit = {
|
|
||||||
statsReceiver
|
|
||||||
.scope(candidateType).scope(candidate.commonRecType.name).stat(
|
|
||||||
"totalEntitiesPerCandidateTypePerCrt").add(numEntities)
|
|
||||||
statsReceiver.scope(candidateType).stat("totalEntitiesPerCandidateType").add(numEntities)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def collectEntities(
|
|
||||||
candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Entities = {
|
|
||||||
candidateDetailsSeq
|
|
||||||
.map { candidateDetails =>
|
|
||||||
val pushCandidate = candidateDetails.candidate
|
|
||||||
|
|
||||||
val userEntities = pushCandidate match {
|
|
||||||
case tweetWithSocialContext: RawCandidate with TweetWithSocialContextTraits =>
|
|
||||||
val authorIdOpt = getAuthorIdFromTweetCandidate(tweetWithSocialContext)
|
|
||||||
val scUserIds = tweetWithSocialContext.socialContextUserIds.toSet
|
|
||||||
updateCandidateAndCrtStats(pushCandidate, "tweetWithSocialContext", scUserIds.size + 1)
|
|
||||||
Entities(users = scUserIds ++ authorIdOpt.toSet)
|
|
||||||
|
|
||||||
case _ => Entities()
|
|
||||||
}
|
|
||||||
|
|
||||||
val relationEntities = {
|
|
||||||
if (isInNetworkTweetType(pushCandidate.commonRecType)) {
|
|
||||||
Entities(
|
|
||||||
relationshipEdges =
|
|
||||||
RelationshipUtil.getPreCandidateRelationshipsForInNetworkTweets(pushCandidate).toSet
|
|
||||||
)
|
|
||||||
} else Entities()
|
|
||||||
}
|
|
||||||
|
|
||||||
userEntities.merge(relationEntities)
|
|
||||||
}
|
|
||||||
.foldLeft(Entities()) { (e1, e2) => e1.merge(e2) }
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This method calls Gizmoduck and Social Graph Service, keep the results in EntitiesMap
|
|
||||||
* and passed onto the update candidate phase in the hydration step
|
|
||||||
*
|
|
||||||
* @param entities contains all userIds and relationEdges for all candidates
|
|
||||||
* @return EntitiesMap contains userMap and relationshipMap
|
|
||||||
*/
|
|
||||||
private def queryEntities(entities: Entities): Future[EntitiesMap] = {
|
|
||||||
|
|
||||||
relationEdgeWithoutDuplicateInQueryStep.incr(entities.relationshipEdges.size)
|
|
||||||
relationEdgeWithoutDuplicateInQueryStepDistribution.add(entities.relationshipEdges.size)
|
|
||||||
|
|
||||||
val relationshipMapFuture = Future
|
|
||||||
.collect(socialGraphServiceProcessStore.multiGet(entities.relationshipEdges))
|
|
||||||
.map { resultMap =>
|
|
||||||
resultMap.collect {
|
|
||||||
case (relationshipEdge, Some(res)) => relationshipEdge -> res
|
|
||||||
case (relationshipEdge, None) => relationshipEdge -> false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
val userMapFuture = Future
|
|
||||||
.collect(safeUserStore.multiGet(entities.users))
|
|
||||||
.map { userMap =>
|
|
||||||
userMap.collect {
|
|
||||||
case (userId, Some(user)) =>
|
|
||||||
userId -> user
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.join(userMapFuture, relationshipMapFuture).map {
|
|
||||||
case (uMap, rMap) => EntitiesMap(userMap = uMap, relationshipMap = rMap)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param candidateDetails: recommendation candidates for a user
|
|
||||||
* @return sequence of candidates tagged with push and ntab copy id
|
|
||||||
*/
|
|
||||||
private def expandCandidatesWithCopy(
|
|
||||||
candidateDetails: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = {
|
|
||||||
candidateCopyCross.expandCandidatesWithCopyId(candidateDetails)
|
|
||||||
}
|
|
||||||
|
|
||||||
def updateCandidates(
|
|
||||||
candidateDetailsWithCopies: Seq[(CandidateDetails[RawCandidate], CopyIds)],
|
|
||||||
entitiesMaps: EntitiesMap
|
|
||||||
): Seq[CandidateDetails[PushCandidate]] = {
|
|
||||||
candidateDetailsWithCopies.map {
|
|
||||||
case (candidateDetail, copyIds) =>
|
|
||||||
val pushCandidate = candidateDetail.candidate
|
|
||||||
val userMap = entitiesMaps.userMap
|
|
||||||
val relationshipMap = entitiesMaps.relationshipMap
|
|
||||||
|
|
||||||
val hydratedCandidate = pushCandidate match {
|
|
||||||
|
|
||||||
case f1TweetCandidate: F1FirstDegree =>
|
|
||||||
getHydratedCandidateForF1FirstDegreeTweet(
|
|
||||||
f1TweetCandidate,
|
|
||||||
userMap,
|
|
||||||
relationshipMap,
|
|
||||||
copyIds)
|
|
||||||
|
|
||||||
case tweetRetweet: TweetRetweetCandidate =>
|
|
||||||
getHydratedCandidateForTweetRetweet(tweetRetweet, userMap, copyIds)
|
|
||||||
|
|
||||||
case tweetFavorite: TweetFavoriteCandidate =>
|
|
||||||
getHydratedCandidateForTweetFavorite(tweetFavorite, userMap, copyIds)
|
|
||||||
|
|
||||||
case tripTweetCandidate: OutOfNetworkTweetCandidate with TripCandidate =>
|
|
||||||
getHydratedCandidateForTripTweetCandidate(tripTweetCandidate, userMap, copyIds)
|
|
||||||
|
|
||||||
case outOfNetworkTweetCandidate: OutOfNetworkTweetCandidate with TopicCandidate =>
|
|
||||||
getHydratedCandidateForOutOfNetworkTweetCandidate(
|
|
||||||
outOfNetworkTweetCandidate,
|
|
||||||
userMap,
|
|
||||||
copyIds)
|
|
||||||
|
|
||||||
case topicProofTweetCandidate: TopicProofTweetCandidate =>
|
|
||||||
getHydratedTopicProofTweetCandidate(topicProofTweetCandidate, userMap, copyIds)
|
|
||||||
|
|
||||||
case subscribedSearchTweetCandidate: SubscribedSearchTweetCandidate =>
|
|
||||||
getHydratedSubscribedSearchTweetCandidate(
|
|
||||||
subscribedSearchTweetCandidate,
|
|
||||||
userMap,
|
|
||||||
copyIds)
|
|
||||||
|
|
||||||
case listRecommendation: ListPushCandidate =>
|
|
||||||
getHydratedListCandidate(apiListStore, listRecommendation, copyIds)
|
|
||||||
|
|
||||||
case discoverTwitterCandidate: DiscoverTwitterCandidate =>
|
|
||||||
getHydratedCandidateForDiscoverTwitterCandidate(discoverTwitterCandidate, copyIds)
|
|
||||||
|
|
||||||
case topTweetImpressionsCandidate: TopTweetImpressionsCandidate =>
|
|
||||||
getHydratedCandidateForTopTweetImpressionsCandidate(
|
|
||||||
topTweetImpressionsCandidate,
|
|
||||||
copyIds)
|
|
||||||
|
|
||||||
case trendTweetCandidate: TrendTweetCandidate =>
|
|
||||||
new TrendTweetPushCandidate(
|
|
||||||
trendTweetCandidate,
|
|
||||||
trendTweetCandidate.authorId.flatMap(userMap.get),
|
|
||||||
copyIds)
|
|
||||||
|
|
||||||
case unknownCandidate =>
|
|
||||||
throw new IllegalArgumentException(
|
|
||||||
s"Incorrect candidate for hydration: ${unknownCandidate.commonRecType}")
|
|
||||||
}
|
|
||||||
|
|
||||||
CandidateDetails(
|
|
||||||
hydratedCandidate,
|
|
||||||
source = candidateDetail.source
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def apply(
|
|
||||||
candidateDetails: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
val isLoggedOutRequest =
|
|
||||||
candidateDetails.headOption.exists(_.candidate.target.isLoggedOutUser)
|
|
||||||
if (!isLoggedOutRequest) {
|
|
||||||
candidateDetails.headOption.map { cd =>
|
|
||||||
MrUserStateUtil.updateMrUserStateStats(cd.candidate.target)(mrUserStateStat)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
expandCandidatesWithCopy(candidateDetails).flatMap { candidateDetailsWithCopy =>
|
|
||||||
candidateWithCopyNumStat.add(candidateDetailsWithCopy.size)
|
|
||||||
val entities = collectEntities(candidateDetailsWithCopy.map(_._1))
|
|
||||||
queryEntities(entities).flatMap { entitiesMap =>
|
|
||||||
val updatedCandidates = updateCandidates(candidateDetailsWithCopy, entitiesMap)
|
|
||||||
updatedCandidates.foreach { cand =>
|
|
||||||
hydratedCandidateStat.counter(cand.candidate.commonRecType.name).incr()
|
|
||||||
}
|
|
||||||
Future.value(updatedCandidates)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,69 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.FeatureMap
|
|
||||||
import com.twitter.frigate.data_pipeline.features_common.MrRequestContextForFeatureStore
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.ml.HydrationContextBuilder
|
|
||||||
import com.twitter.frigate.pushservice.params.PushParams
|
|
||||||
import com.twitter.frigate.pushservice.util.MrUserStateUtil
|
|
||||||
import com.twitter.nrel.heavyranker.FeatureHydrator
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
class RFPHFeatureHydrator(
|
|
||||||
featureHydrator: FeatureHydrator
|
|
||||||
)(
|
|
||||||
implicit globalStats: StatsReceiver) {
|
|
||||||
|
|
||||||
implicit val statsReceiver: StatsReceiver =
|
|
||||||
globalStats.scope("RefreshForPushHandler")
|
|
||||||
|
|
||||||
//stat for feature hydration
|
|
||||||
private val featureHydrationEnabledCounter = statsReceiver.counter("featureHydrationEnabled")
|
|
||||||
private val mrUserStateStat = statsReceiver.scope("mr_user_state")
|
|
||||||
|
|
||||||
private def hydrateFromRelevanceHydrator(
|
|
||||||
candidateDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
mrRequestContextForFeatureStore: MrRequestContextForFeatureStore
|
|
||||||
): Future[Unit] = {
|
|
||||||
val pushCandidates = candidateDetails.map(_.candidate)
|
|
||||||
val candidatesAndContextsFut = Future.collect(pushCandidates.map { pc =>
|
|
||||||
val contextFut = HydrationContextBuilder.build(pc)
|
|
||||||
contextFut.map { ctx => (pc, ctx) }
|
|
||||||
})
|
|
||||||
candidatesAndContextsFut.flatMap { candidatesAndContexts =>
|
|
||||||
val contexts = candidatesAndContexts.map(_._2)
|
|
||||||
val resultsFut = featureHydrator.hydrateCandidate(contexts, mrRequestContextForFeatureStore)
|
|
||||||
resultsFut.map { hydrationResult =>
|
|
||||||
candidatesAndContexts.foreach {
|
|
||||||
case (pushCandidate, context) =>
|
|
||||||
val resultFeatures = hydrationResult.getOrElse(context, FeatureMap())
|
|
||||||
pushCandidate.mergeFeatures(resultFeatures)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def candidateFeatureHydration(
|
|
||||||
candidateDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
mrRequestContextForFeatureStore: MrRequestContextForFeatureStore
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
candidateDetails.headOption match {
|
|
||||||
case Some(cand) =>
|
|
||||||
val target = cand.candidate.target
|
|
||||||
MrUserStateUtil.updateMrUserStateStats(target)(mrUserStateStat)
|
|
||||||
if (target.params(PushParams.DisableAllRelevanceParam)) {
|
|
||||||
Future.value(candidateDetails)
|
|
||||||
} else {
|
|
||||||
featureHydrationEnabledCounter.incr()
|
|
||||||
for {
|
|
||||||
_ <- hydrateFromRelevanceHydrator(candidateDetails, mrRequestContextForFeatureStore)
|
|
||||||
} yield {
|
|
||||||
candidateDetails
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case _ => Future.Nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,104 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.Counter
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base._
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.predicate.PreRankingPredicates
|
|
||||||
import com.twitter.hermit.predicate.NamedPredicate
|
|
||||||
import com.twitter.hermit.predicate.SequentialPredicate
|
|
||||||
import com.twitter.util._
|
|
||||||
|
|
||||||
class RFPHPrerankFilter(
|
|
||||||
)(
|
|
||||||
globalStats: StatsReceiver) {
|
|
||||||
def filter(
|
|
||||||
target: Target,
|
|
||||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[
|
|
||||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
|
||||||
] = {
|
|
||||||
lazy val filterStats: StatsReceiver = globalStats.scope("RefreshForPushHandler/filter")
|
|
||||||
lazy val okFilterCounter: Counter = filterStats.counter("ok")
|
|
||||||
lazy val invalidFilterCounter: Counter = filterStats.counter("invalid")
|
|
||||||
lazy val invalidFilterStat: StatsReceiver = filterStats.scope("invalid")
|
|
||||||
lazy val invalidFilterReasonStat: StatsReceiver = invalidFilterStat.scope("reason")
|
|
||||||
val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered")
|
|
||||||
|
|
||||||
lazy val preRankingPredicates = PreRankingPredicates(
|
|
||||||
filterStats.scope("predicates")
|
|
||||||
)
|
|
||||||
|
|
||||||
lazy val preRankingPredicateChain =
|
|
||||||
new SequentialPredicate[PushCandidate](preRankingPredicates)
|
|
||||||
|
|
||||||
val predicateChain = if (target.pushContext.exists(_.predicatesToEnable.exists(_.nonEmpty))) {
|
|
||||||
val predicatesToEnable = target.pushContext.flatMap(_.predicatesToEnable).getOrElse(Nil)
|
|
||||||
new SequentialPredicate[PushCandidate](preRankingPredicates.filter { pred =>
|
|
||||||
predicatesToEnable.contains(pred.name)
|
|
||||||
})
|
|
||||||
} else preRankingPredicateChain
|
|
||||||
|
|
||||||
predicateChain
|
|
||||||
.track(hydratedCandidates.map(_.candidate))
|
|
||||||
.map { results =>
|
|
||||||
val resultForPreRankFiltering = results
|
|
||||||
.zip(hydratedCandidates)
|
|
||||||
.foldLeft(
|
|
||||||
(
|
|
||||||
Seq.empty[CandidateDetails[PushCandidate]],
|
|
||||||
Seq.empty[CandidateResult[PushCandidate, Result]]
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
case ((goodCandidates, filteredCandidates), (result, candidateDetails)) =>
|
|
||||||
result match {
|
|
||||||
case None =>
|
|
||||||
okFilterCounter.incr()
|
|
||||||
(goodCandidates :+ candidateDetails, filteredCandidates)
|
|
||||||
|
|
||||||
case Some(pred: NamedPredicate[_]) =>
|
|
||||||
invalidFilterCounter.incr()
|
|
||||||
invalidFilterReasonStat.counter(pred.name).incr()
|
|
||||||
invalidFilterReasonStat
|
|
||||||
.scope(candidateDetails.candidate.commonRecType.toString).counter(
|
|
||||||
pred.name).incr()
|
|
||||||
|
|
||||||
val r = Invalid(Some(pred.name))
|
|
||||||
(
|
|
||||||
goodCandidates,
|
|
||||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
|
||||||
candidateDetails.candidate,
|
|
||||||
candidateDetails.source,
|
|
||||||
r
|
|
||||||
)
|
|
||||||
)
|
|
||||||
case Some(_) =>
|
|
||||||
invalidFilterCounter.incr()
|
|
||||||
invalidFilterReasonStat.counter("unknown").incr()
|
|
||||||
invalidFilterReasonStat
|
|
||||||
.scope(candidateDetails.candidate.commonRecType.toString).counter(
|
|
||||||
"unknown").incr()
|
|
||||||
|
|
||||||
val r = Invalid(Some("Filtered by un-named predicate"))
|
|
||||||
(
|
|
||||||
goodCandidates,
|
|
||||||
filteredCandidates :+ CandidateResult[PushCandidate, Result](
|
|
||||||
candidateDetails.candidate,
|
|
||||||
candidateDetails.source,
|
|
||||||
r
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resultForPreRankFiltering match {
|
|
||||||
case (validCandidates, _) if validCandidates.isEmpty && hydratedCandidates.nonEmpty =>
|
|
||||||
allCandidatesFilteredPreRank.incr()
|
|
||||||
case _ => ()
|
|
||||||
}
|
|
||||||
|
|
||||||
resultForPreRankFiltering
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,34 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.Stat
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.TargetUser
|
|
||||||
import com.twitter.frigate.common.candidate.TargetABDecider
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
|
||||||
import com.twitter.frigate.pushservice.target.TargetScoringDetails
|
|
||||||
|
|
||||||
class RFPHRestrictStep()(implicit stats: StatsReceiver) {
|
|
||||||
|
|
||||||
private val statsReceiver: StatsReceiver = stats.scope("RefreshForPushHandler")
|
|
||||||
private val restrictStepStats: StatsReceiver = statsReceiver.scope("restrict")
|
|
||||||
private val restrictStepNumCandidatesDroppedStat: Stat =
|
|
||||||
restrictStepStats.stat("candidates_dropped")
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Limit the number of candidates that enter the Take step
|
|
||||||
*/
|
|
||||||
def restrict(
|
|
||||||
target: TargetUser with TargetABDecider with TargetScoringDetails,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): (Seq[CandidateDetails[PushCandidate]], Seq[CandidateDetails[PushCandidate]]) = {
|
|
||||||
if (target.params(PushFeatureSwitchParams.EnableRestrictStep)) {
|
|
||||||
val restrictSizeParam = PushFeatureSwitchParams.RestrictStepSize
|
|
||||||
val (newCandidates, filteredCandidates) = candidates.splitAt(target.params(restrictSizeParam))
|
|
||||||
val numDropped = candidates.length - newCandidates.length
|
|
||||||
restrictStepNumCandidatesDroppedStat.add(numDropped)
|
|
||||||
(newCandidates, filteredCandidates)
|
|
||||||
} else (candidates, Seq.empty)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,77 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.Stat
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
|
|
||||||
class RFPHStatsRecorder(implicit statsReceiver: StatsReceiver) {
|
|
||||||
|
|
||||||
private val selectedCandidateScoreStats: StatsReceiver =
|
|
||||||
statsReceiver.scope("score_of_sent_candidate_times_10000")
|
|
||||||
|
|
||||||
private val emptyScoreStats: StatsReceiver =
|
|
||||||
statsReceiver.scope("score_of_sent_candidate_empty")
|
|
||||||
|
|
||||||
def trackPredictionScoreStats(candidate: PushCandidate): Unit = {
|
|
||||||
candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach {
|
|
||||||
case Some(s) =>
|
|
||||||
selectedCandidateScoreStats
|
|
||||||
.stat("weighted_open_or_ntab_click_ranking")
|
|
||||||
.add((s * 10000).toFloat)
|
|
||||||
case None =>
|
|
||||||
emptyScoreStats.counter("weighted_open_or_ntab_click_ranking").incr()
|
|
||||||
}
|
|
||||||
candidate.mrWeightedOpenOrNtabClickFilteringProbability.foreach {
|
|
||||||
case Some(s) =>
|
|
||||||
selectedCandidateScoreStats
|
|
||||||
.stat("weighted_open_or_ntab_click_filtering")
|
|
||||||
.add((s * 10000).toFloat)
|
|
||||||
case None =>
|
|
||||||
emptyScoreStats.counter("weighted_open_or_ntab_click_filtering").incr()
|
|
||||||
}
|
|
||||||
candidate.mrWeightedOpenOrNtabClickRankingProbability.foreach {
|
|
||||||
case Some(s) =>
|
|
||||||
selectedCandidateScoreStats
|
|
||||||
.scope(candidate.commonRecType.toString)
|
|
||||||
.stat("weighted_open_or_ntab_click_ranking")
|
|
||||||
.add((s * 10000).toFloat)
|
|
||||||
case None =>
|
|
||||||
emptyScoreStats
|
|
||||||
.scope(candidate.commonRecType.toString)
|
|
||||||
.counter("weighted_open_or_ntab_click_ranking")
|
|
||||||
.incr()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def refreshRequestExceptionStats(
|
|
||||||
exception: Throwable,
|
|
||||||
bStats: StatsReceiver
|
|
||||||
): Unit = {
|
|
||||||
bStats.counter("failures").incr()
|
|
||||||
bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr()
|
|
||||||
}
|
|
||||||
|
|
||||||
def loggedOutRequestExceptionStats(
|
|
||||||
exception: Throwable,
|
|
||||||
bStats: StatsReceiver
|
|
||||||
): Unit = {
|
|
||||||
bStats.counter("logged_out_failures").incr()
|
|
||||||
bStats.scope("failures").counter(exception.getClass.getCanonicalName).incr()
|
|
||||||
}
|
|
||||||
|
|
||||||
def rankDistributionStats(
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
numRecsPerTypeStat: (CommonRecommendationType => Stat)
|
|
||||||
): Unit = {
|
|
||||||
candidatesDetails
|
|
||||||
.groupBy { c =>
|
|
||||||
c.candidate.commonRecType
|
|
||||||
}
|
|
||||||
.mapValues { s =>
|
|
||||||
s.size
|
|
||||||
}
|
|
||||||
.foreach { case (crt, numRecs) => numRecsPerTypeStat(crt).add(numRecs) }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,292 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.Counter
|
|
||||||
import com.twitter.finagle.stats.Stat
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.Stats.track
|
|
||||||
import com.twitter.frigate.common.base.Stats.trackSeq
|
|
||||||
import com.twitter.frigate.common.base._
|
|
||||||
import com.twitter.frigate.common.logger.MRLogger
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.adaptor._
|
|
||||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
|
||||||
import com.twitter.frigate.pushservice.rank.RFPHLightRanker
|
|
||||||
import com.twitter.frigate.pushservice.rank.RFPHRanker
|
|
||||||
import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler
|
|
||||||
import com.twitter.frigate.pushservice.take.candidate_validator.RFPHCandidateValidator
|
|
||||||
import com.twitter.frigate.pushservice.target.PushTargetUserBuilder
|
|
||||||
import com.twitter.frigate.pushservice.target.RFPHTargetPredicates
|
|
||||||
import com.twitter.frigate.pushservice.util.RFPHTakeStepUtil
|
|
||||||
import com.twitter.frigate.pushservice.util.AdhocStatsUtil
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.PushContext
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.RefreshRequest
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.RefreshResponse
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.hermit.predicate.Predicate
|
|
||||||
import com.twitter.timelines.configapi.FeatureValue
|
|
||||||
import com.twitter.util._
|
|
||||||
|
|
||||||
case class ResultWithDebugInfo(result: Result, predicateResults: Seq[PredicateWithResult])
|
|
||||||
|
|
||||||
class RefreshForPushHandler(
|
|
||||||
val pushTargetUserBuilder: PushTargetUserBuilder,
|
|
||||||
val candSourceGenerator: PushCandidateSourceGenerator,
|
|
||||||
rfphRanker: RFPHRanker,
|
|
||||||
candidateHydrator: PushCandidateHydrator,
|
|
||||||
candidateValidator: RFPHCandidateValidator,
|
|
||||||
rfphTakeStepUtil: RFPHTakeStepUtil,
|
|
||||||
rfphRestrictStep: RFPHRestrictStep,
|
|
||||||
val rfphNotifier: RefreshForPushNotifier,
|
|
||||||
rfphStatsRecorder: RFPHStatsRecorder,
|
|
||||||
mrRequestScriberNode: String,
|
|
||||||
rfphFeatureHydrator: RFPHFeatureHydrator,
|
|
||||||
rfphPrerankFilter: RFPHPrerankFilter,
|
|
||||||
rfphLightRanker: RFPHLightRanker
|
|
||||||
)(
|
|
||||||
globalStats: StatsReceiver)
|
|
||||||
extends FetchRankFlowWithHydratedCandidates[Target, RawCandidate, PushCandidate] {
|
|
||||||
|
|
||||||
val log = MRLogger("RefreshForPushHandler")
|
|
||||||
|
|
||||||
implicit val statsReceiver: StatsReceiver =
|
|
||||||
globalStats.scope("RefreshForPushHandler")
|
|
||||||
private val maxCandidatesToBatchInTakeStat: Stat =
|
|
||||||
statsReceiver.stat("max_cands_to_batch_in_take")
|
|
||||||
|
|
||||||
private val rfphRequestCounter = statsReceiver.counter("requests")
|
|
||||||
|
|
||||||
private val buildTargetStats = statsReceiver.scope("build_target")
|
|
||||||
private val processStats = statsReceiver.scope("process")
|
|
||||||
private val notifyStats = statsReceiver.scope("notify")
|
|
||||||
|
|
||||||
private val lightRankingStats: StatsReceiver = statsReceiver.scope("light_ranking")
|
|
||||||
private val reRankingStats: StatsReceiver = statsReceiver.scope("rerank")
|
|
||||||
private val featureHydrationLatency: StatsReceiver =
|
|
||||||
statsReceiver.scope("featureHydrationLatency")
|
|
||||||
private val candidateHydrationStats: StatsReceiver = statsReceiver.scope("candidate_hydration")
|
|
||||||
|
|
||||||
lazy val candSourceEligibleCounter: Counter =
|
|
||||||
candidateStats.counter("cand_source_eligible")
|
|
||||||
lazy val candSourceNotEligibleCounter: Counter =
|
|
||||||
candidateStats.counter("cand_source_not_eligible")
|
|
||||||
|
|
||||||
//pre-ranking stats
|
|
||||||
val allCandidatesFilteredPreRank = filterStats.counter("all_candidates_filtered")
|
|
||||||
|
|
||||||
// total invalid candidates
|
|
||||||
val totalStats: StatsReceiver = statsReceiver.scope("total")
|
|
||||||
val totalInvalidCandidatesStat: Stat = totalStats.stat("candidates_invalid")
|
|
||||||
|
|
||||||
val mrRequestScribeBuiltStats: Counter = statsReceiver.counter("mr_request_scribe_built")
|
|
||||||
|
|
||||||
val mrRequestCandidateScribeStats = statsReceiver.scope("mr_request_scribe_candidates")
|
|
||||||
val mrRequestTargetScribeStats = statsReceiver.scope("mr_request_scribe_target")
|
|
||||||
|
|
||||||
val mrRequestScribeHandler =
|
|
||||||
new MrRequestScribeHandler(mrRequestScriberNode, statsReceiver.scope("mr_request_scribe"))
|
|
||||||
|
|
||||||
val adhocStatsUtil = new AdhocStatsUtil(statsReceiver.scope("adhoc_stats"))
|
|
||||||
|
|
||||||
private def numRecsPerTypeStat(crt: CommonRecommendationType) =
|
|
||||||
fetchStats.scope(crt.toString).stat("dist")
|
|
||||||
|
|
||||||
// static list of target predicates
|
|
||||||
private val targetPredicates = RFPHTargetPredicates(targetStats.scope("predicates"))
|
|
||||||
|
|
||||||
def buildTarget(
|
|
||||||
userId: Long,
|
|
||||||
inputPushContext: Option[PushContext],
|
|
||||||
forcedFeatureValues: Option[Map[String, FeatureValue]] = None
|
|
||||||
): Future[Target] =
|
|
||||||
pushTargetUserBuilder.buildTarget(userId, inputPushContext, forcedFeatureValues)
|
|
||||||
|
|
||||||
override def targetPredicates(target: Target): List[Predicate[Target]] = targetPredicates
|
|
||||||
|
|
||||||
override def isTargetValid(target: Target): Future[Result] = {
|
|
||||||
val resultFut = if (target.skipFilters) {
|
|
||||||
Future.value(trackTargetPredStats(None))
|
|
||||||
} else {
|
|
||||||
predicateSeq(target).track(Seq(target)).map { resultArr =>
|
|
||||||
trackTargetPredStats(resultArr(0))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
track(targetStats)(resultFut)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def candidateSources(
|
|
||||||
target: Target
|
|
||||||
): Future[Seq[CandidateSource[Target, RawCandidate]]] = {
|
|
||||||
Future
|
|
||||||
.collect(candSourceGenerator.sources.map { cs =>
|
|
||||||
cs.isCandidateSourceAvailable(target).map { isEligible =>
|
|
||||||
if (isEligible) {
|
|
||||||
candSourceEligibleCounter.incr()
|
|
||||||
Some(cs)
|
|
||||||
} else {
|
|
||||||
candSourceNotEligibleCounter.incr()
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}).map(_.flatten)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def updateCandidateCounter(
|
|
||||||
candidateResults: Seq[CandidateResult[PushCandidate, Result]]
|
|
||||||
): Unit = {
|
|
||||||
candidateResults.foreach {
|
|
||||||
case candidateResult if candidateResult.result == OK =>
|
|
||||||
okCandidateCounter.incr()
|
|
||||||
case candidateResult if candidateResult.result.isInstanceOf[Invalid] =>
|
|
||||||
invalidCandidateCounter.incr()
|
|
||||||
case _ =>
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override def hydrateCandidates(
|
|
||||||
candidates: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = candidateHydrator(candidates)
|
|
||||||
|
|
||||||
override def filter(
|
|
||||||
target: Target,
|
|
||||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[
|
|
||||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
|
||||||
] = rfphPrerankFilter.filter(target, hydratedCandidates)
|
|
||||||
|
|
||||||
def lightRankAndTake(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
rfphLightRanker.rank(target, candidates)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def rank(
|
|
||||||
target: Target,
|
|
||||||
candidatesDetails: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
val featureHydratedCandidatesFut = trackSeq(featureHydrationLatency)(
|
|
||||||
rfphFeatureHydrator
|
|
||||||
.candidateFeatureHydration(candidatesDetails, target.mrRequestContextForFeatureStore)
|
|
||||||
)
|
|
||||||
featureHydratedCandidatesFut.flatMap { featureHydratedCandidates =>
|
|
||||||
rfphStatsRecorder.rankDistributionStats(featureHydratedCandidates, numRecsPerTypeStat)
|
|
||||||
rfphRanker.initialRank(target, candidatesDetails)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def reRank(
|
|
||||||
target: Target,
|
|
||||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
rfphRanker.reRank(target, rankedCandidates)
|
|
||||||
}
|
|
||||||
|
|
||||||
override def validCandidates(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[PushCandidate]
|
|
||||||
): Future[Seq[Result]] = {
|
|
||||||
Future.collect(candidates.map { candidate =>
|
|
||||||
rfphTakeStepUtil.isCandidateValid(candidate, candidateValidator).map(res => res.result)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
override def desiredCandidateCount(target: Target): Int = target.desiredCandidateCount
|
|
||||||
|
|
||||||
override def batchForCandidatesCheck(target: Target): Int = {
|
|
||||||
val fsParam = PushFeatureSwitchParams.NumberOfMaxCandidatesToBatchInRFPHTakeStep
|
|
||||||
val maxToBatch = target.params(fsParam)
|
|
||||||
maxCandidatesToBatchInTakeStat.add(maxToBatch)
|
|
||||||
maxToBatch
|
|
||||||
}
|
|
||||||
|
|
||||||
override def process(
|
|
||||||
target: Target,
|
|
||||||
externalCandidates: Seq[RawCandidate] = Nil
|
|
||||||
): Future[Response[PushCandidate, Result]] = {
|
|
||||||
isTargetValid(target).flatMap {
|
|
||||||
case OK =>
|
|
||||||
for {
|
|
||||||
candidatesFromSources <- trackSeq(fetchStats)(fetchCandidates(target))
|
|
||||||
externalCandidateDetails = externalCandidates.map(
|
|
||||||
CandidateDetails(_, "refresh_for_push_handler_external_candidate"))
|
|
||||||
allCandidates = candidatesFromSources ++ externalCandidateDetails
|
|
||||||
hydratedCandidatesWithCopy <-
|
|
||||||
trackSeq(candidateHydrationStats)(hydrateCandidates(allCandidates))
|
|
||||||
_ = adhocStatsUtil.getCandidateSourceStats(hydratedCandidatesWithCopy)
|
|
||||||
(candidates, preRankingFilteredCandidates) <-
|
|
||||||
track(filterStats)(filter(target, hydratedCandidatesWithCopy))
|
|
||||||
_ = adhocStatsUtil.getPreRankingFilterStats(preRankingFilteredCandidates)
|
|
||||||
lightRankerFilteredCandidates <-
|
|
||||||
trackSeq(lightRankingStats)(lightRankAndTake(target, candidates))
|
|
||||||
_ = adhocStatsUtil.getLightRankingStats(lightRankerFilteredCandidates)
|
|
||||||
rankedCandidates <- trackSeq(rankingStats)(rank(target, lightRankerFilteredCandidates))
|
|
||||||
_ = adhocStatsUtil.getRankingStats(rankedCandidates)
|
|
||||||
rerankedCandidates <- trackSeq(reRankingStats)(reRank(target, rankedCandidates))
|
|
||||||
_ = adhocStatsUtil.getReRankingStats(rerankedCandidates)
|
|
||||||
(restrictedCandidates, restrictFilteredCandidates) =
|
|
||||||
rfphRestrictStep.restrict(target, rerankedCandidates)
|
|
||||||
allTakeCandidateResults <- track(takeStats)(
|
|
||||||
take(target, restrictedCandidates, desiredCandidateCount(target))
|
|
||||||
)
|
|
||||||
_ = adhocStatsUtil.getTakeCandidateResultStats(allTakeCandidateResults)
|
|
||||||
_ <- track(mrRequestCandidateScribeStats)(
|
|
||||||
mrRequestScribeHandler.scribeForCandidateFiltering(
|
|
||||||
target,
|
|
||||||
hydratedCandidatesWithCopy,
|
|
||||||
preRankingFilteredCandidates,
|
|
||||||
rankedCandidates,
|
|
||||||
rerankedCandidates,
|
|
||||||
restrictFilteredCandidates,
|
|
||||||
allTakeCandidateResults
|
|
||||||
))
|
|
||||||
} yield {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Take processes post restrict step candidates and returns both:
|
|
||||||
* 1. valid + invalid candidates
|
|
||||||
* 2. Candidates that are not processed (more than desired) + restricted candidates
|
|
||||||
* We need #2 only for importance sampling
|
|
||||||
*/
|
|
||||||
val takeCandidateResults =
|
|
||||||
allTakeCandidateResults.filterNot { candResult =>
|
|
||||||
candResult.result == MoreThanDesiredCandidates
|
|
||||||
}
|
|
||||||
|
|
||||||
val totalInvalidCandidates = {
|
|
||||||
preRankingFilteredCandidates.size + //pre-ranking filtered candidates
|
|
||||||
(rerankedCandidates.length - restrictedCandidates.length) + //candidates reject in restrict step
|
|
||||||
takeCandidateResults.count(_.result != OK) //candidates reject in take step
|
|
||||||
}
|
|
||||||
takeInvalidCandidateDist.add(
|
|
||||||
takeCandidateResults
|
|
||||||
.count(_.result != OK)
|
|
||||||
) // take step invalid candidates
|
|
||||||
totalInvalidCandidatesStat.add(totalInvalidCandidates)
|
|
||||||
val allCandidateResults = takeCandidateResults ++ preRankingFilteredCandidates
|
|
||||||
Response(OK, allCandidateResults)
|
|
||||||
}
|
|
||||||
|
|
||||||
case result: Result =>
|
|
||||||
for (_ <- track(mrRequestTargetScribeStats)(
|
|
||||||
mrRequestScribeHandler.scribeForTargetFiltering(target, result))) yield {
|
|
||||||
mrRequestScribeBuiltStats.incr()
|
|
||||||
Response(result, Nil)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def refreshAndSend(request: RefreshRequest): Future[RefreshResponse] = {
|
|
||||||
rfphRequestCounter.incr()
|
|
||||||
for {
|
|
||||||
target <- track(buildTargetStats)(
|
|
||||||
pushTargetUserBuilder
|
|
||||||
.buildTarget(request.userId, request.context))
|
|
||||||
response <- track(processStats)(process(target, externalCandidates = Seq.empty))
|
|
||||||
refreshResponse <- track(notifyStats)(rfphNotifier.checkResponseAndNotify(response, target))
|
|
||||||
} yield {
|
|
||||||
refreshResponse
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,128 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.BroadcastStatsReceiver
|
|
||||||
import com.twitter.finagle.stats.Stat
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.Stats.track
|
|
||||||
import com.twitter.frigate.common.base._
|
|
||||||
import com.twitter.frigate.common.config.CommonConstants
|
|
||||||
import com.twitter.frigate.common.util.PushServiceUtil.FilteredRefreshResponseFut
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.take.CandidateNotifier
|
|
||||||
import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.PushStatus
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.RefreshResponse
|
|
||||||
import com.twitter.util.Future
|
|
||||||
import com.twitter.util.JavaTimer
|
|
||||||
import com.twitter.util.Timer
|
|
||||||
|
|
||||||
class RefreshForPushNotifier(
|
|
||||||
rfphStatsRecorder: RFPHStatsRecorder,
|
|
||||||
candidateNotifier: CandidateNotifier
|
|
||||||
)(
|
|
||||||
globalStats: StatsReceiver) {
|
|
||||||
|
|
||||||
private implicit val statsReceiver: StatsReceiver =
|
|
||||||
globalStats.scope("RefreshForPushHandler")
|
|
||||||
|
|
||||||
private val pushStats: StatsReceiver = statsReceiver.scope("push")
|
|
||||||
private val sendLatency: StatsReceiver = statsReceiver.scope("send_handler")
|
|
||||||
implicit private val timer: Timer = new JavaTimer(true)
|
|
||||||
|
|
||||||
private def notify(
|
|
||||||
candidatesResult: CandidateResult[PushCandidate, Result],
|
|
||||||
target: Target,
|
|
||||||
receivers: Seq[StatsReceiver]
|
|
||||||
): Future[RefreshResponse] = {
|
|
||||||
|
|
||||||
val candidate = candidatesResult.candidate
|
|
||||||
|
|
||||||
val predsResult = candidatesResult.result
|
|
||||||
|
|
||||||
if (predsResult != OK) {
|
|
||||||
val invalidResult = predsResult
|
|
||||||
invalidResult match {
|
|
||||||
case Invalid(Some(reason)) =>
|
|
||||||
Future.value(RefreshResponse(PushStatus.Filtered, Some(reason)))
|
|
||||||
case _ =>
|
|
||||||
Future.value(RefreshResponse(PushStatus.Filtered, None))
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
rfphStatsRecorder.trackPredictionScoreStats(candidate)
|
|
||||||
|
|
||||||
val isQualityUprankingCandidate = candidate.mrQualityUprankingBoost.isDefined
|
|
||||||
val commonRecTypeStats = Seq(
|
|
||||||
statsReceiver.scope(candidate.commonRecType.toString),
|
|
||||||
globalStats.scope(candidate.commonRecType.toString)
|
|
||||||
)
|
|
||||||
val qualityUprankingStats = Seq(
|
|
||||||
statsReceiver.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString),
|
|
||||||
globalStats.scope("QualityUprankingCandidates").scope(candidate.commonRecType.toString)
|
|
||||||
)
|
|
||||||
|
|
||||||
val receiversWithRecTypeStats = {
|
|
||||||
if (isQualityUprankingCandidate) {
|
|
||||||
receivers ++ commonRecTypeStats ++ qualityUprankingStats
|
|
||||||
} else {
|
|
||||||
receivers ++ commonRecTypeStats
|
|
||||||
}
|
|
||||||
}
|
|
||||||
track(sendLatency)(candidateNotifier.notify(candidate).map { res =>
|
|
||||||
trackStatsForResponseToRequest(
|
|
||||||
candidate.commonRecType,
|
|
||||||
candidate.target,
|
|
||||||
res,
|
|
||||||
receiversWithRecTypeStats
|
|
||||||
)(globalStats)
|
|
||||||
RefreshResponse(res.status)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def checkResponseAndNotify(
|
|
||||||
response: Response[PushCandidate, Result],
|
|
||||||
targetUserContext: Target
|
|
||||||
): Future[RefreshResponse] = {
|
|
||||||
val receivers = Seq(statsReceiver)
|
|
||||||
val refreshResponse = response match {
|
|
||||||
case Response(OK, processedCandidates) =>
|
|
||||||
// valid rec candidates
|
|
||||||
val validCandidates = processedCandidates.filter(_.result == OK)
|
|
||||||
|
|
||||||
// top rec candidate
|
|
||||||
validCandidates.headOption match {
|
|
||||||
case Some(candidatesResult) =>
|
|
||||||
candidatesResult.result match {
|
|
||||||
case OK =>
|
|
||||||
notify(candidatesResult, targetUserContext, receivers)
|
|
||||||
.onSuccess { nr =>
|
|
||||||
pushStats.scope("result").counter(nr.status.name).incr()
|
|
||||||
}
|
|
||||||
case _ =>
|
|
||||||
targetUserContext.isTeamMember.flatMap { isTeamMember =>
|
|
||||||
FilteredRefreshResponseFut
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case _ =>
|
|
||||||
FilteredRefreshResponseFut
|
|
||||||
}
|
|
||||||
case Response(Invalid(reason), _) =>
|
|
||||||
// invalid target with known reason
|
|
||||||
FilteredRefreshResponseFut.map(_.copy(targetFilteredBy = reason))
|
|
||||||
case _ =>
|
|
||||||
// invalid target
|
|
||||||
FilteredRefreshResponseFut
|
|
||||||
}
|
|
||||||
|
|
||||||
val bStats = BroadcastStatsReceiver(receivers)
|
|
||||||
Stat
|
|
||||||
.timeFuture(bStats.stat("latency"))(
|
|
||||||
refreshResponse
|
|
||||||
.raiseWithin(CommonConstants.maxPushRequestDuration)
|
|
||||||
)
|
|
||||||
.onFailure { exception =>
|
|
||||||
rfphStatsRecorder.refreshRequestExceptionStats(exception, bStats)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,79 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.util.MRNtabCopy
|
|
||||||
import com.twitter.frigate.common.util.MRPushCopy
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
abstract class BaseCopyFramework(statsReceiver: StatsReceiver) {
|
|
||||||
|
|
||||||
private val NoAvailableCopyStat = statsReceiver.scope("no_copy_for_crt")
|
|
||||||
private val NoAvailableNtabCopyStat = statsReceiver.scope("no_ntab_copy")
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Instantiate push copy filters
|
|
||||||
*/
|
|
||||||
protected final val copyFilters = new CopyFilters(statsReceiver.scope("filters"))
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* The following method fetches all the push copies for a [[com.twitter.frigate.thriftscala.CommonRecommendationType]]
|
|
||||||
* associated with a candidate and then filters the eligible copies based on
|
|
||||||
* [[PushTypes.PushCandidate]] features. These filters are defined in
|
|
||||||
* [[CopyFilters]]
|
|
||||||
*
|
|
||||||
* @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate
|
|
||||||
*
|
|
||||||
* @return - set of eligible push copies for a given candidate
|
|
||||||
*/
|
|
||||||
protected[cross] final def getEligiblePushCopiesFromCandidate(
|
|
||||||
rawCandidate: RawCandidate
|
|
||||||
): Future[Seq[MRPushCopy]] = {
|
|
||||||
val pushCopiesFromRectype = CandidateToCopy.getPushCopiesFromRectype(rawCandidate.commonRecType)
|
|
||||||
|
|
||||||
if (pushCopiesFromRectype.isEmpty) {
|
|
||||||
NoAvailableCopyStat.counter(rawCandidate.commonRecType.name).incr()
|
|
||||||
throw new IllegalStateException(s"No Copy defined for CRT: " + rawCandidate.commonRecType)
|
|
||||||
}
|
|
||||||
pushCopiesFromRectype
|
|
||||||
.map(pushCopySet => copyFilters.execute(rawCandidate, pushCopySet.toSeq))
|
|
||||||
.getOrElse(Future.value(Seq.empty))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* This method essentially forms the base for cross-step for the MagicRecs Copy Framework. Given
|
|
||||||
* a recommendation type this returns a set of tuples wherein each tuple is a pair of push and
|
|
||||||
* ntab copy eligible for the said recommendation type
|
|
||||||
*
|
|
||||||
* @param rawCandidate - [[RawCandidate]] object representing a recommendation candidate
|
|
||||||
* @return - Set of eligible [[MRPushCopy]], Option[[MRNtabCopy]] for a given recommendation type
|
|
||||||
*/
|
|
||||||
protected[cross] final def getEligiblePushAndNtabCopiesFromCandidate(
|
|
||||||
rawCandidate: RawCandidate
|
|
||||||
): Future[Seq[(MRPushCopy, Option[MRNtabCopy])]] = {
|
|
||||||
|
|
||||||
val eligiblePushCopies = getEligiblePushCopiesFromCandidate(rawCandidate)
|
|
||||||
|
|
||||||
eligiblePushCopies.map { pushCopies =>
|
|
||||||
val setBuilder = Set.newBuilder[(MRPushCopy, Option[MRNtabCopy])]
|
|
||||||
pushCopies.foreach { pushCopy =>
|
|
||||||
val ntabCopies = CandidateToCopy.getNtabcopiesFromPushcopy(pushCopy)
|
|
||||||
val pushNtabCopyPairs = ntabCopies match {
|
|
||||||
case Some(ntabCopySet) =>
|
|
||||||
if (ntabCopySet.isEmpty) {
|
|
||||||
NoAvailableNtabCopyStat.counter(s"copy_id: ${pushCopy.copyId}").incr()
|
|
||||||
Set(pushCopy -> None)
|
|
||||||
} // push copy only
|
|
||||||
else ntabCopySet.map(pushCopy -> Some(_))
|
|
||||||
|
|
||||||
case None =>
|
|
||||||
Set.empty[(MRPushCopy, Option[MRNtabCopy])] // no push or ntab copy
|
|
||||||
}
|
|
||||||
setBuilder ++= pushNtabCopyPairs
|
|
||||||
}
|
|
||||||
setBuilder.result().toSeq
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,56 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.util.MRNtabCopy
|
|
||||||
import com.twitter.frigate.common.util.MRPushCopy
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.candidate.CopyIds
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param statsReceiver - stats receiver object
|
|
||||||
*/
|
|
||||||
class CandidateCopyExpansion(statsReceiver: StatsReceiver)
|
|
||||||
extends BaseCopyFramework(statsReceiver) {
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* Given a [[CandidateDetails]] object representing a push recommendation candidate this method
|
|
||||||
* expands it to multiple candidates, each tagged with a push copy id and ntab copy id to
|
|
||||||
* represent the eligible copies for the given recommendation candidate
|
|
||||||
*
|
|
||||||
* @param candidateDetails - [[CandidateDetails]] objects containing a recommendation candidate
|
|
||||||
*
|
|
||||||
* @return - list of tuples of [[PushTypes.RawCandidate]] and [[CopyIds]]
|
|
||||||
*/
|
|
||||||
private final def crossCandidateDetailsWithCopyId(
|
|
||||||
candidateDetails: CandidateDetails[RawCandidate]
|
|
||||||
): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] = {
|
|
||||||
val eligibleCopyPairs = getEligiblePushAndNtabCopiesFromCandidate(candidateDetails.candidate)
|
|
||||||
val copyPairs = eligibleCopyPairs.map(_.map {
|
|
||||||
case (pushCopy: MRPushCopy, ntabCopy: Option[MRNtabCopy]) =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(pushCopy.copyId),
|
|
||||||
ntabCopyId = ntabCopy.map(_.copyId)
|
|
||||||
)
|
|
||||||
})
|
|
||||||
|
|
||||||
copyPairs.map(_.map((candidateDetails, _)))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* This method takes as input a list of [[CandidateDetails]] objects which contain the push
|
|
||||||
* recommendation candidates for a given target user. It expands each input candidate into
|
|
||||||
* multiple candidates, each tagged with a push copy id and ntab copy id to represent the eligible
|
|
||||||
* copies for the given recommendation candidate
|
|
||||||
*
|
|
||||||
* @param candidateDetailsSeq - list of fetched candidates for push recommendation
|
|
||||||
* @return - list of tuples of [[RawCandidate]] and [[CopyIds]]
|
|
||||||
*/
|
|
||||||
final def expandCandidatesWithCopyId(
|
|
||||||
candidateDetailsSeq: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Future[Seq[(CandidateDetails[RawCandidate], CopyIds)]] =
|
|
||||||
Future.collect(candidateDetailsSeq.map(crossCandidateDetailsWithCopyId)).map(_.flatten)
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,11 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.util.MRPushCopy
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param candidate: [[RawCandidate]] is a recommendation candidate
|
|
||||||
* @param pushCopy: [[MRPushCopy]] eligible for candidate
|
|
||||||
*/
|
|
||||||
case class CandidateCopyPair(candidate: RawCandidate, pushCopy: MRPushCopy)
|
|
||||||
Binary file not shown.
@ -1,263 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.util.MrNtabCopyObjects
|
|
||||||
import com.twitter.frigate.common.util.MrPushCopyObjects
|
|
||||||
import com.twitter.frigate.common.util._
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType._
|
|
||||||
|
|
||||||
object CandidateToCopy {
|
|
||||||
|
|
||||||
// Static map from a CommonRecommendationType to set of eligible push notification copies
|
|
||||||
private[cross] val rectypeToPushCopy: Map[CommonRecommendationType, Set[
|
|
||||||
MRPushCopy
|
|
||||||
]] =
|
|
||||||
Map[CommonRecommendationType, Set[MRPushCopy]](
|
|
||||||
F1FirstdegreeTweet -> Set(
|
|
||||||
MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle
|
|
||||||
),
|
|
||||||
F1FirstdegreePhoto -> Set(
|
|
||||||
MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle
|
|
||||||
),
|
|
||||||
F1FirstdegreeVideo -> Set(
|
|
||||||
MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle
|
|
||||||
),
|
|
||||||
TweetRetweet -> Set(
|
|
||||||
MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText,
|
|
||||||
MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText,
|
|
||||||
MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText
|
|
||||||
),
|
|
||||||
TweetRetweetPhoto -> Set(
|
|
||||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText,
|
|
||||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText
|
|
||||||
),
|
|
||||||
TweetRetweetVideo -> Set(
|
|
||||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText,
|
|
||||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText
|
|
||||||
),
|
|
||||||
TweetFavorite -> Set(
|
|
||||||
MrPushCopyObjects.TweetLikeOneSocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetLikeTwoSocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetLikeMultipleSocialContextWithText
|
|
||||||
),
|
|
||||||
TweetFavoritePhoto -> Set(
|
|
||||||
MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText
|
|
||||||
),
|
|
||||||
TweetFavoriteVideo -> Set(
|
|
||||||
MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText,
|
|
||||||
MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText
|
|
||||||
),
|
|
||||||
UnreadBadgeCount -> Set(MrPushCopyObjects.UnreadBadgeCount),
|
|
||||||
InterestBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
InterestBasedPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
|
||||||
InterestBasedVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
|
||||||
UserFollow -> Set(
|
|
||||||
MrPushCopyObjects.UserFollowWithOneSocialContext,
|
|
||||||
MrPushCopyObjects.UserFollowWithTwoSocialContext,
|
|
||||||
MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext
|
|
||||||
),
|
|
||||||
HermitUser -> Set(
|
|
||||||
MrPushCopyObjects.HermitUserWithOneSocialContext,
|
|
||||||
MrPushCopyObjects.HermitUserWithTwoSocialContext,
|
|
||||||
MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
TriangularLoopUser -> Set(
|
|
||||||
MrPushCopyObjects.TriangularLoopUserWithOneSocialContext,
|
|
||||||
MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts,
|
|
||||||
MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext
|
|
||||||
),
|
|
||||||
ForwardAddressbookUserFollow -> Set(MrPushCopyObjects.ForwardAddressBookUserFollow),
|
|
||||||
NewsArticleNewsLanding -> Set(MrPushCopyObjects.NewsArticleNewsLandingCopy),
|
|
||||||
TopicProofTweet -> Set(MrPushCopyObjects.TopicProofTweet),
|
|
||||||
UserInterestinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
UserInterestinPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
|
||||||
UserInterestinVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
|
||||||
TwistlyTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
TwistlyPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
|
||||||
TwistlyVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
|
||||||
ElasticTimelineTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
ElasticTimelinePhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
|
||||||
ElasticTimelineVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
|
||||||
ExploreVideoTweet -> Set(MrPushCopyObjects.ExploreVideoTweet),
|
|
||||||
List -> Set(MrPushCopyObjects.ListRecommendation),
|
|
||||||
InterestBasedUserFollow -> Set(MrPushCopyObjects.UserFollowInterestBasedCopy),
|
|
||||||
PastEmailEngagementTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
PastEmailEngagementPhoto -> Set(MrPushCopyObjects.RecommendedForYouPhoto),
|
|
||||||
PastEmailEngagementVideo -> Set(MrPushCopyObjects.RecommendedForYouVideo),
|
|
||||||
ExplorePush -> Set(MrPushCopyObjects.ExplorePush),
|
|
||||||
ConnectTabPush -> Set(MrPushCopyObjects.ConnectTabPush),
|
|
||||||
ConnectTabWithUserPush -> Set(MrPushCopyObjects.ConnectTabWithUserPush),
|
|
||||||
AddressBookUploadPush -> Set(MrPushCopyObjects.AddressBookPush),
|
|
||||||
InterestPickerPush -> Set(MrPushCopyObjects.InterestPickerPush),
|
|
||||||
CompleteOnboardingPush -> Set(MrPushCopyObjects.CompleteOnboardingPush),
|
|
||||||
GeoPopTweet -> Set(MrPushCopyObjects.GeoPopPushCopy),
|
|
||||||
TagSpaceTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
FrsTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
TwhinTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
MrModelingBasedTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
DetopicTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
TweetImpressions -> Set(MrPushCopyObjects.TopTweetImpressions),
|
|
||||||
TrendTweet -> Set(MrPushCopyObjects.TrendTweet),
|
|
||||||
ReverseAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
ForwardAddressbookTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
SpaceInNetwork -> Set(MrPushCopyObjects.SpaceHost),
|
|
||||||
SpaceOutOfNetwork -> Set(MrPushCopyObjects.SpaceHost),
|
|
||||||
SubscribedSearch -> Set(MrPushCopyObjects.SubscribedSearchTweet),
|
|
||||||
TripGeoTweet -> Set(MrPushCopyObjects.TripGeoTweetPushCopy),
|
|
||||||
CrowdSearchTweet -> Set(MrPushCopyObjects.RecommendedForYouTweet),
|
|
||||||
Digest -> Set(MrPushCopyObjects.Digest),
|
|
||||||
TripHqTweet -> Set(MrPushCopyObjects.TripHqTweetPushCopy)
|
|
||||||
)
|
|
||||||
|
|
||||||
// Static map from a push copy to set of eligible ntab copies
|
|
||||||
private[cross] val pushcopyToNtabcopy: Map[MRPushCopy, Set[MRNtabCopy]] =
|
|
||||||
Map[MRPushCopy, Set[MRNtabCopy]](
|
|
||||||
MrPushCopyObjects.FirstDegreeJustTweetedBoldTitle -> Set(
|
|
||||||
MrNtabCopyObjects.FirstDegreeTweetRecent),
|
|
||||||
MrPushCopyObjects.FirstDegreePhotoJustTweetedBoldTitle -> Set(
|
|
||||||
MrNtabCopyObjects.FirstDegreeTweetRecent
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.FirstDegreeVideoJustTweetedBoldTitle -> Set(
|
|
||||||
MrNtabCopyObjects.FirstDegreeTweetRecent
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetWithOneDisplaySocialContextsWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetWithTwoDisplaySocialContextsWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContextsWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContextsWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetPhotoWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContextsWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetPhotoWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplaySocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetVideoWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContextsWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetVideoWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContextsWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetRetweetVideoWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikeOneSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikeWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikeTwoSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikeWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikeMultipleSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikeWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikePhotoOneSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikePhotoWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikePhotoTwoSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikePhotoWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikePhotoMultipleSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikePhotoWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikeVideoOneSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikeVideoWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikeVideoTwoSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikeVideoWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TweetLikeVideoMultipleSocialContextWithText -> Set(
|
|
||||||
MrNtabCopyObjects.TweetLikeVideoWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.UnreadBadgeCount -> Set.empty[MRNtabCopy],
|
|
||||||
MrPushCopyObjects.RecommendedForYouTweet -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
|
||||||
MrPushCopyObjects.RecommendedForYouPhoto -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
|
||||||
MrPushCopyObjects.RecommendedForYouVideo -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
|
||||||
MrPushCopyObjects.GeoPopPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
|
||||||
MrPushCopyObjects.UserFollowWithOneSocialContext -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.UserFollowWithTwoSocialContext -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.UserFollowOneDisplayAndKOtherSocialContext -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.HermitUserWithOneSocialContext -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowWithOneDisplaySocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.HermitUserWithTwoSocialContext -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowWithTwoDisplaySocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.HermitUserWithOneDisplayAndKOtherSocialContexts -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowWithOneDisplayAndKOtherSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TriangularLoopUserWithOneSocialContext -> Set(
|
|
||||||
MrNtabCopyObjects.TriangularLoopUserWithOneSocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TriangularLoopUserWithTwoSocialContexts -> Set(
|
|
||||||
MrNtabCopyObjects.TriangularLoopUserWithTwoSocialContexts
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.TriangularLoopUserOneDisplayAndKotherSocialContext -> Set(
|
|
||||||
MrNtabCopyObjects.TriangularLoopUserOneDisplayAndKOtherSocialContext
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.NewsArticleNewsLandingCopy -> Set(
|
|
||||||
MrNtabCopyObjects.NewsArticleNewsLandingCopy
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.UserFollowInterestBasedCopy -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowInterestBasedCopy
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.ForwardAddressBookUserFollow -> Set(
|
|
||||||
MrNtabCopyObjects.ForwardAddressBookUserFollow),
|
|
||||||
MrPushCopyObjects.ConnectTabPush -> Set(
|
|
||||||
MrNtabCopyObjects.ConnectTabPush
|
|
||||||
),
|
|
||||||
MrPushCopyObjects.ExplorePush -> Set.empty[MRNtabCopy],
|
|
||||||
MrPushCopyObjects.ConnectTabWithUserPush -> Set(
|
|
||||||
MrNtabCopyObjects.UserFollowInterestBasedCopy),
|
|
||||||
MrPushCopyObjects.AddressBookPush -> Set(MrNtabCopyObjects.AddressBook),
|
|
||||||
MrPushCopyObjects.InterestPickerPush -> Set(MrNtabCopyObjects.InterestPicker),
|
|
||||||
MrPushCopyObjects.CompleteOnboardingPush -> Set(MrNtabCopyObjects.CompleteOnboarding),
|
|
||||||
MrPushCopyObjects.TopicProofTweet -> Set(MrNtabCopyObjects.TopicProofTweet),
|
|
||||||
MrPushCopyObjects.TopTweetImpressions -> Set(MrNtabCopyObjects.TopTweetImpressions),
|
|
||||||
MrPushCopyObjects.TrendTweet -> Set(MrNtabCopyObjects.TrendTweet),
|
|
||||||
MrPushCopyObjects.SpaceHost -> Set(MrNtabCopyObjects.SpaceHost),
|
|
||||||
MrPushCopyObjects.SubscribedSearchTweet -> Set(MrNtabCopyObjects.SubscribedSearchTweet),
|
|
||||||
MrPushCopyObjects.TripGeoTweetPushCopy -> Set(MrNtabCopyObjects.RecommendedForYouCopy),
|
|
||||||
MrPushCopyObjects.Digest -> Set(MrNtabCopyObjects.Digest),
|
|
||||||
MrPushCopyObjects.TripHqTweetPushCopy -> Set(MrNtabCopyObjects.HighQualityTweet),
|
|
||||||
MrPushCopyObjects.ExploreVideoTweet -> Set(MrNtabCopyObjects.ExploreVideoTweet),
|
|
||||||
MrPushCopyObjects.ListRecommendation -> Set(MrNtabCopyObjects.ListRecommendation),
|
|
||||||
MrPushCopyObjects.MagicFanoutCreatorSubscription -> Set(
|
|
||||||
MrNtabCopyObjects.MagicFanoutCreatorSubscription),
|
|
||||||
MrPushCopyObjects.MagicFanoutNewCreator -> Set(MrNtabCopyObjects.MagicFanoutNewCreator)
|
|
||||||
)
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param crt - [[CommonRecommendationType]] used for a frigate push notification
|
|
||||||
*
|
|
||||||
* @return - Set of [[MRPushCopy]] objects representing push copies eligibile for a
|
|
||||||
* [[CommonRecommendationType]]
|
|
||||||
*/
|
|
||||||
def getPushCopiesFromRectype(crt: CommonRecommendationType): Option[Set[MRPushCopy]] =
|
|
||||||
rectypeToPushCopy.get(crt)
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param pushcopy - [[MRPushCopy]] object representing a push notification copy
|
|
||||||
* @return - Set of [[MRNtabCopy]] objects that can be paired with a given [[MRPushCopy]]
|
|
||||||
*/
|
|
||||||
def getNtabcopiesFromPushcopy(pushcopy: MRPushCopy): Option[Set[MRNtabCopy]] =
|
|
||||||
pushcopyToNtabcopy.get(pushcopy)
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,41 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base._
|
|
||||||
import com.twitter.frigate.common.util.MRPushCopy
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.hermit.predicate.Predicate
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
private[cross] class CopyFilters(statsReceiver: StatsReceiver) {
|
|
||||||
|
|
||||||
private val copyPredicates = new CopyPredicates(statsReceiver.scope("copy_predicate"))
|
|
||||||
|
|
||||||
def execute(rawCandidate: RawCandidate, pushCopies: Seq[MRPushCopy]): Future[Seq[MRPushCopy]] = {
|
|
||||||
val candidateCopyPairs: Seq[CandidateCopyPair] =
|
|
||||||
pushCopies.map(CandidateCopyPair(rawCandidate, _))
|
|
||||||
|
|
||||||
val compositePredicate: Predicate[CandidateCopyPair] = rawCandidate match {
|
|
||||||
case _: F1FirstDegree | _: OutOfNetworkTweetCandidate | _: EventCandidate |
|
|
||||||
_: TopicProofTweetCandidate | _: ListPushCandidate | _: HermitInterestBasedUserFollow |
|
|
||||||
_: UserFollowWithoutSocialContextCandidate | _: DiscoverTwitterCandidate |
|
|
||||||
_: TopTweetImpressionsCandidate | _: TrendTweetCandidate |
|
|
||||||
_: SubscribedSearchTweetCandidate | _: DigestCandidate =>
|
|
||||||
copyPredicates.alwaysTruePredicate
|
|
||||||
|
|
||||||
case _: SocialContextActions => copyPredicates.displaySocialContextPredicate
|
|
||||||
|
|
||||||
case _ => copyPredicates.unrecognizedCandidatePredicate // block unrecognised candidates
|
|
||||||
}
|
|
||||||
|
|
||||||
// apply predicate to all [[MRPushCopy]] objects
|
|
||||||
val filterResults: Future[Seq[Boolean]] = compositePredicate(candidateCopyPairs)
|
|
||||||
filterResults.map { results: Seq[Boolean] =>
|
|
||||||
val seqBuilder = Seq.newBuilder[MRPushCopy]
|
|
||||||
results.zip(pushCopies).foreach {
|
|
||||||
case (result, pushCopy) => if (result) seqBuilder += pushCopy
|
|
||||||
}
|
|
||||||
seqBuilder.result()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,36 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.refresh_handler.cross
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.SocialContextActions
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.hermit.predicate.Predicate
|
|
||||||
|
|
||||||
class CopyPredicates(statsReceiver: StatsReceiver) {
|
|
||||||
val alwaysTruePredicate = Predicate
|
|
||||||
.from { _: CandidateCopyPair =>
|
|
||||||
true
|
|
||||||
}.withStats(statsReceiver.scope("always_true_copy_predicate"))
|
|
||||||
|
|
||||||
val unrecognizedCandidatePredicate = alwaysTruePredicate.flip
|
|
||||||
.withStats(statsReceiver.scope("unrecognized_candidate"))
|
|
||||||
|
|
||||||
val displaySocialContextPredicate = Predicate
|
|
||||||
.from { candidateCopyPair: CandidateCopyPair =>
|
|
||||||
candidateCopyPair.candidate match {
|
|
||||||
case candidateWithScActions: RawCandidate with SocialContextActions =>
|
|
||||||
val socialContextUserIds = candidateWithScActions.socialContextActions.map(_.userId)
|
|
||||||
val countSocialContext = socialContextUserIds.size
|
|
||||||
val pushCopy = candidateCopyPair.pushCopy
|
|
||||||
|
|
||||||
countSocialContext match {
|
|
||||||
case 1 => pushCopy.hasOneDisplaySocialContext && !pushCopy.hasOtherSocialContext
|
|
||||||
case 2 => pushCopy.hasTwoDisplayContext && !pushCopy.hasOtherSocialContext
|
|
||||||
case c if c > 2 =>
|
|
||||||
pushCopy.hasOneDisplaySocialContext && pushCopy.hasOtherSocialContext
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
|
|
||||||
case _ => false
|
|
||||||
}
|
|
||||||
}.withStats(statsReceiver.scope("display_social_context_predicate"))
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,388 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.scriber
|
|
||||||
|
|
||||||
import com.twitter.bijection.Base64String
|
|
||||||
import com.twitter.bijection.Injection
|
|
||||||
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
|
||||||
import com.twitter.core_workflows.user_model.thriftscala.{UserState => ThriftUserState}
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.finagle.tracing.Trace
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.CandidateResult
|
|
||||||
import com.twitter.frigate.common.base.Invalid
|
|
||||||
import com.twitter.frigate.common.base.OK
|
|
||||||
import com.twitter.frigate.common.base.Result
|
|
||||||
import com.twitter.frigate.common.rec_types.RecTypes
|
|
||||||
import com.twitter.frigate.data_pipeline.features_common.PushQualityModelFeatureContext
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams
|
|
||||||
import com.twitter.frigate.pushservice.params.PushParams
|
|
||||||
import com.twitter.frigate.scribe.thriftscala.CandidateFilteredOutStep
|
|
||||||
import com.twitter.frigate.scribe.thriftscala.CandidateRequestInfo
|
|
||||||
import com.twitter.frigate.scribe.thriftscala.MrRequestScribe
|
|
||||||
import com.twitter.frigate.scribe.thriftscala.TargetUserInfo
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.frigate.thriftscala.TweetNotification
|
|
||||||
import com.twitter.frigate.thriftscala.{SocialContextAction => TSocialContextAction}
|
|
||||||
import com.twitter.logging.Logger
|
|
||||||
import com.twitter.ml.api.DataRecord
|
|
||||||
import com.twitter.ml.api.Feature
|
|
||||||
import com.twitter.ml.api.FeatureType
|
|
||||||
import com.twitter.ml.api.util.SRichDataRecord
|
|
||||||
import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions
|
|
||||||
import com.twitter.nrel.heavyranker.PushPredictionHelper
|
|
||||||
import com.twitter.util.Future
|
|
||||||
import com.twitter.util.Time
|
|
||||||
import java.util.UUID
|
|
||||||
import scala.collection.mutable
|
|
||||||
|
|
||||||
class MrRequestScribeHandler(mrRequestScriberNode: String, stats: StatsReceiver) {
|
|
||||||
|
|
||||||
private val mrRequestScribeLogger = Logger(mrRequestScriberNode)
|
|
||||||
|
|
||||||
private val mrRequestScribeTargetFilteringStats =
|
|
||||||
stats.counter("MrRequestScribeHandler_target_filtering")
|
|
||||||
private val mrRequestScribeCandidateFilteringStats =
|
|
||||||
stats.counter("MrRequestScribeHandler_candidate_filtering")
|
|
||||||
private val mrRequestScribeInvalidStats =
|
|
||||||
stats.counter("MrRequestScribeHandler_invalid_filtering")
|
|
||||||
private val mrRequestScribeUnsupportedFeatureTypeStats =
|
|
||||||
stats.counter("MrRequestScribeHandler_unsupported_feature_type")
|
|
||||||
private val mrRequestScribeNotIncludedFeatureStats =
|
|
||||||
stats.counter("MrRequestScribeHandler_not_included_features")
|
|
||||||
|
|
||||||
private final val MrRequestScribeInjection: Injection[MrRequestScribe, String] = BinaryScalaCodec(
|
|
||||||
MrRequestScribe
|
|
||||||
) andThen Injection.connect[Array[Byte], Base64String, String]
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param target : Target user id
|
|
||||||
* @param result : Result for target filtering
|
|
||||||
*
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
def scribeForTargetFiltering(target: Target, result: Result): Future[Option[MrRequestScribe]] = {
|
|
||||||
if (target.isLoggedOutUser || !enableTargetFilteringScribing(target)) {
|
|
||||||
Future.None
|
|
||||||
} else {
|
|
||||||
val predicate = result match {
|
|
||||||
case Invalid(reason) => reason
|
|
||||||
case _ =>
|
|
||||||
mrRequestScribeInvalidStats.incr()
|
|
||||||
throw new IllegalStateException("Invalid reason for Target Filtering " + result)
|
|
||||||
}
|
|
||||||
buildScribeThrift(target, predicate, None).map { targetFilteredScribe =>
|
|
||||||
writeAtTargetFilteringStep(target, targetFilteredScribe)
|
|
||||||
Some(targetFilteredScribe)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* @param target : Target user id
|
|
||||||
* @param hydratedCandidates : Candidates hydrated with details: impressionId, frigateNotification and source
|
|
||||||
* @param preRankingFilteredCandidates : Candidates result filtered out at preRanking filtering step
|
|
||||||
* @param rankedCandidates : Sorted candidates details ranked by ranking step
|
|
||||||
* @param rerankedCandidates : Sorted candidates details ranked by reranking step
|
|
||||||
* @param restrictFilteredCandidates : Candidates details filtered out at restrict step
|
|
||||||
* @param allTakeCandidateResults : Candidates results at take step, include the candidates we take and the candidates filtered out at take step [with different result]
|
|
||||||
*
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
def scribeForCandidateFiltering(
|
|
||||||
target: Target,
|
|
||||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]],
|
|
||||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
rerankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]]
|
|
||||||
): Future[Seq[MrRequestScribe]] = {
|
|
||||||
if (target.isLoggedOutUser || target.isEmailUser) {
|
|
||||||
Future.Nil
|
|
||||||
} else if (enableCandidateFilteringScribing(target)) {
|
|
||||||
val hydrateFeature =
|
|
||||||
target.params(PushFeatureSwitchParams.EnableMrRequestScribingWithFeatureHydrating) ||
|
|
||||||
target.scribeFeatureForRequestScribe
|
|
||||||
|
|
||||||
val candidateRequestInfoSeq = generateCandidatesScribeInfo(
|
|
||||||
hydratedCandidates,
|
|
||||||
preRankingFilteredCandidates,
|
|
||||||
rankedCandidates,
|
|
||||||
rerankedCandidates,
|
|
||||||
restrictFilteredCandidates,
|
|
||||||
allTakeCandidateResults,
|
|
||||||
isFeatureHydratingEnabled = hydrateFeature
|
|
||||||
)
|
|
||||||
val flattenStructure =
|
|
||||||
target.params(PushFeatureSwitchParams.EnableFlattenMrRequestScribing) || hydrateFeature
|
|
||||||
candidateRequestInfoSeq.flatMap { candidateRequestInfos =>
|
|
||||||
if (flattenStructure) {
|
|
||||||
Future.collect {
|
|
||||||
candidateRequestInfos.map { candidateRequestInfo =>
|
|
||||||
buildScribeThrift(target, None, Some(Seq(candidateRequestInfo)))
|
|
||||||
.map { mrRequestScribe =>
|
|
||||||
writeAtCandidateFilteringStep(target, mrRequestScribe)
|
|
||||||
mrRequestScribe
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
buildScribeThrift(target, None, Some(candidateRequestInfos))
|
|
||||||
.map { mrRequestScribe =>
|
|
||||||
writeAtCandidateFilteringStep(target, mrRequestScribe)
|
|
||||||
Seq(mrRequestScribe)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else Future.Nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private def buildScribeThrift(
|
|
||||||
target: Target,
|
|
||||||
targetFilteredOutPredicate: Option[String],
|
|
||||||
candidatesRequestInfo: Option[Seq[CandidateRequestInfo]]
|
|
||||||
): Future[MrRequestScribe] = {
|
|
||||||
Future
|
|
||||||
.join(
|
|
||||||
target.targetUserState,
|
|
||||||
generateTargetFeatureScribeInfo(target),
|
|
||||||
target.targetUser).map {
|
|
||||||
case (userStateOption, targetFeatureOption, gizmoduckUserOpt) =>
|
|
||||||
val userState = userStateOption.map(userState => ThriftUserState(userState.id))
|
|
||||||
val targetFeatures =
|
|
||||||
targetFeatureOption.map(ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord)
|
|
||||||
val traceId = Trace.id.traceId.toLong
|
|
||||||
|
|
||||||
MrRequestScribe(
|
|
||||||
requestId = UUID.randomUUID.toString.replaceAll("-", ""),
|
|
||||||
scribedTimeMs = Time.now.inMilliseconds,
|
|
||||||
targetUserId = target.targetId,
|
|
||||||
targetUserInfo = Some(
|
|
||||||
TargetUserInfo(
|
|
||||||
userState,
|
|
||||||
features = targetFeatures,
|
|
||||||
userType = gizmoduckUserOpt.map(_.userType))
|
|
||||||
),
|
|
||||||
targetFilteredOutPredicate = targetFilteredOutPredicate,
|
|
||||||
candidates = candidatesRequestInfo,
|
|
||||||
traceId = Some(traceId)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def generateTargetFeatureScribeInfo(
|
|
||||||
target: Target
|
|
||||||
): Future[Option[DataRecord]] = {
|
|
||||||
val featureList =
|
|
||||||
target.params(PushFeatureSwitchParams.TargetLevelFeatureListForMrRequestScribing)
|
|
||||||
if (featureList.nonEmpty) {
|
|
||||||
PushPredictionHelper
|
|
||||||
.getDataRecordFromTargetFeatureMap(
|
|
||||||
target.targetId,
|
|
||||||
target.featureMap,
|
|
||||||
stats
|
|
||||||
).map { dataRecord =>
|
|
||||||
val richRecord =
|
|
||||||
new SRichDataRecord(dataRecord, PushQualityModelFeatureContext.featureContext)
|
|
||||||
|
|
||||||
val selectedRecord =
|
|
||||||
SRichDataRecord(new DataRecord(), PushQualityModelFeatureContext.featureContext)
|
|
||||||
featureList.map { featureName =>
|
|
||||||
val feature: Feature[_] = {
|
|
||||||
try {
|
|
||||||
PushQualityModelFeatureContext.featureContext.getFeature(featureName)
|
|
||||||
} catch {
|
|
||||||
case _: Exception =>
|
|
||||||
mrRequestScribeNotIncludedFeatureStats.incr()
|
|
||||||
throw new IllegalStateException(
|
|
||||||
"Scribing features not included in FeatureContext: " + featureName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
richRecord.getFeatureValueOpt(feature).foreach { featureVal =>
|
|
||||||
feature.getFeatureType() match {
|
|
||||||
case FeatureType.BINARY =>
|
|
||||||
selectedRecord.setFeatureValue(
|
|
||||||
feature.asInstanceOf[Feature[Boolean]],
|
|
||||||
featureVal.asInstanceOf[Boolean])
|
|
||||||
case FeatureType.CONTINUOUS =>
|
|
||||||
selectedRecord.setFeatureValue(
|
|
||||||
feature.asInstanceOf[Feature[Double]],
|
|
||||||
featureVal.asInstanceOf[Double])
|
|
||||||
case FeatureType.STRING =>
|
|
||||||
selectedRecord.setFeatureValue(
|
|
||||||
feature.asInstanceOf[Feature[String]],
|
|
||||||
featureVal.asInstanceOf[String])
|
|
||||||
case FeatureType.DISCRETE =>
|
|
||||||
selectedRecord.setFeatureValue(
|
|
||||||
feature.asInstanceOf[Feature[Long]],
|
|
||||||
featureVal.asInstanceOf[Long])
|
|
||||||
case _ =>
|
|
||||||
mrRequestScribeUnsupportedFeatureTypeStats.incr()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(selectedRecord.getRecord)
|
|
||||||
}
|
|
||||||
} else Future.None
|
|
||||||
}
|
|
||||||
|
|
||||||
private def generateCandidatesScribeInfo(
|
|
||||||
hydratedCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
preRankingFilteredCandidates: Seq[CandidateResult[PushCandidate, Result]],
|
|
||||||
rankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
rerankedCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
restrictFilteredCandidates: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
allTakeCandidateResults: Seq[CandidateResult[PushCandidate, Result]],
|
|
||||||
isFeatureHydratingEnabled: Boolean
|
|
||||||
): Future[Seq[CandidateRequestInfo]] = {
|
|
||||||
val candidatesMap = new mutable.HashMap[String, CandidateRequestInfo]
|
|
||||||
|
|
||||||
hydratedCandidates.foreach { hydratedCandidate =>
|
|
||||||
val frgNotif = hydratedCandidate.candidate.frigateNotification
|
|
||||||
val simplifiedTweetNotificationOpt = frgNotif.tweetNotification.map { tweetNotification =>
|
|
||||||
TweetNotification(
|
|
||||||
tweetNotification.tweetId,
|
|
||||||
Seq.empty[TSocialContextAction],
|
|
||||||
tweetNotification.tweetAuthorId)
|
|
||||||
}
|
|
||||||
val simplifiedFrigateNotification = FrigateNotification(
|
|
||||||
frgNotif.commonRecommendationType,
|
|
||||||
frgNotif.notificationDisplayLocation,
|
|
||||||
tweetNotification = simplifiedTweetNotificationOpt
|
|
||||||
)
|
|
||||||
candidatesMap(hydratedCandidate.candidate.impressionId) = CandidateRequestInfo(
|
|
||||||
candidateId = "",
|
|
||||||
candidateSource = hydratedCandidate.source.substring(
|
|
||||||
0,
|
|
||||||
Math.min(6, hydratedCandidate.source.length)
|
|
||||||
),
|
|
||||||
frigateNotification = Some(simplifiedFrigateNotification),
|
|
||||||
modelScore = None,
|
|
||||||
rankPosition = None,
|
|
||||||
rerankPosition = None,
|
|
||||||
features = None,
|
|
||||||
isSent = Some(false)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
preRankingFilteredCandidates.foreach { preRankingFilteredCandidateResult =>
|
|
||||||
candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId) =
|
|
||||||
candidatesMap(preRankingFilteredCandidateResult.candidate.impressionId)
|
|
||||||
.copy(
|
|
||||||
candidateFilteredOutPredicate = preRankingFilteredCandidateResult.result match {
|
|
||||||
case Invalid(reason) => reason
|
|
||||||
case _ => {
|
|
||||||
mrRequestScribeInvalidStats.incr()
|
|
||||||
throw new IllegalStateException(
|
|
||||||
"Invalid reason for Candidate Filtering " + preRankingFilteredCandidateResult.result)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
candidateFilteredOutStep = Some(CandidateFilteredOutStep.PreRankFiltering)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
_ <- Future.collectToTry {
|
|
||||||
rankedCandidates.zipWithIndex.map {
|
|
||||||
case (rankedCandidateDetail, index) =>
|
|
||||||
val modelScoresFut = {
|
|
||||||
val crt = rankedCandidateDetail.candidate.commonRecType
|
|
||||||
if (RecTypes.notEligibleForModelScoreTracking.contains(crt)) Future.None
|
|
||||||
else rankedCandidateDetail.candidate.modelScores.map(Some(_))
|
|
||||||
}
|
|
||||||
|
|
||||||
modelScoresFut.map { modelScores =>
|
|
||||||
candidatesMap(rankedCandidateDetail.candidate.impressionId) =
|
|
||||||
candidatesMap(rankedCandidateDetail.candidate.impressionId).copy(
|
|
||||||
rankPosition = Some(index),
|
|
||||||
modelScore = modelScores
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = rerankedCandidates.zipWithIndex.foreach {
|
|
||||||
case (rerankedCandidateDetail, index) => {
|
|
||||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId) =
|
|
||||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy(
|
|
||||||
rerankPosition = Some(index)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_ <- Future.collectToTry {
|
|
||||||
rerankedCandidates.map { rerankedCandidateDetail =>
|
|
||||||
if (isFeatureHydratingEnabled) {
|
|
||||||
PushPredictionHelper
|
|
||||||
.getDataRecord(
|
|
||||||
rerankedCandidateDetail.candidate.target.targetHydrationContext,
|
|
||||||
rerankedCandidateDetail.candidate.target.featureMap,
|
|
||||||
rerankedCandidateDetail.candidate.candidateHydrationContext,
|
|
||||||
rerankedCandidateDetail.candidate.candidateFeatureMap(),
|
|
||||||
stats
|
|
||||||
).map { features =>
|
|
||||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId) =
|
|
||||||
candidatesMap(rerankedCandidateDetail.candidate.impressionId).copy(
|
|
||||||
features = Some(
|
|
||||||
ScalaToJavaDataRecordConversions.javaDataRecord2ScalaDataRecord(features))
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} else Future.Unit
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = restrictFilteredCandidates.foreach { restrictFilteredCandidateDetatil =>
|
|
||||||
candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId) =
|
|
||||||
candidatesMap(restrictFilteredCandidateDetatil.candidate.impressionId)
|
|
||||||
.copy(candidateFilteredOutStep = Some(CandidateFilteredOutStep.Restrict))
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = allTakeCandidateResults.foreach { allTakeCandidateResult =>
|
|
||||||
allTakeCandidateResult.result match {
|
|
||||||
case OK =>
|
|
||||||
candidatesMap(allTakeCandidateResult.candidate.impressionId) =
|
|
||||||
candidatesMap(allTakeCandidateResult.candidate.impressionId).copy(isSent = Some(true))
|
|
||||||
case Invalid(reason) =>
|
|
||||||
candidatesMap(allTakeCandidateResult.candidate.impressionId) =
|
|
||||||
candidatesMap(allTakeCandidateResult.candidate.impressionId).copy(
|
|
||||||
candidateFilteredOutPredicate = reason,
|
|
||||||
candidateFilteredOutStep = Some(CandidateFilteredOutStep.PostRankFiltering))
|
|
||||||
case _ =>
|
|
||||||
mrRequestScribeInvalidStats.incr()
|
|
||||||
throw new IllegalStateException(
|
|
||||||
"Invalid reason for Candidate Filtering " + allTakeCandidateResult.result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} yield candidatesMap.values.toSeq
|
|
||||||
}
|
|
||||||
|
|
||||||
private def enableTargetFilteringScribing(target: Target): Boolean = {
|
|
||||||
target.params(PushParams.EnableMrRequestScribing) && target.params(
|
|
||||||
PushFeatureSwitchParams.EnableMrRequestScribingForTargetFiltering)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def enableCandidateFilteringScribing(target: Target): Boolean = {
|
|
||||||
target.params(PushParams.EnableMrRequestScribing) && target.params(
|
|
||||||
PushFeatureSwitchParams.EnableMrRequestScribingForCandidateFiltering)
|
|
||||||
}
|
|
||||||
|
|
||||||
private def writeAtTargetFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = {
|
|
||||||
logToScribe(mrRequestScribe)
|
|
||||||
mrRequestScribeTargetFilteringStats.incr()
|
|
||||||
}
|
|
||||||
|
|
||||||
private def writeAtCandidateFilteringStep(target: Target, mrRequestScribe: MrRequestScribe) = {
|
|
||||||
logToScribe(mrRequestScribe)
|
|
||||||
mrRequestScribeCandidateFilteringStats.incr()
|
|
||||||
}
|
|
||||||
|
|
||||||
private def logToScribe(mrRequestScribe: MrRequestScribe): Unit = {
|
|
||||||
val logEntry: String = MrRequestScribeInjection(mrRequestScribe)
|
|
||||||
mrRequestScribeLogger.info(logEntry)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,250 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.BroadcastStatsReceiver
|
|
||||||
import com.twitter.finagle.stats.Stat
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base.CandidateDetails
|
|
||||||
import com.twitter.frigate.common.base.CandidateFilteringOnlyFlow
|
|
||||||
import com.twitter.frigate.common.base.CandidateResult
|
|
||||||
import com.twitter.frigate.common.base.FeatureMap
|
|
||||||
import com.twitter.frigate.common.base.OK
|
|
||||||
import com.twitter.frigate.common.base.Response
|
|
||||||
import com.twitter.frigate.common.base.Result
|
|
||||||
import com.twitter.frigate.common.base.Stats.track
|
|
||||||
import com.twitter.frigate.common.config.CommonConstants
|
|
||||||
import com.twitter.frigate.common.logger.MRLogger
|
|
||||||
import com.twitter.frigate.common.rec_types.RecTypes
|
|
||||||
import com.twitter.frigate.common.util.InvalidRequestException
|
|
||||||
import com.twitter.frigate.common.util.MrNtabCopyObjects
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.config.Config
|
|
||||||
import com.twitter.frigate.pushservice.ml.HydrationContextBuilder
|
|
||||||
import com.twitter.frigate.pushservice.params.PushFeatureSwitchParams.EnableMagicFanoutNewsForYouNtabCopy
|
|
||||||
import com.twitter.frigate.pushservice.scriber.MrRequestScribeHandler
|
|
||||||
import com.twitter.frigate.pushservice.send_handler.generator.PushRequestToCandidate
|
|
||||||
import com.twitter.frigate.pushservice.take.SendHandlerNotifier
|
|
||||||
import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPostCandidateValidator
|
|
||||||
import com.twitter.frigate.pushservice.take.candidate_validator.SendHandlerPreCandidateValidator
|
|
||||||
import com.twitter.frigate.pushservice.target.PushTargetUserBuilder
|
|
||||||
import com.twitter.frigate.pushservice.util.ResponseStatsTrackUtils.trackStatsForResponseToRequest
|
|
||||||
import com.twitter.frigate.pushservice.util.SendHandlerPredicateUtil
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.PushRequest
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.PushRequestScribe
|
|
||||||
import com.twitter.frigate.pushservice.thriftscala.PushResponse
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.nrel.heavyranker.FeatureHydrator
|
|
||||||
import com.twitter.util._
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A handler for sending PushRequests
|
|
||||||
*/
|
|
||||||
class SendHandler(
|
|
||||||
pushTargetUserBuilder: PushTargetUserBuilder,
|
|
||||||
preCandidateValidator: SendHandlerPreCandidateValidator,
|
|
||||||
postCandidateValidator: SendHandlerPostCandidateValidator,
|
|
||||||
sendHandlerNotifier: SendHandlerNotifier,
|
|
||||||
candidateHydrator: SendHandlerPushCandidateHydrator,
|
|
||||||
featureHydrator: FeatureHydrator,
|
|
||||||
sendHandlerPredicateUtil: SendHandlerPredicateUtil,
|
|
||||||
mrRequestScriberNode: String
|
|
||||||
)(
|
|
||||||
implicit val statsReceiver: StatsReceiver,
|
|
||||||
implicit val config: Config)
|
|
||||||
extends CandidateFilteringOnlyFlow[Target, RawCandidate, PushCandidate] {
|
|
||||||
|
|
||||||
implicit private val timer: Timer = new JavaTimer(true)
|
|
||||||
val stats = statsReceiver.scope("SendHandler")
|
|
||||||
val log = MRLogger("SendHandler")
|
|
||||||
|
|
||||||
private val buildTargetStats = stats.scope("build_target")
|
|
||||||
|
|
||||||
private val candidateHydrationLatency: Stat =
|
|
||||||
stats.stat("candidateHydrationLatency")
|
|
||||||
|
|
||||||
private val candidatePreValidatorLatency: Stat =
|
|
||||||
stats.stat("candidatePreValidatorLatency")
|
|
||||||
|
|
||||||
private val candidatePostValidatorLatency: Stat =
|
|
||||||
stats.stat("candidatePostValidatorLatency")
|
|
||||||
|
|
||||||
private val featureHydrationLatency: StatsReceiver =
|
|
||||||
stats.scope("featureHydrationLatency")
|
|
||||||
|
|
||||||
private val mrRequestScribeHandler =
|
|
||||||
new MrRequestScribeHandler(mrRequestScriberNode, stats.scope("mr_request_scribe"))
|
|
||||||
|
|
||||||
def apply(request: PushRequest): Future[PushResponse] = {
|
|
||||||
val receivers = Seq(
|
|
||||||
stats,
|
|
||||||
stats.scope(request.notification.commonRecommendationType.toString)
|
|
||||||
)
|
|
||||||
val bStats = BroadcastStatsReceiver(receivers)
|
|
||||||
bStats.counter("requests").incr()
|
|
||||||
Stat
|
|
||||||
.timeFuture(bStats.stat("latency"))(
|
|
||||||
process(request).raiseWithin(CommonConstants.maxPushRequestDuration))
|
|
||||||
.onSuccess {
|
|
||||||
case (pushResp, rawCandidate) =>
|
|
||||||
trackStatsForResponseToRequest(
|
|
||||||
rawCandidate.commonRecType,
|
|
||||||
rawCandidate.target,
|
|
||||||
pushResp,
|
|
||||||
receivers)(statsReceiver)
|
|
||||||
if (!request.context.exists(_.darkWrite.contains(true))) {
|
|
||||||
config.requestScribe(PushRequestScribe(request, pushResp))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.onFailure { ex =>
|
|
||||||
bStats.counter("failures").incr()
|
|
||||||
bStats.scope("failures").counter(ex.getClass.getCanonicalName).incr()
|
|
||||||
}
|
|
||||||
.map {
|
|
||||||
case (pushResp, _) => pushResp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def process(request: PushRequest): Future[(PushResponse, RawCandidate)] = {
|
|
||||||
val recType = request.notification.commonRecommendationType
|
|
||||||
|
|
||||||
track(buildTargetStats)(
|
|
||||||
pushTargetUserBuilder
|
|
||||||
.buildTarget(
|
|
||||||
request.userId,
|
|
||||||
request.context
|
|
||||||
)
|
|
||||||
).flatMap { targetUser =>
|
|
||||||
val responseWithScribedInfo = request.context.exists { context =>
|
|
||||||
context.responseWithScribedInfo.contains(true)
|
|
||||||
}
|
|
||||||
val newRequest =
|
|
||||||
if (request.notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent &&
|
|
||||||
targetUser.params(EnableMagicFanoutNewsForYouNtabCopy)) {
|
|
||||||
val newNotification = request.notification.copy(ntabCopyId =
|
|
||||||
Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId))
|
|
||||||
request.copy(notification = newNotification)
|
|
||||||
} else request
|
|
||||||
|
|
||||||
if (RecTypes.isSendHandlerType(recType) || newRequest.context.exists(
|
|
||||||
_.allowCRT.contains(true))) {
|
|
||||||
|
|
||||||
val rawCandidateFut = PushRequestToCandidate.generatePushCandidate(
|
|
||||||
newRequest.notification,
|
|
||||||
targetUser
|
|
||||||
)
|
|
||||||
|
|
||||||
rawCandidateFut.flatMap { rawCandidate =>
|
|
||||||
val pushResponse = process(targetUser, Seq(rawCandidate)).flatMap {
|
|
||||||
sendHandlerNotifier.checkResponseAndNotify(_, responseWithScribedInfo)
|
|
||||||
}
|
|
||||||
|
|
||||||
pushResponse.map { pushResponse =>
|
|
||||||
(pushResponse, rawCandidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Future.exception(InvalidRequestException(s"${recType.name} not supported in SendHandler"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def hydrateFeatures(
|
|
||||||
candidateDetails: Seq[CandidateDetails[PushCandidate]],
|
|
||||||
target: Target,
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
candidateDetails.headOption match {
|
|
||||||
case Some(candidateDetail)
|
|
||||||
if RecTypes.notEligibleForModelScoreTracking(candidateDetail.candidate.commonRecType) =>
|
|
||||||
Future.value(candidateDetails)
|
|
||||||
|
|
||||||
case Some(candidateDetail) =>
|
|
||||||
val hydrationContextFut = HydrationContextBuilder.build(candidateDetail.candidate)
|
|
||||||
hydrationContextFut.flatMap { hc =>
|
|
||||||
featureHydrator
|
|
||||||
.hydrateCandidate(Seq(hc), target.mrRequestContextForFeatureStore)
|
|
||||||
.map { hydrationResult =>
|
|
||||||
val features = hydrationResult.getOrElse(hc, FeatureMap())
|
|
||||||
candidateDetail.candidate.mergeFeatures(features)
|
|
||||||
candidateDetails
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case _ => Future.Nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override def process(
|
|
||||||
target: Target,
|
|
||||||
externalCandidates: Seq[RawCandidate]
|
|
||||||
): Future[Response[PushCandidate, Result]] = {
|
|
||||||
val candidate = externalCandidates.map(CandidateDetails(_, "realtime"))
|
|
||||||
|
|
||||||
for {
|
|
||||||
hydratedCandidatesWithCopy <- hydrateCandidates(candidate)
|
|
||||||
|
|
||||||
(candidates, preHydrationFilteredCandidates) <- track(filterStats)(
|
|
||||||
filter(target, hydratedCandidatesWithCopy)
|
|
||||||
)
|
|
||||||
|
|
||||||
featureHydratedCandidates <-
|
|
||||||
track(featureHydrationLatency)(hydrateFeatures(candidates, target))
|
|
||||||
|
|
||||||
allTakeCandidateResults <- track(takeStats)(
|
|
||||||
take(target, featureHydratedCandidates, desiredCandidateCount(target))
|
|
||||||
)
|
|
||||||
|
|
||||||
_ <- mrRequestScribeHandler.scribeForCandidateFiltering(
|
|
||||||
target = target,
|
|
||||||
hydratedCandidates = hydratedCandidatesWithCopy,
|
|
||||||
preRankingFilteredCandidates = preHydrationFilteredCandidates,
|
|
||||||
rankedCandidates = featureHydratedCandidates,
|
|
||||||
rerankedCandidates = Seq.empty,
|
|
||||||
restrictFilteredCandidates = Seq.empty, // no restrict step
|
|
||||||
allTakeCandidateResults = allTakeCandidateResults
|
|
||||||
)
|
|
||||||
} yield {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* We combine the results for all filtering steps and pass on in sequence to next step
|
|
||||||
*
|
|
||||||
* This is done to ensure the filtering reason for the candidate from multiple levels of
|
|
||||||
* filtering is carried all the way until [[PushResponse]] is built and returned from
|
|
||||||
* frigate-pushservice-send
|
|
||||||
*/
|
|
||||||
Response(OK, allTakeCandidateResults ++ preHydrationFilteredCandidates)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override def hydrateCandidates(
|
|
||||||
candidates: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
Stat.timeFuture(candidateHydrationLatency)(candidateHydrator(candidates))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Filter Step - pre-predicates and app specific predicates
|
|
||||||
override def filter(
|
|
||||||
target: Target,
|
|
||||||
hydratedCandidatesDetails: Seq[CandidateDetails[PushCandidate]]
|
|
||||||
): Future[
|
|
||||||
(Seq[CandidateDetails[PushCandidate]], Seq[CandidateResult[PushCandidate, Result]])
|
|
||||||
] = {
|
|
||||||
Stat.timeFuture(candidatePreValidatorLatency)(
|
|
||||||
sendHandlerPredicateUtil.preValidationForCandidate(
|
|
||||||
hydratedCandidatesDetails,
|
|
||||||
preCandidateValidator
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Post Validation - Take step
|
|
||||||
override def validCandidates(
|
|
||||||
target: Target,
|
|
||||||
candidates: Seq[PushCandidate]
|
|
||||||
): Future[Seq[Result]] = {
|
|
||||||
Stat.timeFuture(candidatePostValidatorLatency)(Future.collect(candidates.map { candidate =>
|
|
||||||
sendHandlerPredicateUtil
|
|
||||||
.postValidationForCandidate(candidate, postCandidateValidator)
|
|
||||||
.map(res => res.result)
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,184 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler
|
|
||||||
|
|
||||||
import com.twitter.escherbird.metadata.thriftscala.EntityMegadata
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.base._
|
|
||||||
import com.twitter.frigate.common.store.interests.InterestsLookupRequestWithContext
|
|
||||||
import com.twitter.frigate.common.util.MrNtabCopyObjects
|
|
||||||
import com.twitter.frigate.common.util.MrPushCopyObjects
|
|
||||||
import com.twitter.frigate.magic_events.thriftscala.FanoutEvent
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.ml.PushMLModelScorer
|
|
||||||
import com.twitter.frigate.pushservice.model.candidate.CopyIds
|
|
||||||
import com.twitter.frigate.pushservice.store.EventRequest
|
|
||||||
import com.twitter.frigate.pushservice.store.UttEntityHydrationStore
|
|
||||||
import com.twitter.frigate.pushservice.util.CandidateHydrationUtil._
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.gizmoduck.thriftscala.User
|
|
||||||
import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery
|
|
||||||
import com.twitter.interests.thriftscala.UserInterests
|
|
||||||
import com.twitter.livevideo.timeline.domain.v2.{Event => LiveEvent}
|
|
||||||
import com.twitter.simclusters_v2.thriftscala.SimClustersInferredEntities
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.strato.client.UserId
|
|
||||||
import com.twitter.ubs.thriftscala.AudioSpace
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
case class SendHandlerPushCandidateHydrator(
|
|
||||||
lexServiceStore: ReadableStore[EventRequest, LiveEvent],
|
|
||||||
fanoutMetadataStore: ReadableStore[(Long, Long), FanoutEvent],
|
|
||||||
semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata],
|
|
||||||
safeUserStore: ReadableStore[Long, User],
|
|
||||||
simClusterToEntityStore: ReadableStore[Int, SimClustersInferredEntities],
|
|
||||||
audioSpaceStore: ReadableStore[String, AudioSpace],
|
|
||||||
interestsLookupStore: ReadableStore[InterestsLookupRequestWithContext, UserInterests],
|
|
||||||
uttEntityHydrationStore: UttEntityHydrationStore,
|
|
||||||
superFollowCreatorTweetCountStore: ReadableStore[UserId, Int]
|
|
||||||
)(
|
|
||||||
implicit statsReceiver: StatsReceiver,
|
|
||||||
implicit val weightedOpenOrNtabClickModelScorer: PushMLModelScorer) {
|
|
||||||
|
|
||||||
lazy val candidateWithCopyNumStat = statsReceiver.stat("candidate_with_copy_num")
|
|
||||||
lazy val hydratedCandidateStat = statsReceiver.scope("hydrated_candidates")
|
|
||||||
|
|
||||||
def updateCandidates(
|
|
||||||
candidateDetails: Seq[CandidateDetails[RawCandidate]],
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
|
|
||||||
Future.collect {
|
|
||||||
candidateDetails.map { candidateDetail =>
|
|
||||||
val pushCandidate = candidateDetail.candidate
|
|
||||||
|
|
||||||
val copyIds = getCopyIdsByCRT(pushCandidate.commonRecType)
|
|
||||||
|
|
||||||
val hydratedCandidateFut = pushCandidate match {
|
|
||||||
case magicFanoutNewsEventCandidate: MagicFanoutNewsEventCandidate =>
|
|
||||||
getHydratedCandidateForMagicFanoutNewsEvent(
|
|
||||||
magicFanoutNewsEventCandidate,
|
|
||||||
copyIds,
|
|
||||||
lexServiceStore,
|
|
||||||
fanoutMetadataStore,
|
|
||||||
semanticCoreMegadataStore,
|
|
||||||
simClusterToEntityStore,
|
|
||||||
interestsLookupStore,
|
|
||||||
uttEntityHydrationStore
|
|
||||||
)
|
|
||||||
|
|
||||||
case scheduledSpaceSubscriberCandidate: ScheduledSpaceSubscriberCandidate =>
|
|
||||||
getHydratedCandidateForScheduledSpaceSubscriber(
|
|
||||||
scheduledSpaceSubscriberCandidate,
|
|
||||||
safeUserStore,
|
|
||||||
copyIds,
|
|
||||||
audioSpaceStore
|
|
||||||
)
|
|
||||||
case scheduledSpaceSpeakerCandidate: ScheduledSpaceSpeakerCandidate =>
|
|
||||||
getHydratedCandidateForScheduledSpaceSpeaker(
|
|
||||||
scheduledSpaceSpeakerCandidate,
|
|
||||||
safeUserStore,
|
|
||||||
copyIds,
|
|
||||||
audioSpaceStore
|
|
||||||
)
|
|
||||||
case magicFanoutSportsEventCandidate: MagicFanoutSportsEventCandidate with MagicFanoutSportsScoreInformation =>
|
|
||||||
getHydratedCandidateForMagicFanoutSportsEvent(
|
|
||||||
magicFanoutSportsEventCandidate,
|
|
||||||
copyIds,
|
|
||||||
lexServiceStore,
|
|
||||||
fanoutMetadataStore,
|
|
||||||
semanticCoreMegadataStore,
|
|
||||||
interestsLookupStore,
|
|
||||||
uttEntityHydrationStore
|
|
||||||
)
|
|
||||||
case magicFanoutProductLaunchCandidate: MagicFanoutProductLaunchCandidate =>
|
|
||||||
getHydratedCandidateForMagicFanoutProductLaunch(
|
|
||||||
magicFanoutProductLaunchCandidate,
|
|
||||||
copyIds)
|
|
||||||
case creatorEventCandidate: MagicFanoutCreatorEventCandidate =>
|
|
||||||
getHydratedCandidateForMagicFanoutCreatorEvent(
|
|
||||||
creatorEventCandidate,
|
|
||||||
safeUserStore,
|
|
||||||
copyIds,
|
|
||||||
superFollowCreatorTweetCountStore)
|
|
||||||
case _ =>
|
|
||||||
throw new IllegalArgumentException("Incorrect candidate type when update candidates")
|
|
||||||
}
|
|
||||||
|
|
||||||
hydratedCandidateFut.map { hydratedCandidate =>
|
|
||||||
hydratedCandidateStat.counter(hydratedCandidate.commonRecType.name).incr()
|
|
||||||
CandidateDetails(
|
|
||||||
hydratedCandidate,
|
|
||||||
source = candidateDetail.source
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def getCopyIdsByCRT(crt: CommonRecommendationType): CopyIds = {
|
|
||||||
crt match {
|
|
||||||
case CommonRecommendationType.MagicFanoutNewsEvent =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewsPushCopy.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewsForYouCopy.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
|
|
||||||
case CommonRecommendationType.ScheduledSpaceSubscriber =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSubscriber.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSubscriber.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case CommonRecommendationType.ScheduledSpaceSpeaker =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.ScheduledSpaceSpeaker.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.ScheduledSpaceSpeakerNow.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case CommonRecommendationType.SpaceSpeaker =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.SpaceSpeaker.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.SpaceSpeaker.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case CommonRecommendationType.SpaceHost =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.SpaceHost.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.SpaceHost.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case CommonRecommendationType.MagicFanoutSportsEvent =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutSportsPushCopy.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutSportsCopy.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case CommonRecommendationType.MagicFanoutProductLaunch =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutProductLaunch.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.ProductLaunch.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case CommonRecommendationType.CreatorSubscriber =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutCreatorSubscription.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutCreatorSubscription.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case CommonRecommendationType.NewCreator =>
|
|
||||||
CopyIds(
|
|
||||||
pushCopyId = Some(MrPushCopyObjects.MagicFanoutNewCreator.copyId),
|
|
||||||
ntabCopyId = Some(MrNtabCopyObjects.MagicFanoutNewCreator.copyId),
|
|
||||||
aggregationId = None
|
|
||||||
)
|
|
||||||
case _ =>
|
|
||||||
throw new IllegalArgumentException("Incorrect candidate type when fetch copy ids")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def apply(
|
|
||||||
candidateDetails: Seq[CandidateDetails[RawCandidate]]
|
|
||||||
): Future[Seq[CandidateDetails[PushCandidate]]] = {
|
|
||||||
updateCandidates(candidateDetails)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,17 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
trait CandidateGenerator {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build RawCandidate from FrigateNotification
|
|
||||||
* @param target
|
|
||||||
* @param frigateNotification
|
|
||||||
* @return RawCandidate
|
|
||||||
*/
|
|
||||||
def getCandidate(target: Target, frigateNotification: FrigateNotification): Future[RawCandidate]
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,70 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.MagicFanoutCreatorEventCandidate
|
|
||||||
import com.twitter.frigate.magic_events.thriftscala.CreatorFanoutType
|
|
||||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object MagicFanoutCreatorEventCandidateGenerator extends CandidateGenerator {
|
|
||||||
override def getCandidate(
|
|
||||||
targetUser: PushTypes.Target,
|
|
||||||
notification: FrigateNotification
|
|
||||||
): Future[PushTypes.RawCandidate] = {
|
|
||||||
|
|
||||||
require(
|
|
||||||
notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber || notification.commonRecommendationType == CommonRecommendationType.NewCreator,
|
|
||||||
"MagicFanoutCreatorEvent: unexpected CRT " + notification.commonRecommendationType
|
|
||||||
)
|
|
||||||
require(
|
|
||||||
notification.creatorSubscriptionNotification.isDefined,
|
|
||||||
"MagicFanoutCreatorEvent: creatorSubscriptionNotification is not defined")
|
|
||||||
require(
|
|
||||||
notification.creatorSubscriptionNotification.exists(_.magicFanoutPushId.isDefined),
|
|
||||||
"MagicFanoutCreatorEvent: magicFanoutPushId is not defined")
|
|
||||||
require(
|
|
||||||
notification.creatorSubscriptionNotification.exists(_.fanoutReasons.isDefined),
|
|
||||||
"MagicFanoutCreatorEvent: fanoutReasons is not defined")
|
|
||||||
require(
|
|
||||||
notification.creatorSubscriptionNotification.exists(_.creatorId.isDefined),
|
|
||||||
"MagicFanoutCreatorEvent: creatorId is not defined")
|
|
||||||
if (notification.commonRecommendationType == CommonRecommendationType.CreatorSubscriber) {
|
|
||||||
require(
|
|
||||||
notification.creatorSubscriptionNotification
|
|
||||||
.exists(_.subscriberId.isDefined),
|
|
||||||
"MagicFanoutCreatorEvent: subscriber id is not defined"
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
val creatorSubscriptionNotification = notification.creatorSubscriptionNotification.get
|
|
||||||
|
|
||||||
val candidate = new RawCandidate with MagicFanoutCreatorEventCandidate {
|
|
||||||
|
|
||||||
override val target: Target = targetUser
|
|
||||||
|
|
||||||
override val pushId: Long =
|
|
||||||
creatorSubscriptionNotification.magicFanoutPushId.get
|
|
||||||
|
|
||||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
|
||||||
creatorSubscriptionNotification.fanoutReasons.get
|
|
||||||
|
|
||||||
override val creatorFanoutType: CreatorFanoutType =
|
|
||||||
creatorSubscriptionNotification.creatorFanoutType
|
|
||||||
|
|
||||||
override val commonRecType: CommonRecommendationType =
|
|
||||||
notification.commonRecommendationType
|
|
||||||
|
|
||||||
override val frigateNotification: FrigateNotification = notification
|
|
||||||
|
|
||||||
override val subscriberId: Option[Long] = creatorSubscriptionNotification.subscriberId
|
|
||||||
|
|
||||||
override val creatorId: Long = creatorSubscriptionNotification.creatorId.get
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.value(candidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,57 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.MagicFanoutNewsEventCandidate
|
|
||||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object MagicFanoutNewsEventCandidateGenerator extends CandidateGenerator {
|
|
||||||
|
|
||||||
override def getCandidate(
|
|
||||||
targetUser: Target,
|
|
||||||
notification: FrigateNotification
|
|
||||||
): Future[RawCandidate] = {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutNewsEvent]]
|
|
||||||
* AND pushId field should be set
|
|
||||||
**/
|
|
||||||
require(
|
|
||||||
notification.commonRecommendationType == CommonRecommendationType.MagicFanoutNewsEvent,
|
|
||||||
"MagicFanoutNewsEvent: unexpected CRT " + notification.commonRecommendationType
|
|
||||||
)
|
|
||||||
|
|
||||||
require(
|
|
||||||
notification.magicFanoutEventNotification.exists(_.pushId.isDefined),
|
|
||||||
"MagicFanoutNewsEvent: pushId is not defined")
|
|
||||||
|
|
||||||
val magicFanoutEventNotification = notification.magicFanoutEventNotification.get
|
|
||||||
|
|
||||||
val candidate = new RawCandidate with MagicFanoutNewsEventCandidate {
|
|
||||||
|
|
||||||
override val target: Target = targetUser
|
|
||||||
|
|
||||||
override val eventId: Long = magicFanoutEventNotification.eventId
|
|
||||||
|
|
||||||
override val pushId: Long = magicFanoutEventNotification.pushId.get
|
|
||||||
|
|
||||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
|
||||||
magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty)
|
|
||||||
|
|
||||||
override val momentId: Option[Long] = magicFanoutEventNotification.momentId
|
|
||||||
|
|
||||||
override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage
|
|
||||||
|
|
||||||
override val details: Option[MagicFanoutEventNotificationDetails] =
|
|
||||||
magicFanoutEventNotification.details
|
|
||||||
|
|
||||||
override val frigateNotification: FrigateNotification = notification
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.value(candidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,54 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.MagicFanoutProductLaunchCandidate
|
|
||||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
|
||||||
import com.twitter.frigate.magic_events.thriftscala.ProductType
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object MagicFanoutProductLaunchCandidateGenerator extends CandidateGenerator {
|
|
||||||
|
|
||||||
override def getCandidate(
|
|
||||||
targetUser: PushTypes.Target,
|
|
||||||
notification: FrigateNotification
|
|
||||||
): Future[PushTypes.RawCandidate] = {
|
|
||||||
|
|
||||||
require(
|
|
||||||
notification.commonRecommendationType == CommonRecommendationType.MagicFanoutProductLaunch,
|
|
||||||
"MagicFanoutProductLaunch: unexpected CRT " + notification.commonRecommendationType
|
|
||||||
)
|
|
||||||
require(
|
|
||||||
notification.magicFanoutProductLaunchNotification.isDefined,
|
|
||||||
"MagicFanoutProductLaunch: magicFanoutProductLaunchNotification is not defined")
|
|
||||||
require(
|
|
||||||
notification.magicFanoutProductLaunchNotification.exists(_.magicFanoutPushId.isDefined),
|
|
||||||
"MagicFanoutProductLaunch: magicFanoutPushId is not defined")
|
|
||||||
require(
|
|
||||||
notification.magicFanoutProductLaunchNotification.exists(_.fanoutReasons.isDefined),
|
|
||||||
"MagicFanoutProductLaunch: fanoutReasons is not defined")
|
|
||||||
|
|
||||||
val magicFanoutProductLaunchNotification = notification.magicFanoutProductLaunchNotification.get
|
|
||||||
|
|
||||||
val candidate = new RawCandidate with MagicFanoutProductLaunchCandidate {
|
|
||||||
|
|
||||||
override val target: Target = targetUser
|
|
||||||
|
|
||||||
override val pushId: Long =
|
|
||||||
magicFanoutProductLaunchNotification.magicFanoutPushId.get
|
|
||||||
|
|
||||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
|
||||||
magicFanoutProductLaunchNotification.fanoutReasons.get
|
|
||||||
|
|
||||||
override val productLaunchType: ProductType =
|
|
||||||
magicFanoutProductLaunchNotification.productLaunchType
|
|
||||||
|
|
||||||
override val frigateNotification: FrigateNotification = notification
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.value(candidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,153 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.BaseballGameLiveUpdate
|
|
||||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.BasketballGameLiveUpdate
|
|
||||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.CricketMatchLiveUpdate
|
|
||||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.NflFootballGameLiveUpdate
|
|
||||||
import com.twitter.datatools.entityservice.entities.sports.thriftscala.SoccerMatchLiveUpdate
|
|
||||||
import com.twitter.escherbird.common.thriftscala.Domains
|
|
||||||
import com.twitter.escherbird.common.thriftscala.QualifiedId
|
|
||||||
import com.twitter.escherbird.metadata.thriftscala.EntityMegadata
|
|
||||||
import com.twitter.frigate.common.base.BaseGameScore
|
|
||||||
import com.twitter.frigate.common.base.MagicFanoutSportsEventCandidate
|
|
||||||
import com.twitter.frigate.common.base.MagicFanoutSportsScoreInformation
|
|
||||||
import com.twitter.frigate.common.base.TeamInfo
|
|
||||||
import com.twitter.frigate.magic_events.thriftscala.MagicEventsReason
|
|
||||||
import com.twitter.frigate.pushservice.exception.InvalidSportDomainException
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.params.PushConstants
|
|
||||||
import com.twitter.frigate.pushservice.predicate.magic_fanout.MagicFanoutSportsUtil
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.frigate.thriftscala.MagicFanoutEventNotificationDetails
|
|
||||||
import com.twitter.hermit.store.semantic_core.SemanticEntityForQuery
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object MagicFanoutSportsEventCandidateGenerator {
|
|
||||||
|
|
||||||
final def getCandidate(
|
|
||||||
targetUser: Target,
|
|
||||||
notification: FrigateNotification,
|
|
||||||
basketballGameScoreStore: ReadableStore[QualifiedId, BasketballGameLiveUpdate],
|
|
||||||
baseballGameScoreStore: ReadableStore[QualifiedId, BaseballGameLiveUpdate],
|
|
||||||
cricketMatchScoreStore: ReadableStore[QualifiedId, CricketMatchLiveUpdate],
|
|
||||||
soccerMatchScoreStore: ReadableStore[QualifiedId, SoccerMatchLiveUpdate],
|
|
||||||
nflGameScoreStore: ReadableStore[QualifiedId, NflFootballGameLiveUpdate],
|
|
||||||
semanticCoreMegadataStore: ReadableStore[SemanticEntityForQuery, EntityMegadata],
|
|
||||||
): Future[RawCandidate] = {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* frigateNotification recommendation type should be [[CommonRecommendationType.MagicFanoutSportsEvent]]
|
|
||||||
* AND pushId field should be set
|
|
||||||
*
|
|
||||||
* */
|
|
||||||
require(
|
|
||||||
notification.commonRecommendationType == CommonRecommendationType.MagicFanoutSportsEvent,
|
|
||||||
"MagicFanoutSports: unexpected CRT " + notification.commonRecommendationType
|
|
||||||
)
|
|
||||||
|
|
||||||
require(
|
|
||||||
notification.magicFanoutEventNotification.exists(_.pushId.isDefined),
|
|
||||||
"MagicFanoutSportsEvent: pushId is not defined")
|
|
||||||
|
|
||||||
val magicFanoutEventNotification = notification.magicFanoutEventNotification.get
|
|
||||||
val eventId = magicFanoutEventNotification.eventId
|
|
||||||
val _isScoreUpdate = magicFanoutEventNotification.isScoreUpdate.getOrElse(false)
|
|
||||||
|
|
||||||
val gameScoresFut: Future[Option[BaseGameScore]] = {
|
|
||||||
if (_isScoreUpdate) {
|
|
||||||
semanticCoreMegadataStore
|
|
||||||
.get(SemanticEntityForQuery(PushConstants.SportsEventDomainId, eventId))
|
|
||||||
.flatMap {
|
|
||||||
case Some(megadata) =>
|
|
||||||
if (megadata.domains.contains(Domains.BasketballGame)) {
|
|
||||||
basketballGameScoreStore
|
|
||||||
.get(QualifiedId(Domains.BasketballGame.value, eventId)).map {
|
|
||||||
case Some(game) if game.status.isDefined =>
|
|
||||||
val status = game.status.get
|
|
||||||
MagicFanoutSportsUtil.transformToGameScore(game.score, status)
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
} else if (megadata.domains.contains(Domains.BaseballGame)) {
|
|
||||||
baseballGameScoreStore
|
|
||||||
.get(QualifiedId(Domains.BaseballGame.value, eventId)).map {
|
|
||||||
case Some(game) if game.status.isDefined =>
|
|
||||||
val status = game.status.get
|
|
||||||
MagicFanoutSportsUtil.transformToGameScore(game.runs, status)
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
} else if (megadata.domains.contains(Domains.NflFootballGame)) {
|
|
||||||
nflGameScoreStore
|
|
||||||
.get(QualifiedId(Domains.NflFootballGame.value, eventId)).map {
|
|
||||||
case Some(game) if game.status.isDefined =>
|
|
||||||
val nflScore = MagicFanoutSportsUtil.transformNFLGameScore(game)
|
|
||||||
nflScore
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
} else if (megadata.domains.contains(Domains.SoccerMatch)) {
|
|
||||||
soccerMatchScoreStore
|
|
||||||
.get(QualifiedId(Domains.SoccerMatch.value, eventId)).map {
|
|
||||||
case Some(game) if game.status.isDefined =>
|
|
||||||
val soccerScore = MagicFanoutSportsUtil.transformSoccerGameScore(game)
|
|
||||||
soccerScore
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// The domains are not in our list of supported sports
|
|
||||||
throw new InvalidSportDomainException(
|
|
||||||
s"Domain for entity ${eventId} is not supported")
|
|
||||||
}
|
|
||||||
case _ => Future.None
|
|
||||||
}
|
|
||||||
} else Future.None
|
|
||||||
}
|
|
||||||
|
|
||||||
val homeTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap {
|
|
||||||
case Some(gameScore) =>
|
|
||||||
MagicFanoutSportsUtil.getTeamInfo(gameScore.home, semanticCoreMegadataStore)
|
|
||||||
case _ => Future.None
|
|
||||||
}
|
|
||||||
|
|
||||||
val awayTeamInfoFut: Future[Option[TeamInfo]] = gameScoresFut.flatMap {
|
|
||||||
case Some(gameScore) =>
|
|
||||||
MagicFanoutSportsUtil.getTeamInfo(gameScore.away, semanticCoreMegadataStore)
|
|
||||||
case _ => Future.None
|
|
||||||
}
|
|
||||||
|
|
||||||
val candidate = new RawCandidate
|
|
||||||
with MagicFanoutSportsEventCandidate
|
|
||||||
with MagicFanoutSportsScoreInformation {
|
|
||||||
|
|
||||||
override val target: Target = targetUser
|
|
||||||
|
|
||||||
override val eventId: Long = magicFanoutEventNotification.eventId
|
|
||||||
|
|
||||||
override val pushId: Long = magicFanoutEventNotification.pushId.get
|
|
||||||
|
|
||||||
override val candidateMagicEventsReasons: Seq[MagicEventsReason] =
|
|
||||||
magicFanoutEventNotification.eventReasons.getOrElse(Seq.empty)
|
|
||||||
|
|
||||||
override val momentId: Option[Long] = magicFanoutEventNotification.momentId
|
|
||||||
|
|
||||||
override val eventLanguage: Option[String] = magicFanoutEventNotification.eventLanguage
|
|
||||||
|
|
||||||
override val details: Option[MagicFanoutEventNotificationDetails] =
|
|
||||||
magicFanoutEventNotification.details
|
|
||||||
|
|
||||||
override val frigateNotification: FrigateNotification = notification
|
|
||||||
|
|
||||||
override val homeTeamInfo: Future[Option[TeamInfo]] = homeTeamInfoFut
|
|
||||||
|
|
||||||
override val awayTeamInfo: Future[Option[TeamInfo]] = awayTeamInfoFut
|
|
||||||
|
|
||||||
override val gameScores: Future[Option[BaseGameScore]] = gameScoresFut
|
|
||||||
|
|
||||||
override val isScoreUpdate: Boolean = _isScoreUpdate
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.value(candidate)
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,49 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.pushservice.config.Config
|
|
||||||
import com.twitter.frigate.pushservice.exception.UnsupportedCrtException
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.frigate.thriftscala.{CommonRecommendationType => CRT}
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object PushRequestToCandidate {
|
|
||||||
final def generatePushCandidate(
|
|
||||||
frigateNotification: FrigateNotification,
|
|
||||||
target: Target
|
|
||||||
)(
|
|
||||||
implicit config: Config
|
|
||||||
): Future[RawCandidate] = {
|
|
||||||
|
|
||||||
val candidateGenerator: (Target, FrigateNotification) => Future[RawCandidate] = {
|
|
||||||
frigateNotification.commonRecommendationType match {
|
|
||||||
case CRT.MagicFanoutNewsEvent => MagicFanoutNewsEventCandidateGenerator.getCandidate
|
|
||||||
case CRT.ScheduledSpaceSubscriber => ScheduledSpaceSubscriberCandidateGenerator.getCandidate
|
|
||||||
case CRT.ScheduledSpaceSpeaker => ScheduledSpaceSpeakerCandidateGenerator.getCandidate
|
|
||||||
case CRT.MagicFanoutSportsEvent =>
|
|
||||||
MagicFanoutSportsEventCandidateGenerator.getCandidate(
|
|
||||||
_,
|
|
||||||
_,
|
|
||||||
config.basketballGameScoreStore,
|
|
||||||
config.baseballGameScoreStore,
|
|
||||||
config.cricketMatchScoreStore,
|
|
||||||
config.soccerMatchScoreStore,
|
|
||||||
config.nflGameScoreStore,
|
|
||||||
config.semanticCoreMegadataStore
|
|
||||||
)
|
|
||||||
case CRT.MagicFanoutProductLaunch =>
|
|
||||||
MagicFanoutProductLaunchCandidateGenerator.getCandidate
|
|
||||||
case CRT.NewCreator =>
|
|
||||||
MagicFanoutCreatorEventCandidateGenerator.getCandidate
|
|
||||||
case CRT.CreatorSubscriber =>
|
|
||||||
MagicFanoutCreatorEventCandidateGenerator.getCandidate
|
|
||||||
case _ =>
|
|
||||||
throw new UnsupportedCrtException(
|
|
||||||
"UnsupportedCrtException for SendHandler: " + frigateNotification.commonRecommendationType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
candidateGenerator(target, frigateNotification)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,55 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.ScheduledSpaceSpeakerCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object ScheduledSpaceSpeakerCandidateGenerator extends CandidateGenerator {
|
|
||||||
|
|
||||||
override def getCandidate(
|
|
||||||
targetUser: Target,
|
|
||||||
notification: FrigateNotification
|
|
||||||
): Future[RawCandidate] = {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSpeaker]]
|
|
||||||
*
|
|
||||||
**/
|
|
||||||
require(
|
|
||||||
notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSpeaker,
|
|
||||||
"ScheduledSpaceSpeaker: unexpected CRT " + notification.commonRecommendationType
|
|
||||||
)
|
|
||||||
|
|
||||||
val spaceNotification = notification.spaceNotification.getOrElse(
|
|
||||||
throw new IllegalStateException("ScheduledSpaceSpeaker notification object not defined"))
|
|
||||||
|
|
||||||
require(
|
|
||||||
spaceNotification.hostUserId.isDefined,
|
|
||||||
"ScheduledSpaceSpeaker notification - hostUserId not defined"
|
|
||||||
)
|
|
||||||
|
|
||||||
val spaceHostId = spaceNotification.hostUserId
|
|
||||||
|
|
||||||
require(
|
|
||||||
spaceNotification.scheduledStartTime.isDefined,
|
|
||||||
"ScheduledSpaceSpeaker notification - scheduledStartTime not defined"
|
|
||||||
)
|
|
||||||
|
|
||||||
val scheduledStartTime = spaceNotification.scheduledStartTime.get
|
|
||||||
|
|
||||||
val candidate = new RawCandidate with ScheduledSpaceSpeakerCandidate {
|
|
||||||
override val target: Target = targetUser
|
|
||||||
override val frigateNotification: FrigateNotification = notification
|
|
||||||
override val spaceId: String = spaceNotification.broadcastId
|
|
||||||
override val hostId: Option[Long] = spaceHostId
|
|
||||||
override val startTime: Long = scheduledStartTime
|
|
||||||
override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers
|
|
||||||
override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.value(candidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,55 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.send_handler.generator
|
|
||||||
|
|
||||||
import com.twitter.frigate.common.base.ScheduledSpaceSubscriberCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.RawCandidate
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.Target
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
object ScheduledSpaceSubscriberCandidateGenerator extends CandidateGenerator {
|
|
||||||
|
|
||||||
override def getCandidate(
|
|
||||||
targetUser: Target,
|
|
||||||
notification: FrigateNotification
|
|
||||||
): Future[RawCandidate] = {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* frigateNotification recommendation type should be [[CommonRecommendationType.ScheduledSpaceSubscriber]]
|
|
||||||
*
|
|
||||||
**/
|
|
||||||
require(
|
|
||||||
notification.commonRecommendationType == CommonRecommendationType.ScheduledSpaceSubscriber,
|
|
||||||
"ScheduledSpaceSubscriber: unexpected CRT " + notification.commonRecommendationType
|
|
||||||
)
|
|
||||||
|
|
||||||
val spaceNotification = notification.spaceNotification.getOrElse(
|
|
||||||
throw new IllegalStateException("ScheduledSpaceSubscriber notification object not defined"))
|
|
||||||
|
|
||||||
require(
|
|
||||||
spaceNotification.hostUserId.isDefined,
|
|
||||||
"ScheduledSpaceSubscriber notification - hostUserId not defined"
|
|
||||||
)
|
|
||||||
|
|
||||||
val spaceHostId = spaceNotification.hostUserId
|
|
||||||
|
|
||||||
require(
|
|
||||||
spaceNotification.scheduledStartTime.isDefined,
|
|
||||||
"ScheduledSpaceSubscriber notification - scheduledStartTime not defined"
|
|
||||||
)
|
|
||||||
|
|
||||||
val scheduledStartTime = spaceNotification.scheduledStartTime.get
|
|
||||||
|
|
||||||
val candidate = new RawCandidate with ScheduledSpaceSubscriberCandidate {
|
|
||||||
override val target: Target = targetUser
|
|
||||||
override val frigateNotification: FrigateNotification = notification
|
|
||||||
override val spaceId: String = spaceNotification.broadcastId
|
|
||||||
override val hostId: Option[Long] = spaceHostId
|
|
||||||
override val startTime: Long = scheduledStartTime
|
|
||||||
override val speakerIds: Option[Seq[Long]] = spaceNotification.speakers
|
|
||||||
override val listenerIds: Option[Seq[Long]] = spaceNotification.listeners
|
|
||||||
}
|
|
||||||
|
|
||||||
Future.value(candidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,17 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.content_mixer.thriftscala.ContentMixer
|
|
||||||
import com.twitter.content_mixer.thriftscala.ContentMixerRequest
|
|
||||||
import com.twitter.content_mixer.thriftscala.ContentMixerResponse
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
case class ContentMixerStore(contentMixer: ContentMixer.MethodPerEndpoint)
|
|
||||||
extends ReadableStore[ContentMixerRequest, ContentMixerResponse] {
|
|
||||||
|
|
||||||
override def get(request: ContentMixerRequest): Future[Option[ContentMixerResponse]] = {
|
|
||||||
contentMixer.getCandidates(request).map { response =>
|
|
||||||
Some(response)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,15 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.copyselectionservice.thriftscala._
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
class CopySelectionServiceStore(copySelectionServiceClient: CopySelectionService.FinagledClient)
|
|
||||||
extends ReadableStore[CopySelectionRequestV1, Copy] {
|
|
||||||
override def get(k: CopySelectionRequestV1): Future[Option[Copy]] =
|
|
||||||
copySelectionServiceClient.getSelectedCopy(CopySelectionRequest.V1(k)).map {
|
|
||||||
case CopySelectionResponse.V1(response) =>
|
|
||||||
Some(response.selectedCopy)
|
|
||||||
case _ => throw CopyServiceException(CopyServiceErrorCode.VersionNotFound)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,58 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.cr_mixer.thriftscala.CrMixer
|
|
||||||
import com.twitter.cr_mixer.thriftscala.CrMixerTweetRequest
|
|
||||||
import com.twitter.cr_mixer.thriftscala.CrMixerTweetResponse
|
|
||||||
import com.twitter.cr_mixer.thriftscala.FrsTweetRequest
|
|
||||||
import com.twitter.cr_mixer.thriftscala.FrsTweetResponse
|
|
||||||
import com.twitter.finagle.stats.NullStatsReceiver
|
|
||||||
import com.twitter.finagle.stats.Stat
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Store to get content recs from content recommender.
|
|
||||||
*/
|
|
||||||
case class CrMixerTweetStore(
|
|
||||||
crMixer: CrMixer.MethodPerEndpoint
|
|
||||||
)(
|
|
||||||
implicit statsReceiver: StatsReceiver = NullStatsReceiver) {
|
|
||||||
|
|
||||||
private val requestsCounter = statsReceiver.counter("requests")
|
|
||||||
private val successCounter = statsReceiver.counter("success")
|
|
||||||
private val failuresCounter = statsReceiver.counter("failures")
|
|
||||||
private val nonEmptyCounter = statsReceiver.counter("non_empty")
|
|
||||||
private val emptyCounter = statsReceiver.counter("empty")
|
|
||||||
private val failuresScope = statsReceiver.scope("failures")
|
|
||||||
private val latencyStat = statsReceiver.stat("latency")
|
|
||||||
|
|
||||||
private def updateStats[T](f: => Future[Option[T]]): Future[Option[T]] = {
|
|
||||||
requestsCounter.incr()
|
|
||||||
Stat
|
|
||||||
.timeFuture(latencyStat)(f)
|
|
||||||
.onSuccess { r =>
|
|
||||||
if (r.isDefined) nonEmptyCounter.incr() else emptyCounter.incr()
|
|
||||||
successCounter.incr()
|
|
||||||
}
|
|
||||||
.onFailure { e =>
|
|
||||||
{
|
|
||||||
failuresCounter.incr()
|
|
||||||
failuresScope.counter(e.getClass.getName).incr()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def getTweetRecommendations(
|
|
||||||
request: CrMixerTweetRequest
|
|
||||||
): Future[Option[CrMixerTweetResponse]] = {
|
|
||||||
updateStats(crMixer.getTweetRecommendations(request).map { response =>
|
|
||||||
Some(response)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
def getFRSTweetCandidates(request: FrsTweetRequest): Future[Option[FrsTweetResponse]] = {
|
|
||||||
updateStats(crMixer.getFrsBasedTweetRecommendations(request).map { response =>
|
|
||||||
Some(response)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,28 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.explore_ranker.thriftscala.ExploreRanker
|
|
||||||
import com.twitter.explore_ranker.thriftscala.ExploreRankerResponse
|
|
||||||
import com.twitter.explore_ranker.thriftscala.ExploreRankerRequest
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
/** A Store for Video Tweet Recommendations from Explore
|
|
||||||
*
|
|
||||||
* @param exploreRankerService
|
|
||||||
*/
|
|
||||||
case class ExploreRankerStore(exploreRankerService: ExploreRanker.MethodPerEndpoint)
|
|
||||||
extends ReadableStore[ExploreRankerRequest, ExploreRankerResponse] {
|
|
||||||
|
|
||||||
/** Method to get video recommendations
|
|
||||||
*
|
|
||||||
* @param request explore ranker request object
|
|
||||||
* @return
|
|
||||||
*/
|
|
||||||
override def get(
|
|
||||||
request: ExploreRankerRequest
|
|
||||||
): Future[Option[ExploreRankerResponse]] = {
|
|
||||||
exploreRankerService.getRankedResults(request).map { response =>
|
|
||||||
Some(response)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,46 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService
|
|
||||||
import com.twitter.follow_recommendations.thriftscala.Recommendation
|
|
||||||
import com.twitter.follow_recommendations.thriftscala.RecommendationRequest
|
|
||||||
import com.twitter.follow_recommendations.thriftscala.RecommendationResponse
|
|
||||||
import com.twitter.follow_recommendations.thriftscala.UserRecommendation
|
|
||||||
import com.twitter.inject.Logging
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
case class FollowRecommendationsStore(
|
|
||||||
frsClient: FollowRecommendationsThriftService.MethodPerEndpoint,
|
|
||||||
statsReceiver: StatsReceiver)
|
|
||||||
extends ReadableStore[RecommendationRequest, RecommendationResponse]
|
|
||||||
with Logging {
|
|
||||||
|
|
||||||
private val scopedStats = statsReceiver.scope(getClass.getSimpleName)
|
|
||||||
private val requests = scopedStats.counter("requests")
|
|
||||||
private val valid = scopedStats.counter("valid")
|
|
||||||
private val invalid = scopedStats.counter("invalid")
|
|
||||||
private val numTotalResults = scopedStats.stat("total_results")
|
|
||||||
private val numValidResults = scopedStats.stat("valid_results")
|
|
||||||
|
|
||||||
override def get(request: RecommendationRequest): Future[Option[RecommendationResponse]] = {
|
|
||||||
requests.incr()
|
|
||||||
frsClient.getRecommendations(request).map { response =>
|
|
||||||
numTotalResults.add(response.recommendations.size)
|
|
||||||
val validRecs = response.recommendations.filter {
|
|
||||||
case Recommendation.User(_: UserRecommendation) =>
|
|
||||||
valid.incr()
|
|
||||||
true
|
|
||||||
case _ =>
|
|
||||||
invalid.incr()
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
numValidResults.add(validRecs.size)
|
|
||||||
Some(
|
|
||||||
RecommendationResponse(
|
|
||||||
recommendations = validRecs
|
|
||||||
))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,190 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.finagle.stats.BroadcastStatsReceiver
|
|
||||||
import com.twitter.finagle.stats.NullStatsReceiver
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.logger.MRLogger
|
|
||||||
import com.twitter.frigate.common.store
|
|
||||||
import com.twitter.frigate.common.store.Fail
|
|
||||||
import com.twitter.frigate.common.store.IbisRequestInfo
|
|
||||||
import com.twitter.frigate.common.store.IbisResponse
|
|
||||||
import com.twitter.frigate.common.store.Sent
|
|
||||||
import com.twitter.frigate.pushservice.model.PushTypes.PushCandidate
|
|
||||||
import com.twitter.frigate.thriftscala.CommonRecommendationType
|
|
||||||
import com.twitter.ibis2.service.thriftscala.Flags
|
|
||||||
import com.twitter.ibis2.service.thriftscala.FlowControl
|
|
||||||
import com.twitter.ibis2.service.thriftscala.Ibis2Request
|
|
||||||
import com.twitter.ibis2.service.thriftscala.Ibis2Response
|
|
||||||
import com.twitter.ibis2.service.thriftscala.Ibis2ResponseStatus
|
|
||||||
import com.twitter.ibis2.service.thriftscala.Ibis2Service
|
|
||||||
import com.twitter.ibis2.service.thriftscala.NotificationNotSentCode
|
|
||||||
import com.twitter.ibis2.service.thriftscala.TargetFanoutResult.NotSentReason
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
trait Ibis2Store extends store.Ibis2Store {
|
|
||||||
def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse]
|
|
||||||
}
|
|
||||||
|
|
||||||
case class PushIbis2Store(
|
|
||||||
ibisClient: Ibis2Service.MethodPerEndpoint
|
|
||||||
)(
|
|
||||||
implicit val statsReceiver: StatsReceiver = NullStatsReceiver)
|
|
||||||
extends Ibis2Store {
|
|
||||||
private val log = MRLogger(this.getClass.getSimpleName)
|
|
||||||
private val stats = statsReceiver.scope("ibis_v2_store")
|
|
||||||
private val statsByCrt = stats.scope("byCrt")
|
|
||||||
private val requestsByCrt = statsByCrt.scope("requests")
|
|
||||||
private val failuresByCrt = statsByCrt.scope("failures")
|
|
||||||
private val successByCrt = statsByCrt.scope("success")
|
|
||||||
|
|
||||||
private val statsByIbisModel = stats.scope("byIbisModel")
|
|
||||||
private val requestsByIbisModel = statsByIbisModel.scope("requests")
|
|
||||||
private val failuresByIbisModel = statsByIbisModel.scope("failures")
|
|
||||||
private val successByIbisModel = statsByIbisModel.scope("success")
|
|
||||||
|
|
||||||
private[this] def ibisSend(
|
|
||||||
ibis2Request: Ibis2Request,
|
|
||||||
commonRecommendationType: CommonRecommendationType
|
|
||||||
): Future[IbisResponse] = {
|
|
||||||
val ibisModel = ibis2Request.modelName
|
|
||||||
|
|
||||||
val bStats = if (ibis2Request.flags.getOrElse(Flags()).darkWrite.contains(true)) {
|
|
||||||
BroadcastStatsReceiver(
|
|
||||||
Seq(
|
|
||||||
stats,
|
|
||||||
stats.scope("dark_write")
|
|
||||||
)
|
|
||||||
)
|
|
||||||
} else BroadcastStatsReceiver(Seq(stats))
|
|
||||||
|
|
||||||
bStats.counter("requests").incr()
|
|
||||||
requestsByCrt.counter(commonRecommendationType.name).incr()
|
|
||||||
requestsByIbisModel.counter(ibisModel).incr()
|
|
||||||
|
|
||||||
retry(ibisClient, ibis2Request, 3, bStats)
|
|
||||||
.map { response =>
|
|
||||||
bStats.counter(response.status.status.name).incr()
|
|
||||||
successByCrt.counter(response.status.status.name, commonRecommendationType.name).incr()
|
|
||||||
successByIbisModel.counter(response.status.status.name, ibisModel).incr()
|
|
||||||
response.status.status match {
|
|
||||||
case Ibis2ResponseStatus.SuccessWithDeliveries |
|
|
||||||
Ibis2ResponseStatus.SuccessNoDeliveries =>
|
|
||||||
IbisResponse(Sent, Some(response))
|
|
||||||
case _ =>
|
|
||||||
IbisResponse(Fail, Some(response))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.onFailure { ex =>
|
|
||||||
bStats.counter("failures").incr()
|
|
||||||
val exceptionName = ex.getClass.getCanonicalName
|
|
||||||
bStats.scope("failures").counter(exceptionName).incr()
|
|
||||||
failuresByCrt.counter(exceptionName, commonRecommendationType.name).incr()
|
|
||||||
failuresByIbisModel.counter(exceptionName, ibisModel).incr()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def getNotifNotSentReason(
|
|
||||||
ibis2Response: Ibis2Response
|
|
||||||
): Option[NotificationNotSentCode] = {
|
|
||||||
ibis2Response.status.fanoutResults match {
|
|
||||||
case Some(fanoutResult) =>
|
|
||||||
fanoutResult.pushResult.flatMap { pushResult =>
|
|
||||||
pushResult.results.headOption match {
|
|
||||||
case Some(NotSentReason(notSentInfo)) => Some(notSentInfo.notSentCode)
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = {
|
|
||||||
val requestWithIID = if (ibis2Request.flowControl.exists(_.externalIid.isDefined)) {
|
|
||||||
ibis2Request
|
|
||||||
} else {
|
|
||||||
ibis2Request.copy(
|
|
||||||
flowControl = Some(
|
|
||||||
ibis2Request.flowControl
|
|
||||||
.getOrElse(FlowControl())
|
|
||||||
.copy(externalIid = Some(candidate.impressionId))
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
val commonRecommendationType = candidate.frigateNotification.commonRecommendationType
|
|
||||||
|
|
||||||
ibisSend(requestWithIID, commonRecommendationType)
|
|
||||||
.onSuccess { response =>
|
|
||||||
response.ibis2Response.foreach { ibis2Response =>
|
|
||||||
getNotifNotSentReason(ibis2Response).foreach { notifNotSentCode =>
|
|
||||||
stats.scope(ibis2Response.status.status.name).counter(s"$notifNotSentCode").incr()
|
|
||||||
}
|
|
||||||
if (ibis2Response.status.status != Ibis2ResponseStatus.SuccessWithDeliveries) {
|
|
||||||
log.warning(
|
|
||||||
s"Request dropped on ibis for ${ibis2Request.recipientSelector.recipientId}: $ibis2Response")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
.onFailure { ex =>
|
|
||||||
log.warning(
|
|
||||||
s"Ibis Request failure: ${ex.getClass.getCanonicalName} \n For IbisRequest: $ibis2Request")
|
|
||||||
log.error(ex, ex.getMessage)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// retry request when Ibis2ResponseStatus is PreFanoutError
|
|
||||||
def retry(
|
|
||||||
ibisClient: Ibis2Service.MethodPerEndpoint,
|
|
||||||
request: Ibis2Request,
|
|
||||||
retryCount: Int,
|
|
||||||
bStats: StatsReceiver
|
|
||||||
): Future[Ibis2Response] = {
|
|
||||||
ibisClient.sendNotification(request).flatMap { response =>
|
|
||||||
response.status.status match {
|
|
||||||
case Ibis2ResponseStatus.PreFanoutError if retryCount > 0 =>
|
|
||||||
bStats.scope("requests").counter("retry").incr()
|
|
||||||
bStats.counter(response.status.status.name).incr()
|
|
||||||
retry(ibisClient, request, retryCount - 1, bStats)
|
|
||||||
case _ =>
|
|
||||||
Future.value(response)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override def send(
|
|
||||||
ibis2Request: Ibis2Request,
|
|
||||||
requestInfo: IbisRequestInfo
|
|
||||||
): Future[IbisResponse] = {
|
|
||||||
ibisSend(ibis2Request, requestInfo.commonRecommendationType)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case class StagingIbis2Store(remoteIbis2Store: PushIbis2Store) extends Ibis2Store {
|
|
||||||
|
|
||||||
final def addDarkWriteFlagIbis2Request(
|
|
||||||
isTeamMember: Boolean,
|
|
||||||
ibis2Request: Ibis2Request
|
|
||||||
): Ibis2Request = {
|
|
||||||
val flags =
|
|
||||||
ibis2Request.flags.getOrElse(Flags())
|
|
||||||
val darkWrite: Boolean = !isTeamMember || flags.darkWrite.getOrElse(false)
|
|
||||||
ibis2Request.copy(flags = Some(flags.copy(darkWrite = Some(darkWrite))))
|
|
||||||
}
|
|
||||||
|
|
||||||
override def send(ibis2Request: Ibis2Request, candidate: PushCandidate): Future[IbisResponse] = {
|
|
||||||
candidate.target.isTeamMember.flatMap { isTeamMember =>
|
|
||||||
val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request)
|
|
||||||
remoteIbis2Store.send(ibis2Req, candidate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override def send(
|
|
||||||
ibis2Request: Ibis2Request,
|
|
||||||
requestInfo: IbisRequestInfo
|
|
||||||
): Future[IbisResponse] = {
|
|
||||||
requestInfo.isTeamMember.flatMap { isTeamMember =>
|
|
||||||
val ibis2Req = addDarkWriteFlagIbis2Request(isTeamMember, ibis2Request)
|
|
||||||
remoteIbis2Store.send(ibis2Req, requestInfo)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,16 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.interests_discovery.thriftscala.InterestsDiscoveryService
|
|
||||||
import com.twitter.interests_discovery.thriftscala.RecommendedListsRequest
|
|
||||||
import com.twitter.interests_discovery.thriftscala.RecommendedListsResponse
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
case class InterestDiscoveryStore(
|
|
||||||
client: InterestsDiscoveryService.MethodPerEndpoint)
|
|
||||||
extends ReadableStore[RecommendedListsRequest, RecommendedListsResponse] {
|
|
||||||
|
|
||||||
override def get(request: RecommendedListsRequest): Future[Option[RecommendedListsResponse]] = {
|
|
||||||
client.getListRecos(request).map(Some(_))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,156 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.frigate.common.candidate.TargetDecider
|
|
||||||
import com.twitter.frigate.common.history.History
|
|
||||||
import com.twitter.frigate.common.history.HistoryStoreKeyContext
|
|
||||||
import com.twitter.frigate.common.history.PushServiceHistoryStore
|
|
||||||
import com.twitter.frigate.data_pipeline.thriftscala._
|
|
||||||
import com.twitter.frigate.thriftscala.FrigateNotification
|
|
||||||
import com.twitter.hermit.store.labeled_push_recs.LabeledPushRecsJoinedWithNotificationHistoryStore
|
|
||||||
import com.twitter.logging.Logger
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Future
|
|
||||||
import com.twitter.util.Time
|
|
||||||
|
|
||||||
case class LabeledPushRecsVerifyingStoreKey(
|
|
||||||
historyStoreKey: HistoryStoreKeyContext,
|
|
||||||
useHydratedDataset: Boolean,
|
|
||||||
verifyHydratedDatasetResults: Boolean) {
|
|
||||||
def userId: Long = historyStoreKey.targetUserId
|
|
||||||
}
|
|
||||||
|
|
||||||
case class LabeledPushRecsVerifyingStoreResponse(
|
|
||||||
userHistory: UserHistoryValue,
|
|
||||||
unequalNotificationsUnhydratedToHydrated: Option[
|
|
||||||
Map[(Time, FrigateNotification), FrigateNotification]
|
|
||||||
],
|
|
||||||
missingFromHydrated: Option[Map[Time, FrigateNotification]])
|
|
||||||
|
|
||||||
case class LabeledPushRecsVerifyingStore(
|
|
||||||
labeledPushRecsStore: ReadableStore[UserHistoryKey, UserHistoryValue],
|
|
||||||
historyStore: PushServiceHistoryStore
|
|
||||||
)(
|
|
||||||
implicit stats: StatsReceiver)
|
|
||||||
extends ReadableStore[LabeledPushRecsVerifyingStoreKey, LabeledPushRecsVerifyingStoreResponse] {
|
|
||||||
|
|
||||||
private def getByJoiningWithRealHistory(
|
|
||||||
key: HistoryStoreKeyContext
|
|
||||||
): Future[Option[UserHistoryValue]] = {
|
|
||||||
val historyFut = historyStore.get(key, Some(365.days))
|
|
||||||
val toJoinWithRealHistoryFut = labeledPushRecsStore.get(UserHistoryKey.UserId(key.targetUserId))
|
|
||||||
Future.join(historyFut, toJoinWithRealHistoryFut).map {
|
|
||||||
case (_, None) => None
|
|
||||||
case (History(realtimeHistoryMap), Some(uhValue)) =>
|
|
||||||
Some(
|
|
||||||
LabeledPushRecsJoinedWithNotificationHistoryStore
|
|
||||||
.joinLabeledPushRecsSentWithNotificationHistory(uhValue, realtimeHistoryMap, stats)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private def processUserHistoryValue(uhValue: UserHistoryValue): Map[Time, FrigateNotification] = {
|
|
||||||
uhValue.events
|
|
||||||
.getOrElse(Nil)
|
|
||||||
.collect {
|
|
||||||
case Event(
|
|
||||||
EventType.LabeledPushRecSend,
|
|
||||||
Some(tsMillis),
|
|
||||||
Some(EventUnion.LabeledPushRecSendEvent(lprs: LabeledPushRecSendEvent))
|
|
||||||
) if lprs.pushRecSendEvent.frigateNotification.isDefined =>
|
|
||||||
Time.fromMilliseconds(tsMillis) -> lprs.pushRecSendEvent.frigateNotification.get
|
|
||||||
}
|
|
||||||
.toMap
|
|
||||||
}
|
|
||||||
|
|
||||||
override def get(
|
|
||||||
key: LabeledPushRecsVerifyingStoreKey
|
|
||||||
): Future[Option[LabeledPushRecsVerifyingStoreResponse]] = {
|
|
||||||
val uhKey = UserHistoryKey.UserId(key.userId)
|
|
||||||
if (!key.useHydratedDataset) {
|
|
||||||
getByJoiningWithRealHistory(key.historyStoreKey).map { uhValueOpt =>
|
|
||||||
uhValueOpt.map { uhValue => LabeledPushRecsVerifyingStoreResponse(uhValue, None, None) }
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
labeledPushRecsStore.get(uhKey).flatMap { hydratedValueOpt: Option[UserHistoryValue] =>
|
|
||||||
if (!key.verifyHydratedDatasetResults) {
|
|
||||||
Future.value(hydratedValueOpt.map { uhValue =>
|
|
||||||
LabeledPushRecsVerifyingStoreResponse(uhValue, None, None)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
getByJoiningWithRealHistory(key.historyStoreKey).map {
|
|
||||||
joinedWithRealHistoryOpt: Option[UserHistoryValue] =>
|
|
||||||
val joinedWithRealHistoryMap =
|
|
||||||
joinedWithRealHistoryOpt.map(processUserHistoryValue).getOrElse(Map.empty)
|
|
||||||
val hydratedMap = hydratedValueOpt.map(processUserHistoryValue).getOrElse(Map.empty)
|
|
||||||
val unequal = joinedWithRealHistoryMap.flatMap {
|
|
||||||
case (time, frigateNotif) =>
|
|
||||||
hydratedMap.get(time).collect {
|
|
||||||
case n if n != frigateNotif => ((time, frigateNotif), n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
val missing = joinedWithRealHistoryMap.filter {
|
|
||||||
case (time, frigateNotif) => !hydratedMap.contains(time)
|
|
||||||
}
|
|
||||||
hydratedValueOpt.map { hydratedValue =>
|
|
||||||
LabeledPushRecsVerifyingStoreResponse(hydratedValue, Some(unequal), Some(missing))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case class LabeledPushRecsStoreKey(target: TargetDecider, historyStoreKey: HistoryStoreKeyContext) {
|
|
||||||
def userId: Long = historyStoreKey.targetUserId
|
|
||||||
}
|
|
||||||
|
|
||||||
case class LabeledPushRecsDecideredStore(
|
|
||||||
verifyingStore: ReadableStore[
|
|
||||||
LabeledPushRecsVerifyingStoreKey,
|
|
||||||
LabeledPushRecsVerifyingStoreResponse
|
|
||||||
],
|
|
||||||
useHydratedLabeledSendsDatasetDeciderKey: String,
|
|
||||||
verifyHydratedLabeledSendsForHistoryDeciderKey: String
|
|
||||||
)(
|
|
||||||
implicit globalStats: StatsReceiver)
|
|
||||||
extends ReadableStore[LabeledPushRecsStoreKey, UserHistoryValue] {
|
|
||||||
private val log = Logger()
|
|
||||||
private val stats = globalStats.scope("LabeledPushRecsDecideredStore")
|
|
||||||
private val numComparisons = stats.counter("num_comparisons")
|
|
||||||
private val numMissingStat = stats.stat("num_missing")
|
|
||||||
private val numUnequalStat = stats.stat("num_unequal")
|
|
||||||
|
|
||||||
override def get(key: LabeledPushRecsStoreKey): Future[Option[UserHistoryValue]] = {
|
|
||||||
val useHydrated = key.target.isDeciderEnabled(
|
|
||||||
useHydratedLabeledSendsDatasetDeciderKey,
|
|
||||||
stats,
|
|
||||||
useRandomRecipient = true
|
|
||||||
)
|
|
||||||
|
|
||||||
val verifyHydrated = if (useHydrated) {
|
|
||||||
key.target.isDeciderEnabled(
|
|
||||||
verifyHydratedLabeledSendsForHistoryDeciderKey,
|
|
||||||
stats,
|
|
||||||
useRandomRecipient = true
|
|
||||||
)
|
|
||||||
} else false
|
|
||||||
|
|
||||||
val newKey = LabeledPushRecsVerifyingStoreKey(key.historyStoreKey, useHydrated, verifyHydrated)
|
|
||||||
verifyingStore.get(newKey).map {
|
|
||||||
case None => None
|
|
||||||
case Some(LabeledPushRecsVerifyingStoreResponse(uhValue, unequalOpt, missingOpt)) =>
|
|
||||||
(unequalOpt, missingOpt) match {
|
|
||||||
case (Some(unequal), Some(missing)) =>
|
|
||||||
numComparisons.incr()
|
|
||||||
numMissingStat.add(missing.size)
|
|
||||||
numUnequalStat.add(unequal.size)
|
|
||||||
case _ => //no-op
|
|
||||||
}
|
|
||||||
Some(uhValue)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,26 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.livevideo.common.ids.EventId
|
|
||||||
import com.twitter.livevideo.timeline.client.v2.LiveVideoTimelineClient
|
|
||||||
import com.twitter.livevideo.timeline.domain.v2.Event
|
|
||||||
import com.twitter.livevideo.timeline.domain.v2.LookupContext
|
|
||||||
import com.twitter.stitch.storehaus.ReadableStoreOfStitch
|
|
||||||
import com.twitter.stitch.NotFound
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
|
|
||||||
case class EventRequest(eventId: Long, lookupContext: LookupContext = LookupContext.default)
|
|
||||||
|
|
||||||
object LexServiceStore {
|
|
||||||
def apply(
|
|
||||||
liveVideoTimelineClient: LiveVideoTimelineClient
|
|
||||||
): ReadableStore[EventRequest, Event] = {
|
|
||||||
ReadableStoreOfStitch { eventRequest =>
|
|
||||||
liveVideoTimelineClient.getEvent(
|
|
||||||
EventId(eventRequest.eventId),
|
|
||||||
eventRequest.lookupContext) rescue {
|
|
||||||
case NotFound => Stitch.NotFound
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,45 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.hermit.store.common.ReadableWritableStore
|
|
||||||
import com.twitter.notificationservice.thriftscala.GenericNotificationOverrideKey
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryCompactScalaInjection
|
|
||||||
import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection
|
|
||||||
import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection
|
|
||||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpoint
|
|
||||||
import com.twitter.storage.client.manhattan.kv.impl.Component
|
|
||||||
import com.twitter.storage.client.manhattan.kv.impl.DescriptorP1L1
|
|
||||||
import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor
|
|
||||||
import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor
|
|
||||||
import com.twitter.util.Future
|
|
||||||
|
|
||||||
case class NTabHistoryStore(mhEndpoint: ManhattanKVEndpoint, dataset: String)
|
|
||||||
extends ReadableWritableStore[(Long, String), GenericNotificationOverrideKey] {
|
|
||||||
|
|
||||||
private val keyDesc: DescriptorP1L1.EmptyKey[Long, String] =
|
|
||||||
KeyDescriptor(Component(LongInjection), Component(StringInjection))
|
|
||||||
|
|
||||||
private val genericNotifKeyValDesc: ValueDescriptor.EmptyValue[GenericNotificationOverrideKey] =
|
|
||||||
ValueDescriptor[GenericNotificationOverrideKey](
|
|
||||||
BinaryCompactScalaInjection(GenericNotificationOverrideKey)
|
|
||||||
)
|
|
||||||
|
|
||||||
override def get(key: (Long, String)): Future[Option[GenericNotificationOverrideKey]] = {
|
|
||||||
val (userId, impressionId) = key
|
|
||||||
val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId)
|
|
||||||
|
|
||||||
Stitch
|
|
||||||
.run(mhEndpoint.get(mhKey, genericNotifKeyValDesc))
|
|
||||||
.map { optionMhValue =>
|
|
||||||
optionMhValue.map(_.contents)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
override def put(keyValue: ((Long, String), GenericNotificationOverrideKey)): Future[Unit] = {
|
|
||||||
val ((userId, impressionId), genericNotifOverrideKey) = keyValue
|
|
||||||
val mhKey = keyDesc.withDataset(dataset).withPkey(userId).withLkey(impressionId)
|
|
||||||
val mhVal = genericNotifKeyValDesc.withValue(genericNotifOverrideKey)
|
|
||||||
Stitch.run(mhEndpoint.insert(mhKey, mhVal))
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,73 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.finagle.stats.StatsReceiver
|
|
||||||
import com.twitter.onboarding.task.service.thriftscala.FatigueFlowEnrollment
|
|
||||||
import com.twitter.stitch.Stitch
|
|
||||||
import com.twitter.storage.client.manhattan.bijections.Bijections.BinaryScalaInjection
|
|
||||||
import com.twitter.storage.client.manhattan.bijections.Bijections.LongInjection
|
|
||||||
import com.twitter.storage.client.manhattan.bijections.Bijections.StringInjection
|
|
||||||
import com.twitter.storage.client.manhattan.kv.impl.Component
|
|
||||||
import com.twitter.storage.client.manhattan.kv.impl.KeyDescriptor
|
|
||||||
import com.twitter.storage.client.manhattan.kv.impl.ValueDescriptor
|
|
||||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClient
|
|
||||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
|
||||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVEndpointBuilder
|
|
||||||
import com.twitter.storage.client.manhattan.kv.NoMtlsParams
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.storehaus_internal.manhattan.Omega
|
|
||||||
import com.twitter.util.Duration
|
|
||||||
import com.twitter.util.Future
|
|
||||||
import com.twitter.util.Time
|
|
||||||
|
|
||||||
case class OCFHistoryStoreKey(userId: Long, fatigueDuration: Duration, fatigueGroup: String)
|
|
||||||
|
|
||||||
class OCFPromptHistoryStore(
|
|
||||||
manhattanAppId: String,
|
|
||||||
dataset: String,
|
|
||||||
mtlsParams: ManhattanKVClientMtlsParams = NoMtlsParams
|
|
||||||
)(
|
|
||||||
implicit stats: StatsReceiver)
|
|
||||||
extends ReadableStore[OCFHistoryStoreKey, FatigueFlowEnrollment] {
|
|
||||||
|
|
||||||
import ManhattanInjections._
|
|
||||||
|
|
||||||
private val client = ManhattanKVClient(
|
|
||||||
appId = manhattanAppId,
|
|
||||||
dest = Omega.wilyName,
|
|
||||||
mtlsParams = mtlsParams,
|
|
||||||
label = "ocf_history_store"
|
|
||||||
)
|
|
||||||
private val endpoint = ManhattanKVEndpointBuilder(client, defaultMaxTimeout = 5.seconds)
|
|
||||||
.statsReceiver(stats.scope("ocf_history_store"))
|
|
||||||
.build()
|
|
||||||
|
|
||||||
private val limitResultsTo = 1
|
|
||||||
|
|
||||||
private val datasetKey = keyDesc.withDataset(dataset)
|
|
||||||
|
|
||||||
override def get(storeKey: OCFHistoryStoreKey): Future[Option[FatigueFlowEnrollment]] = {
|
|
||||||
val userId = storeKey.userId
|
|
||||||
val fatigueGroup = storeKey.fatigueGroup
|
|
||||||
val fatigueLength = storeKey.fatigueDuration.inMilliseconds
|
|
||||||
val currentTime = Time.now.inMilliseconds
|
|
||||||
val fullKey = datasetKey
|
|
||||||
.withPkey(userId)
|
|
||||||
.from(fatigueGroup)
|
|
||||||
.to(fatigueGroup, fatigueLength - currentTime)
|
|
||||||
|
|
||||||
Stitch
|
|
||||||
.run(endpoint.slice(fullKey, valDesc, limit = Some(limitResultsTo)))
|
|
||||||
.map { results =>
|
|
||||||
if (results.nonEmpty) {
|
|
||||||
val (_, mhValue) = results.head
|
|
||||||
Some(mhValue.contents)
|
|
||||||
} else None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
object ManhattanInjections {
|
|
||||||
val keyDesc = KeyDescriptor(Component(LongInjection), Component(StringInjection, LongInjection))
|
|
||||||
val valDesc = ValueDescriptor(BinaryScalaInjection(FatigueFlowEnrollment))
|
|
||||||
}
|
|
||||||
Binary file not shown.
@ -1,81 +0,0 @@
|
|||||||
package com.twitter.frigate.pushservice.store
|
|
||||||
|
|
||||||
import com.twitter.conversions.DurationOps._
|
|
||||||
import com.twitter.frigate.common.history.History
|
|
||||||
import com.twitter.frigate.common.store.RealTimeClientEventStore
|
|
||||||
import com.twitter.frigate.data_pipeline.common.HistoryJoin
|
|
||||||
import com.twitter.frigate.data_pipeline.thriftscala.Event
|
|
||||||
import com.twitter.frigate.data_pipeline.thriftscala.EventUnion
|
|
||||||
import com.twitter.frigate.data_pipeline.thriftscala.PushRecSendEvent
|
|
||||||
import com.twitter.frigate.data_pipeline.thriftscala.UserHistoryValue
|
|
||||||
import com.twitter.storehaus.ReadableStore
|
|
||||||
import com.twitter.util.Duration
|
|
||||||
import com.twitter.util.Future
|
|
||||||
import com.twitter.util.Time
|
|
||||||
|
|
||||||
case class OnlineUserHistoryKey(
|
|
||||||
userId: Long,
|
|
||||||
offlineUserHistory: Option[UserHistoryValue],
|
|
||||||
history: Option[History])
|
|
||||||
|
|
||||||
case class OnlineUserHistoryStore(
|
|
||||||
realTimeClientEventStore: RealTimeClientEventStore,
|
|
||||||
duration: Duration = 3.days)
|
|
||||||
extends ReadableStore[OnlineUserHistoryKey, UserHistoryValue] {
|
|
||||||
|
|
||||||
override def get(key: OnlineUserHistoryKey): Future[Option[UserHistoryValue]] = {
|
|
||||||
val now = Time.now
|
|
||||||
|
|
||||||
val pushRecSends = key.history
|
|
||||||
.getOrElse(History(Nil.toMap))
|
|
||||||
.sortedPushDmHistory
|
|
||||||
.filter(_._1 > now - (duration + 1.day))
|
|
||||||
.map {
|
|
||||||
case (time, frigateNotification) =>
|
|
||||||
val pushRecSendEvent = PushRecSendEvent(
|
|
||||||
frigateNotification = Some(frigateNotification),
|
|
||||||
impressionId = frigateNotification.impressionId
|
|
||||||
)
|
|
||||||
pushRecSendEvent -> time
|
|
||||||
}
|
|
||||||
|
|
||||||
realTimeClientEventStore
|
|
||||||
.get(key.userId, now - duration, now)
|
|
||||||
.map { attributedEventHistory =>
|
|
||||||
val attributedClientEvents = attributedEventHistory.sortedHistory.flatMap {
|
|
||||||
case (time, event) =>
|
|
||||||
event.eventUnion match {
|
|
||||||
case Some(eventUnion: EventUnion.AttributedPushRecClientEvent) =>
|
|
||||||
Some((eventUnion.attributedPushRecClientEvent, event.eventType, time))
|
|
||||||
case _ => None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
val realtimeLabeledSends: Seq[Event] = HistoryJoin.getLabeledPushRecSends(
|
|
||||||
pushRecSends,
|
|
||||||
attributedClientEvents,
|
|
||||||
Seq(),
|
|
||||||
Seq(),
|
|
||||||
Seq(),
|
|
||||||
now
|
|
||||||
)
|
|
||||||
|
|
||||||
key.offlineUserHistory.map { offlineUserHistory =>
|
|
||||||
val combinedEvents = offlineUserHistory.events.map { offlineEvents =>
|
|
||||||
(offlineEvents ++ realtimeLabeledSends)
|
|
||||||
.map { event =>
|
|
||||||
event.timestampMillis -> event
|
|
||||||
}
|
|
||||||
.toMap
|
|
||||||
.values
|
|
||||||
.toSeq
|
|
||||||
.sortBy { event =>
|
|
||||||
-1 * event.timestampMillis.getOrElse(0L)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
offlineUserHistory.copy(events = combinedEvents)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user