[docx] split commit for file 2200
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
2488f40edf
commit
b471ac86b4
|
|
@ -1,29 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"stitch/stitch-core",
|
||||
"stitch/stitch-tweetypie",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"stitch/stitch-core",
|
||||
"stitch/stitch-tweetypie",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,241 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_tweetypie
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.spam.rtf.thriftscala.SafetyLevel
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
||||
import com.twitter.tweetypie.thriftscala.TweetVisibilityPolicy
|
||||
import com.twitter.tweetypie.{thriftscala => TP}
|
||||
|
||||
// Candidate Features
|
||||
object IsCommunityTweetFeature extends Feature[TweetCandidate, Boolean]
|
||||
|
||||
// Tweetypie VF Features
|
||||
object HasTakedownFeature extends Feature[TweetCandidate, Boolean]
|
||||
object HasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean]
|
||||
object IsHydratedFeature extends Feature[TweetCandidate, Boolean]
|
||||
object IsNarrowcastFeature extends Feature[TweetCandidate, Boolean]
|
||||
object IsNsfwAdminFeature extends Feature[TweetCandidate, Boolean]
|
||||
object IsNsfwFeature extends Feature[TweetCandidate, Boolean]
|
||||
object IsNsfwUserFeature extends Feature[TweetCandidate, Boolean]
|
||||
object IsNullcastFeature extends Feature[TweetCandidate, Boolean]
|
||||
object QuotedTweetDroppedFeature extends Feature[TweetCandidate, Boolean]
|
||||
object QuotedTweetHasTakedownFeature extends Feature[TweetCandidate, Boolean]
|
||||
object QuotedTweetHasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean]
|
||||
object QuotedTweetIdFeature extends Feature[TweetCandidate, Option[Long]]
|
||||
object SourceTweetHasTakedownFeature extends Feature[TweetCandidate, Boolean]
|
||||
object SourceTweetHasTakedownForLocaleFeature extends Feature[TweetCandidate, Boolean]
|
||||
object TakedownCountryCodesFeature extends Feature[TweetCandidate, Set[String]]
|
||||
object IsReplyFeature extends Feature[TweetCandidate, Boolean]
|
||||
object InReplyToFeature extends Feature[TweetCandidate, Option[Long]]
|
||||
object IsRetweetFeature extends Feature[TweetCandidate, Boolean]
|
||||
|
||||
object TweetTweetypieCandidateFeatureHydrator {
|
||||
val CoreTweetFields: Set[TP.TweetInclude] = Set[TP.TweetInclude](
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.IdField.id),
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id)
|
||||
)
|
||||
|
||||
val NsfwLabelFields: Set[TP.TweetInclude] = Set[TP.TweetInclude](
|
||||
// Tweet fields containing NSFW related attributes, in addition to what exists in coreData.
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.NsfwHighRecallLabelField.id),
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.NsfwHighPrecisionLabelField.id),
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.NsfaHighRecallLabelField.id)
|
||||
)
|
||||
|
||||
val SafetyLabelFields: Set[TP.TweetInclude] = Set[TP.TweetInclude](
|
||||
// Tweet fields containing RTF labels for abuse and spam.
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.SpamLabelField.id),
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.AbusiveLabelField.id)
|
||||
)
|
||||
|
||||
val OrganicTweetTPHydrationFields: Set[TP.TweetInclude] = CoreTweetFields ++
|
||||
NsfwLabelFields ++
|
||||
SafetyLabelFields ++
|
||||
Set(
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.TakedownCountryCodesField.id),
|
||||
// QTs imply a TweetyPie -> SGS request dependency
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.QuotedTweetField.id),
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.EscherbirdEntityAnnotationsField.id),
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.CommunitiesField.id),
|
||||
// Field required for determining if a Tweet was created via News Camera.
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.ComposerSourceField.id)
|
||||
)
|
||||
|
||||
val InjectedTweetTPHydrationFields: Set[TP.TweetInclude] =
|
||||
OrganicTweetTPHydrationFields ++ Set(
|
||||
// Mentions imply a TweetyPie -> Gizmoduck request dependency
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.MentionsField.id),
|
||||
TP.TweetInclude.TweetFieldId(TP.Tweet.HashtagsField.id)
|
||||
)
|
||||
|
||||
val DefaultFeatureMap = FeatureMapBuilder()
|
||||
.add(IsNsfwAdminFeature, false)
|
||||
.add(IsNsfwUserFeature, false)
|
||||
.add(IsNsfwFeature, false)
|
||||
.add(IsNullcastFeature, false)
|
||||
.add(IsNarrowcastFeature, false)
|
||||
.add(HasTakedownFeature, false)
|
||||
.add(IsCommunityTweetFeature, false)
|
||||
.add(TakedownCountryCodesFeature, Set.empty: Set[String])
|
||||
.add(IsHydratedFeature, false)
|
||||
.add(HasTakedownForLocaleFeature, false)
|
||||
.add(QuotedTweetDroppedFeature, false)
|
||||
.add(SourceTweetHasTakedownFeature, false)
|
||||
.add(QuotedTweetHasTakedownFeature, false)
|
||||
.add(SourceTweetHasTakedownForLocaleFeature, false)
|
||||
.add(QuotedTweetHasTakedownForLocaleFeature, false)
|
||||
.add(IsReplyFeature, false)
|
||||
.add(InReplyToFeature, None)
|
||||
.add(IsRetweetFeature, false)
|
||||
.build()
|
||||
}
|
||||
|
||||
class TweetTweetypieCandidateFeatureHydrator(
|
||||
tweetypieStitchClient: TweetypieStitchClient,
|
||||
safetyLevelPredicate: PipelineQuery => SafetyLevel)
|
||||
extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] {
|
||||
|
||||
import TweetTweetypieCandidateFeatureHydrator._
|
||||
|
||||
override val features: Set[Feature[_, _]] =
|
||||
Set(
|
||||
IsNsfwFeature,
|
||||
IsNsfwAdminFeature,
|
||||
IsNsfwUserFeature,
|
||||
IsNullcastFeature,
|
||||
IsNarrowcastFeature,
|
||||
HasTakedownFeature,
|
||||
IsCommunityTweetFeature,
|
||||
TakedownCountryCodesFeature,
|
||||
IsHydratedFeature,
|
||||
HasTakedownForLocaleFeature,
|
||||
QuotedTweetDroppedFeature,
|
||||
SourceTweetHasTakedownFeature,
|
||||
QuotedTweetHasTakedownFeature,
|
||||
SourceTweetHasTakedownForLocaleFeature,
|
||||
QuotedTweetHasTakedownForLocaleFeature,
|
||||
IsReplyFeature,
|
||||
InReplyToFeature,
|
||||
IsRetweetFeature
|
||||
)
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier =
|
||||
FeatureHydratorIdentifier("TweetTweetypie")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidate: BaseTweetCandidate,
|
||||
existingFeatures: FeatureMap
|
||||
): Stitch[FeatureMap] = {
|
||||
val countryCode = query.getCountryCode.getOrElse("")
|
||||
|
||||
tweetypieStitchClient
|
||||
.getTweetFields(
|
||||
tweetId = candidate.id,
|
||||
options = TP.GetTweetFieldsOptions(
|
||||
tweetIncludes = OrganicTweetTPHydrationFields,
|
||||
includeRetweetedTweet = true,
|
||||
includeQuotedTweet = true,
|
||||
visibilityPolicy = TweetVisibilityPolicy.UserVisible,
|
||||
safetyLevel = Some(safetyLevelPredicate(query))
|
||||
)
|
||||
).map {
|
||||
case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), quoteOpt, _) =>
|
||||
val coreData = found.tweet.coreData
|
||||
val isNsfwAdmin = coreData.exists(_.nsfwAdmin)
|
||||
val isNsfwUser = coreData.exists(_.nsfwUser)
|
||||
val hasTakedown = coreData.exists(_.hasTakedown)
|
||||
val isReply = coreData.exists(_.reply.nonEmpty)
|
||||
val ancestorId = coreData.flatMap(_.reply).flatMap(_.inReplyToStatusId)
|
||||
val isRetweet = coreData.exists(_.share.nonEmpty)
|
||||
val takedownCountryCodes =
|
||||
found.tweet.takedownCountryCodes.getOrElse(Seq.empty).map(_.toLowerCase).toSet
|
||||
|
||||
val quotedTweetDropped = quoteOpt.exists {
|
||||
case _: TP.TweetFieldsResultState.Filtered =>
|
||||
true
|
||||
case _: TP.TweetFieldsResultState.NotFound =>
|
||||
true
|
||||
case _ => false
|
||||
}
|
||||
val quotedTweetIsNsfw = quoteOpt.exists {
|
||||
case quoteTweet: TP.TweetFieldsResultState.Found =>
|
||||
quoteTweet.found.tweet.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)
|
||||
case _ => false
|
||||
}
|
||||
val quotedTweetHasTakedown = quoteOpt.exists {
|
||||
case quoteTweet: TP.TweetFieldsResultState.Found =>
|
||||
quoteTweet.found.tweet.coreData.exists(_.hasTakedown)
|
||||
case _ => false
|
||||
}
|
||||
val quotedTweetTakedownCountryCodes = quoteOpt
|
||||
.collect {
|
||||
case quoteTweet: TP.TweetFieldsResultState.Found =>
|
||||
quoteTweet.found.tweet.takedownCountryCodes
|
||||
.getOrElse(Seq.empty).map(_.toLowerCase).toSet
|
||||
}.getOrElse(Set.empty[String])
|
||||
|
||||
val sourceTweetIsNsfw =
|
||||
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
|
||||
val sourceTweetHasTakedown =
|
||||
found.retweetedTweet.exists(_.coreData.exists(_.hasTakedown))
|
||||
val sourceTweetTakedownCountryCodes = found.retweetedTweet
|
||||
.map { sourceTweet: TP.Tweet =>
|
||||
sourceTweet.takedownCountryCodes.getOrElse(Seq.empty).map(_.toLowerCase).toSet
|
||||
}.getOrElse(Set.empty)
|
||||
|
||||
FeatureMapBuilder()
|
||||
.add(IsNsfwAdminFeature, isNsfwAdmin)
|
||||
.add(IsNsfwUserFeature, isNsfwUser)
|
||||
.add(IsNsfwFeature, isNsfwAdmin || isNsfwUser || sourceTweetIsNsfw || quotedTweetIsNsfw)
|
||||
.add(IsNullcastFeature, coreData.exists(_.nullcast))
|
||||
.add(IsNarrowcastFeature, coreData.exists(_.narrowcast.nonEmpty))
|
||||
.add(HasTakedownFeature, hasTakedown)
|
||||
.add(
|
||||
HasTakedownForLocaleFeature,
|
||||
hasTakedownForLocale(hasTakedown, countryCode, takedownCountryCodes))
|
||||
.add(QuotedTweetDroppedFeature, quotedTweetDropped)
|
||||
.add(SourceTweetHasTakedownFeature, sourceTweetHasTakedown)
|
||||
.add(QuotedTweetHasTakedownFeature, quotedTweetHasTakedown)
|
||||
.add(
|
||||
SourceTweetHasTakedownForLocaleFeature,
|
||||
hasTakedownForLocale(
|
||||
sourceTweetHasTakedown,
|
||||
countryCode,
|
||||
sourceTweetTakedownCountryCodes))
|
||||
.add(
|
||||
QuotedTweetHasTakedownForLocaleFeature,
|
||||
hasTakedownForLocale(
|
||||
quotedTweetHasTakedown,
|
||||
countryCode,
|
||||
quotedTweetTakedownCountryCodes))
|
||||
.add(IsCommunityTweetFeature, found.tweet.communities.exists(_.communityIds.nonEmpty))
|
||||
.add(
|
||||
TakedownCountryCodesFeature,
|
||||
found.tweet.takedownCountryCodes.getOrElse(Seq.empty).map(_.toLowerCase).toSet)
|
||||
.add(IsHydratedFeature, true)
|
||||
.add(IsReplyFeature, isReply)
|
||||
.add(InReplyToFeature, ancestorId)
|
||||
.add(IsRetweetFeature, isRetweet)
|
||||
.build()
|
||||
|
||||
// If no tweet result found, return default features
|
||||
case _ =>
|
||||
DefaultFeatureMap
|
||||
}
|
||||
}
|
||||
|
||||
private def hasTakedownForLocale(
|
||||
hasTakedown: Boolean,
|
||||
countryCode: String,
|
||||
takedownCountryCodes: Set[String]
|
||||
) = hasTakedown && takedownCountryCodes.contains(countryCode)
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"stitch/stitch-core",
|
||||
"stitch/stitch-tweetypie",
|
||||
"util/util-slf4j-api",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
|
||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"stitch/stitch-core",
|
||||
"stitch/stitch-tweetypie",
|
||||
"util/util-slf4j-api",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,98 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.spam.rtf.{thriftscala => SPAM}
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
||||
import com.twitter.tweetypie.{thriftscala => TP}
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Throw
|
||||
import com.twitter.util.Try
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
object VisibilityReason
|
||||
extends FeatureWithDefaultOnFailure[TweetCandidate, Option[SPAM.FilteredReason]] {
|
||||
override val defaultValue = None
|
||||
}
|
||||
|
||||
/**
|
||||
* A [[BulkCandidateFeatureHydrator]] that hydrates TweetCandidates with VisibilityReason features
|
||||
* by [[SPAM.SafetyLevel]] when present. The [[VisibilityReason]] feature represents a VisibilityFiltering
|
||||
* [[SPAM.FilteredReason]], which contains safety filtering verdict information including action (e.g.
|
||||
* Drop, Avoid) and reason (e.g. Misinformation, Abuse). This feature can inform downstream services'
|
||||
* handling and presentation of Tweets (e.g. ad avoidance).
|
||||
*
|
||||
* @param tweetypieStitchClient used to retrieve Tweet fields for BaseTweetCandidates
|
||||
* @param safetyLevel specifies VisibilityFiltering SafetyLabel
|
||||
*/
|
||||
|
||||
@Singleton
|
||||
case class TweetVisibilityReasonBulkCandidateFeatureHydrator @Inject() (
|
||||
tweetypieStitchClient: TweetypieStitchClient,
|
||||
safetyLevel: SPAM.SafetyLevel)
|
||||
extends BulkCandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate]
|
||||
with Logging {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"TweetVisibilityReason")
|
||||
|
||||
override def features: Set[Feature[_, _]] = Set(VisibilityReason)
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[BaseTweetCandidate]]
|
||||
): Stitch[Seq[FeatureMap]] = {
|
||||
Stitch
|
||||
.traverse(candidates.map(_.candidate.id)) { tweetId =>
|
||||
tweetypieStitchClient
|
||||
.getTweetFields(
|
||||
tweetId = tweetId,
|
||||
options = TP.GetTweetFieldsOptions(
|
||||
forUserId = query.getOptionalUserId,
|
||||
tweetIncludes = Set.empty,
|
||||
doNotCache = true,
|
||||
visibilityPolicy = TP.TweetVisibilityPolicy.UserVisible,
|
||||
safetyLevel = Some(safetyLevel)
|
||||
)
|
||||
).liftToTry
|
||||
}.map { getTweetFieldsResults: Seq[Try[TP.GetTweetFieldsResult]] =>
|
||||
val tweetFields: Seq[Try[TP.TweetFieldsResultFound]] = getTweetFieldsResults.map {
|
||||
case Return(TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), _, _)) =>
|
||||
Return(found)
|
||||
case Return(TP.GetTweetFieldsResult(_, resultState, _, _)) =>
|
||||
Throw(
|
||||
VisibilityReasonFeatureHydrationFailure(
|
||||
s"Unexpected tweet result state: ${resultState}"))
|
||||
case Throw(e) =>
|
||||
Throw(e)
|
||||
}
|
||||
|
||||
tweetFields.map { tweetFieldTry =>
|
||||
val tweetFilteredReason = tweetFieldTry.map { tweetField =>
|
||||
tweetField.suppressReason match {
|
||||
case Some(suppressReason) => Some(suppressReason)
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
|
||||
FeatureMapBuilder()
|
||||
.add(VisibilityReason, tweetFilteredReason)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case class VisibilityReasonFeatureHydrationFailure(message: String)
|
||||
extends Exception(s"VisibilityReasonFeatureHydrationFailure($message)")
|
||||
Binary file not shown.
|
|
@ -1,97 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.async
|
||||
|
||||
import com.twitter.ml.featurestore.lib.EntityId
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featurestorev1.BaseFeatureStoreV1QueryFeature
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1DynamicClientBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[AsyncQueryFeatureHydrator]] that hydrated asynchronously for features
|
||||
* to be before the step identified in [[hydrateBefore]]
|
||||
*
|
||||
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before.
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run asynchronously
|
||||
* @tparam Query The domain model for the query or request
|
||||
*/
|
||||
case class AsyncQueryFeatureHydrator[-Query <: PipelineQuery] private[async] (
|
||||
override val hydrateBefore: PipelineStepIdentifier,
|
||||
queryFeatureHydrator: QueryFeatureHydrator[Query])
|
||||
extends QueryFeatureHydrator[Query]
|
||||
with AsyncHydrator {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"Async" + queryFeatureHydrator.identifier.name)
|
||||
override val alerts: Seq[Alert] = queryFeatureHydrator.alerts
|
||||
override val features: Set[Feature[_, _]] = queryFeatureHydrator.features
|
||||
|
||||
override def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
|
||||
}
|
||||
|
||||
/**
|
||||
* A [[FeatureStoreV1QueryFeatureHydrator]] with [[AsyncHydrator]] that hydrated asynchronously for features
|
||||
* to be before the step identified in [[hydrateBefore]]. We need a standalone class for feature store,
|
||||
* different from the above as FStore hydrators are exempt from validations at run time.
|
||||
*
|
||||
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before.
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run asynchronously
|
||||
* @tparam Query The domain model for the query or request
|
||||
*/
|
||||
case class AsyncFeatureStoreV1QueryFeatureHydrator[Query <: PipelineQuery] private[async] (
|
||||
override val hydrateBefore: PipelineStepIdentifier,
|
||||
featureStoreV1QueryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query])
|
||||
extends FeatureStoreV1QueryFeatureHydrator[
|
||||
Query
|
||||
]
|
||||
with AsyncHydrator {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"Async" + featureStoreV1QueryFeatureHydrator.identifier.name)
|
||||
override val alerts: Seq[Alert] = featureStoreV1QueryFeatureHydrator.alerts
|
||||
|
||||
override val features: Set[BaseFeatureStoreV1QueryFeature[Query, _ <: EntityId, _]] =
|
||||
featureStoreV1QueryFeatureHydrator.features
|
||||
|
||||
override val clientBuilder: FeatureStoreV1DynamicClientBuilder =
|
||||
featureStoreV1QueryFeatureHydrator.clientBuilder
|
||||
}
|
||||
|
||||
object AsyncQueryFeatureHydrator {
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[AsyncQueryFeatureHydrator]] that hydrated asynchronously for features
|
||||
* to be before the step identified in [[hydrateBefore]]
|
||||
*
|
||||
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before.
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run asynchronously
|
||||
* @tparam Query The domain model for the query or request
|
||||
*/
|
||||
def apply[Query <: PipelineQuery](
|
||||
hydrateBefore: PipelineStepIdentifier,
|
||||
queryFeatureHydrator: QueryFeatureHydrator[Query]
|
||||
): AsyncQueryFeatureHydrator[Query] =
|
||||
new AsyncQueryFeatureHydrator(hydrateBefore, queryFeatureHydrator)
|
||||
|
||||
/**
|
||||
* A [[FeatureStoreV1QueryFeatureHydrator]] with [[AsyncHydrator]] that hydrated asynchronously for features
|
||||
* to be before the step identified in [[hydrateBefore]]. We need a standalone class for feature store,
|
||||
* different from the above as FStore hydrators are exempt from validations at run time.
|
||||
*
|
||||
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before.
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run asynchronously
|
||||
* @tparam Query The domain model for the query or request
|
||||
*/
|
||||
def apply[Query <: PipelineQuery](
|
||||
hydrateBefore: PipelineStepIdentifier,
|
||||
featureStoreV1QueryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query]
|
||||
): AsyncFeatureStoreV1QueryFeatureHydrator[Query] =
|
||||
new AsyncFeatureStoreV1QueryFeatureHydrator(hydrateBefore, featureStoreV1QueryFeatureHydrator)
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
|
|
@ -1,19 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"cr-ml-ranker/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"cr-ml-ranker/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,37 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.cr_ml_ranker
|
||||
|
||||
import com.twitter.cr_ml_ranker.{thriftscala => t}
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
object CrMlRankerCommonFeatures extends Feature[PipelineQuery, t.CommonFeatures]
|
||||
object CrMlRankerRankingConfig extends Feature[PipelineQuery, t.RankingConfig]
|
||||
|
||||
private[cr_ml_ranker] class CrMlRankerCommonQueryFeatureHydrator(
|
||||
crMlRanker: t.CrMLRanker.MethodPerEndpoint,
|
||||
rankingConfigSelector: RankingConfigBuilder)
|
||||
extends QueryFeatureHydrator[PipelineQuery] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("CrMlRanker")
|
||||
|
||||
override val features: Set[Feature[_, _]] =
|
||||
Set(CrMlRankerCommonFeatures, CrMlRankerRankingConfig)
|
||||
|
||||
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
|
||||
val rankingConfig = rankingConfigSelector.apply(query)
|
||||
Stitch
|
||||
.callFuture(
|
||||
crMlRanker.getCommonFeatures(
|
||||
t.RankingRequestContext(query.getRequiredUserId, rankingConfig))).map { commonFeatures =>
|
||||
FeatureMapBuilder()
|
||||
.add(CrMlRankerRankingConfig, rankingConfig)
|
||||
.add(CrMlRankerCommonFeatures, commonFeatures)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,18 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.cr_ml_ranker
|
||||
|
||||
import com.twitter.cr_ml_ranker.{thriftscala => t}
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* Builds a query hydrator that hydrates Common Features for the given Query from CR ML Ranker
|
||||
* to be later used to call CR ML Ranker for scoring using the desired [[RankingConfigBuilder]]
|
||||
* for building the ranking config.
|
||||
*/
|
||||
@Singleton
|
||||
class CrMlRankerCommonQueryFeatureHydratorBuilder @Inject() (
|
||||
crMlRanker: t.CrMLRanker.MethodPerEndpoint) {
|
||||
|
||||
def build(rankingConfigSelector: RankingConfigBuilder): CrMlRankerCommonQueryFeatureHydrator =
|
||||
new CrMlRankerCommonQueryFeatureHydrator(crMlRanker, rankingConfigSelector)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,11 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.cr_ml_ranker
|
||||
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.cr_ml_ranker.{thriftscala => t}
|
||||
|
||||
/**
|
||||
* Builder for constructing a ranking config from a query
|
||||
*/
|
||||
trait RankingConfigBuilder {
|
||||
def apply(query: PipelineQuery): t.RankingConfig
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/src/jvm/com/twitter/storehaus:core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"src/thrift/com/twitter/timelines/impression_store:thrift-scala",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/src/jvm/com/twitter/storehaus:core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"src/thrift/com/twitter/timelines/impression_store:thrift-scala",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,57 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.timelines.impressionstore.thriftscala.ImpressionList
|
||||
import com.twitter.util.Future
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* Query Feature to store ids of the tweets impressed by the user.
|
||||
*/
|
||||
case object ImpressedTweets extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] {
|
||||
override val defaultValue: Seq[Long] = Seq.empty
|
||||
}
|
||||
|
||||
/**
|
||||
* Enrich the query with a list of tweet ids that the user has already seen.
|
||||
*/
|
||||
@Singleton
|
||||
case class ImpressedTweetsQueryFeatureHydrator @Inject() (
|
||||
tweetImpressionStore: ReadableStore[Long, ImpressionList])
|
||||
extends QueryFeatureHydrator[PipelineQuery] {
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetsToExclude")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(ImpressedTweets)
|
||||
|
||||
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
|
||||
query.getOptionalUserId match {
|
||||
case Some(userId) =>
|
||||
val featureMapResult: Future[FeatureMap] = tweetImpressionStore
|
||||
.get(userId).map { impressionListOpt =>
|
||||
val tweetIdsOpt = for {
|
||||
impressionList <- impressionListOpt
|
||||
impressions <- impressionList.impressions
|
||||
} yield {
|
||||
impressions.map(_.tweetId)
|
||||
}
|
||||
val tweetIds = tweetIdsOpt.getOrElse(Seq.empty)
|
||||
FeatureMapBuilder().add(ImpressedTweets, tweetIds).build()
|
||||
}
|
||||
Stitch.callFuture(featureMapResult)
|
||||
// Non-logged-in users do not have userId, returns empty feature
|
||||
|
||||
case None =>
|
||||
val featureMapResult = FeatureMapBuilder().add(ImpressedTweets, Seq.empty).build()
|
||||
Stitch.value(featureMapResult)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,31 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.logged_in_only
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[Conditionally]] to run only for logged in users
|
||||
*
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run when query.isLoggedOut is false
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Result The type of the candidates
|
||||
*/
|
||||
case class LoggedInOnlyQueryFeatureHydrator[-Query <: PipelineQuery, Result <: UniversalNoun[Any]](
|
||||
queryFeatureHydrator: QueryFeatureHydrator[Query])
|
||||
extends QueryFeatureHydrator[Query]
|
||||
with Conditionally[Query] {
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"LoggedInOnly" + queryFeatureHydrator.identifier.name)
|
||||
override val alerts: Seq[Alert] = queryFeatureHydrator.alerts
|
||||
override val features: Set[Feature[_, _]] = queryFeatureHydrator.features
|
||||
override def onlyIf(query: Query): Boolean =
|
||||
Conditionally.and(query, queryFeatureHydrator, !query.isLoggedOut)
|
||||
override def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,48 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[Conditionally]] based on a [[Param]] that hydrates asynchronously for features
|
||||
* to be before the step identified in [[hydrateBefore]]
|
||||
*
|
||||
* @param enabledParam the param to turn this [[QueryFeatureHydrator]] on and off
|
||||
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before.
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run when `enabledParam` is true
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Result The type of the candidates
|
||||
*/
|
||||
case class AsyncParamGatedQueryFeatureHydrator[
|
||||
-Query <: PipelineQuery,
|
||||
Result <: UniversalNoun[Any]
|
||||
](
|
||||
enabledParam: Param[Boolean],
|
||||
override val hydrateBefore: PipelineStepIdentifier,
|
||||
queryFeatureHydrator: QueryFeatureHydrator[Query])
|
||||
extends QueryFeatureHydrator[Query]
|
||||
with Conditionally[Query]
|
||||
with AsyncHydrator {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"AsyncParamGated" + queryFeatureHydrator.identifier.name)
|
||||
|
||||
override val alerts: Seq[Alert] = queryFeatureHydrator.alerts
|
||||
|
||||
override val features: Set[Feature[_, _]] = queryFeatureHydrator.features
|
||||
|
||||
override def onlyIf(query: Query): Boolean =
|
||||
Conditionally.and(query, queryFeatureHydrator, query.params(enabledParam))
|
||||
|
||||
override def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,39 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[Conditionally]] based on a [[Param]]
|
||||
*
|
||||
* @param enabledParam the param to turn this [[QueryFeatureHydrator]] on and off
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run when `enabledParam` is true
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Result The type of the candidates
|
||||
*/
|
||||
case class ParamGatedQueryFeatureHydrator[-Query <: PipelineQuery, Result <: UniversalNoun[Any]](
|
||||
enabledParam: Param[Boolean],
|
||||
queryFeatureHydrator: QueryFeatureHydrator[Query])
|
||||
extends QueryFeatureHydrator[Query]
|
||||
with Conditionally[Query] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"ParamGated" + queryFeatureHydrator.identifier.name)
|
||||
|
||||
override val alerts: Seq[Alert] = queryFeatureHydrator.alerts
|
||||
|
||||
override val features: Set[Feature[_, _]] = queryFeatureHydrator.features
|
||||
|
||||
override def onlyIf(query: Query): Boolean =
|
||||
Conditionally.and(query, queryFeatureHydrator, query.params(enabledParam))
|
||||
|
||||
override def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,53 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.featurestorev1
|
||||
|
||||
import com.twitter.ml.featurestore.lib.EntityId
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featurestorev1.BaseFeatureStoreV1QueryFeature
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.AsyncHydrator
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1DynamicClientBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.PipelineStepIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[Conditionally]] based on a [[Param]] that hydrates asynchronously for features
|
||||
* to be before the step identified in [[hydrateBefore]]
|
||||
*
|
||||
* @param enabledParam the param to turn this [[QueryFeatureHydrator]] on and off
|
||||
* @param hydrateBefore the [[PipelineStepIdentifier]] step to make sure this feature is hydrated before.
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run when `enabledParam` is true
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Result The type of the candidates
|
||||
*/
|
||||
case class AsyncParamGatedFeatureStoreV1QueryFeatureHydrator[
|
||||
Query <: PipelineQuery,
|
||||
Result <: UniversalNoun[Any]
|
||||
](
|
||||
enabledParam: Param[Boolean],
|
||||
override val hydrateBefore: PipelineStepIdentifier,
|
||||
queryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query])
|
||||
extends FeatureStoreV1QueryFeatureHydrator[Query]
|
||||
with Conditionally[Query]
|
||||
with AsyncHydrator {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"AsyncParamGated" + queryFeatureHydrator.identifier.name)
|
||||
|
||||
override val alerts: Seq[Alert] = queryFeatureHydrator.alerts
|
||||
|
||||
override val features: Set[BaseFeatureStoreV1QueryFeature[Query, _ <: EntityId, _]] =
|
||||
queryFeatureHydrator.features
|
||||
|
||||
override val clientBuilder: FeatureStoreV1DynamicClientBuilder =
|
||||
queryFeatureHydrator.clientBuilder
|
||||
override def onlyIf(query: Query): Boolean =
|
||||
Conditionally.and(query, queryFeatureHydrator, query.params(enabledParam))
|
||||
|
||||
override def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
exports = [
|
||||
"configapi/configapi-core",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common/alert",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator/featurestorev1",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,47 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.featurestorev1
|
||||
|
||||
import com.twitter.ml.featurestore.lib.EntityId
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featurestorev1.BaseFeatureStoreV1QueryFeature
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1DynamicClientBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.featurestorev1.FeatureStoreV1QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[Conditionally]] based on a [[Param]]
|
||||
*
|
||||
* @param enabledParam the param to turn this [[QueryFeatureHydrator]] on and off
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run when `enabledParam` is true
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Result The type of the candidates
|
||||
*/
|
||||
case class ParamGatedFeatureStoreV1QueryFeatureHydrator[
|
||||
Query <: PipelineQuery,
|
||||
Result <: UniversalNoun[Any]
|
||||
](
|
||||
enabledParam: Param[Boolean],
|
||||
queryFeatureHydrator: FeatureStoreV1QueryFeatureHydrator[Query])
|
||||
extends FeatureStoreV1QueryFeatureHydrator[Query]
|
||||
with Conditionally[Query] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
"ParamGated" + queryFeatureHydrator.identifier.name)
|
||||
|
||||
override val alerts: Seq[Alert] = queryFeatureHydrator.alerts
|
||||
|
||||
override val features: Set[BaseFeatureStoreV1QueryFeature[Query, _ <: EntityId, _]] =
|
||||
queryFeatureHydrator.features
|
||||
|
||||
override val clientBuilder: FeatureStoreV1DynamicClientBuilder =
|
||||
queryFeatureHydrator.clientBuilder
|
||||
override def onlyIf(query: Query): Boolean =
|
||||
Conditionally.and(query, queryFeatureHydrator, query.params(enabledParam))
|
||||
|
||||
override def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,54 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.feature_hydrator.query.qualityfactor_gated
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
|
||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
object QualityFactorGatedQueryFeatureHydrator {
|
||||
val IdentifierPrefix = "QfGated"
|
||||
}
|
||||
|
||||
/**
|
||||
* A [[QueryFeatureHydrator]] with [[Conditionally]] based on a qualityFactor threshold.
|
||||
* @param pipelineIdentifier identifier of the pipeline that associated with observed quality factor
|
||||
* @param qualityFactorInclusiveThreshold the threshold of the quality factor that results in the hydrator being turned off
|
||||
* @param queryFeatureHydrator the underlying [[QueryFeatureHydrator]] to run when quality factor value
|
||||
* is above the given inclusive threshold
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Result The type of the candidates
|
||||
*/
|
||||
case class QualityFactorGatedQueryFeatureHydrator[
|
||||
-Query <: PipelineQuery with HasQualityFactorStatus,
|
||||
Result <: UniversalNoun[Any]
|
||||
](
|
||||
pipelineIdentifier: ComponentIdentifier,
|
||||
qualityFactorInclusiveThreshold: Param[Double],
|
||||
queryFeatureHydrator: QueryFeatureHydrator[Query])
|
||||
extends QueryFeatureHydrator[Query]
|
||||
with Conditionally[Query] {
|
||||
import QualityFactorGatedQueryFeatureHydrator._
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
|
||||
IdentifierPrefix + queryFeatureHydrator.identifier.name)
|
||||
|
||||
override val alerts: Seq[Alert] = queryFeatureHydrator.alerts
|
||||
|
||||
override val features: Set[Feature[_, _]] = queryFeatureHydrator.features
|
||||
|
||||
override def onlyIf(query: Query): Boolean = Conditionally.and(
|
||||
query,
|
||||
queryFeatureHydrator,
|
||||
query.getQualityFactorCurrentValue(pipelineIdentifier) >= query.params(
|
||||
qualityFactorInclusiveThreshold))
|
||||
|
||||
override def hydrate(query: Query): Stitch[FeatureMap] = queryFeatureHydrator.hydrate(query)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,40 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.search.common.util.bloomfilter.AdaptiveLongIntBloomFilter
|
||||
|
||||
trait GetAdaptiveLongIntBloomFilter[Query <: PipelineQuery] {
|
||||
def apply(query: Query): Option[AdaptiveLongIntBloomFilter]
|
||||
}
|
||||
|
||||
case class AdaptiveLongIntBloomFilterDedupFilter[
|
||||
Query <: PipelineQuery,
|
||||
Candidate <: UniversalNoun[Long]
|
||||
](
|
||||
getBloomFilter: GetAdaptiveLongIntBloomFilter[Query])
|
||||
extends Filter[Query, Candidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier(
|
||||
"AdaptiveLongIntBloomFilterDedupFilter")
|
||||
|
||||
override def apply(
|
||||
query: Query,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
|
||||
val filterResult = getBloomFilter(query)
|
||||
.map { bloomFilter =>
|
||||
val (kept, removed) =
|
||||
candidates.map(_.candidate).partition(candidate => !bloomFilter.contains(candidate.id))
|
||||
FilterResult(kept, removed)
|
||||
}.getOrElse(FilterResult(candidates.map(_.candidate), Seq.empty))
|
||||
|
||||
Stitch.value(filterResult)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/identifier",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||
"src/java/com/twitter/search/common/util/bloomfilter",
|
||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"stitch/stitch-core",
|
||||
"stitch/stitch-tweetypie",
|
||||
"stitch/stitch-tweetypie/src/main/scala",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"snowflake/src/main/scala/com/twitter/snowflake/id",
|
||||
"src/java/com/twitter/search/common/util/bloomfilter",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,28 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasExcludedIds
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
case class ExcludedIdsFilter[
|
||||
Query <: PipelineQuery with HasExcludedIds,
|
||||
Candidate <: UniversalNoun[Long]
|
||||
]() extends Filter[Query, Candidate] {
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("ExcludedIds")
|
||||
|
||||
override def apply(
|
||||
query: Query,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
val (kept, removed) =
|
||||
candidates.map(_.candidate).partition(candidate => !query.excludedIds.contains(candidate.id))
|
||||
|
||||
val filterResult = FilterResult(kept = kept, removed = removed)
|
||||
Stitch.value(filterResult)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,63 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
object FeatureFilter {
|
||||
|
||||
/**
|
||||
* Builds a Filter using the Feature name as the FilterIdentifier
|
||||
*
|
||||
* @see [[FeatureFilter.fromFeature(identifier, feature)]]
|
||||
*/
|
||||
def fromFeature[Candidate <: UniversalNoun[Any]](
|
||||
feature: Feature[Candidate, Boolean]
|
||||
): Filter[PipelineQuery, Candidate] =
|
||||
FeatureFilter.fromFeature(FilterIdentifier(feature.toString), feature)
|
||||
|
||||
/**
|
||||
* Builds a Filter that keeps candidates when the provided Boolean Feature is present and True.
|
||||
* If the Feature is missing or False, the candidate is removed.
|
||||
*
|
||||
* {{{
|
||||
* Filter.fromFeature(
|
||||
* FilterIdentifier("SomeFilter"),
|
||||
* feature = SomeFeature
|
||||
* )
|
||||
* }}}
|
||||
*
|
||||
* @param identifier A FilterIdentifier for the new filter
|
||||
* @param feature A feature of [Candidate, Boolean] type used to determine whether Candidates will be kept
|
||||
* when this feature is present and true otherwise they will be removed.
|
||||
*/
|
||||
def fromFeature[Candidate <: UniversalNoun[Any]](
|
||||
identifier: FilterIdentifier,
|
||||
feature: Feature[Candidate, Boolean]
|
||||
): Filter[PipelineQuery, Candidate] = {
|
||||
val i = identifier
|
||||
|
||||
new Filter[PipelineQuery, Candidate] {
|
||||
override val identifier: FilterIdentifier = i
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate =>
|
||||
filterCandidate.features.getOrElse(feature, false)
|
||||
}
|
||||
|
||||
Stitch.value(
|
||||
FilterResult(
|
||||
kept = keptCandidates.map(_.candidate),
|
||||
removed = removedCandidates.map(_.candidate)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,64 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.component_library.filter.FeatureConditionalFilter.IdentifierInfix
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Predicate to apply to candidate feature, to determine whether to apply filter.
|
||||
* True indicates we will apply the filter. False indicates to keep candidate and not apply filter.
|
||||
* @tparam FeatureValue
|
||||
*/
|
||||
trait ShouldApplyFilter[FeatureValue] {
|
||||
def apply(feature: FeatureValue): Boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* A filter that applies the [[filter]] for candidates for which [[shouldApplyFilter]] is true, and keeps the others
|
||||
* @param feature feature to determine whether to apply underyling filter
|
||||
* @param shouldApplyFilter function to determine whether to apply filter
|
||||
* @param filter the actual filter to apply if shouldApplyFilter is True
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Candidate The type of the candidates
|
||||
* @tparam FeatureValueType
|
||||
*/
|
||||
case class FeatureValueConditionalFilter[
|
||||
-Query <: PipelineQuery,
|
||||
Candidate <: UniversalNoun[Any],
|
||||
FeatureValueType
|
||||
](
|
||||
feature: Feature[Candidate, FeatureValueType],
|
||||
shouldApplyFilter: ShouldApplyFilter[FeatureValueType],
|
||||
filter: Filter[Query, Candidate])
|
||||
extends Filter[Query, Candidate] {
|
||||
override val identifier: FilterIdentifier = FilterIdentifier(
|
||||
feature.toString + IdentifierInfix + filter.identifier.name
|
||||
)
|
||||
|
||||
override val alerts: Seq[Alert] = filter.alerts
|
||||
|
||||
override def apply(
|
||||
query: Query,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
val (candidatesToFilter, candidatesToKeep) = candidates.partition { candidate =>
|
||||
shouldApplyFilter(candidate.features.get(feature))
|
||||
}
|
||||
filter.apply(query, candidatesToFilter).map { filterResult =>
|
||||
FilterResult(
|
||||
kept = filterResult.kept ++ candidatesToKeep.map(_.candidate),
|
||||
removed = filterResult.removed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object FeatureConditionalFilter {
|
||||
val IdentifierInfix = "FeatureConditional"
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,27 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetAuthorIdFeature
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A filter that checks for presence of a successfully hydrated [[TweetAuthorIdFeature]]
|
||||
*/
|
||||
case class HasAuthorIdFeatureFilter[Candidate <: TweetCandidate]()
|
||||
extends Filter[PipelineQuery, Candidate] {
|
||||
|
||||
override val identifier = FilterIdentifier("HasAuthorIdFeature")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
val (kept, removed) = candidates.partition(_.features.getTry(TweetAuthorIdFeature).isReturn)
|
||||
Stitch.value(FilterResult(kept.map(_.candidate), removed.map(_.candidate)))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,41 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.component_library.filter.ParamGatedFilter.IdentifierPrefix
|
||||
import com.twitter.product_mixer.core.functional_component.common.alert.Alert
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.Conditionally
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
/**
|
||||
* A [[Filter]] with [[Conditionally]] based on a [[Param]]
|
||||
*
|
||||
* @param enabledParam the param to turn this filter on and off
|
||||
* @param filter the underlying filter to run when `enabledParam` is true
|
||||
* @tparam Query The domain model for the query or request
|
||||
* @tparam Candidate The type of the candidates
|
||||
*/
|
||||
case class ParamGatedFilter[-Query <: PipelineQuery, Candidate <: UniversalNoun[Any]](
|
||||
enabledParam: Param[Boolean],
|
||||
filter: Filter[Query, Candidate])
|
||||
extends Filter[Query, Candidate]
|
||||
with Filter.Conditionally[Query, Candidate] {
|
||||
override val identifier: FilterIdentifier = FilterIdentifier(
|
||||
IdentifierPrefix + filter.identifier.name)
|
||||
override val alerts: Seq[Alert] = filter.alerts
|
||||
override def onlyIf(query: Query, candidates: Seq[CandidateWithFeatures[Candidate]]): Boolean =
|
||||
Conditionally.and(Filter.Input(query, candidates), filter, query.params(enabledParam))
|
||||
override def apply(
|
||||
query: Query,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = filter.apply(query, candidates)
|
||||
}
|
||||
|
||||
object ParamGatedFilter {
|
||||
val IdentifierPrefix = "ParamGated"
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,63 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Predicate which will be applied to each candidate. True indicates that the candidate will be
|
||||
* @tparam Candidate - the type of the candidate
|
||||
*/
|
||||
trait ShouldKeepCandidate[Candidate] {
|
||||
def apply(candidate: Candidate): Boolean
|
||||
}
|
||||
|
||||
object PredicateFilter {
|
||||
|
||||
/**
|
||||
* Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
|
||||
* we recommend including the name of the shouldKeepCandidate parameter.
|
||||
*
|
||||
* {{{
|
||||
* Filter.fromPredicate(
|
||||
* FilterIdentifier("SomeFilter"),
|
||||
* shouldKeepCandidate = { candidate: UserCandidate => candidate.id % 2 == 0L }
|
||||
* )
|
||||
* }}}
|
||||
*
|
||||
* @param identifier A FilterIdentifier for the new filter
|
||||
* @param shouldKeepCandidate A predicate function from the candidate. Candidates will be kept
|
||||
* when this function returns True.
|
||||
*/
|
||||
def fromPredicate[Candidate <: UniversalNoun[Any]](
|
||||
identifier: FilterIdentifier,
|
||||
shouldKeepCandidate: ShouldKeepCandidate[Candidate]
|
||||
): Filter[PipelineQuery, Candidate] = {
|
||||
val i = identifier
|
||||
|
||||
new Filter[PipelineQuery, Candidate] {
|
||||
override val identifier: FilterIdentifier = i
|
||||
|
||||
/**
|
||||
* Filter the list of candidates
|
||||
*
|
||||
* @return a FilterResult including both the list of kept candidate and the list of removed candidates
|
||||
*/
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
|
||||
filterCandidate =>
|
||||
shouldKeepCandidate(filterCandidate)
|
||||
}
|
||||
|
||||
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,42 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import com.twitter.util.Duration
|
||||
|
||||
/**
|
||||
* @param maxAgeParam Feature Switch configurable for convenience
|
||||
* @tparam Candidate The type of the candidates
|
||||
*/
|
||||
case class SnowflakeIdAgeFilter[Candidate <: UniversalNoun[Long]](
|
||||
maxAgeParam: Param[Duration])
|
||||
extends Filter[PipelineQuery, Candidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("SnowflakeIdAge")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
val maxAge = query.params(maxAgeParam)
|
||||
|
||||
val (keptCandidates, removedCandidates) = candidates
|
||||
.map(_.candidate)
|
||||
.partition { filterCandidate =>
|
||||
SnowflakeId.timeFromIdOpt(filterCandidate.id) match {
|
||||
case Some(creationTime) =>
|
||||
query.queryTime.since(creationTime) <= maxAge
|
||||
case _ => false
|
||||
}
|
||||
}
|
||||
|
||||
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,47 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A [[filter]] that filters candidates based on a country code feature
|
||||
*
|
||||
* @param countryCodeFeature the feature to filter candidates on
|
||||
*/
|
||||
case class TweetAuthorCountryFilter[Candidate <: BaseTweetCandidate](
|
||||
countryCodeFeature: Feature[Candidate, Option[String]])
|
||||
extends Filter[PipelineQuery, Candidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("TweetAuthorCountry")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
|
||||
val userCountry = query.getCountryCode
|
||||
|
||||
val (keptCandidates, removedCandidates) = candidates.partition { filteredCandidate =>
|
||||
val authorCountry = filteredCandidate.features.get(countryCodeFeature)
|
||||
|
||||
(authorCountry, userCountry) match {
|
||||
case (Some(authorCountryCode), Some(userCountryCode)) =>
|
||||
authorCountryCode.equalsIgnoreCase(userCountryCode)
|
||||
case _ => true
|
||||
}
|
||||
}
|
||||
|
||||
Stitch.value(
|
||||
FilterResult(
|
||||
kept = keptCandidates.map(_.candidate),
|
||||
removed = removedCandidates.map(_.candidate)
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,37 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetAuthorIdFeature
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A [[filter]] that filters based on whether query user is the author of the tweet. This will NOT filter empty user ids
|
||||
* @note It is recommended to apply [[HasAuthorIdFeatureFilter]] before this, as this will FAIL if feature is unavailable
|
||||
*
|
||||
* @tparam Candidate The type of the candidates
|
||||
*/
|
||||
case class TweetAuthorIsSelfFilter[Candidate <: BaseTweetCandidate]()
|
||||
extends Filter[PipelineQuery, Candidate] {
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("TweetAuthorIsSelf")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
val (kept, removed) = candidates.partition { candidate =>
|
||||
val authorId = candidate.features.get(TweetAuthorIdFeature)
|
||||
!query.getOptionalUserId.contains(authorId)
|
||||
}
|
||||
|
||||
val filterResult = FilterResult(
|
||||
kept = kept.map(_.candidate),
|
||||
removed = removed.map(_.candidate)
|
||||
)
|
||||
Stitch.value(filterResult)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,36 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_tweetypie.IsReplyFeature
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Filters out tweets that is a reply to a tweet
|
||||
*/
|
||||
case class TweetIsNotReplyFilter[Candidate <: BaseTweetCandidate]()
|
||||
extends Filter[PipelineQuery, Candidate] {
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("TweetIsNotReply")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
|
||||
val (kept, removed) = candidates
|
||||
.partition { candidate =>
|
||||
!candidate.features.get(IsReplyFeature)
|
||||
}
|
||||
|
||||
val filterResult = FilterResult(
|
||||
kept = kept.map(_.candidate),
|
||||
removed = removed.map(_.candidate)
|
||||
)
|
||||
|
||||
Stitch.value(filterResult)
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,40 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
case class TweetLanguageFilter[Candidate <: BaseTweetCandidate](
|
||||
languageCodeFeature: Feature[Candidate, Option[String]])
|
||||
extends Filter[PipelineQuery, Candidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("TweetLanguage")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
|
||||
val userAppLanguage = query.getLanguageCode
|
||||
|
||||
val (keptCandidates, removedCandidates) = candidates.partition { filterCandidate =>
|
||||
val tweetLanguage = filterCandidate.features.get(languageCodeFeature)
|
||||
|
||||
(tweetLanguage, userAppLanguage) match {
|
||||
case (Some(tweetLanguageCode), Some(userAppLanguageCode)) =>
|
||||
tweetLanguageCode.equalsIgnoreCase(userAppLanguageCode)
|
||||
case _ => true
|
||||
}
|
||||
}
|
||||
|
||||
Stitch.value(
|
||||
FilterResult(
|
||||
kept = keptCandidates.map(_.candidate),
|
||||
removed = removedCandidates.map(_.candidate)))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,71 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.util.logging.Logging
|
||||
import com.twitter.product_mixer.component_library.filter.TweetVisibilityFilter._
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.spam.rtf.thriftscala.SafetyLevel
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
|
||||
import com.twitter.tweetypie.{thriftscala => TP}
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Try
|
||||
|
||||
object TweetVisibilityFilter {
|
||||
val DefaultTweetIncludes = Set(TP.TweetInclude.TweetFieldId(TP.Tweet.IdField.id))
|
||||
private final val getTweetFieldsFailureMessage = "TweetyPie.getTweetFields failed: "
|
||||
}
|
||||
|
||||
case class TweetVisibilityFilter[Candidate <: BaseTweetCandidate](
|
||||
tweetypieStitchClient: TweetypieStitchClient,
|
||||
tweetVisibilityPolicy: TP.TweetVisibilityPolicy,
|
||||
safetyLevel: SafetyLevel,
|
||||
tweetIncludes: Set[TP.TweetInclude.TweetFieldId] = DefaultTweetIncludes)
|
||||
extends Filter[PipelineQuery, Candidate]
|
||||
with Logging {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("TweetVisibility")
|
||||
|
||||
def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
Stitch
|
||||
.traverse(candidates.map(_.candidate.id)) { tweetId =>
|
||||
tweetypieStitchClient
|
||||
.getTweetFields(tweetId, getTweetFieldsOptions(query.getOptionalUserId))
|
||||
.liftToTry
|
||||
}
|
||||
.map { getTweetFieldsResults: Seq[Try[TP.GetTweetFieldsResult]] =>
|
||||
val (checkedSucceeded, checkFailed) = getTweetFieldsResults.partition(_.isReturn)
|
||||
checkFailed.foreach(e => warn(() => getTweetFieldsFailureMessage, e.throwable))
|
||||
if (checkFailed.nonEmpty) {
|
||||
warn(() =>
|
||||
s"TweetVisibilityFilter dropped ${checkFailed.size} candidates due to tweetypie failure.")
|
||||
}
|
||||
|
||||
val allowedTweets = checkedSucceeded.collect {
|
||||
case Return(TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), _, _)) =>
|
||||
found.tweet.id
|
||||
}.toSet
|
||||
|
||||
val (kept, removed) =
|
||||
candidates.map(_.candidate).partition(candidate => allowedTweets.contains(candidate.id))
|
||||
|
||||
FilterResult(kept = kept, removed = removed)
|
||||
}
|
||||
}
|
||||
|
||||
private def getTweetFieldsOptions(userId: Option[Long]) =
|
||||
TP.GetTweetFieldsOptions(
|
||||
forUserId = userId,
|
||||
tweetIncludes = tweetIncludes.toSet,
|
||||
doNotCache = true,
|
||||
visibilityPolicy = tweetVisibilityPolicy,
|
||||
safetyLevel = Some(safetyLevel)
|
||||
)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,32 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
case class UrtUnorderedExcludeIdsCursorFilter[
|
||||
Candidate <: UniversalNoun[Long],
|
||||
Query <: PipelineQuery with HasPipelineCursor[UrtUnorderedExcludeIdsCursor]
|
||||
]() extends Filter[Query, Candidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("UnorderedExcludeIdsCursor")
|
||||
|
||||
override def apply(
|
||||
query: Query,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
|
||||
val excludeIds = query.pipelineCursor.map(_.excludedIds.toSet).getOrElse(Set.empty)
|
||||
val (kept, removed) =
|
||||
candidates.map(_.candidate).partition(candidate => !excludeIds.contains(candidate.id))
|
||||
|
||||
val filterResult = FilterResult(kept = kept, removed = removed)
|
||||
Stitch.value(filterResult)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
||||
"strato/config/columns/lists/reads:core-strato-client",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"strato/config/columns/lists/reads:core-strato-client",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,52 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter.list_visibility
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TwitterListCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.UniversalNoun
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.socialgraph.thriftscala.SocialgraphList
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.strato.catalog.Fetch
|
||||
import com.twitter.strato.generated.client.lists.reads.CoreOnListClientColumn
|
||||
|
||||
/* This Filter queries the core.List.strato column
|
||||
* on Strato, and filters out any lists that are not
|
||||
* returned. core.List.strato performs an authorization
|
||||
* check, and does not return lists the viewer is not authorized
|
||||
* to have access to. */
|
||||
class ListVisibilityFilter[Candidate <: UniversalNoun[Long]](
|
||||
listsColumn: CoreOnListClientColumn)
|
||||
extends Filter[PipelineQuery, Candidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("ListVisibility")
|
||||
|
||||
def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
|
||||
val listCandidates = candidates.collect {
|
||||
case CandidateWithFeatures(candidate: TwitterListCandidate, _) => candidate
|
||||
}
|
||||
|
||||
Stitch
|
||||
.traverse(
|
||||
listCandidates.map(_.id)
|
||||
) { listId =>
|
||||
listsColumn.fetcher.fetch(listId)
|
||||
}.map { fetchResults =>
|
||||
fetchResults.collect {
|
||||
case Fetch.Result(Some(list: SocialgraphList), _) => list.id
|
||||
}
|
||||
}.map { allowedListIds =>
|
||||
val (kept, excluded) = candidates.map(_.candidate).partition {
|
||||
case candidate: TwitterListCandidate => allowedListIds.contains(candidate.id)
|
||||
case _ => true
|
||||
}
|
||||
FilterResult(kept, excluded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,37 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.filter.tweet_impression
|
||||
|
||||
import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweets
|
||||
import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Filters out tweets that the user has seen
|
||||
*/
|
||||
case class TweetImpressionFilter[Candidate <: BaseTweetCandidate](
|
||||
) extends Filter[PipelineQuery, Candidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("TweetImpression")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[Candidate]]
|
||||
): Stitch[FilterResult[Candidate]] = {
|
||||
|
||||
// Set of Tweets that have impressed the user
|
||||
val impressedTweetsSet: Set[Long] = query.features match {
|
||||
case Some(featureMap) => featureMap.getOrElse(ImpressedTweets, Seq.empty).toSet
|
||||
case None => Set.empty
|
||||
}
|
||||
|
||||
val (keptCandidates, removedCandidates) = candidates.partition { filteredCandidate =>
|
||||
!impressedTweetsSet.contains(filteredCandidate.candidate.id)
|
||||
}
|
||||
|
||||
Stitch.value(FilterResult(keptCandidates.map(_.candidate), removedCandidates.map(_.candidate)))
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor",
|
||||
"src/scala/com/twitter/ml/featurestore/lib",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,13 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
object DefinedCountryCodeGate extends Gate[PipelineQuery] {
|
||||
override val identifier: GateIdentifier = GateIdentifier("DefinedCountryCode")
|
||||
|
||||
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
|
||||
Stitch.value(query.getCountryCode.isDefined)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,83 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.MissingFeatureException
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.functional_component.gate.GateResult
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.MisconfiguredFeatureMapFailure
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.util.Return
|
||||
import com.twitter.util.Throw
|
||||
|
||||
trait ShouldContinue[Value] {
|
||||
|
||||
/** Given the [[Feature]] value, returns whether the execution should continue */
|
||||
def apply(featureValue: Value): Boolean
|
||||
|
||||
/** If the [[Feature]] is a failure, use this value */
|
||||
def onFailedFeature(t: Throwable): GateResult = GateResult.Stop
|
||||
|
||||
/**
|
||||
* If the [[Feature]], or [[com.twitter.product_mixer.core.feature.featuremap.FeatureMap]],
|
||||
* is missing use this value
|
||||
*/
|
||||
def onMissingFeature: GateResult = GateResult.Stop
|
||||
}
|
||||
|
||||
object FeatureGate {
|
||||
|
||||
def fromFeature(
|
||||
feature: Feature[_, Boolean]
|
||||
): FeatureGate[Boolean] =
|
||||
FeatureGate.fromFeature(GateIdentifier(feature.toString), feature)
|
||||
|
||||
def fromNegatedFeature(
|
||||
feature: Feature[_, Boolean]
|
||||
): FeatureGate[Boolean] =
|
||||
FeatureGate.fromNegatedFeature(GateIdentifier(feature.toString), feature)
|
||||
|
||||
def fromFeature(
|
||||
gateIdentifier: GateIdentifier,
|
||||
feature: Feature[_, Boolean]
|
||||
): FeatureGate[Boolean] =
|
||||
FeatureGate[Boolean](gateIdentifier, feature, identity)
|
||||
|
||||
def fromNegatedFeature(
|
||||
gateIdentifier: GateIdentifier,
|
||||
feature: Feature[_, Boolean]
|
||||
): FeatureGate[Boolean] =
|
||||
FeatureGate[Boolean](gateIdentifier, feature, !identity(_))
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* A [[Gate]] that is actuated based upon the value of the provided feature
|
||||
*/
|
||||
case class FeatureGate[Value](
|
||||
gateIdentifier: GateIdentifier,
|
||||
feature: Feature[_, Value],
|
||||
continue: ShouldContinue[Value])
|
||||
extends Gate[PipelineQuery] {
|
||||
|
||||
override val identifier: GateIdentifier = gateIdentifier
|
||||
|
||||
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = {
|
||||
Stitch
|
||||
.value(
|
||||
query.features.map(_.getTry(feature)) match {
|
||||
case Some(Return(value)) => continue(value)
|
||||
case Some(Throw(_: MissingFeatureException)) => continue.onMissingFeature.continue
|
||||
case Some(Throw(t)) => continue.onFailedFeature(t).continue
|
||||
case None =>
|
||||
throw PipelineFailure(
|
||||
MisconfiguredFeatureMapFailure,
|
||||
"Expected a FeatureMap to be present but none was found, ensure that your" +
|
||||
"PipelineQuery has a FeatureMap configured before gating on Feature values"
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,19 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Gate used in first page. Use request cursor to determine if the gate should be open or closed.
|
||||
*/
|
||||
object FirstPageGate extends Gate[PipelineQuery with HasPipelineCursor[_]] {
|
||||
|
||||
override val identifier: GateIdentifier = GateIdentifier("FirstPage")
|
||||
|
||||
// If cursor is first page, then gate should return continue, otherwise return stop
|
||||
override def shouldContinue(query: PipelineQuery with HasPipelineCursor[_]): Stitch[Boolean] =
|
||||
Stitch.value(query.isFirstPage)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,21 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.common.CandidateScope
|
||||
import com.twitter.product_mixer.core.functional_component.gate.QueryAndCandidateGate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A Gate that only continues if the previously returned candidates are empty. This is useful
|
||||
* for gating dependent candidate pipelines that are intedned to be used as a backfill when there
|
||||
* are no candidates available.
|
||||
*/
|
||||
case class NoCandidatesGate(scope: CandidateScope) extends QueryAndCandidateGate[PipelineQuery] {
|
||||
override val identifier: GateIdentifier = GateIdentifier("NoCandidates")
|
||||
override def shouldContinue(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithDetails]
|
||||
): Stitch[Boolean] = Stitch.value(scope.partition(candidates).candidatesInScope.isEmpty)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,16 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate
|
||||
|
||||
import com.twitter.product_mixer.component_library.model.query.ads.AdsQuery
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
object NonEmptyAdsQueryStringGate extends Gate[PipelineQuery with AdsQuery] {
|
||||
override val identifier: GateIdentifier = GateIdentifier("NonEmptyAdsQueryString")
|
||||
|
||||
override def shouldContinue(query: PipelineQuery with AdsQuery): Stitch[Boolean] = {
|
||||
val queryString = query.searchRequestContext.flatMap(_.queryString)
|
||||
Stitch.value(queryString.exists(_.trim.nonEmpty))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,22 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.common.CandidateScope
|
||||
import com.twitter.product_mixer.core.functional_component.gate.QueryAndCandidateGate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A Gate that only continues if the previously returned candidates are not empty. This is useful
|
||||
* for gating dependent candidate pipelines that are intended to only be used if a previous pipeline
|
||||
* completed successfully.
|
||||
*/
|
||||
case class NonEmptyCandidatesGate(scope: CandidateScope)
|
||||
extends QueryAndCandidateGate[PipelineQuery] {
|
||||
override val identifier: GateIdentifier = GateIdentifier("NonEmptyCandidates")
|
||||
override def shouldContinue(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithDetails]
|
||||
): Stitch[Boolean] = Stitch.value(scope.partition(candidates).candidatesInScope.nonEmpty)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,25 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate
|
||||
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A Gate that only continues if the quality factor value of the pipeline is above the given
|
||||
* threshold. This is useful for disabling an expensive function when the pipeline is under pressure
|
||||
* (quality factor is low).
|
||||
*/
|
||||
case class QualityFactorGate(pipelineIdentifier: ComponentIdentifier, threshold: Double)
|
||||
extends Gate[PipelineQuery with HasQualityFactorStatus] {
|
||||
|
||||
override val identifier: GateIdentifier = GateIdentifier(
|
||||
s"${pipelineIdentifier.name}QualityFactor")
|
||||
|
||||
override def shouldContinue(
|
||||
query: PipelineQuery with HasQualityFactorStatus
|
||||
): Stitch[Boolean] =
|
||||
Stitch.value(query.getQualityFactorCurrentValue(pipelineIdentifier) >= threshold)
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,34 +0,0 @@
|
|||
package com.twitter.product_mixer.component_library.gate.any_candidates_without_feature
|
||||
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.functional_component.common.CandidateScope
|
||||
import com.twitter.product_mixer.core.functional_component.gate.QueryAndCandidateGate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* A gate that enables a component only if any candidates are missing a specific feature.
|
||||
* You can restrict which candidates to check with the scope parameter.
|
||||
* This is most commonly used to do backfill scoring, where you can have one Scoring Pipeline that
|
||||
* might return a score feature "FeatureA" and another sequential pipeline that you only want to run
|
||||
* if the previous scoring pipeline fails to hydrate for all candidates.
|
||||
* @param identifier Unique identifier for this gate. Typically, AnyCandidatesWithout{YourFeature}.
|
||||
* @param scope A [[CandidateScope]] to specify which candidates to check.
|
||||
* @param missingFeature The feature that should be missing for any of the candidates for this gate to continue
|
||||
*/
|
||||
case class AnyCandidatesWithoutFeatureGate(
|
||||
override val identifier: GateIdentifier,
|
||||
scope: CandidateScope,
|
||||
missingFeature: Feature[_, _])
|
||||
extends QueryAndCandidateGate[PipelineQuery] {
|
||||
|
||||
override def shouldContinue(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithDetails]
|
||||
): Stitch[Boolean] =
|
||||
Stitch.value(scope.partition(candidates).candidatesInScope.exists { candidateWithDetails =>
|
||||
!candidateWithDetails.features.getSuccessfulFeatures.contains(missingFeature)
|
||||
})
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue