mirror of
https://github.com/twitter/the-algorithm.git
synced 2024-11-14 07:35:10 +01:00
[docx] split commit for file 1800
Signed-off-by: Ari Archer <ari.web.xyz@gmail.com>
This commit is contained in:
Ari Archer
parent
94812b2b2c
commit
65c3a3fe90
Binary file not shown.
@ -1,58 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetCountryEngagementCache
|
||||
import com.twitter.home_mixer.util.CandidatesUtil
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.servo.cache.ReadCache
|
||||
import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup
|
||||
import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
object TweetCountryEngagementRealTimeAggregateFeature
|
||||
extends DataRecordInAFeature[TweetCandidate]
|
||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
||||
override def defaultValue: DataRecord = new DataRecord()
|
||||
}
|
||||
|
||||
@Singleton
|
||||
class TweetCountryEngagementRealTimeAggregateFeatureHydrator @Inject() (
|
||||
@Named(TweetCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord],
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier =
|
||||
FeatureHydratorIdentifier("TweetCountryEngagementRealTimeAggregate")
|
||||
|
||||
override val outputFeature: DataRecordInAFeature[TweetCandidate] =
|
||||
TweetCountryEngagementRealTimeAggregateFeature
|
||||
|
||||
override val aggregateGroups: Seq[AggregateGroup] = Seq(
|
||||
tweetCountryRealTimeAggregates,
|
||||
tweetCountryPrivateEngagementsRealTimeAggregates
|
||||
)
|
||||
|
||||
override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map(
|
||||
tweetCountryRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_engagement_real_time_aggregates.",
|
||||
tweetCountryPrivateEngagementsRealTimeAggregates -> "tweet-country_code.timelines.tweet_country_private_engagement_real_time_aggregates."
|
||||
)
|
||||
|
||||
override def keysFromQueryAndCandidates(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Seq[Option[(Long, String)]] = {
|
||||
val countryCode = query.clientContext.countryCode
|
||||
candidates.map { candidate =>
|
||||
val originalTweetId = CandidatesUtil.getOriginalTweetId(candidate)
|
||||
countryCode.map((originalTweetId, _))
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,61 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetEngagementCache
|
||||
import com.twitter.home_mixer.util.CandidatesUtil
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.servo.cache.ReadCache
|
||||
import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup
|
||||
import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
object TweetEngagementRealTimeAggregateFeature
|
||||
extends DataRecordInAFeature[TweetCandidate]
|
||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
||||
override def defaultValue: DataRecord = new DataRecord()
|
||||
}
|
||||
|
||||
@Singleton
|
||||
class TweetEngagementRealTimeAggregateFeatureHydrator @Inject() (
|
||||
@Named(TweetEngagementCache) override val client: ReadCache[Long, DataRecord],
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier =
|
||||
FeatureHydratorIdentifier("TweetEngagementRealTimeAggregate")
|
||||
|
||||
override val outputFeature: DataRecordInAFeature[TweetCandidate] =
|
||||
TweetEngagementRealTimeAggregateFeature
|
||||
|
||||
override val aggregateGroups: Seq[AggregateGroup] = Seq(
|
||||
tweetEngagement30MinuteCountsProd,
|
||||
tweetEngagementTotalCountsProd,
|
||||
tweetEngagementUserStateRealTimeAggregatesProd,
|
||||
tweetNegativeEngagementUserStateRealTimeAggregates,
|
||||
tweetNegativeEngagement6HourCounts,
|
||||
tweetNegativeEngagementTotalCounts,
|
||||
tweetShareEngagementsRealTimeAggregates,
|
||||
tweetBCEDwellEngagementsRealTimeAggregates
|
||||
)
|
||||
|
||||
override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map(
|
||||
tweetShareEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_share_engagements_real_time_aggregates.",
|
||||
tweetBCEDwellEngagementsRealTimeAggregates -> "original_tweet.timelines.tweet_bce_dwell_engagements_real_time_aggregates."
|
||||
)
|
||||
|
||||
override def keysFromQueryAndCandidates(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Seq[Option[Long]] = {
|
||||
candidates
|
||||
.map(candidate => Some(CandidatesUtil.getOriginalTweetId(candidate)))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,57 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.home_mixer.model.HomeFeatures.TwitterListIdFeature
|
||||
import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwitterListEngagementCache
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.servo.cache.ReadCache
|
||||
import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup
|
||||
import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
object TwitterListEngagementRealTimeAggregateFeature
|
||||
extends DataRecordInAFeature[TweetCandidate]
|
||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
||||
override def defaultValue: DataRecord = new DataRecord()
|
||||
}
|
||||
|
||||
@Singleton
|
||||
class TwitterListEngagementRealTimeAggregateFeatureHydrator @Inject() (
|
||||
@Named(TwitterListEngagementCache) override val client: ReadCache[Long, DataRecord],
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier =
|
||||
FeatureHydratorIdentifier("TwitterListEngagementRealTimeAggregate")
|
||||
|
||||
override val outputFeature: DataRecordInAFeature[TweetCandidate] =
|
||||
TwitterListEngagementRealTimeAggregateFeature
|
||||
|
||||
override val aggregateGroups: Seq[AggregateGroup] = Seq(
|
||||
listEngagementRealTimeAggregatesProd
|
||||
)
|
||||
|
||||
override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map(
|
||||
listEngagementRealTimeAggregatesProd -> "twitter_list.timelines.twitter_list_engagement_real_time_aggregates."
|
||||
)
|
||||
|
||||
override def keysFromQueryAndCandidates(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Seq[Option[Long]] = {
|
||||
candidates.map { candidate =>
|
||||
candidate.features
|
||||
.getTry(TwitterListIdFeature)
|
||||
.toOption
|
||||
.flatten
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,59 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserAuthorEngagementCache
|
||||
import com.twitter.home_mixer.util.CandidatesUtil
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.servo.cache.ReadCache
|
||||
import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup
|
||||
import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
object UserAuthorEngagementRealTimeAggregateFeature
|
||||
extends DataRecordInAFeature[TweetCandidate]
|
||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
||||
override def defaultValue: DataRecord = new DataRecord()
|
||||
}
|
||||
|
||||
@Singleton
|
||||
class UserAuthorEngagementRealTimeAggregateFeatureHydrator @Inject() (
|
||||
@Named(UserAuthorEngagementCache) override val client: ReadCache[(Long, Long), DataRecord],
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, Long)] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier =
|
||||
FeatureHydratorIdentifier("UserAuthorEngagementRealTimeAggregate")
|
||||
|
||||
override val outputFeature: DataRecordInAFeature[TweetCandidate] =
|
||||
UserAuthorEngagementRealTimeAggregateFeature
|
||||
|
||||
override val aggregateGroups: Seq[AggregateGroup] = Seq(
|
||||
userAuthorEngagementRealTimeAggregatesProd,
|
||||
userAuthorShareEngagementsRealTimeAggregates
|
||||
)
|
||||
|
||||
override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map(
|
||||
userAuthorEngagementRealTimeAggregatesProd -> "user-author.timelines.user_author_engagement_real_time_aggregates.",
|
||||
userAuthorShareEngagementsRealTimeAggregates -> "user-author.timelines.user_author_share_engagements_real_time_aggregates."
|
||||
)
|
||||
|
||||
override def keysFromQueryAndCandidates(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Seq[Option[(Long, Long)]] = {
|
||||
val userId = query.getRequiredUserId
|
||||
candidates.map { candidate =>
|
||||
CandidatesUtil
|
||||
.getOriginalAuthorId(candidate.features)
|
||||
.map((userId, _))
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,56 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates
|
||||
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserEngagementCache
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.servo.cache.ReadCache
|
||||
import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup
|
||||
import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
object UserEngagementRealTimeAggregateFeature
|
||||
extends DataRecordInAFeature[PipelineQuery]
|
||||
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] {
|
||||
override def defaultValue: DataRecord = new DataRecord()
|
||||
}
|
||||
|
||||
@Singleton
|
||||
class UserEngagementRealTimeAggregatesFeatureHydrator @Inject() (
|
||||
@Named(UserEngagementCache) override val client: ReadCache[Long, DataRecord],
|
||||
override val statsReceiver: StatsReceiver)
|
||||
extends BaseRealTimeAggregateQueryFeatureHydrator[Long] {
|
||||
|
||||
override val identifier: FeatureHydratorIdentifier =
|
||||
FeatureHydratorIdentifier("UserEngagementRealTimeAggregates")
|
||||
|
||||
override val outputFeature: DataRecordInAFeature[PipelineQuery] =
|
||||
UserEngagementRealTimeAggregateFeature
|
||||
|
||||
val aggregateGroups: Seq[AggregateGroup] = Seq(
|
||||
userEngagementRealTimeAggregatesProd,
|
||||
userShareEngagementsRealTimeAggregates,
|
||||
userBCEDwellEngagementsRealTimeAggregates,
|
||||
userEngagement48HourRealTimeAggregatesProd,
|
||||
userNegativeEngagementAuthorUserState72HourRealTimeAggregates,
|
||||
userNegativeEngagementAuthorUserStateRealTimeAggregates,
|
||||
userProfileEngagementRealTimeAggregates,
|
||||
)
|
||||
|
||||
override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map(
|
||||
userShareEngagementsRealTimeAggregates -> "user.timelines.user_share_engagements_real_time_aggregates.",
|
||||
userBCEDwellEngagementsRealTimeAggregates -> "user.timelines.user_bce_dwell_engagements_real_time_aggregates.",
|
||||
userEngagement48HourRealTimeAggregatesProd -> "user.timelines.user_engagement_48_hour_real_time_aggregates.",
|
||||
userNegativeEngagementAuthorUserState72HourRealTimeAggregates -> "user.timelines.user_negative_engagement_author_user_state_72_hour_real_time_aggregates.",
|
||||
userProfileEngagementRealTimeAggregates -> "user.timelines.user_profile_engagement_real_time_aggregates."
|
||||
)
|
||||
|
||||
override def keysFromQueryAndCandidates(query: PipelineQuery): Option[Long] = {
|
||||
Some(query.getRequiredUserId)
|
||||
}
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter",
|
||||
"stitch/stitch-core",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,37 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.filter
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
|
||||
import com.twitter.home_mixer.util.CandidatesUtil
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Remove any candidate that is in the ancestor list of any reply, including retweets of ancestors.
|
||||
*
|
||||
* E.g. if B replied to A and D was a retweet of A, we would prefer to drop D since otherwise
|
||||
* we may end up serving the same tweet twice in the timeline (e.g. serving both A->B and D).
|
||||
*/
|
||||
object DuplicateConversationTweetsFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("DuplicateConversationTweets")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Stitch[FilterResult[TweetCandidate]] = {
|
||||
val allAncestors = candidates
|
||||
.flatMap(_.features.getOrElse(AncestorsFeature, Seq.empty))
|
||||
.map(_.tweetId).toSet
|
||||
|
||||
val (kept, removed) = candidates.partition { candidate =>
|
||||
!allAncestors.contains(CandidatesUtil.getOriginalTweetId(candidate))
|
||||
}
|
||||
|
||||
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,38 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.filter
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CompetitorSetParam
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
object OutOfNetworkCompetitorFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("OutOfNetworkCompetitor")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Stitch[FilterResult[TweetCandidate]] = {
|
||||
val competitorAuthors = query.params(CompetitorSetParam)
|
||||
val (removed, kept) =
|
||||
candidates.partition(isOutOfNetworkTweetFromCompetitor(_, competitorAuthors))
|
||||
|
||||
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))
|
||||
}
|
||||
|
||||
def isOutOfNetworkTweetFromCompetitor(
|
||||
candidate: CandidateWithFeatures[TweetCandidate],
|
||||
competitorAuthors: Set[Long]
|
||||
): Boolean = {
|
||||
!candidate.features.getOrElse(InNetworkFeature, true) &&
|
||||
!candidate.features.getOrElse(IsRetweetFeature, false) &&
|
||||
candidate.features.getOrElse(AuthorIdFeature, None).exists(competitorAuthors.contains)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,38 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.filter
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature
|
||||
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CompetitorURLSeqParam
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
object OutOfNetworkCompetitorURLFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("OutOfNetworkCompetitorURL")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Stitch[FilterResult[TweetCandidate]] = {
|
||||
val competitorUrls = query.params(CompetitorURLSeqParam).toSet
|
||||
val (removed, kept) = candidates.partition(hasOutOfNetworkUrlFromCompetitor(_, competitorUrls))
|
||||
|
||||
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))
|
||||
}
|
||||
|
||||
def hasOutOfNetworkUrlFromCompetitor(
|
||||
candidate: CandidateWithFeatures[TweetCandidate],
|
||||
competitorUrls: Set[String]
|
||||
): Boolean = {
|
||||
!candidate.features.getOrElse(InNetworkFeature, true) &&
|
||||
!candidate.features.getOrElse(IsRetweetFeature, false) &&
|
||||
candidate.features
|
||||
.getOrElse(TweetUrlsFeature, Seq.empty).toSet.intersect(competitorUrls).nonEmpty
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,40 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.filter
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||
import com.twitter.home_mixer.util.ReplyRetweetUtil
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* This filter removes source tweets of retweets, added via second EB call in TLR
|
||||
*/
|
||||
object RetweetSourceTweetRemovingFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("RetweetSourceTweetRemoving")
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Stitch[FilterResult[TweetCandidate]] = {
|
||||
val (kept, removed) =
|
||||
candidates.partition(
|
||||
_.features.getOrElse(EarlybirdFeature, None).exists(_.isSourceTweet)) match {
|
||||
case (sourceTweets, nonSourceTweets) =>
|
||||
val inReplyToTweetIds: Set[Long] =
|
||||
nonSourceTweets
|
||||
.filter(ReplyRetweetUtil.isEligibleReply(_)).flatMap(
|
||||
_.features.getOrElse(InReplyToTweetIdFeature, None)).toSet
|
||||
val (keptSourceTweets, removedSourceTweets) = sourceTweets
|
||||
.map(_.candidate)
|
||||
.partition(candidate => inReplyToTweetIds.contains(candidate.id))
|
||||
(nonSourceTweets.map(_.candidate) ++ keptSourceTweets, removedSourceTweets)
|
||||
}
|
||||
Stitch.value(FilterResult(kept = kept, removed = removed))
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,61 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.filter
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures._
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.filter.Filter
|
||||
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsSocialContextFilter extends Filter[PipelineQuery, TweetCandidate] {
|
||||
|
||||
override val identifier: FilterIdentifier = FilterIdentifier("ScoredTweetsSocialContext")
|
||||
|
||||
// Tweets from candidate sources which don't need generic like/follow/topic proof
|
||||
private val AllowedSources: Set[st.SuggestType] = Set(
|
||||
st.SuggestType.RankedListTweet,
|
||||
st.SuggestType.RecommendedTrendTweet,
|
||||
st.SuggestType.MediaTweet
|
||||
)
|
||||
|
||||
override def apply(
|
||||
query: PipelineQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Stitch[FilterResult[TweetCandidate]] = {
|
||||
val validTweetIds = candidates
|
||||
.filter { candidate =>
|
||||
candidate.features.getOrElse(InNetworkFeature, true) ||
|
||||
candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) ||
|
||||
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isDefined ||
|
||||
hasLikedBySocialContext(candidate.features) ||
|
||||
hasFollowedBySocialContext(candidate.features) ||
|
||||
hasTopicSocialContext(candidate.features)
|
||||
}.map(_.candidate.id).toSet
|
||||
|
||||
val (kept, removed) =
|
||||
candidates.map(_.candidate).partition(candidate => validTweetIds.contains(candidate.id))
|
||||
|
||||
Stitch.value(FilterResult(kept = kept, removed = removed))
|
||||
}
|
||||
|
||||
private def hasLikedBySocialContext(candidateFeatures: FeatureMap): Boolean =
|
||||
candidateFeatures
|
||||
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
|
||||
.exists(
|
||||
candidateFeatures
|
||||
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty)
|
||||
.toSet.contains
|
||||
)
|
||||
|
||||
private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean =
|
||||
candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty
|
||||
|
||||
private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = {
|
||||
candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined &&
|
||||
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined
|
||||
}
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,34 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.gate
|
||||
|
||||
import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate.identifierSuffix
|
||||
import com.twitter.home_mixer.util.CachedScoredTweetsHelper
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
case class MinCachedTweetsGate(
|
||||
candidatePipelineIdentifier: CandidatePipelineIdentifier,
|
||||
minCachedTweetsParam: Param[Int])
|
||||
extends Gate[PipelineQuery] {
|
||||
|
||||
override val identifier: GateIdentifier =
|
||||
GateIdentifier(candidatePipelineIdentifier + identifierSuffix)
|
||||
|
||||
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = {
|
||||
val minCachedTweets = query.params(minCachedTweetsParam)
|
||||
val cachedScoredTweets =
|
||||
query.features.map(CachedScoredTweetsHelper.unseenCachedScoredTweets).getOrElse(Seq.empty)
|
||||
val numCachedTweets = cachedScoredTweets.count { tweet =>
|
||||
tweet.candidatePipelineIdentifier.exists(
|
||||
CandidatePipelineIdentifier(_).equals(candidatePipelineIdentifier))
|
||||
}
|
||||
Stitch.value(numCachedTweets < minCachedTweets)
|
||||
}
|
||||
}
|
||||
|
||||
object MinCachedTweetsGate {
|
||||
val identifierSuffix = "MinCachedTweets"
|
||||
}
|
||||
Binary file not shown.
@ -1,27 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.gate
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature
|
||||
import com.twitter.product_mixer.core.functional_component.gate.Gate
|
||||
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Gate continues if the amount of time passed since the previous request is greater
|
||||
* than the configured amount or if the previous request time in not available
|
||||
*/
|
||||
object MinTimeSinceLastRequestGate extends Gate[PipelineQuery] {
|
||||
|
||||
override val identifier: GateIdentifier = GateIdentifier("TimeSinceLastRequest")
|
||||
|
||||
private val MinTimeSinceLastRequest = 24.hours
|
||||
|
||||
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] = Stitch.value {
|
||||
query.features.exists { features =>
|
||||
features
|
||||
.getOrElse(LastNonPollingTimeFeature, None)
|
||||
.forall(lnpt => (query.queryTime - lnpt) > MinTimeSinceLastRequest)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
|
||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/premarshaller",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,22 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.marshaller
|
||||
|
||||
import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery
|
||||
import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse
|
||||
import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller
|
||||
import com.twitter.product_mixer.core.model.common.identifier.DomainMarshallerIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
|
||||
/**
|
||||
* Creates a domain model of the Scored Tweets product response from the set of candidates selected
|
||||
*/
|
||||
object ScoredTweetsResponseDomainMarshaller
|
||||
extends DomainMarshaller[ScoredTweetsQuery, ScoredTweetsResponse] {
|
||||
|
||||
override val identifier: DomainMarshallerIdentifier =
|
||||
DomainMarshallerIdentifier("ScoredTweetsResponse")
|
||||
|
||||
override def apply(
|
||||
query: ScoredTweetsQuery,
|
||||
selections: Seq[CandidateWithDetails]
|
||||
): ScoredTweetsResponse = ScoredTweetsResponse(scoredTweets = selections)
|
||||
}
|
||||
Binary file not shown.
@ -1,70 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.marshaller
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures._
|
||||
import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse
|
||||
import com.twitter.home_mixer.{thriftscala => t}
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller
|
||||
import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.metadata.TopicContextFunctionalityTypeMarshaller
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransportMarshallerIdentifier
|
||||
|
||||
/**
|
||||
* Marshall the domain model into our transport (Thrift) model.
|
||||
*/
|
||||
object ScoredTweetsResponseTransportMarshaller
|
||||
extends TransportMarshaller[ScoredTweetsResponse, t.ScoredTweetsResponse] {
|
||||
|
||||
override val identifier: TransportMarshallerIdentifier =
|
||||
TransportMarshallerIdentifier("ScoredTweetsResponse")
|
||||
|
||||
override def apply(input: ScoredTweetsResponse): t.ScoredTweetsResponse = {
|
||||
val scoredTweets = input.scoredTweets.map { tweet =>
|
||||
mkScoredTweet(tweet.candidateIdLong, tweet.features)
|
||||
}
|
||||
t.ScoredTweetsResponse(scoredTweets)
|
||||
}
|
||||
|
||||
private def mkScoredTweet(tweetId: Long, features: FeatureMap): t.ScoredTweet = {
|
||||
val topicFunctionalityType = features
|
||||
.getOrElse(TopicContextFunctionalityTypeFeature, None)
|
||||
.map(TopicContextFunctionalityTypeMarshaller(_))
|
||||
|
||||
t.ScoredTweet(
|
||||
tweetId = tweetId,
|
||||
authorId = features.get(AuthorIdFeature).get,
|
||||
score = features.get(ScoreFeature),
|
||||
suggestType = features.get(SuggestTypeFeature),
|
||||
sourceTweetId = features.getOrElse(SourceTweetIdFeature, None),
|
||||
sourceUserId = features.getOrElse(SourceUserIdFeature, None),
|
||||
quotedTweetId = features.getOrElse(QuotedTweetIdFeature, None),
|
||||
quotedUserId = features.getOrElse(QuotedUserIdFeature, None),
|
||||
inReplyToTweetId = features.getOrElse(InReplyToTweetIdFeature, None),
|
||||
inReplyToUserId = features.getOrElse(InReplyToUserIdFeature, None),
|
||||
directedAtUserId = features.getOrElse(DirectedAtUserIdFeature, None),
|
||||
inNetwork = Some(features.getOrElse(InNetworkFeature, true)),
|
||||
sgsValidLikedByUserIds = Some(features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)),
|
||||
sgsValidFollowedByUserIds =
|
||||
Some(features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)),
|
||||
topicId = features.getOrElse(TopicIdSocialContextFeature, None),
|
||||
topicFunctionalityType = topicFunctionalityType,
|
||||
ancestors = Some(features.getOrElse(AncestorsFeature, Seq.empty)),
|
||||
isReadFromCache = Some(features.getOrElse(IsReadFromCacheFeature, false)),
|
||||
streamToKafka = Some(features.getOrElse(StreamToKafkaFeature, false)),
|
||||
exclusiveConversationAuthorId =
|
||||
features.getOrElse(ExclusiveConversationAuthorIdFeature, None),
|
||||
authorMetadata = Some(
|
||||
t.AuthorMetadata(
|
||||
blueVerified = features.getOrElse(AuthorIsBlueVerifiedFeature, false),
|
||||
goldVerified = features.getOrElse(AuthorIsGoldVerifiedFeature, false),
|
||||
grayVerified = features.getOrElse(AuthorIsGrayVerifiedFeature, false),
|
||||
legacyVerified = features.getOrElse(AuthorIsLegacyVerifiedFeature, false),
|
||||
creator = features.getOrElse(AuthorIsCreatorFeature, false)
|
||||
)),
|
||||
lastScoredTimestampMs = None,
|
||||
candidatePipelineIdentifier = None,
|
||||
tweetUrls = None,
|
||||
perspectiveFilteredLikedByUserIds =
|
||||
Some(features.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty)),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||
],
|
||||
exports = [
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,39 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.model
|
||||
|
||||
import com.twitter.home_mixer.model.request.DeviceContext
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.model.request.HasSeenTweetIds
|
||||
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
|
||||
import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.model.marshalling.request._
|
||||
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorStatus
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
case class ScoredTweetsQuery(
|
||||
override val params: Params,
|
||||
override val clientContext: ClientContext,
|
||||
override val pipelineCursor: Option[UrtOrderedCursor],
|
||||
override val requestedMaxResults: Option[Int],
|
||||
override val debugOptions: Option[DebugOptions],
|
||||
override val features: Option[FeatureMap],
|
||||
override val deviceContext: Option[DeviceContext],
|
||||
override val seenTweetIds: Option[Seq[Long]],
|
||||
override val qualityFactorStatus: Option[QualityFactorStatus])
|
||||
extends PipelineQuery
|
||||
with HasPipelineCursor[UrtOrderedCursor]
|
||||
with HasDeviceContext
|
||||
with HasSeenTweetIds
|
||||
with HasQualityFactorStatus {
|
||||
override val product: Product = ScoredTweetsProduct
|
||||
|
||||
override def withFeatureMap(features: FeatureMap): ScoredTweetsQuery =
|
||||
copy(features = Some(features))
|
||||
|
||||
override def withQualityFactorStatus(
|
||||
qualityFactorStatus: QualityFactorStatus
|
||||
): ScoredTweetsQuery = copy(qualityFactorStatus = Some(qualityFactorStatus))
|
||||
}
|
||||
Binary file not shown.
@ -1,6 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.model
|
||||
|
||||
import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
|
||||
case class ScoredTweetsResponse(scoredTweets: Seq[CandidateWithDetails]) extends HasMarshalling
|
||||
@ -1,11 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,361 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.param
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.home_mixer.param.decider.DeciderKey
|
||||
import com.twitter.timelines.configapi.DurationConversion
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.HasDurationConversion
|
||||
import com.twitter.timelines.configapi.decider.BooleanDeciderParam
|
||||
import com.twitter.util.Duration
|
||||
|
||||
object ScoredTweetsParam {
|
||||
val SupportedClientFSName = "scored_tweets_supported_client"
|
||||
|
||||
object CandidatePipeline {
|
||||
object EnableInNetworkParam
|
||||
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsInNetworkCandidatePipeline)
|
||||
|
||||
object EnableTweetMixerParam
|
||||
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsTweetMixerCandidatePipeline)
|
||||
|
||||
object EnableUtegParam
|
||||
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsUtegCandidatePipeline)
|
||||
|
||||
object EnableFrsParam
|
||||
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsFrsCandidatePipeline)
|
||||
|
||||
object EnableListsParam
|
||||
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsListsCandidatePipeline)
|
||||
|
||||
object EnablePopularVideosParam
|
||||
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsPopularVideosCandidatePipeline)
|
||||
|
||||
object EnableBackfillParam
|
||||
extends BooleanDeciderParam(DeciderKey.EnableScoredTweetsBackfillCandidatePipeline)
|
||||
}
|
||||
|
||||
object EnableBackfillCandidatePipelineParam
|
||||
extends FSParam[Boolean](
|
||||
name = "scored_tweets_enable_backfill_candidate_pipeline",
|
||||
default = true
|
||||
)
|
||||
|
||||
object QualityFactor {
|
||||
object InNetworkMaxTweetsToScoreParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_quality_factor_earlybird_max_tweets_to_score",
|
||||
default = 500,
|
||||
min = 0,
|
||||
max = 10000
|
||||
)
|
||||
|
||||
object UtegMaxTweetsToScoreParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_quality_factor_uteg_max_tweets_to_score",
|
||||
default = 500,
|
||||
min = 0,
|
||||
max = 10000
|
||||
)
|
||||
|
||||
object FrsMaxTweetsToScoreParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_quality_factor_frs_max_tweets_to_score",
|
||||
default = 500,
|
||||
min = 0,
|
||||
max = 10000
|
||||
)
|
||||
|
||||
object TweetMixerMaxTweetsToScoreParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_quality_factor_tweet_mixer_max_tweets_to_score",
|
||||
default = 500,
|
||||
min = 0,
|
||||
max = 10000
|
||||
)
|
||||
|
||||
object ListsMaxTweetsToScoreParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_quality_factor_lists_max_tweets_to_score",
|
||||
default = 500,
|
||||
min = 0,
|
||||
max = 100
|
||||
)
|
||||
|
||||
object PopularVideosMaxTweetsToScoreParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_quality_factor_popular_videos_max_tweets_to_score",
|
||||
default = 40,
|
||||
min = 0,
|
||||
max = 10000
|
||||
)
|
||||
|
||||
object BackfillMaxTweetsToScoreParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_quality_factor_backfill_max_tweets_to_score",
|
||||
default = 500,
|
||||
min = 0,
|
||||
max = 10000
|
||||
)
|
||||
}
|
||||
|
||||
object ServerMaxResultsParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_server_max_results",
|
||||
default = 120,
|
||||
min = 1,
|
||||
max = 500
|
||||
)
|
||||
|
||||
object MaxInNetworkResultsParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_max_in_network_results",
|
||||
default = 60,
|
||||
min = 1,
|
||||
max = 500
|
||||
)
|
||||
|
||||
object MaxOutOfNetworkResultsParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_max_out_of_network_results",
|
||||
default = 60,
|
||||
min = 1,
|
||||
max = 500
|
||||
)
|
||||
|
||||
object CachedScoredTweets {
|
||||
object TTLParam
|
||||
extends FSBoundedParam[Duration](
|
||||
name = "scored_tweets_cached_scored_tweets_ttl_minutes",
|
||||
default = 3.minutes,
|
||||
min = 0.minute,
|
||||
max = 60.minutes
|
||||
)
|
||||
with HasDurationConversion {
|
||||
override val durationConversion: DurationConversion = DurationConversion.FromMinutes
|
||||
}
|
||||
|
||||
object MinCachedTweetsParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "scored_tweets_cached_scored_tweets_min_cached_tweets",
|
||||
default = 30,
|
||||
min = 0,
|
||||
max = 1000
|
||||
)
|
||||
}
|
||||
|
||||
object Scoring {
|
||||
object HomeModelParam
|
||||
extends FSParam[String](name = "scored_tweets_home_model", default = "Home")
|
||||
|
||||
object ModelWeights {
|
||||
|
||||
object FavParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_fav",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object RetweetParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_retweet",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object ReplyParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_reply",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object GoodProfileClickParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_good_profile_click",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 1000000.0
|
||||
)
|
||||
|
||||
object VideoPlayback50Param
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_video_playback50",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object ReplyEngagedByAuthorParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_reply_engaged_by_author",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 200.0
|
||||
)
|
||||
|
||||
object GoodClickParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_good_click",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 1000000.0
|
||||
)
|
||||
|
||||
object GoodClickV2Param
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_good_click_v2",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 1000000.0
|
||||
)
|
||||
|
||||
object TweetDetailDwellParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_tweet_detail_dwell",
|
||||
default = 0.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object ProfileDwelledParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_profile_dwelled",
|
||||
default = 0.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object BookmarkParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_bookmark",
|
||||
default = 0.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object ShareParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_share",
|
||||
default = 0.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object ShareMenuClickParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_share_menu_click",
|
||||
default = 0.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object NegativeFeedbackV2Param
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_negative_feedback_v2",
|
||||
default = 1.0,
|
||||
min = -1000.0,
|
||||
max = 0.0
|
||||
)
|
||||
|
||||
object ReportParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_report",
|
||||
default = 1.0,
|
||||
min = -20000.0,
|
||||
max = 0.0
|
||||
)
|
||||
|
||||
object WeakNegativeFeedbackParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_weak_negative_feedback",
|
||||
default = 0.0,
|
||||
min = -1000.0,
|
||||
max = 0.0
|
||||
)
|
||||
|
||||
object StrongNegativeFeedbackParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_model_weight_strong_negative_feedback",
|
||||
default = 0.0,
|
||||
min = -1000.0,
|
||||
max = 0.0
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
object EnableSimClustersSimilarityFeatureHydrationDeciderParam
|
||||
extends BooleanDeciderParam(decider = DeciderKey.EnableSimClustersSimilarityFeatureHydration)
|
||||
|
||||
object CompetitorSetParam
|
||||
extends FSParam[Set[Long]](name = "scored_tweets_competitor_list", default = Set.empty)
|
||||
|
||||
object CompetitorURLSeqParam
|
||||
extends FSParam[Seq[String]](name = "scored_tweets_competitor_url_list", default = Seq.empty)
|
||||
|
||||
object BlueVerifiedAuthorInNetworkMultiplierParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_blue_verified_author_in_network_multiplier",
|
||||
default = 4.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object BlueVerifiedAuthorOutOfNetworkMultiplierParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_blue_verified_author_out_of_network_multiplier",
|
||||
default = 2.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object CreatorInNetworkMultiplierParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_creator_in_network_multiplier",
|
||||
default = 1.1,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object CreatorOutOfNetworkMultiplierParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_creator_out_of_network_multiplier",
|
||||
default = 1.3,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object OutOfNetworkScaleFactorParam
|
||||
extends FSBoundedParam[Double](
|
||||
name = "scored_tweets_out_of_network_scale_factor",
|
||||
default = 1.0,
|
||||
min = 0.0,
|
||||
max = 100.0
|
||||
)
|
||||
|
||||
object EnableScribeScoredCandidatesParam
|
||||
extends FSParam[Boolean](name = "scored_tweets_enable_scribing", default = false)
|
||||
|
||||
object EarlybirdTensorflowModel {
|
||||
|
||||
object InNetworkParam
|
||||
extends FSParam[String](
|
||||
name = "scored_tweets_in_network_earlybird_tensorflow_model",
|
||||
default = "timelines_recap_replica")
|
||||
|
||||
object FrsParam
|
||||
extends FSParam[String](
|
||||
name = "scored_tweets_frs_earlybird_tensorflow_model",
|
||||
default = "timelines_rectweet_replica")
|
||||
|
||||
object UtegParam
|
||||
extends FSParam[String](
|
||||
name = "scored_tweets_uteg_earlybird_tensorflow_model",
|
||||
default = "timelines_rectweet_replica")
|
||||
}
|
||||
|
||||
}
|
||||
Binary file not shown.
@ -1,89 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.param
|
||||
|
||||
import com.twitter.home_mixer.param.decider.DeciderKey
|
||||
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam._
|
||||
import com.twitter.product_mixer.core.product.ProductParamConfig
|
||||
import com.twitter.servo.decider.DeciderKeyName
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ScoredTweetsParamConfig @Inject() () extends ProductParamConfig {
|
||||
override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableScoredTweetsProduct
|
||||
override val supportedClientFSName: String = SupportedClientFSName
|
||||
|
||||
override val booleanDeciderOverrides = Seq(
|
||||
CandidatePipeline.EnableBackfillParam,
|
||||
CandidatePipeline.EnableTweetMixerParam,
|
||||
CandidatePipeline.EnableFrsParam,
|
||||
CandidatePipeline.EnableInNetworkParam,
|
||||
CandidatePipeline.EnableListsParam,
|
||||
CandidatePipeline.EnablePopularVideosParam,
|
||||
CandidatePipeline.EnableUtegParam,
|
||||
ScoredTweetsParam.EnableSimClustersSimilarityFeatureHydrationDeciderParam
|
||||
)
|
||||
|
||||
override val booleanFSOverrides = Seq(
|
||||
EnableBackfillCandidatePipelineParam,
|
||||
EnableScribeScoredCandidatesParam
|
||||
)
|
||||
|
||||
override val boundedIntFSOverrides = Seq(
|
||||
CachedScoredTweets.MinCachedTweetsParam,
|
||||
MaxInNetworkResultsParam,
|
||||
MaxOutOfNetworkResultsParam,
|
||||
QualityFactor.BackfillMaxTweetsToScoreParam,
|
||||
QualityFactor.TweetMixerMaxTweetsToScoreParam,
|
||||
QualityFactor.FrsMaxTweetsToScoreParam,
|
||||
QualityFactor.InNetworkMaxTweetsToScoreParam,
|
||||
QualityFactor.ListsMaxTweetsToScoreParam,
|
||||
QualityFactor.PopularVideosMaxTweetsToScoreParam,
|
||||
QualityFactor.UtegMaxTweetsToScoreParam,
|
||||
ServerMaxResultsParam
|
||||
)
|
||||
|
||||
override val boundedDurationFSOverrides = Seq(
|
||||
CachedScoredTweets.TTLParam
|
||||
)
|
||||
|
||||
override val stringFSOverrides = Seq(
|
||||
Scoring.HomeModelParam,
|
||||
EarlybirdTensorflowModel.InNetworkParam,
|
||||
EarlybirdTensorflowModel.FrsParam,
|
||||
EarlybirdTensorflowModel.UtegParam
|
||||
)
|
||||
|
||||
override val boundedDoubleFSOverrides = Seq(
|
||||
BlueVerifiedAuthorInNetworkMultiplierParam,
|
||||
BlueVerifiedAuthorOutOfNetworkMultiplierParam,
|
||||
CreatorInNetworkMultiplierParam,
|
||||
CreatorOutOfNetworkMultiplierParam,
|
||||
OutOfNetworkScaleFactorParam,
|
||||
// Model Weights
|
||||
Scoring.ModelWeights.FavParam,
|
||||
Scoring.ModelWeights.ReplyParam,
|
||||
Scoring.ModelWeights.RetweetParam,
|
||||
Scoring.ModelWeights.GoodClickParam,
|
||||
Scoring.ModelWeights.GoodClickV2Param,
|
||||
Scoring.ModelWeights.GoodProfileClickParam,
|
||||
Scoring.ModelWeights.ReplyEngagedByAuthorParam,
|
||||
Scoring.ModelWeights.VideoPlayback50Param,
|
||||
Scoring.ModelWeights.ReportParam,
|
||||
Scoring.ModelWeights.NegativeFeedbackV2Param,
|
||||
Scoring.ModelWeights.TweetDetailDwellParam,
|
||||
Scoring.ModelWeights.ProfileDwelledParam,
|
||||
Scoring.ModelWeights.BookmarkParam,
|
||||
Scoring.ModelWeights.ShareParam,
|
||||
Scoring.ModelWeights.ShareMenuClickParam,
|
||||
Scoring.ModelWeights.StrongNegativeFeedbackParam,
|
||||
Scoring.ModelWeights.WeakNegativeFeedbackParam
|
||||
)
|
||||
|
||||
override val longSetFSOverrides = Seq(
|
||||
CompetitorSetParam
|
||||
)
|
||||
|
||||
override val stringSeqFSOverrides = Seq(
|
||||
CompetitorURLSeqParam
|
||||
)
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelines:util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/common/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/candidate",
|
||||
"timelineservice/common:model",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,64 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.query_transformer
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.core_workflows.user_model.{thriftscala => um}
|
||||
import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature
|
||||
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam
|
||||
import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer._
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.timelineranker.{thriftscala => t}
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
import com.twitter.timelines.model.candidate.CandidateTweetSourceId
|
||||
|
||||
object TimelineRankerFrsQueryTransformer {
|
||||
private val DefaultSinceDuration = 24.hours
|
||||
private val ExpandedSinceDuration = 48.hours
|
||||
private val MaxTweetsToFetch = 100
|
||||
|
||||
private val tweetKindOptions: TweetKindOption.ValueSet =
|
||||
TweetKindOption(includeOriginalTweetsAndQuotes = true)
|
||||
|
||||
private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set(
|
||||
um.UserState.Light,
|
||||
um.UserState.MediumNonTweeter,
|
||||
um.UserState.MediumTweeter,
|
||||
um.UserState.NearZero,
|
||||
um.UserState.New,
|
||||
um.UserState.VeryLight
|
||||
)
|
||||
}
|
||||
|
||||
case class TimelineRankerFrsQueryTransformer[
|
||||
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
|
||||
](
|
||||
override val candidatePipelineIdentifier: CandidatePipelineIdentifier,
|
||||
override val maxTweetsToFetch: Int = MaxTweetsToFetch)
|
||||
extends CandidatePipelineQueryTransformer[Query, t.RecapQuery]
|
||||
with TimelineRankerQueryTransformer[Query] {
|
||||
|
||||
override val candidateTweetSourceId = CandidateTweetSourceId.FrsTweet
|
||||
override val options = tweetKindOptions
|
||||
|
||||
override def getTensorflowModel(query: Query): Option[String] = {
|
||||
Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.FrsParam))
|
||||
}
|
||||
|
||||
override def seedAuthorIds(query: Query): Option[Seq[Long]] = {
|
||||
query.features.flatMap(_.getOrElse(FrsSeedUserIdsFeature, None))
|
||||
}
|
||||
|
||||
override def transform(input: Query): t.RecapQuery = {
|
||||
val userState = input.features.get.getOrElse(UserStateFeature, None)
|
||||
|
||||
val sinceDuration =
|
||||
if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration
|
||||
else DefaultSinceDuration
|
||||
|
||||
buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,63 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.query_transformer
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.core_workflows.user_model.{thriftscala => um}
|
||||
import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam
|
||||
import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer._
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.timelineranker.{thriftscala => t}
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
import com.twitter.timelines.model.candidate.CandidateTweetSourceId
|
||||
|
||||
object TimelineRankerInNetworkQueryTransformer {
|
||||
private val DefaultSinceDuration = 24.hours
|
||||
private val ExpandedSinceDuration = 48.hours
|
||||
private val MaxTweetsToFetch = 600
|
||||
|
||||
private val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOption(
|
||||
includeReplies = true,
|
||||
includeRetweets = true,
|
||||
includeOriginalTweetsAndQuotes = true,
|
||||
includeExtendedReplies = true
|
||||
)
|
||||
|
||||
private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set(
|
||||
um.UserState.Light,
|
||||
um.UserState.MediumNonTweeter,
|
||||
um.UserState.MediumTweeter,
|
||||
um.UserState.NearZero,
|
||||
um.UserState.New,
|
||||
um.UserState.VeryLight
|
||||
)
|
||||
}
|
||||
|
||||
case class TimelineRankerInNetworkQueryTransformer[
|
||||
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
|
||||
](
|
||||
override val candidatePipelineIdentifier: CandidatePipelineIdentifier,
|
||||
override val maxTweetsToFetch: Int = MaxTweetsToFetch)
|
||||
extends CandidatePipelineQueryTransformer[Query, t.RecapQuery]
|
||||
with TimelineRankerQueryTransformer[Query] {
|
||||
|
||||
override val candidateTweetSourceId = CandidateTweetSourceId.RecycledTweet
|
||||
override val options = tweetKindOptions
|
||||
|
||||
override def getTensorflowModel(query: Query): Option[String] = {
|
||||
Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.InNetworkParam))
|
||||
}
|
||||
|
||||
override def transform(input: Query): t.RecapQuery = {
|
||||
val userState = input.features.get.getOrElse(UserStateFeature, None)
|
||||
|
||||
val sinceDuration =
|
||||
if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration
|
||||
else DefaultSinceDuration
|
||||
|
||||
buildTimelineRankerQuery(input, sinceDuration).toThriftRecapQuery
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,108 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.query_transformer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerQueryTransformer._
|
||||
import com.twitter.home_mixer.util.CachedScoredTweetsHelper
|
||||
import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.timelineranker.{model => tlr}
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
import com.twitter.timelines.earlybird.common.options.EarlybirdOptions
|
||||
import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig
|
||||
import com.twitter.timelines.earlybird.common.utils.SearchOperator
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.timelines.model.candidate.CandidateTweetSourceId
|
||||
import com.twitter.timelines.util.SnowflakeSortIndexHelper
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Time
|
||||
|
||||
object TimelineRankerQueryTransformer {
|
||||
|
||||
/**
|
||||
* Specifies the maximum number of excluded tweet ids to include in the search index query.
|
||||
* Earlybird's named multi term disjunction map feature supports up to 1500 tweet ids.
|
||||
*/
|
||||
private val EarlybirdMaxExcludedTweets = 1500
|
||||
|
||||
/**
|
||||
* Maximum number of query hits each earlybird shard is allowed to accumulate before
|
||||
* early-terminating the query and reducing the hits to MaxNumEarlybirdResults.
|
||||
*/
|
||||
private val EarlybirdMaxHits = 1000
|
||||
|
||||
/**
|
||||
* Maximum number of results TLR should retrieve from each earlybird shard.
|
||||
*/
|
||||
private val EarlybirdMaxResults = 300
|
||||
}
|
||||
|
||||
trait TimelineRankerQueryTransformer[
|
||||
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] {
|
||||
def maxTweetsToFetch: Int
|
||||
def options: TweetKindOption.ValueSet = TweetKindOption.Default
|
||||
def candidateTweetSourceId: CandidateTweetSourceId.Value
|
||||
def utegLikedByTweetsOptions(query: Query): Option[tlr.UtegLikedByTweetsOptions] = None
|
||||
def seedAuthorIds(query: Query): Option[Seq[Long]] = None
|
||||
def candidatePipelineIdentifier: CandidatePipelineIdentifier
|
||||
def earlybirdModels: Seq[EarlybirdScoringModelConfig] =
|
||||
EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementProd
|
||||
def getTensorflowModel(query: Query): Option[String] = None
|
||||
|
||||
def buildTimelineRankerQuery(query: Query, sinceDuration: Duration): tlr.RecapQuery = {
|
||||
val sinceTime: Time = sinceDuration.ago
|
||||
val untilTime: Time = Time.now
|
||||
|
||||
val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime)
|
||||
val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime)
|
||||
val range = tlr.TweetIdRange(Some(fromTweetIdExclusive), Some(toTweetIdExclusive))
|
||||
|
||||
val excludedTweetIds = query.features.map { featureMap =>
|
||||
CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange(
|
||||
featureMap,
|
||||
candidatePipelineIdentifier,
|
||||
EarlybirdMaxExcludedTweets,
|
||||
sinceTime,
|
||||
untilTime)
|
||||
}
|
||||
|
||||
val maxCount =
|
||||
(query.getQualityFactorCurrentValue(candidatePipelineIdentifier) * maxTweetsToFetch).toInt
|
||||
|
||||
val authorScoreMap = query.features
|
||||
.map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[UserId, Double]))
|
||||
.getOrElse(Map.empty)
|
||||
|
||||
val deviceContext =
|
||||
query.deviceContext.map(_.toTimelineServiceDeviceContext(query.clientContext))
|
||||
|
||||
val tensorflowModel = getTensorflowModel(query)
|
||||
|
||||
val earlyBirdOptions = EarlybirdOptions(
|
||||
maxNumHitsPerShard = EarlybirdMaxHits,
|
||||
maxNumResultsPerShard = EarlybirdMaxResults,
|
||||
models = earlybirdModels,
|
||||
authorScoreMap = authorScoreMap,
|
||||
skipVeryRecentTweets = true,
|
||||
tensorflowModel = tensorflowModel
|
||||
)
|
||||
|
||||
tlr.RecapQuery(
|
||||
userId = query.getRequiredUserId,
|
||||
maxCount = Some(maxCount),
|
||||
range = Some(range),
|
||||
options = options,
|
||||
searchOperator = SearchOperator.Exclude,
|
||||
earlybirdOptions = Some(earlyBirdOptions),
|
||||
deviceContext = deviceContext,
|
||||
authorIds = seedAuthorIds(query),
|
||||
excludedTweetIds = excludedTweetIds,
|
||||
utegLikedByTweetsOptions = utegLikedByTweetsOptions(query),
|
||||
searchClientSubId = None,
|
||||
candidateTweetSourceId = Some(candidateTweetSourceId),
|
||||
hydratesContentFeatures = Some(false)
|
||||
)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,59 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.query_transformer
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam
|
||||
import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer._
|
||||
import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.timelineranker.{model => tlr}
|
||||
import com.twitter.timelineranker.{thriftscala => t}
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
import com.twitter.timelines.earlybird.common.options.EarlybirdScoringModelConfig
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.timelines.model.candidate.CandidateTweetSourceId
|
||||
|
||||
object TimelineRankerUtegQueryTransformer {
|
||||
private val SinceDuration = 24.hours
|
||||
private val MaxTweetsToFetch = 300
|
||||
private val MaxUtegCandidates = 800
|
||||
|
||||
private val tweetKindOptions =
|
||||
TweetKindOption(includeOriginalTweetsAndQuotes = true, includeReplies = true)
|
||||
|
||||
def utegEarlybirdModels: Seq[EarlybirdScoringModelConfig] =
|
||||
EarlybirdRequestUtil.EarlybirdScoringModels.UnifiedEngagementRectweet
|
||||
}
|
||||
|
||||
case class TimelineRankerUtegQueryTransformer[
|
||||
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
|
||||
](
|
||||
override val candidatePipelineIdentifier: CandidatePipelineIdentifier,
|
||||
override val maxTweetsToFetch: Int = MaxTweetsToFetch)
|
||||
extends CandidatePipelineQueryTransformer[Query, t.UtegLikedByTweetsQuery]
|
||||
with TimelineRankerQueryTransformer[Query] {
|
||||
|
||||
override val candidateTweetSourceId = CandidateTweetSourceId.RecommendedTweet
|
||||
override val options = tweetKindOptions
|
||||
override val earlybirdModels = utegEarlybirdModels
|
||||
override def getTensorflowModel(query: Query): Option[String] = {
|
||||
Some(query.params(ScoredTweetsParam.EarlybirdTensorflowModel.UtegParam))
|
||||
}
|
||||
|
||||
override def utegLikedByTweetsOptions(input: Query): Option[tlr.UtegLikedByTweetsOptions] = Some(
|
||||
tlr.UtegLikedByTweetsOptions(
|
||||
utegCount = MaxUtegCandidates,
|
||||
isInNetwork = false,
|
||||
weightedFollowings = input.features
|
||||
.map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[UserId, Double]))
|
||||
.getOrElse(Map.empty)
|
||||
)
|
||||
)
|
||||
|
||||
override def transform(input: Query): t.UtegLikedByTweetsQuery =
|
||||
buildTimelineRankerQuery(input, SinceDuration).toThriftUtegLikedByTweetsQuery
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
|
||||
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"timelines:util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
|
||||
"timelines/src/main/scala/com/twitter/timelines/common/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/candidate",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,41 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUserIdsFeature
|
||||
import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer._
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.search.earlybird.{thriftscala => eb}
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
|
||||
object EarlybirdFrsQueryTransformer {
|
||||
private val SinceDuration = 24.hours
|
||||
private val MaxTweetsToFetch = 100
|
||||
private val TensorflowModel = Some("timelines_rectweet_replica")
|
||||
|
||||
private val TweetKindOptions: TweetKindOption.ValueSet =
|
||||
TweetKindOption(includeOriginalTweetsAndQuotes = true)
|
||||
}
|
||||
|
||||
case class EarlybirdFrsQueryTransformer[
|
||||
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
|
||||
](
|
||||
candidatePipelineIdentifier: CandidatePipelineIdentifier,
|
||||
override val clientId: Option[String])
|
||||
extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest]
|
||||
with EarlybirdQueryTransformer[Query] {
|
||||
|
||||
override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions
|
||||
override val maxTweetsToFetch: Int = MaxTweetsToFetch
|
||||
override val tensorflowModel: Option[String] = TensorflowModel
|
||||
|
||||
override def transform(query: Query): eb.EarlybirdRequest = {
|
||||
val seedUserIds = query.features
|
||||
.flatMap(_.getOrElse(FrsSeedUserIdsFeature, None))
|
||||
.getOrElse(Seq.empty).toSet
|
||||
buildEarlybirdQuery(query, SinceDuration, seedUserIds)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,68 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.core_workflows.user_model.{thriftscala => um}
|
||||
import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer._
|
||||
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.search.earlybird.{thriftscala => eb}
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
|
||||
object EarlybirdInNetworkQueryTransformer {
|
||||
private val DefaultSinceDuration = 24.hours
|
||||
private val ExpandedSinceDuration = 48.hours
|
||||
private val MaxTweetsToFetch = 600
|
||||
private val TensorflowModel = Some("timelines_recap_replica")
|
||||
|
||||
private val TweetKindOptions: TweetKindOption.ValueSet = TweetKindOption(
|
||||
includeReplies = true,
|
||||
includeRetweets = true,
|
||||
includeOriginalTweetsAndQuotes = true,
|
||||
includeExtendedReplies = true
|
||||
)
|
||||
|
||||
private val UserStatesForExtendedSinceDuration: Set[um.UserState] = Set(
|
||||
um.UserState.Light,
|
||||
um.UserState.MediumNonTweeter,
|
||||
um.UserState.MediumTweeter,
|
||||
um.UserState.NearZero,
|
||||
um.UserState.New,
|
||||
um.UserState.VeryLight
|
||||
)
|
||||
}
|
||||
|
||||
case class EarlybirdInNetworkQueryTransformer[
|
||||
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext
|
||||
](
|
||||
candidatePipelineIdentifier: CandidatePipelineIdentifier,
|
||||
override val clientId: Option[String])
|
||||
extends CandidatePipelineQueryTransformer[Query, eb.EarlybirdRequest]
|
||||
with EarlybirdQueryTransformer[Query] {
|
||||
|
||||
override val tweetKindOptions: TweetKindOption.ValueSet = TweetKindOptions
|
||||
override val maxTweetsToFetch: Int = MaxTweetsToFetch
|
||||
override val tensorflowModel: Option[String] = TensorflowModel
|
||||
|
||||
override def transform(query: Query): eb.EarlybirdRequest = {
|
||||
|
||||
val userState = query.features.get.getOrElse(UserStateFeature, None)
|
||||
|
||||
val sinceDuration =
|
||||
if (userState.exists(UserStatesForExtendedSinceDuration.contains)) ExpandedSinceDuration
|
||||
else DefaultSinceDuration
|
||||
|
||||
val followedUserIds =
|
||||
query.features
|
||||
.map(
|
||||
_.getOrElse(
|
||||
SGSFollowedUsersFeature,
|
||||
Seq.empty)).toSeq.flatten.toSet + query.getRequiredUserId
|
||||
|
||||
buildEarlybirdQuery(query, sinceDuration, followedUserIds)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,70 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature
|
||||
import com.twitter.home_mixer.model.request.HasDeviceContext
|
||||
import com.twitter.home_mixer.util.CachedScoredTweetsHelper
|
||||
import com.twitter.home_mixer.util.earlybird.EarlybirdRequestUtil
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.quality_factor.HasQualityFactorStatus
|
||||
import com.twitter.search.earlybird.{thriftscala => eb}
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
import com.twitter.timelines.util.SnowflakeSortIndexHelper
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Time
|
||||
|
||||
trait EarlybirdQueryTransformer[
|
||||
Query <: PipelineQuery with HasQualityFactorStatus with HasDeviceContext] {
|
||||
|
||||
def candidatePipelineIdentifier: CandidatePipelineIdentifier
|
||||
def clientId: Option[String] = None
|
||||
def maxTweetsToFetch: Int = 100
|
||||
def tweetKindOptions: TweetKindOption.ValueSet
|
||||
def tensorflowModel: Option[String] = None
|
||||
|
||||
private val EarlybirdMaxExcludedTweets = 1500
|
||||
|
||||
def buildEarlybirdQuery(
|
||||
query: Query,
|
||||
sinceDuration: Duration,
|
||||
followedUserIds: Set[Long] = Set.empty
|
||||
): eb.EarlybirdRequest = {
|
||||
val sinceTime: Time = sinceDuration.ago
|
||||
val untilTime: Time = Time.now
|
||||
|
||||
val fromTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(sinceTime)
|
||||
val toTweetIdExclusive = SnowflakeSortIndexHelper.timestampToFakeId(untilTime)
|
||||
|
||||
val excludedTweetIds = query.features.map { featureMap =>
|
||||
CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweetsInRange(
|
||||
featureMap,
|
||||
candidatePipelineIdentifier,
|
||||
EarlybirdMaxExcludedTweets,
|
||||
sinceTime,
|
||||
untilTime)
|
||||
}
|
||||
|
||||
val maxCount =
|
||||
(query.getQualityFactorCurrentValue(candidatePipelineIdentifier) * maxTweetsToFetch).toInt
|
||||
|
||||
val authorScoreMap = query.features
|
||||
.map(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]))
|
||||
.getOrElse(Map.empty)
|
||||
|
||||
EarlybirdRequestUtil.getTweetsRequest(
|
||||
userId = Some(query.getRequiredUserId),
|
||||
clientId = clientId,
|
||||
skipVeryRecentTweets = true,
|
||||
followedUserIds = followedUserIds,
|
||||
retweetsMutedUserIds = Set.empty,
|
||||
beforeTweetIdExclusive = Some(toTweetIdExclusive),
|
||||
afterTweetIdExclusive = Some(fromTweetIdExclusive),
|
||||
excludedTweetIds = excludedTweetIds.map(_.toSet),
|
||||
maxCount = maxCount,
|
||||
tweetTypes = TweetTypes.fromTweetKindOption(tweetKindOptions),
|
||||
authorScoreMap = Some(authorScoreMap),
|
||||
tensorflowModel = tensorflowModel
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
dependencies = [
|
||||
"explore/explore-ranker/thrift/src/main/thrift:thrift-scala",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
|
||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
||||
"topic-social-proof/server/src/main/thrift:thrift-scala",
|
||||
"tweet-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,119 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.home_mixer.marshaller.timelines.TopicContextFunctionalityTypeUnmarshaller
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGoldVerifiedFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsGrayVerifiedFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsLegacyVerifiedFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CachedCandidatePipelineIdentifierFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.LastScoredTimestampMsFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature
|
||||
import com.twitter.home_mixer.{thriftscala => hmt}
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
|
||||
object CachedScoredTweetsResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[hmt.ScoredTweet] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("CachedScoredTweetsResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
AncestorsFeature,
|
||||
AuthorIdFeature,
|
||||
AuthorIsBlueVerifiedFeature,
|
||||
AuthorIsCreatorFeature,
|
||||
AuthorIsGoldVerifiedFeature,
|
||||
AuthorIsGrayVerifiedFeature,
|
||||
AuthorIsLegacyVerifiedFeature,
|
||||
CachedCandidatePipelineIdentifierFeature,
|
||||
DirectedAtUserIdFeature,
|
||||
ExclusiveConversationAuthorIdFeature,
|
||||
InNetworkFeature,
|
||||
InReplyToTweetIdFeature,
|
||||
InReplyToUserIdFeature,
|
||||
IsReadFromCacheFeature,
|
||||
IsRetweetFeature,
|
||||
LastScoredTimestampMsFeature,
|
||||
PerspectiveFilteredLikedByUserIdsFeature,
|
||||
QuotedTweetIdFeature,
|
||||
QuotedUserIdFeature,
|
||||
SGSValidFollowedByUserIdsFeature,
|
||||
SGSValidLikedByUserIdsFeature,
|
||||
ScoreFeature,
|
||||
SourceTweetIdFeature,
|
||||
SourceUserIdFeature,
|
||||
StreamToKafkaFeature,
|
||||
SuggestTypeFeature,
|
||||
TopicContextFunctionalityTypeFeature,
|
||||
TopicIdSocialContextFeature,
|
||||
TweetUrlsFeature,
|
||||
WeightedModelScoreFeature
|
||||
)
|
||||
|
||||
override def transform(candidate: hmt.ScoredTweet): FeatureMap =
|
||||
FeatureMapBuilder()
|
||||
.add(AncestorsFeature, candidate.ancestors.getOrElse(Seq.empty))
|
||||
.add(AuthorIdFeature, Some(candidate.authorId))
|
||||
.add(AuthorIsBlueVerifiedFeature, candidate.authorMetadata.exists(_.blueVerified))
|
||||
.add(AuthorIsGoldVerifiedFeature, candidate.authorMetadata.exists(_.goldVerified))
|
||||
.add(AuthorIsGrayVerifiedFeature, candidate.authorMetadata.exists(_.grayVerified))
|
||||
.add(AuthorIsLegacyVerifiedFeature, candidate.authorMetadata.exists(_.legacyVerified))
|
||||
.add(AuthorIsCreatorFeature, candidate.authorMetadata.exists(_.creator))
|
||||
.add(CachedCandidatePipelineIdentifierFeature, candidate.candidatePipelineIdentifier)
|
||||
.add(DirectedAtUserIdFeature, candidate.directedAtUserId)
|
||||
.add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId)
|
||||
.add(InNetworkFeature, candidate.inNetwork.getOrElse(true))
|
||||
.add(InReplyToTweetIdFeature, candidate.inReplyToTweetId)
|
||||
.add(InReplyToUserIdFeature, candidate.inReplyToUserId)
|
||||
.add(IsReadFromCacheFeature, true)
|
||||
.add(IsRetweetFeature, candidate.sourceTweetId.isDefined)
|
||||
.add(LastScoredTimestampMsFeature, candidate.lastScoredTimestampMs)
|
||||
.add(
|
||||
PerspectiveFilteredLikedByUserIdsFeature,
|
||||
candidate.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty))
|
||||
.add(QuotedTweetIdFeature, candidate.quotedTweetId)
|
||||
.add(QuotedUserIdFeature, candidate.quotedUserId)
|
||||
.add(ScoreFeature, candidate.score)
|
||||
.add(SGSValidLikedByUserIdsFeature, candidate.sgsValidLikedByUserIds.getOrElse(Seq.empty))
|
||||
.add(
|
||||
SGSValidFollowedByUserIdsFeature,
|
||||
candidate.sgsValidFollowedByUserIds.getOrElse(Seq.empty))
|
||||
.add(SourceTweetIdFeature, candidate.sourceTweetId)
|
||||
.add(SourceUserIdFeature, candidate.sourceUserId)
|
||||
.add(StreamToKafkaFeature, false)
|
||||
.add(SuggestTypeFeature, candidate.suggestType)
|
||||
.add(
|
||||
TopicContextFunctionalityTypeFeature,
|
||||
candidate.topicFunctionalityType.map(TopicContextFunctionalityTypeUnmarshaller(_)))
|
||||
.add(TopicIdSocialContextFeature, candidate.topicId)
|
||||
.add(TweetUrlsFeature, candidate.tweetUrls.getOrElse(Seq.empty))
|
||||
.add(WeightedModelScoreFeature, candidate.score)
|
||||
.build()
|
||||
}
|
||||
Binary file not shown.
@ -1,30 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsBackfillResponseFeatureTransformer extends CandidateFeatureTransformer[Long] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ScoredTweetsBackfillResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
CandidateSourceIdFeature,
|
||||
FromInNetworkSourceFeature,
|
||||
SuggestTypeFeature
|
||||
)
|
||||
|
||||
override def transform(candidate: Long): FeatureMap = FeatureMapBuilder()
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.BackfillOrganicTweet))
|
||||
.add(FromInNetworkSourceFeature, true)
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.RankedOrganicTweet))
|
||||
.build()
|
||||
}
|
||||
Binary file not shown.
@ -1,31 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineranker.{thriftscala => tlr}
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsFrsResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[tlr.CandidateTweet] {
|
||||
|
||||
override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsFrsResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features
|
||||
|
||||
override def transform(candidate: tlr.CandidateTweet): FeatureMap = {
|
||||
val baseFeatures = TimelineRankerResponseTransformer.transform(candidate)
|
||||
|
||||
val features = FeatureMapBuilder()
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.FrsTweet))
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.FrsTweet))
|
||||
.build()
|
||||
|
||||
baseFeatures ++ features
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,34 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineranker.{thriftscala => tlr}
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsInNetworkResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[tlr.CandidateTweet] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ScoredTweetsInNetworkResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features
|
||||
|
||||
override def transform(candidate: tlr.CandidateTweet): FeatureMap = {
|
||||
val baseFeatures = TimelineRankerResponseTransformer.transform(candidate)
|
||||
|
||||
val features = FeatureMapBuilder()
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet))
|
||||
.add(FromInNetworkSourceFeature, true)
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.RankedTimelineTweet))
|
||||
.build()
|
||||
|
||||
baseFeatures ++ features
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,45 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineservice.{thriftscala => t}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
|
||||
object ScoredTweetsListsResponseFeatureTransformer extends CandidateFeatureTransformer[t.Tweet] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ScoredTweetsListsResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
AuthorIdFeature,
|
||||
CandidateSourceIdFeature,
|
||||
FromInNetworkSourceFeature,
|
||||
IsRetweetFeature,
|
||||
SuggestTypeFeature,
|
||||
SourceTweetIdFeature,
|
||||
SourceUserIdFeature,
|
||||
)
|
||||
|
||||
override def transform(candidate: t.Tweet): FeatureMap = {
|
||||
FeatureMapBuilder()
|
||||
.add(AuthorIdFeature, candidate.userId)
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.ListTweet))
|
||||
.add(FromInNetworkSourceFeature, false)
|
||||
.add(IsRetweetFeature, candidate.sourceStatusId.isDefined)
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.RankedListTweet))
|
||||
.add(SourceTweetIdFeature, candidate.sourceStatusId)
|
||||
.add(SourceUserIdFeature, candidate.sourceUserId)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,46 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.explore_ranker.{thriftscala => ert}
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsPopularVideosResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[ert.ExploreTweetRecommendation] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ScoredTweetsPopularVideosResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
AuthorIdFeature,
|
||||
CandidateSourceIdFeature,
|
||||
FromInNetworkSourceFeature,
|
||||
HasVideoFeature,
|
||||
IsRandomTweetFeature,
|
||||
StreamToKafkaFeature,
|
||||
SuggestTypeFeature
|
||||
)
|
||||
|
||||
override def transform(candidate: ert.ExploreTweetRecommendation): FeatureMap = {
|
||||
FeatureMapBuilder()
|
||||
.add(AuthorIdFeature, candidate.authorId)
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.MediaTweet))
|
||||
.add(FromInNetworkSourceFeature, false)
|
||||
.add(HasVideoFeature, candidate.mediaType.contains(ert.MediaType.Video))
|
||||
.add(IsRandomTweetFeature, false)
|
||||
.add(StreamToKafkaFeature, true)
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.MediaTweet))
|
||||
.build()
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,52 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.tweet_mixer.{thriftscala => tmt}
|
||||
import com.twitter.home_mixer.model.HomeFeatures._
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
import com.twitter.tsp.{thriftscala => tsp}
|
||||
|
||||
object ScoredTweetsTweetMixerResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[tmt.TweetResult] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ScoredTweetsTweetMixerResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
CandidateSourceIdFeature,
|
||||
FromInNetworkSourceFeature,
|
||||
IsRandomTweetFeature,
|
||||
StreamToKafkaFeature,
|
||||
SuggestTypeFeature,
|
||||
TSPMetricTagFeature
|
||||
)
|
||||
|
||||
override def transform(candidate: tmt.TweetResult): FeatureMap = {
|
||||
val tweetMixerMetricTags = candidate.metricTags.getOrElse(Seq.empty)
|
||||
val tspMetricTag = tweetMixerMetricTags
|
||||
.map(TweetMixerMetricTagToTspMetricTag)
|
||||
.filter(_.nonEmpty).map(_.get).toSet
|
||||
|
||||
FeatureMapBuilder()
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.Simcluster))
|
||||
.add(FromInNetworkSourceFeature, false)
|
||||
.add(IsRandomTweetFeature, false)
|
||||
.add(StreamToKafkaFeature, true)
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.ScTweet))
|
||||
.add(TSPMetricTagFeature, tspMetricTag)
|
||||
.build()
|
||||
}
|
||||
|
||||
private def TweetMixerMetricTagToTspMetricTag(
|
||||
tweetMixerMetricTag: tmt.MetricTag
|
||||
): Option[tsp.MetricTag] = tweetMixerMetricTag match {
|
||||
case tmt.MetricTag.TweetFavorite => Some(tsp.MetricTag.TweetFavorite)
|
||||
case tmt.MetricTag.Retweet => Some(tsp.MetricTag.Retweet)
|
||||
case _ => None
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,31 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.timelineranker.{thriftscala => tlr}
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsUtegResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[tlr.CandidateTweet] {
|
||||
|
||||
override val identifier: TransformerIdentifier = TransformerIdentifier("ScoredTweetsUtegResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = TimelineRankerResponseTransformer.features
|
||||
|
||||
override def transform(candidate: tlr.CandidateTweet): FeatureMap = {
|
||||
val baseFeatures = TimelineRankerResponseTransformer.transform(candidate)
|
||||
|
||||
val features = FeatureMapBuilder()
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecommendedTweet))
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.ActivityTweet))
|
||||
.build()
|
||||
|
||||
baseFeatures ++ features
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,91 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature
|
||||
import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.timelineranker.{thriftscala => tlr}
|
||||
|
||||
object TimelineRankerResponseTransformer {
|
||||
|
||||
val features: Set[Feature[_, _]] = Set(
|
||||
AuthorIdFeature,
|
||||
CandidateSourceIdFeature,
|
||||
DirectedAtUserIdFeature,
|
||||
EarlybirdFeature,
|
||||
EarlybirdScoreFeature,
|
||||
ExclusiveConversationAuthorIdFeature,
|
||||
FromInNetworkSourceFeature,
|
||||
HasImageFeature,
|
||||
HasVideoFeature,
|
||||
InReplyToTweetIdFeature,
|
||||
InReplyToUserIdFeature,
|
||||
IsRandomTweetFeature,
|
||||
IsRetweetFeature,
|
||||
MentionScreenNameFeature,
|
||||
MentionUserIdFeature,
|
||||
StreamToKafkaFeature,
|
||||
QuotedTweetIdFeature,
|
||||
QuotedUserIdFeature,
|
||||
SourceTweetIdFeature,
|
||||
SourceUserIdFeature,
|
||||
SuggestTypeFeature,
|
||||
TweetUrlsFeature
|
||||
)
|
||||
|
||||
def transform(candidate: tlr.CandidateTweet): FeatureMap = {
|
||||
val tweet = candidate.tweet
|
||||
val quotedTweet = tweet.filter(_.quotedTweet.exists(_.tweetId != 0)).flatMap(_.quotedTweet)
|
||||
val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty)
|
||||
val coreData = tweet.flatMap(_.coreData)
|
||||
val share = coreData.flatMap(_.share)
|
||||
val reply = coreData.flatMap(_.reply)
|
||||
|
||||
FeatureMapBuilder()
|
||||
.add(AuthorIdFeature, coreData.map(_.userId))
|
||||
.add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId)))
|
||||
.add(EarlybirdFeature, candidate.features)
|
||||
.add(EarlybirdScoreFeature, candidate.features.map(_.earlybirdScore))
|
||||
.add(
|
||||
ExclusiveConversationAuthorIdFeature,
|
||||
tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId)))
|
||||
.add(FromInNetworkSourceFeature, false)
|
||||
.add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage))
|
||||
.add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo))
|
||||
.add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId))
|
||||
.add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId))
|
||||
.add(IsRandomTweetFeature, candidate.features.exists(_.isRandomTweet.getOrElse(false)))
|
||||
.add(IsRetweetFeature, share.isDefined)
|
||||
.add(MentionScreenNameFeature, mentions.map(_.screenName))
|
||||
.add(MentionUserIdFeature, mentions.flatMap(_.userId))
|
||||
.add(StreamToKafkaFeature, true)
|
||||
.add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId))
|
||||
.add(QuotedUserIdFeature, quotedTweet.map(_.userId))
|
||||
.add(SourceTweetIdFeature, share.map(_.sourceStatusId))
|
||||
.add(SourceUserIdFeature, share.map(_.sourceUserId))
|
||||
.add(TweetUrlsFeature, candidate.features.flatMap(_.urlsList).getOrElse(Seq.empty))
|
||||
.build()
|
||||
}
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
|
||||
"home-mixer/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,92 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdScoreFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRandomTweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature
|
||||
import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.search.earlybird.{thriftscala => eb}
|
||||
|
||||
object EarlybirdResponseTransformer {
|
||||
|
||||
val features: Set[Feature[_, _]] = Set(
|
||||
AuthorIdFeature,
|
||||
CandidateSourceIdFeature,
|
||||
DirectedAtUserIdFeature,
|
||||
EarlybirdScoreFeature,
|
||||
EarlybirdSearchResultFeature,
|
||||
ExclusiveConversationAuthorIdFeature,
|
||||
FromInNetworkSourceFeature,
|
||||
HasImageFeature,
|
||||
HasVideoFeature,
|
||||
InReplyToTweetIdFeature,
|
||||
InReplyToUserIdFeature,
|
||||
IsRandomTweetFeature,
|
||||
IsRetweetFeature,
|
||||
MentionScreenNameFeature,
|
||||
MentionUserIdFeature,
|
||||
StreamToKafkaFeature,
|
||||
QuotedTweetIdFeature,
|
||||
QuotedUserIdFeature,
|
||||
SourceTweetIdFeature,
|
||||
SourceUserIdFeature,
|
||||
SuggestTypeFeature,
|
||||
TweetUrlsFeature
|
||||
)
|
||||
|
||||
def transform(candidate: eb.ThriftSearchResult): FeatureMap = {
|
||||
val tweet = candidate.tweetypieTweet
|
||||
val quotedTweet = tweet.flatMap(_.quotedTweet)
|
||||
val mentions = tweet.flatMap(_.mentions).getOrElse(Seq.empty)
|
||||
val coreData = tweet.flatMap(_.coreData)
|
||||
val share = coreData.flatMap(_.share)
|
||||
val reply = coreData.flatMap(_.reply)
|
||||
FeatureMapBuilder()
|
||||
.add(AuthorIdFeature, coreData.map(_.userId))
|
||||
.add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId)))
|
||||
.add(EarlybirdSearchResultFeature, Some(candidate))
|
||||
.add(EarlybirdScoreFeature, candidate.metadata.flatMap(_.score))
|
||||
.add(
|
||||
ExclusiveConversationAuthorIdFeature,
|
||||
tweet.flatMap(_.exclusiveTweetControl.map(_.conversationAuthorId)))
|
||||
.add(FromInNetworkSourceFeature, false)
|
||||
.add(HasImageFeature, tweet.exists(TweetMediaFeaturesExtractor.hasImage))
|
||||
.add(HasVideoFeature, tweet.exists(TweetMediaFeaturesExtractor.hasVideo))
|
||||
.add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId))
|
||||
.add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId))
|
||||
.add(IsRandomTweetFeature, candidate.tweetFeatures.exists(_.isRandomTweet.getOrElse(false)))
|
||||
.add(IsRetweetFeature, share.isDefined)
|
||||
.add(MentionScreenNameFeature, mentions.map(_.screenName))
|
||||
.add(MentionUserIdFeature, mentions.flatMap(_.userId))
|
||||
.add(StreamToKafkaFeature, true)
|
||||
.add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId))
|
||||
.add(QuotedUserIdFeature, quotedTweet.map(_.userId))
|
||||
.add(SourceTweetIdFeature, share.map(_.sourceStatusId))
|
||||
.add(SourceUserIdFeature, share.map(_.sourceUserId))
|
||||
.add(
|
||||
TweetUrlsFeature,
|
||||
candidate.metadata.flatMap(_.tweetUrls.map(_.map(_.originalUrl))).getOrElse(Seq.empty))
|
||||
.build()
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,33 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.search.earlybird.{thriftscala => eb}
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsEarlybirdFrsResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[eb.ThriftSearchResult] {
|
||||
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ScoredTweetsEarlybirdFrsResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features
|
||||
|
||||
override def transform(candidate: eb.ThriftSearchResult): FeatureMap = {
|
||||
|
||||
val baseFeatures = EarlybirdResponseTransformer.transform(candidate)
|
||||
|
||||
val features = FeatureMapBuilder()
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.FrsTweet))
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.FrsTweet))
|
||||
.build()
|
||||
|
||||
baseFeatures ++ features
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,32 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer
|
||||
import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier
|
||||
import com.twitter.search.earlybird.{thriftscala => eb}
|
||||
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => cts}
|
||||
import com.twitter.timelineservice.suggests.{thriftscala => st}
|
||||
|
||||
object ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer
|
||||
extends CandidateFeatureTransformer[eb.ThriftSearchResult] {
|
||||
override val identifier: TransformerIdentifier =
|
||||
TransformerIdentifier("ScoredTweetsEarlybirdInNetworkResponse")
|
||||
|
||||
override val features: Set[Feature[_, _]] = EarlybirdResponseTransformer.features
|
||||
|
||||
override def transform(candidate: eb.ThriftSearchResult): FeatureMap = {
|
||||
|
||||
val baseFeatures = EarlybirdResponseTransformer.transform(candidate)
|
||||
|
||||
val features = FeatureMapBuilder()
|
||||
.add(CandidateSourceIdFeature, Some(cts.CandidateTweetSourceId.RecycledTweet))
|
||||
.add(SuggestTypeFeature, Some(st.SuggestType.RecycledTweet))
|
||||
.build()
|
||||
|
||||
baseFeatures ++ features
|
||||
}
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
dependencies = [
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
|
||||
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
|
||||
"src/scala/com/twitter/timelines/prediction/features/recap",
|
||||
"timelineservice/common:model",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
@ -1,63 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.scorer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
|
||||
trait DiversityDiscountProvider {
|
||||
|
||||
/**
|
||||
* Fetch the ID of the entity to diversify
|
||||
*/
|
||||
def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long]
|
||||
|
||||
/**
|
||||
* Compute discount factor for each candidate based on position (zero-based)
|
||||
* relative to other candidates associated with the same entity
|
||||
*/
|
||||
def discount(position: Int): Double
|
||||
|
||||
/**
|
||||
* Return candidate IDs sorted by score in descending order
|
||||
*/
|
||||
def sort(candidates: Seq[CandidateWithFeatures[TweetCandidate]]): Seq[Long] = candidates
|
||||
.map { candidate =>
|
||||
(candidate.candidate.id, candidate.features.getOrElse(ScoreFeature, None).getOrElse(0.0))
|
||||
}
|
||||
.sortBy(_._2)(Ordering.Double.reverse)
|
||||
.map(_._1)
|
||||
|
||||
/**
|
||||
* Group by the specified entity ID (e.g. authors, likers, followers)
|
||||
* Sort each group by score in descending order
|
||||
* Determine the discount factor based on the position of each candidate
|
||||
*/
|
||||
def apply(
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Map[Long, Double] = candidates
|
||||
.groupBy(entityId)
|
||||
.flatMap {
|
||||
case (entityIdOpt, entityCandidates) =>
|
||||
val sortedCandidateIds = sort(entityCandidates)
|
||||
|
||||
if (entityIdOpt.isDefined) {
|
||||
sortedCandidateIds.zipWithIndex.map {
|
||||
case (candidateId, index) =>
|
||||
candidateId -> discount(index)
|
||||
}
|
||||
} else sortedCandidateIds.map(_ -> 1.0)
|
||||
}
|
||||
}
|
||||
|
||||
object AuthorDiversityDiscountProvider extends DiversityDiscountProvider {
|
||||
private val Decay = 0.5
|
||||
private val Floor = 0.25
|
||||
|
||||
override def entityId(candidate: CandidateWithFeatures[TweetCandidate]): Option[Long] =
|
||||
candidate.features.getOrElse(AuthorIdFeature, None)
|
||||
|
||||
// Provides an exponential decay based discount by position (with a floor)
|
||||
override def discount(position: Int): Double =
|
||||
(1 - Floor) * Math.pow(Decay, position) + Floor
|
||||
}
|
||||
Binary file not shown.
@ -1,46 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.scorer
|
||||
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||
import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* Apply various heuristics to the model score
|
||||
*/
|
||||
object HeuristicScorer extends Scorer[ScoredTweetsQuery, TweetCandidate] {
|
||||
|
||||
override val identifier: ScorerIdentifier = ScorerIdentifier("Heuristic")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(ScoreFeature)
|
||||
|
||||
override def apply(
|
||||
query: ScoredTweetsQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Stitch[Seq[FeatureMap]] = {
|
||||
val rescorers = Seq(
|
||||
RescoreOutOfNetwork,
|
||||
RescoreReplies,
|
||||
RescoreBlueVerified,
|
||||
RescoreCreators,
|
||||
RescoreMTLNormalization,
|
||||
RescoreAuthorDiversity(AuthorDiversityDiscountProvider(candidates)),
|
||||
RescoreFeedbackFatigue(query)
|
||||
)
|
||||
|
||||
val updatedScores = candidates.map { candidate =>
|
||||
val score = candidate.features.getOrElse(ScoreFeature, None)
|
||||
val scaleFactor = rescorers.map(_(query, candidate)).product
|
||||
val updatedScore = score.map(_ * scaleFactor)
|
||||
FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
|
||||
}
|
||||
|
||||
Stitch.value(updatedScores)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@ -1,179 +0,0 @@
|
||||
package com.twitter.home_mixer.product.scored_tweets.scorer
|
||||
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
|
||||
import com.twitter.home_mixer.model.HomeFeatures.WeightedModelScoreFeature
|
||||
import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery
|
||||
import com.twitter.home_mixer.product.scored_tweets.scorer.PredictedScoreFeature.PredictedScoreFeatures
|
||||
import com.twitter.ml.api.DataRecord
|
||||
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
|
||||
import com.twitter.product_mixer.core.feature.Feature
|
||||
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
|
||||
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
|
||||
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
|
||||
import com.twitter.product_mixer.core.feature.featuremap.datarecord.AllFeatures
|
||||
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordConverter
|
||||
import com.twitter.product_mixer.core.feature.featuremap.datarecord.DataRecordExtractor
|
||||
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
|
||||
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
|
||||
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
|
||||
import com.twitter.product_mixer.core.pipeline.PipelineQuery
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.IllegalStateFailure
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
import com.twitter.product_mixer.core.util.OffloadFuturePools
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.clients.predictionservice.PredictionGRPCService
|
||||
import com.twitter.timelines.clients.predictionservice.PredictionServiceGRPCClient
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Return
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
object CommonFeaturesDataRecordFeature
|
||||
extends DataRecordInAFeature[PipelineQuery]
|
||||
with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] {
|
||||
override def defaultValue: DataRecord = new DataRecord()
|
||||
}
|
||||
|
||||
object CandidateFeaturesDataRecordFeature
|
||||
extends DataRecordInAFeature[TweetCandidate]
|
||||
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
|
||||
override def defaultValue: DataRecord = new DataRecord()
|
||||
}
|
||||
|
||||
@Singleton
|
||||
case class NaviModelScorer @Inject() (
|
||||
predictionGRPCService: PredictionGRPCService,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends Scorer[ScoredTweetsQuery, TweetCandidate] {
|
||||
|
||||
override val identifier: ScorerIdentifier = ScorerIdentifier("NaviModel")
|
||||
|
||||
override val features: Set[Feature[_, _]] = Set(
|
||||
CommonFeaturesDataRecordFeature,
|
||||
CandidateFeaturesDataRecordFeature,
|
||||
WeightedModelScoreFeature,
|
||||
ScoreFeature
|
||||
) ++ PredictedScoreFeatures.asInstanceOf[Set[Feature[_, _]]]
|
||||
|
||||
private val queryDataRecordAdapter = new DataRecordConverter(AllFeatures())
|
||||
private val candidatesDataRecordAdapter = new DataRecordConverter(AllFeatures())
|
||||
private val resultDataRecordExtractor = new DataRecordExtractor(PredictedScoreFeatures)
|
||||
|
||||
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
||||
private val failuresStat = scopedStatsReceiver.stat("failures")
|
||||
private val responsesStat = scopedStatsReceiver.stat("responses")
|
||||
private val invalidResponsesCounter = scopedStatsReceiver.counter("invalidResponses")
|
||||
private val candidatesDataRecordAdapterLatencyStat =
|
||||
scopedStatsReceiver.scope("candidatesDataRecordAdapter").stat("latency_ms")
|
||||
|
||||
private val StatsReadabilityMultiplier = 1000
|
||||
private val Epsilon = 0.001
|
||||
private val PredictedScoreStatName = f"predictedScore${StatsReadabilityMultiplier}x"
|
||||
private val MissingScoreStatName = "missingScore"
|
||||
private val scoreStat = scopedStatsReceiver.stat(f"score${StatsReadabilityMultiplier}x")
|
||||
|
||||
private val RequestBatchSize = 64
|
||||
private val DataRecordConstructionParallelism = 32
|
||||
private val ModelId = "Home"
|
||||
|
||||
private val modelClient = new PredictionServiceGRPCClient(
|
||||
service = predictionGRPCService,
|
||||
statsReceiver = statsReceiver,
|
||||
requestBatchSize = RequestBatchSize,
|
||||
useCompact = false
|
||||
)
|
||||
|
||||
override def apply(
|
||||
query: ScoredTweetsQuery,
|
||||
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
|
||||
): Stitch[Seq[FeatureMap]] = {
|
||||
val commonRecord = query.features.map(queryDataRecordAdapter.toDataRecord)
|
||||
val candidateRecords: Future[Seq[DataRecord]] =
|
||||
Stat.time(candidatesDataRecordAdapterLatencyStat) {
|
||||
OffloadFuturePools.parallelize[FeatureMap, DataRecord](
|
||||
inputSeq = candidates.map(_.features),
|
||||
transformer = candidatesDataRecordAdapter.toDataRecord(_),
|
||||
parallelism = DataRecordConstructionParallelism,
|
||||
default = new DataRecord
|
||||
)
|
||||
}
|
||||
|
||||
val scoreFeatureMaps = candidateRecords.flatMap { records =>
|
||||
val predictionResponses =
|
||||
modelClient.getPredictions(records, commonRecord, modelId = Some(ModelId))
|
||||
|
||||
predictionResponses.map { responses =>
|
||||
failuresStat.add(responses.count(_.isThrow))
|
||||
responsesStat.add(responses.size)
|
||||
|
||||
if (responses.size == candidates.size) {
|
||||
val predictedScoreFeatureMaps = responses.map {
|
||||
case Return(dataRecord) => resultDataRecordExtractor.fromDataRecord(dataRecord)
|
||||
case _ => resultDataRecordExtractor.fromDataRecord(new DataRecord())
|
||||
}
|
||||
|
||||
// Add Data Record to candidate Feature Map for logging in later stages
|
||||
predictedScoreFeatureMaps.zip(records).map {
|
||||
case (predictedScoreFeatureMap, candidateRecord) =>
|
||||
val weightedModelScore = computeWeightedModelScore(query, predictedScoreFeatureMap)
|
||||
scoreStat.add((weightedModelScore * StatsReadabilityMultiplier).toFloat)
|
||||
|
||||
predictedScoreFeatureMap +
|
||||
(CandidateFeaturesDataRecordFeature, candidateRecord) +
|
||||
(CommonFeaturesDataRecordFeature, commonRecord.getOrElse(new DataRecord())) +
|
||||
(ScoreFeature, Some(weightedModelScore)) +
|
||||
(WeightedModelScoreFeature, Some(weightedModelScore))
|
||||
}
|
||||
} else {
|
||||
invalidResponsesCounter.incr()
|
||||
throw PipelineFailure(IllegalStateFailure, "Result size mismatched candidates size")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Stitch.callFuture(scoreFeatureMaps)
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the weighted sum of predicted scores of all engagements
|
||||
* Convert negative score to positive, if needed
|
||||
*/
|
||||
private def computeWeightedModelScore(
|
||||
query: PipelineQuery,
|
||||
features: FeatureMap
|
||||
): Double = {
|
||||
val weightedScoreAndModelWeightSeq = PredictedScoreFeatures.toSeq.map { predictedScoreFeature =>
|
||||
val predictedScoreOpt = predictedScoreFeature.extractScore(features)
|
||||
|
||||
predictedScoreOpt match {
|
||||
case Some(predictedScore) =>
|
||||
scopedStatsReceiver
|
||||
.stat(predictedScoreFeature.statName, PredictedScoreStatName)
|
||||
.add((predictedScore * StatsReadabilityMultiplier).toFloat)
|
||||
case None =>
|
||||
scopedStatsReceiver.counter(predictedScoreFeature.statName, MissingScoreStatName).incr()
|
||||
}
|
||||
|
||||
val weight = query.params(predictedScoreFeature.modelWeightParam)
|
||||
val weightedScore = predictedScoreOpt.getOrElse(0.0) * weight
|
||||
(weightedScore, weight)
|
||||
}
|
||||
|
||||
val (weightedScores, modelWeights) = weightedScoreAndModelWeightSeq.unzip
|
||||
val combinedScoreSum = weightedScores.sum
|
||||
|
||||
val positiveModelWeightsSum = modelWeights.filter(_ > 0.0).sum
|
||||
val negativeModelWeightsSum = modelWeights.filter(_ < 0).sum.abs
|
||||
val modelWeightsSum = positiveModelWeightsSum + negativeModelWeightsSum
|
||||
|
||||
val weightedScoresSum =
|
||||
if (modelWeightsSum == 0) combinedScoreSum.max(0.0)
|
||||
else if (combinedScoreSum < 0)
|
||||
(combinedScoreSum + negativeModelWeightsSum) / modelWeightsSum * Epsilon
|
||||
else combinedScoreSum + Epsilon
|
||||
|
||||
weightedScoresSum
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user