Compare commits

...

5 Commits

Author SHA1 Message Date
petemihaylov
5a7eb08ec3
Merge cbbf9e0a01 into 72eda9a24f 2023-07-17 21:40:11 -05:00
twitter-team
72eda9a24f [opensource] Update home mixer with latest changes 2023-07-13 16:33:04 +05:30
petemihaylov
cbbf9e0a01 Fix formatting and comments 2023-04-01 17:03:32 +02:00
petemihaylov
76ace4fea3
Fix comments in decider.yml 2023-04-01 15:34:38 +02:00
petemihaylov
65872b1963
Fix typos in CR-Mixer Readme.md 2023-04-01 15:30:54 +02:00
415 changed files with 9429 additions and 3971 deletions

View File

@ -1,6 +1,6 @@
# CR-Mixer
CR-Mixer is a candidate generation service proposed as part of the Personalization Strategy vision for Twitter. Its aim is to speed up the iteration and development of candidate generation and light ranking. The service acts as a lightweight coordinating layer that delegates candidate generation tasks to underlying compute services. It focuses on Twitter's candidate generation use cases and offers a centralized platform for fetching, mixing, and managing candidate sources and light rankers. The overarching goal is to increase the speed and ease of testing and developing candidate generation pipelines, ultimately delivering more value to Twitter users.
CR-Mixer is a candidate generation service proposed as part of the Personalization Strategy vision for Twitter. It aims to speed up the iteration and development of candidate generation and light ranking. The service acts as a lightweight coordinating layer that delegates candidate generation tasks to underlying compute services. It focuses on Twitter's candidate generation use cases and offers a centralized platform for fetching, mixing, and managing candidate sources and light rankers. The overarching goal is to increase the speed and ease of testing and developing candidate generation pipelines, ultimately delivering more value to Twitter users.
CR-Mixer acts as a configurator and delegator, providing abstractions for the challenging parts of candidate generation and handling performance issues. It will offer a 1-stop-shop for fetching and mixing candidate sources, a managed and shared performant platform, a light ranking layer, a common filtering layer, a version control system, a co-owned feature switch set, and peripheral tooling.

View File

@ -2,11 +2,11 @@
# https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/cr-mixer/server/src/main/scala/com/twitter/cr_mixer/param/decider/DeciderKey.scala
dark_traffic_filter:
comment: Proportion of the requests that are forwarded as dark traffic to the proxy
comment: "Proportion of the requests that are forwarded as dark traffic to the proxy"
default_availability: 0
enable_tweet_recommendations_home_product:
comment: Proportion of requests where we return an actual response for TweetRecommendations Home product
comment: "Proportion of requests where we return an actual response for TweetRecommendations Home product"
default_availability: 10000
enable_tweet_health_score:

View File

@ -26,18 +26,17 @@ case class AdsBlender @Inject() (globalStats: StatsReceiver) {
def blend(
inputCandidates: Seq[Seq[InitialAdsCandidate]],
): Future[Seq[BlendedAdsCandidate]] = {
// Filter out empty candidate sequence
// Filter out empty candidate sequence.
val candidates = inputCandidates.filter(_.nonEmpty)
val (interestedInCandidates, twistlyCandidates) =
candidates.partition(_.head.candidateGenerationInfo.sourceInfoOpt.isEmpty)
// First interleave twistly candidates
// Interleave twistly candidates.
val interleavedTwistlyCandidates = InterleaveUtil.interleave(twistlyCandidates)
val twistlyAndInterestedInCandidates =
Seq(interestedInCandidates.flatten, interleavedTwistlyCandidates)
// then interleave twistly candidates with interested in to make them even
// Interleave twistly candidates with interested in to make them even.
val interleavedCandidates = InterleaveUtil.interleave(twistlyAndInterestedInCandidates)
stats.stat("candidates").add(interleavedCandidates.size)
@ -45,6 +44,7 @@ case class AdsBlender @Inject() (globalStats: StatsReceiver) {
val blendedCandidates = buildBlendedAdsCandidate(inputCandidates, interleavedCandidates)
Future.value(blendedCandidates)
}
private def buildBlendedAdsCandidate(
inputCandidates: Seq[Seq[InitialAdsCandidate]],
interleavedCandidates: Seq[InitialAdsCandidate]
@ -73,5 +73,4 @@ case class AdsBlender @Inject() (globalStats: StatsReceiver) {
}
tweetIdMap.toMap
}
}

View File

@ -9,7 +9,7 @@ import scala.collection.mutable
object BlendedCandidatesBuilder {
/**
* @param inputCandidates input candidate prior to interleaving
* @param inputCandidates input candidate prior to interleaving.
* @param interleavedCandidates after interleaving. These tweets are de-duplicated.
*/
def build(
@ -23,8 +23,8 @@ object BlendedCandidatesBuilder {
}
/**
* This function tells you which CandidateGenerationInfo generated a given tweet.
* The same tweet can be generated by different sources.
* This function tells you which CandidateGenerationInfo generated a given tweet
*/
private def buildCandidateToCGInfosMap(
candidateSeq: Seq[Seq[InitialCandidate]],
@ -44,5 +44,4 @@ object BlendedCandidatesBuilder {
}
tweetIdMap.toMap
}
}

View File

@ -17,7 +17,7 @@ case class ContentSignalBlender @Inject() (globalStats: StatsReceiver) {
private val stats: StatsReceiver = globalStats.scope(name)
/**
* Exposes multiple types of sorting relying only on Content Based signals
* Exposes multiple types of sorting relying only on Content Based signals.
* Candidate Recency, Random, FavoriteCount and finally Standardized, which standardizes the scores
* that come from the active SimilarityEngine and then sort on the standardized scores.
*/
@ -25,7 +25,7 @@ case class ContentSignalBlender @Inject() (globalStats: StatsReceiver) {
params: Params,
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]] = {
// Filter out empty candidate sequence
// Filter out empty candidate sequence.
val candidates = inputCandidates.filter(_.nonEmpty)
val sortedCandidates = params(BlenderParams.ContentBlenderTypeSortingAlgorithmParam) match {
case BlenderParams.ContentBasedSortingAlgorithmEnum.CandidateRecency =>

View File

@ -70,9 +70,9 @@ object CountWeightedInterleaveBlender {
* We pass two parameters to the weighted interleaver:
* @param rankerWeightShrinkage shrinkage parameter between [0, 1] that determines how close we
* stay to uniform sampling. The bigger the shrinkage the
* closer we are to uniform round robin
* closer we are to uniform round robin.
* @param maxWeightAdjustments max number of weighted sampling to do prior to defaulting to
* uniform. Set so that we avoid infinite loops (e.g. if weights are
* uniform. Set so that we avoid infinite loops. (e.g. if weights are
* 0)
*/
case class WeightedBlenderQuery(

View File

@ -18,22 +18,22 @@ case class SourceTypeBackFillBlender @Inject() (globalStats: StatsReceiver) {
private val stats: StatsReceiver = globalStats.scope(name)
/**
* Partition the candidates based on source type
* Partition the candidates based on source type.
* Interleave the two partitions of candidates separately
* Then append the back fill candidates to the end
* and then append the back fill candidates to the end.
*/
def blend(
params: Params,
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]] = {
// Filter out empty candidate sequence
// Filter out empty candidate sequence.
val candidates = inputCandidates.filter(_.nonEmpty)
val backFillSourceTypes =
if (params(BlenderParams.SourceTypeBackFillEnableVideoBackFill)) BackFillSourceTypesWithVideo
else BackFillSourceTypes
// partition candidates based on their source types
// Partition candidates based on their source types.
val (backFillCandidates, regularCandidates) =
candidates.partition(
_.head.candidateGenerationInfo.sourceInfoOpt
@ -43,7 +43,7 @@ case class SourceTypeBackFillBlender @Inject() (globalStats: StatsReceiver) {
val interleavedBackFillCandidates =
InterleaveUtil.interleave(backFillCandidates)
stats.stat("backFillCandidates").add(interleavedBackFillCandidates.size)
// Append interleaved backfill candidates to the end
// Append interleaved backfill candidates to the end.
val interleavedCandidates = interleavedRegularCandidates ++ interleavedBackFillCandidates
stats.stat("candidates").add(interleavedCandidates.size)
@ -51,7 +51,6 @@ case class SourceTypeBackFillBlender @Inject() (globalStats: StatsReceiver) {
val blendedCandidates = BlendedCandidatesBuilder.build(inputCandidates, interleavedCandidates)
Future.value(blendedCandidates)
}
}
object ImplicitSignalBackFillBlender {

View File

@ -27,7 +27,7 @@ case class SwitchBlender @Inject() (
userState: UserState,
inputCandidates: Seq[Seq[InitialCandidate]],
): Future[Seq[BlendedCandidate]] = {
// Take out empty seq
// Take out the empty seq.
val nonEmptyCandidates = inputCandidates.collect {
case candidates if candidates.nonEmpty =>
candidates
@ -43,7 +43,7 @@ case class SwitchBlender @Inject() (
}
val candidatesToBlend = nonEmptyCandidates.sortBy(_.head)(innerSignalSorting)
// Blend based on specified blender rules
// Blend based on specified blender rules.
params(BlenderParams.BlendingAlgorithmParam) match {
case BlendingAlgorithmEnum.RoundRobin =>
defaultBlender.blend(candidatesToBlend)

View File

@ -39,24 +39,25 @@ class AdsCandidateGenerator @Inject() (
def get(query: AdsCandidateGeneratorQuery): Future[Seq[RankedAdsCandidate]] = {
val allStats = stats.scope("all")
val perProductStats = stats.scope("perProduct", query.product.toString)
StatsUtil.trackItemsStats(allStats) {
StatsUtil.trackItemsStats(perProductStats) {
for {
// fetch source signals
// Fetch source signals.
sourceSignals <- StatsUtil.trackBlockStats(fetchSourcesStats) {
fetchSources(query)
}
realGraphSeeds <- StatsUtil.trackItemMapStats(fetchRealGraphSeedsStats) {
fetchSeeds(query)
}
// get initial candidates from similarity engines
// hydrate lineItemInfo and filter out non active ads
// Get initial candidates from similarity engines.
// Hydrate lineItemInfo and filter out non active ads.
initialCandidates <- StatsUtil.trackBlockStats(fetchCandidatesStats) {
fetchCandidates(query, sourceSignals, realGraphSeeds)
}
// blend candidates
// Blend candidates.
blendedCandidates <- StatsUtil.trackItemsStats(interleaveStats) {
interleave(initialCandidates)
}
@ -73,7 +74,6 @@ class AdsCandidateGenerator @Inject() (
}
}
}
}
def fetchSources(
@ -95,7 +95,6 @@ class AdsCandidateGenerator @Inject() (
.fetchCandidates(query.userId, sourceSignals, realGraphSeeds, query.params),
query.params(AdsParams.EnableScribe)
)
}
private def fetchSeeds(
@ -121,7 +120,6 @@ class AdsCandidateGenerator @Inject() (
scoreBoostFactor: Double,
statsReceiver: StatsReceiver,
): Future[Seq[RankedAdsCandidate]] = {
val candidateSize = candidates.size
val rankedCandidates = candidates.zipWithIndex.map {
case (candidate, index) =>

View File

@ -90,7 +90,6 @@ case class AdsCandidateSourcesRouter @Inject() (
realGraphSeeds: Map[UserId, Double],
params: configapi.Params
): Future[Seq[Seq[InitialAdsCandidate]]] = {
val simClustersANN1ConfigId = params(SimClustersANNParams.SimClustersANN1ConfigId)
val tweetBasedSANNMinScore = params(
@ -296,7 +295,7 @@ case class AdsCandidateSourcesRouter @Inject() (
params
)
// dark traffic to simclusters-ann-2
// Dark traffic to simclusters-ann-2
if (decider.isAvailable(DeciderConstants.enableSimClustersANN2DarkTrafficDeciderKey)) {
val simClustersANN2ConfigId = params(SimClustersANNParams.SimClustersANN2ConfigId)
val sann2Query = SimClustersANNSimilarityEngine.fromParams(
@ -329,7 +328,6 @@ case class AdsCandidateSourcesRouter @Inject() (
sourceInfo: Option[SourceInfo],
params: configapi.Params
) = {
val query = ProducerBasedUserAdGraphSimilarityEngine.fromParams(
sourceInfo.get.internalId,
params
@ -352,7 +350,6 @@ case class AdsCandidateSourcesRouter @Inject() (
sourceInfo: Option[SourceInfo],
params: configapi.Params
) = {
val query = TweetBasedUserAdGraphSimilarityEngine.fromParams(
sourceInfo.get.internalId,
params
@ -375,7 +372,6 @@ case class AdsCandidateSourcesRouter @Inject() (
realGraphSeeds: Map[UserId, Double],
params: configapi.Params
) = {
val query = ConsumersBasedUserAdGraphSimilarityEngine
.fromParams(realGraphSeeds, params)
@ -394,7 +390,7 @@ case class AdsCandidateSourcesRouter @Inject() (
CandidateGenerationInfo(
Some(sourceInfo),
similarityEngineInfo,
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs.
)
)
})
@ -404,7 +400,7 @@ case class AdsCandidateSourcesRouter @Inject() (
similarityEngine: HnswANNSimilarityEngine,
similarityEngineType: SimilarityEngineType,
requestUserId: UserId,
sourceInfo: Option[SourceInfo], // if none, then it's consumer-based similarity engine
sourceInfo: Option[SourceInfo], // If none, then it's consumer-based similarity engine.
model: String
): Future[Seq[TweetWithCandidateGenerationInfo]] = {
val internalId =
@ -455,7 +451,7 @@ case class AdsCandidateSourcesRouter @Inject() (
CandidateGenerationInfo(
None,
similarityEngineInfo,
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs.
)
)
}

View File

@ -96,7 +96,6 @@ case class CandidateSourcesRouter @Inject() (
sourceGraphs: Map[String, Option[GraphSourceInfo]],
params: configapi.Params,
): Future[Seq[Seq[InitialCandidate]]] = {
val tweetBasedCandidatesFuture = getCandidates(
getTweetBasedSourceInfo(sourceSignals),
params,
@ -225,7 +224,7 @@ case class CandidateSourcesRouter @Inject() (
consumersBasedUvgRealGraphInCandidatesFuture,
customizedRetrievalBasedCandidatesFuture
)).map { candidatesList =>
// remove empty innerSeq
// Remove empty innerSeq.
val result = candidatesList.flatten.filter(_.nonEmpty)
stats.stat("numOfSequences").add(result.size)
stats.stat("flattenCandidatesWithDup").add(result.flatten.size)
@ -262,7 +261,7 @@ case class CandidateSourcesRouter @Inject() (
CandidateGenerationInfo(
sourceInfo,
similarityEngineInfo,
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs.
)
)
}
@ -330,7 +329,7 @@ case class CandidateSourcesRouter @Inject() (
CandidateGenerationInfo(
None,
similarityEngineInfo,
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs.
)
)
}
@ -358,7 +357,7 @@ case class CandidateSourcesRouter @Inject() (
engine.getCandidates(EngineQuery(query, params)).map {
_.map {
_.map { tweetWithScore =>
// define filters
// Define filters.
TweetWithCandidateGenerationInfo(
tweetWithScore.tweetId,
CandidateGenerationInfo(
@ -401,7 +400,7 @@ case class CandidateSourcesRouter @Inject() (
CandidateGenerationInfo(
None,
similarityEngineInfo,
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs
Seq.empty // Atomic Similarity Engine. Hence it has no contributing SEs.
)
)
}

View File

@ -48,7 +48,6 @@ class CrCandidateGenerator @Inject() (
timeoutConfig: TimeoutConfig,
globalStats: StatsReceiver) {
private val timer: Timer = new JavaTimer(true)
private val stats: StatsReceiver = globalStats.scope(this.getClass.getCanonicalName)
private val fetchSourcesStats = stats.scope("fetchSources")
@ -78,14 +77,14 @@ class CrCandidateGenerator @Inject() (
fetchSources(query)
}
initialCandidates <- StatsUtil.trackBlockStats(fetchCandidatesAfterFilterStats) {
// find the positive and negative signals
// Find the positive and negative signals.
val (positiveSignals, negativeSignals) = sourceSignals.partition { signal =>
!EnabledNegativeSourceTypes.contains(signal.sourceType)
}
fetchPositiveSourcesStats.stat("size").add(positiveSignals.size)
fetchNegativeSourcesStats.stat("size").add(negativeSignals.size)
// find the positive signals to keep, removing block and muted users
// Find the positive signals to keep, removing block and muted users.
val filteredSourceInfo =
if (negativeSignals.nonEmpty && query.params(
RecentNegativeSignalParams.EnableSourceParam)) {
@ -94,7 +93,7 @@ class CrCandidateGenerator @Inject() (
positiveSignals
}
// fetch candidates from the positive signals
// Fetch candidates from the positive signals.
StatsUtil.trackBlockStats(fetchCandidatesStats) {
fetchCandidates(query, filteredSourceInfo, sourceGraphsMap)
}

View File

@ -28,7 +28,7 @@ import javax.inject.Singleton
import scala.collection.mutable.ArrayBuffer
/**
* A candidate generator that fetches similar tweets from multiple customized retrieval based candidate sources
* A candidate generator that fetches similar tweets from multiple customized retrieval based candidate sources.
*
* Different from [[TweetBasedCandidateGeneration]], this store returns candidates from different
* similarity engines without blending. In other words, this class shall not be thought of as a

View File

@ -105,7 +105,7 @@ class FrsTweetCandidateGenerator @Inject() (
}
/**
* Fetch recommended seed users from FRS
* Fetch recommended seed users from FRS.
*/
private def fetchSeeds(
userId: UserId,
@ -131,7 +131,7 @@ class FrsTweetCandidateGenerator @Inject() (
}
/**
* Fetch tweet candidates from Earlybird
* Fetch tweet candidates from Earlybird.
*/
private def fetchCandidates(
searcherUserId: UserId,
@ -141,7 +141,7 @@ class FrsTweetCandidateGenerator @Inject() (
params: Params
): Future[Option[Seq[TweetWithAuthor]]] = {
if (seedAuthors.nonEmpty) {
// call earlybird
// Call Earlybird.
val query = EarlybirdSimilarityEngineRouter.queryFromParams(
Some(searcherUserId),
seedAuthors,
@ -154,7 +154,7 @@ class FrsTweetCandidateGenerator @Inject() (
}
/**
* Filter candidates that do not pass visibility filter policy
* Filter candidates that do not pass visibility filter policy.
*/
private def filterCandidates(
candidates: Option[Seq[TweetWithAuthor]],
@ -175,7 +175,7 @@ class FrsTweetCandidateGenerator @Inject() (
}
/**
* Hydrate the candidates with the FRS candidate sources and scores
* Hydrate the candidates with the FRS candidate sources and scores.
*/
private def hydrateCandidates(
frsAuthorWithScores: Option[Map[UserId, FrsQueryResult]],
@ -193,8 +193,8 @@ class FrsTweetCandidateGenerator @Inject() (
frsCandidateSourceScores = frsQueryResult.flatMap { result =>
result.sourceWithScores.map {
_.collect {
// see TokenStrToAlgorithmMap @ https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/hermit/hermit-core/src/main/scala/com/twitter/hermit/constants/AlgorithmFeedbackTokens.scala
// see Algorithm @ https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/hermit/hermit-core/src/main/scala/com/twitter/hermit/model/Algorithm.scala
// See TokenStrToAlgorithmMap @ https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/hermit/hermit-core/src/main/scala/com/twitter/hermit/constants/AlgorithmFeedbackTokens.scala
// See Algorithm @ https://sourcegraph.twitter.biz/git.twitter.biz/source/-/blob/hermit/hermit-core/src/main/scala/com/twitter/hermit/model/Algorithm.scala
case (candidateSourceAlgoStr, score)
if AlgorithmFeedbackTokens.TokenStrToAlgorithmMap.contains(
candidateSourceAlgoStr) =>
@ -210,7 +210,6 @@ class FrsTweetCandidateGenerator @Inject() (
}
}
}
}
object FrsTweetCandidateGenerator {

View File

@ -43,7 +43,6 @@ class RelatedTweetCandidateGenerator @Inject() (
def get(
query: RelatedTweetCandidateGeneratorQuery
): Future[Seq[InitialCandidate]] = {
val allStats = stats.scope("all")
val perProductStats = stats.scope("perProduct", query.product.toString)
StatsUtil.trackItemsStats(allStats) {
@ -90,9 +89,9 @@ class RelatedTweetCandidateGenerator @Inject() (
}
/***
* fetch Candidates from TweetBased/ProducerBased Unified Similarity Engine,
* and apply VF filter based on TweetInfoStore
* To align with the downstream processing (filter, rank), we tend to return a Seq[Seq[InitialCandidate]]
* Fetch Candidates from TweetBased/ProducerBased Unified Similarity Engine,
* and apply VF filter based on TweetInfoStore.
* To align with the downstream processing (filter, rank). We tend to return a Seq[Seq[InitialCandidate]]
* instead of a Seq[Candidate] even though we only have a Seq in it.
*/
private def getCandidatesFromSimilarityEngine[QueryType](
@ -103,7 +102,7 @@ class RelatedTweetCandidateGenerator @Inject() (
/***
* We wrap the query to be a Seq of queries for the Sim Engine to ensure evolvability of candidate generation
* and as a result, it will return Seq[Seq[InitialCandidate]]
* and as a result, it will return Seq[Seq[InitialCandidate]].
*/
val engineQueries =
Seq(fromParamsForRelatedTweet(query.internalId, query.params))
@ -138,7 +137,7 @@ class RelatedTweetCandidateGenerator @Inject() (
Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos =>
/***
* If tweetInfo does not exist, we will filter out this tweet candidate.
* This tweetInfo filter also acts as the VF filter
* This tweetInfo filter also acts as the VF filter.
*/
candidates.collect {
case candidate if tweetInfos.getOrElse(candidate.tweetId, None).isDefined =>

View File

@ -36,7 +36,6 @@ class RelatedVideoTweetCandidateGenerator @Inject() (
def get(
query: RelatedVideoTweetCandidateGeneratorQuery
): Future[Seq[InitialCandidate]] = {
val allStats = stats.scope("all")
val perProductStats = stats.scope("perProduct", query.product.toString)
StatsUtil.trackItemsStats(allStats) {
@ -75,8 +74,8 @@ class RelatedVideoTweetCandidateGenerator @Inject() (
}
/***
* fetch Candidates from TweetBased/ProducerBased Unified Similarity Engine,
* and apply VF filter based on TweetInfoStore
* Fetch Candidates from TweetBased/ProducerBased Unified Similarity Engine,
* and apply VF filter based on TweetInfoStore.
* To align with the downstream processing (filter, rank), we tend to return a Seq[Seq[InitialCandidate]]
* instead of a Seq[Candidate] even though we only have a Seq in it.
*/
@ -88,7 +87,7 @@ class RelatedVideoTweetCandidateGenerator @Inject() (
/***
* We wrap the query to be a Seq of queries for the Sim Engine to ensure evolvability of candidate generation
* and as a result, it will return Seq[Seq[InitialCandidate]]
* and as a result, it will return Seq[Seq[InitialCandidate]].
*/
val engineQueries =
Seq(fromParamsForRelatedVideoTweet(query.internalId, query.params))
@ -121,7 +120,7 @@ class RelatedVideoTweetCandidateGenerator @Inject() (
Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos =>
/***
* If tweetInfo does not exist, we will filter out this tweet candidate.
* This tweetInfo filter also acts as the VF filter
* This tweetInfo filter also acts as the VF filter.
*/
candidates.collect {
case candidate if tweetInfos.getOrElse(candidate.tweetId, None).isDefined =>

View File

@ -179,7 +179,7 @@ case class SimClustersInterestedInCandidateGeneration @Inject() (
else
Future.None
// AddressBookInterestedIn Queries
// AddressBookInterestedIn Queries.
val userAddressBookInterestedInCandidateResultFut =
if (query.enableAddressBookNextInterestedIn && query.enableProdSimClustersANNSimilarityEngine)
getInterestedInCandidateResult(
@ -397,7 +397,7 @@ object SimClustersInterestedInCandidateGeneration {
internalId: InternalId,
params: configapi.Params,
): Query = {
// SimClusters common configs
// SimClusters common configs.
val simClustersModelVersion =
ModelVersions.Enum.enumToSimClustersModelVersionMap(params(GlobalParams.ModelVersionParam))
val simClustersANNConfigId = params(SimClustersANNParams.SimClustersANNConfigId)
@ -415,13 +415,13 @@ object SimClustersInterestedInCandidateGeneration {
val simClustersAddressBookInterestedInMinScore = params(
InterestedInParams.MinScoreAddressBookParam)
// InterestedIn embeddings parameters
// InterestedIn embeddings parameters.
val interestedInEmbedding = params(InterestedInParams.InterestedInEmbeddingIdParam)
val nextInterestedInEmbedding = params(InterestedInParams.NextInterestedInEmbeddingIdParam)
val addressbookInterestedInEmbedding = params(
InterestedInParams.AddressBookInterestedInEmbeddingIdParam)
// Prod SimClustersANN Query
// Prod SimClustersANN Query.
val interestedInSimClustersANNQuery =
SimClustersANNSimilarityEngine.fromParams(
internalId,
@ -446,7 +446,7 @@ object SimClustersInterestedInCandidateGeneration {
simClustersANNConfigId,
params)
// Experimental SANN cluster Query
// Experimental SANN cluster Query.
val interestedInExperimentalSimClustersANNQuery =
SimClustersANNSimilarityEngine.fromParams(
internalId,
@ -471,7 +471,7 @@ object SimClustersInterestedInCandidateGeneration {
experimentalSimClustersANNConfigId,
params)
// SimClusters ANN cluster 1 Query
// SimClusters ANN cluster 1 Query.
val interestedInSimClustersANN1Query =
SimClustersANNSimilarityEngine.fromParams(
internalId,
@ -496,7 +496,7 @@ object SimClustersInterestedInCandidateGeneration {
simClustersANN1ConfigId,
params)
// SimClusters ANN cluster 2 Query
// SimClusters ANN cluster 2 Query.
val interestedInSimClustersANN2Query =
SimClustersANNSimilarityEngine.fromParams(
internalId,
@ -521,7 +521,7 @@ object SimClustersInterestedInCandidateGeneration {
simClustersANN2ConfigId,
params)
// SimClusters ANN cluster 3 Query
// SimClusters ANN cluster 3 Query.
val interestedInSimClustersANN3Query =
SimClustersANNSimilarityEngine.fromParams(
internalId,
@ -546,7 +546,7 @@ object SimClustersInterestedInCandidateGeneration {
simClustersANN3ConfigId,
params)
// SimClusters ANN cluster 5 Query
// SimClusters ANN cluster 5 Query.
val interestedInSimClustersANN5Query =
SimClustersANNSimilarityEngine.fromParams(
internalId,
@ -554,7 +554,8 @@ object SimClustersInterestedInCandidateGeneration {
simClustersModelVersion,
simClustersANN5ConfigId,
params)
// SimClusters ANN cluster 4 Query
// SimClusters ANN cluster 4 Query.
val interestedInSimClustersANN4Query =
SimClustersANNSimilarityEngine.fromParams(
internalId,

View File

@ -116,7 +116,7 @@ class TopicTweetCandidateGenerator @Inject() (
val tweetIds = candidates.map(_.tweetId).toSet
val numTweetsPreFilter = tweetIds.size
Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos =>
/** *
/**
* If tweetInfo does not exist, we will filter out this tweet candidate.
*/
val tweetyPieFilteredInitialCandidates = candidates.collect {
@ -142,7 +142,6 @@ class TopicTweetCandidateGenerator @Inject() (
topicId -> tweetyPieFilteredInitialCandidates
}
}
Future.collect(initialCandidates.toSeq).map(_.toMap)
}
@ -152,7 +151,6 @@ class TopicTweetCandidateGenerator @Inject() (
isVideoOnly: Boolean,
excludeTweetIds: Set[TweetId]
): Future[Map[TopicId, Seq[InitialCandidate]]] = {
val earliestTweetId = SnowflakeId.firstIdFor(Time.now - maxTweetAge)
val filteredResults = topicTweetMap.map {

View File

@ -46,18 +46,16 @@ class UtegTweetCandidateGenerator @Inject() (
def get(
query: UtegTweetCandidateGeneratorQuery
): Future[Seq[TweetWithScoreAndSocialProof]] = {
val allStats = stats.scope("all")
val perProductStats = stats.scope("perProduct", query.product.toString)
StatsUtil.trackItemsStats(allStats) {
StatsUtil.trackItemsStats(perProductStats) {
/**
* The candidate we return in the end needs a social proof field, which isn't
* supported by the any existing Candidate type, so we created TweetWithScoreAndSocialProof
* instead.
*
* However, filters and light ranker expect Candidate-typed param to work. In order to minimise the
* However, filters and light ranker expect Candidate-typed param to work. In order to minimize the
* changes to them, we are doing conversions from/to TweetWithScoreAndSocialProof to/from Candidate
* in this method.
*/
@ -111,7 +109,6 @@ class UtegTweetCandidateGenerator @Inject() (
candidate.toRankedCandidate(score)
}
)
}
def fetchCandidates(
@ -136,7 +133,7 @@ class UtegTweetCandidateGenerator @Inject() (
): Future[Seq[InitialCandidate]] = {
val tweetIds = candidates.map(_.tweetId).toSet
Future.collect(tweetInfoStore.multiGet(tweetIds)).map { tweetInfos =>
/** *
/**
* If tweetInfo does not exist, we will filter out this tweet candidate.
*/
candidates.collect {
@ -172,7 +169,7 @@ class UtegTweetCandidateGenerator @Inject() (
candidate.predictionScore,
tweet.socialProofByType
)
// The exception should never be thrown
// The exception should never be thrown.
}.getOrElse(throw new Exception("Cannot find ranked candidate in original UTEG tweets"))
}
}

View File

@ -41,12 +41,11 @@ object SimClustersANNConfig {
annAlgorithm = ScoringAlgorithm.CosineSimilarity,
)
/*
SimClustersANNConfigId: String
Format: Prod - EmbeddingType_ModelVersion_Default
Format: Experiment - EmbeddingType_ModelVersion_Date_Two-Digit-Serial-Number. Date : YYYYMMDD
/**
* SimClustersANNConfigId: String
* Format: Prod - EmbeddingType_ModelVersion_Default
* Format: Experiment - EmbeddingType_ModelVersion_Date_Two-Digit-Serial-Number. Date : YYYYMMDD
*/
private val FavBasedProducer_Model20m145k2020_Default = DefaultConfig.copy()
// Chunnan's exp on maxTweetCandidateAgeDays 2
@ -142,12 +141,14 @@ object SimClustersANNConfig {
candidateEmbeddingType = EmbeddingType.LogFavBasedRealTimeTweet,
maxTweetCandidateAge = 1.hours
)
// SANN-4 config
private val LogFavLongestL2EmbeddingTweet_Model20m145k2020_20221220 =
LogFavLongestL2EmbeddingTweet_Model20m145k2020_Default.copy(
candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet,
maxTweetCandidateAge = 48.hours
)
private val UnfilteredUserInterestedIn_Model20m145k2020_Default = DefaultConfig.copy()
// Chunnan's exp on maxTweetCandidateAgeDays 2
@ -199,6 +200,7 @@ object SimClustersANNConfig {
candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet,
maxTweetCandidateAge = 48.hours
)
private val LogFavBasedUserInterestedInFromAPE_Model20m145k2020_Default = DefaultConfig.copy()
// Chunnan's exp on maxTweetCandidateAgeDays 2
@ -302,6 +304,7 @@ object SimClustersANNConfig {
candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet,
maxTweetCandidateAge = 48.hours
)
private val UserNextInterestedIn_Model20m145k2020_Default = DefaultConfig.copy()
// Chunnan's exp on maxTweetCandidateAgeDays 2
@ -353,7 +356,8 @@ object SimClustersANNConfig {
candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet,
maxTweetCandidateAge = 48.hours
)
// Vincent's experiment on using FollowBasedProducer as query embedding type for UserFollow
// Vincent's experiment on using FollowBasedProducer as query embedding type for UserFollow.
private val FollowBasedProducer_Model20m145k2020_Default =
FavBasedProducer_Model20m145k2020_Default.copy()
@ -400,6 +404,7 @@ object SimClustersANNConfig {
candidateEmbeddingType = EmbeddingType.LogFavBasedEvergreenTweet,
maxTweetCandidateAge = 48.hours
)
val DefaultConfigMappings: Map[String, SimClustersANNConfig] = Map(
"FavBasedProducer_Model20m145k2020_Default" -> FavBasedProducer_Model20m145k2020_Default,
"FavBasedProducer_Model20m145k2020_20220617_06" -> FavBasedProducer_Model20m145k2020_20220617_06,

View File

@ -3,22 +3,22 @@ package com.twitter.cr_mixer.config
import com.twitter.util.Duration
case class TimeoutConfig(
/* Default timeouts for candidate generator */
// Default timeouts for candidate generator.
serviceTimeout: Duration,
signalFetchTimeout: Duration,
similarityEngineTimeout: Duration,
annServiceClientTimeout: Duration,
/* For Uteg Candidate Generator */
// For Uteg Candidate Generator.
utegSimilarityEngineTimeout: Duration,
/* For User State Store */
// For User State Store
userStateUnderlyingStoreTimeout: Duration,
userStateStoreTimeout: Duration,
/* For FRS based tweets */
// Timeout passed to EarlyBird server
// For FRS based tweets .
// Timeout passed to EarlyBird server.
earlybirdServerTimeout: Duration,
// Timeout set on CrMixer side
earlybirdSimilarityEngineTimeout: Duration,
frsBasedTweetEndpointTimeout: Duration,
topicTweetEndpointTimeout: Duration,
// Timeout Settings for Navi gRPC Client
// Timeout Settings for Navi gRPC Client.
naviRequestTimeout: Duration)

View File

@ -101,7 +101,7 @@ class CrMixerThriftController @Inject() (
ExceptionUtils.getStackTrace(e)
).mkString("\n")
/** *
/**
* We chose logger.info() here to print message instead of logger.error since that
* logger.error sometimes suppresses detailed stacktrace.
*/
@ -109,8 +109,7 @@ class CrMixerThriftController @Inject() (
}
private def generateRequestUUID(): Long = {
/** *
/**
* We generate unique UUID via bitwise operations. See the below link for more:
* https://stackoverflow.com/questions/15184820/how-to-generate-unique-positive-long-using-uuid
*/
@ -119,7 +118,6 @@ class CrMixerThriftController @Inject() (
handle(t.CrMixer.GetTweetRecommendations) { args: t.CrMixer.GetTweetRecommendations.Args =>
val endpointName = "getTweetRecommendations"
val requestUUID = generateRequestUUID()
val startTime = Time.now.inMilliseconds
val userId = args.request.clientContext.userId.getOrElse(
@ -168,10 +166,9 @@ class CrMixerThriftController @Inject() (
Future(CrMixerTweetResponse(Seq.empty))
}
}
}
/** *
/**
* GetRelatedTweetsForQueryTweet and GetRelatedTweetsForQueryAuthor are essentially
* doing very similar things, except that one passes in TweetId which calls TweetBased engine,
* and the other passes in AuthorId which calls ProducerBased engine.
@ -221,7 +218,6 @@ class CrMixerThriftController @Inject() (
Future(RelatedTweetResponse(Seq.empty))
}
}
}
private def getRelatedVideoTweets(
@ -330,7 +326,6 @@ class CrMixerThriftController @Inject() (
Future(AdsResponse(Seq.empty))
}
}
}
private def buildCrCandidateGeneratorQuery(
@ -338,7 +333,6 @@ class CrMixerThriftController @Inject() (
requestUUID: Long,
userId: Long
): Future[CrCandidateGeneratorQuery] = {
val product = thriftRequest.product
val productContext = thriftRequest.productContext
val scopedStats = statsReceiver
@ -357,7 +351,7 @@ class CrMixerThriftController @Inject() (
userState
)
// Specify product-specific behavior mapping here
// Specify product-specific behavior mapping here.
val maxNumResults = (product, productContext) match {
case (t.Product.Home, Some(t.ProductContext.HomeContext(homeContext))) =>
homeContext.maxResults.getOrElse(9999)
@ -392,7 +386,6 @@ class CrMixerThriftController @Inject() (
thriftRequest: RelatedTweetRequest,
requestUUID: Long
): Future[RelatedTweetCandidateGeneratorQuery] = {
val product = thriftRequest.product
val scopedStats = statsReceiver
.scope(product.toString).scope("RelatedTweetRequest")
@ -409,8 +402,8 @@ class CrMixerThriftController @Inject() (
thriftRequest.product,
userState)
// Specify product-specific behavior mapping here
// Currently, Home takes 10, and RUX takes 100
// Specify product-specific behavior mapping here.
// Currently, Home takes 10, and RUX takes 100.
val maxNumResults = params(RelatedTweetGlobalParams.MaxCandidatesPerRequestParam)
RelatedTweetCandidateGeneratorQuery(
@ -458,7 +451,6 @@ class CrMixerThriftController @Inject() (
thriftRequest: RelatedVideoTweetRequest,
requestUUID: Long
): Future[RelatedVideoTweetCandidateGeneratorQuery] = {
val product = thriftRequest.product
val scopedStats = statsReceiver
.scope(product.toString).scope("RelatedVideoTweetRequest")
@ -487,14 +479,12 @@ class CrMixerThriftController @Inject() (
requestUUID = requestUUID
)
}
}
private def buildUtegTweetQuery(
thriftRequest: UtegTweetRequest,
requestUUID: Long
): Future[UtegTweetCandidateGeneratorQuery] = {
val userId = thriftRequest.clientContext.userId.getOrElse(
throw new IllegalArgumentException("userId must be present in the Thrift clientContext")
)
@ -536,7 +526,6 @@ class CrMixerThriftController @Inject() (
requestUUID = requestUUID
)
}
}
private def buildTopicTweetQuery(
@ -550,7 +539,7 @@ class CrMixerThriftController @Inject() (
val product = thriftRequest.product
val productContext = thriftRequest.productContext
// Specify product-specific behavior mapping here
// Specify product-specific behavior mapping here.
val isVideoOnly = (product, productContext) match {
case (t.Product.ExploreTopics, Some(t.ProductContext.ExploreContext(context))) =>
context.isVideoOnly
@ -646,7 +635,6 @@ class CrMixerThriftController @Inject() (
private def buildThriftResponse(
candidates: Seq[RankedCandidate]
): CrMixerTweetResponse = {
val tweets = candidates.map { candidate =>
TweetRecommendation(
tweetId = candidate.tweetId,
@ -663,7 +651,7 @@ class CrMixerThriftController @Inject() (
private def scribeTweetScoreFunnelSeries(
candidates: Seq[RankedCandidate]
): Seq[RankedCandidate] = {
// 202210210901 is a random number for code search of Lensview
// 202210210901 is a random number for code search of Lensview.
tweetScoreFunnelSeries.startNewSpan(
name = "GetTweetRecommendationsTopLevelTweetSimilarityEngineType",
codePtr = 202210210901L) {
@ -734,7 +722,6 @@ class CrMixerThriftController @Inject() (
request: CrMixerTweetRequest,
response: Future[CrMixerTweetResponse]
): Unit = {
val userId = request.clientContext.userId.getOrElse(
throw new IllegalArgumentException(
"userId must be present in getTweetRecommendations() Thrift clientContext"))

View File

@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
- Fetch Tweet Candidates
- ScoredTweetsInNetworkCandidatePipelineConfig
- ScoredTweetsCrMixerCandidatePipelineConfig
- ScoredTweetsTweetMixerCandidatePipelineConfig
- ScoredTweetsUtegCandidatePipelineConfig
- ScoredTweetsFrsCandidatePipelineConfig
- Feature Hydration and Scoring
@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
- ListTweetsAdsCandidatePipelineConfig (fetch ads)

View File

@ -21,6 +21,7 @@ scala_library(
"finatra/inject/inject-utils/src/main/scala",
"home-mixer/server/src/main/resources",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@ -31,6 +32,10 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"stringcenter/client",
"stringcenter/client/src/main/java",
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

View File

@ -2,7 +2,7 @@ package com.twitter.home_mixer
import com.twitter.finatra.http.routing.HttpWarmup
import com.twitter.finatra.httpclient.RequestBuilder._
import com.twitter.inject.Logging
import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler
import com.twitter.util.Try
import javax.inject.Inject

View File

@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
import com.twitter.finatra.thrift.filters._
import com.twitter.finatra.thrift.routing.ThriftRouter
import com.twitter.home_mixer.controller.HomeThriftController
import com.twitter.home_mixer.federated.HomeMixerColumn
import com.twitter.home_mixer.module._
import com.twitter.home_mixer.param.GlobalParamConfigModule
import com.twitter.home_mixer.product.HomeMixerProductModule
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
import com.twitter.product_mixer.component_library.module.EarlybirdModule
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
import com.twitter.product_mixer.core.controllers.ProductMixerController
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
import com.twitter.product_mixer.core.module.ProductMixerModule
import com.twitter.product_mixer.core.module.StratoClientModule
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.fed.server.StratoFedServer
object HomeMixerServerMain extends HomeMixerServer
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls {
class HomeMixerServer
extends StratoFedServer
with ThriftServer
with Mtls
with HttpServer
with HttpMtls {
override val name = "home-mixer-server"
override val modules: Seq[Module] = Seq(
AccountRecommendationsMixerModule,
AdvertiserBrandSafetySettingsStoreModule,
BlenderClientModule,
ClientSentImpressionsPublisherModule,
ConversationServiceModule,
CrMixerClientModule,
EarlybirdModule,
ExploreRankerClientModule,
FeedbackHistoryClientModule,
GizmoduckClientModule,
GlobalParamConfigModule,
HomeAdsCandidateSourceModule,
HomeMixerFlagsModule,
HomeMixerProductModule,
HomeMixerResourcesModule,
HomeNaviModelClientModule,
ImpressionBloomFilterModule,
InjectionHistoryClientModule,
FeedbackHistoryClientModule,
ManhattanClientsModule,
ManhattanFeatureRepositoryModule,
ManhattanTweetImpressionStoreModule,
MemcachedFeatureRepositoryModule,
NaviModelClientModule,
OnboardingTaskServiceModule,
OptimizedStratoClientModule,
PeopleDiscoveryServiceModule,
@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
SimClustersRecentEngagementsClientModule,
SocialGraphServiceModule,
StaleTweetsCacheModule,
StratoClientModule,
ThriftFeatureRepositoryModule,
TimelineMixerClientModule,
TimelineRankerClientModule,
TimelineScorerClientModule,
TimelineServiceClientModule,
TimelinesPersistenceStoreClientModule,
TopicSocialProofClientModule,
TweetImpressionStoreModule,
TweetyPieClientModule,
TweetMixerClientModule,
TweetypieClientModule,
TweetypieStaticEntitiesCacheClientModule,
UserMetadataStoreModule,
UserSessionStoreModule,
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
new ProductScopeStringCenterModule()
)
def configureThrift(router: ThriftRouter): Unit = {
override def configureThrift(router: ThriftRouter): Unit = {
router
.filter[LoggingMDCFilter]
.filter[TraceIdMDCFilter]
@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
this.injector,
st.HomeMixer.ExecutePipeline))
override val dest: String = "/s/home-mixer/home-mixer:strato"
override val columns: Seq[Class[_ <: StratoFed.Column]] =
Seq(classOf[HomeMixerColumn])
override protected def warmup(): Unit = {
handle[HomeMixerThriftServerWarmupHandler]()
handle[HomeMixerHttpServerWarmupHandler]()

View File

@ -3,7 +3,7 @@ package com.twitter.home_mixer
import com.twitter.finagle.thrift.ClientId
import com.twitter.finatra.thrift.routing.ThriftWarmup
import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.inject.Logging
import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler
import com.twitter.product_mixer.core.{thriftscala => pt}
import com.twitter.scrooge.Request

View File

@ -5,19 +5,13 @@ scala_library(
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@ -25,10 +19,6 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
],
exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

View File

@ -1,18 +1,23 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
import com.twitter.product_mixer.component_library.filter.FeatureFilter
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
override val gates: Seq[BaseGate[Query]],
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
extends DependentCandidatePipelineConfig[
@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
val tweetsWithConversationMetadata = candidates.map { candidate =>
TweetWithConversationMetadata(
tweetId = candidate.candidateIdLong,
userId = None,
sourceTweetId = None,
sourceUserId = None,
inReplyToTweetId = None,
userId = candidate.features.getOrElse(AuthorIdFeature, None),
sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
conversationId = None,
ancestors = Seq.empty
)
@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
override val preFilterFeatureHydrationPhase1: Seq[
BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator)
] = Seq(
tweetypieFeatureHydrator,
InNetworkFeatureHydrator,
)
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
RetweetDeduplicationFilter,
@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
),
invalidSubscriptionTweetFilter,
InvalidConversationModuleFilter
)

View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@ -15,7 +15,7 @@ import javax.inject.Singleton
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
namesFeatureHydrator: NamesFeatureHydrator) {
def build(
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
new ConversationServiceCandidatePipelineConfig(
conversationServiceCandidateSource,
tweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator,
namesFeatureHydrator,
invalidSubscriptionTweetFilter,
gates,
decorator
)

View File

@ -1,7 +1,7 @@
package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
import com.twitter.home_mixer.service.HomeMixerAlertConfig

View File

@ -13,8 +13,5 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/thrift/com/twitter/context:twitter-context-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"twitter-context/src/main/scala",
],
)

View File

@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.controllers.DebugTwitterContext
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
import com.twitter.product_mixer.core.service.urt.UrtService
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch

View File

@ -0,0 +1,24 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"stitch/stitch-repo/src/main/scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
"strato/src/main/scala/com/twitter/strato/callcontext",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
],
)

View File

@ -0,0 +1,217 @@
package com.twitter.home_mixer.federated
import com.twitter.gizmoduck.{thriftscala => gd}
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
import com.twitter.home_mixer.model.request.HomeMixerRequest
import com.twitter.home_mixer.{thriftscala => hm}
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
import com.twitter.product_mixer.core.{thriftscala => pm}
import com.twitter.stitch.Arrow
import com.twitter.stitch.Stitch
import com.twitter.strato.callcontext.CallContext
import com.twitter.strato.catalog.OpMetadata
import com.twitter.strato.config._
import com.twitter.strato.data._
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.timelines.render.{thriftscala => tr}
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class HomeMixerColumn @Inject() (
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn,
paramsBuilder: ParamsBuilder,
productPipelineRegistry: ProductPipelineRegistry)
extends StratoFed.Column(HomeMixerColumn.Path)
with StratoFed.Fetch.Arrow {
override val contactInfo: ContactInfo = ContactInfo(
contactEmail = "",
ldapGroup = "",
slackRoomId = ""
)
override val metadata: OpMetadata =
OpMetadata(
lifecycle = Some(Lifecycle.Production),
description =
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
)
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
ServiceIdentifierPattern(
role = "",
service = "",
env = "",
zone = Seq(""))
)
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
override type Key = gql.TimelineKey
override type View = gql.HomeTimelineView
override type Value = tr.Timeline
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
private def createHomeMixerRequestArrow(
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn
): Arrow[(Key, View), hm.HomeMixerRequest] = {
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
(gd.LookupContext(), Set(gd.QueryFields.Roles))
val populateUserRoles = Arrow
.flatMap[(Key, View), Option[Set[String]]] { _ =>
Stitch.collect {
CallContext.twitterUserId.map { userId =>
compositeOnUserClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch(userId, gizmoduckView).map(_.v)
.map {
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
}
}
}
}
val populateIpAddress = Arrow
.flatMap[(Key, View), Option[String]](_ =>
auditIpClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch((), ()).map(_.v))
Arrow.join(
populateUserRoles,
populateIpAddress
)
}
Arrow.zipWithArg(populateUserRolesAndIp).map {
case ((key, view), (roles, ipAddress)) =>
val deviceContextOpt = Some(
hm.DeviceContext(
isPolling = CallContext.isPolling,
requestContext = view.requestContext,
latestControlAvailable = view.latestControlAvailable,
autoplayEnabled = view.autoplayEnabled
))
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
val (product, productContext) = key match {
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
(
hm.Product.ForYou,
hm.ProductContext.ForYou(
hm.ForYou(
deviceContextOpt,
seenTweetIds,
view.dspClientContext,
view.pushToHomeTweetId
)
))
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
(
hm.Product.Following,
hm.ProductContext.Following(
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
(
hm.Product.Subscribed,
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
}
val clientContext = pm.ClientContext(
userId = CallContext.twitterUserId,
guestId = CallContext.guestId,
guestIdAds = CallContext.guestIdAds,
guestIdMarketing = CallContext.guestIdMarketing,
appId = CallContext.clientApplicationId,
ipAddress = ipAddress,
userAgent = CallContext.userAgent,
countryCode = CallContext.requestCountryCode,
languageCode = CallContext.requestLanguageCode,
isTwoffice = CallContext.isInternalOrTwoffice,
userRoles = roles,
deviceId = CallContext.deviceId,
mobileDeviceId = CallContext.mobileDeviceId,
mobileDeviceAdId = CallContext.adId,
limitAdTracking = CallContext.limitAdTracking
)
hm.HomeMixerRequest(
clientContext = clientContext,
product = product,
productContext = Some(productContext),
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
cursor = view.cursor.filter(_.nonEmpty)
)
}
}
override val fetch: Arrow[(Key, View), Result[Value]] = {
val transformThriftIntoPipelineRequest: Arrow[
(Key, View),
ProductPipelineRequest[HomeMixerRequest]
] = {
Arrow
.identity[(Key, View)]
.andThen {
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
}
.map {
case thriftRequest =>
val request = homeMixerRequestUnmarshaller(thriftRequest)
val params = paramsBuilder.build(
clientContext = request.clientContext,
product = request.product,
featureOverrides =
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
)
ProductPipelineRequest(request, params)
}
}
val underlyingProduct: Arrow[
ProductPipelineRequest[HomeMixerRequest],
ProductPipelineResult[tr.TimelineResponse]
] = Arrow
.identity[ProductPipelineRequest[HomeMixerRequest]]
.map { pipelineRequest =>
val pipelineArrow = productPipelineRegistry
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
pipelineRequest.request.product)
.arrow
(pipelineArrow, pipelineRequest)
}.applyArrow
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
_.result match {
case Some(result) => found(result.timeline)
case _ => missing
}
}
}
}
object HomeMixerColumn {
val Path = "home-mixer/homeMixer.Timeline"
private val FetchCallstack = s"$Path:fetch"
private val MaxCount: Option[Int] = Some(100)
}

View File

@ -10,11 +10,8 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"stitch/stitch-timelineservice/src/main/scala",
"strato/config/columns/recommendations/similarity:similarity-strato-client",
"strato/src/main/scala/com/twitter/strato/client",
],
exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

View File

@ -6,13 +6,11 @@ scala_library(
dependencies = [
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@ -25,8 +23,6 @@ scala_library(
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"stringcenter/client",
"stringcenter/client/src/main/java",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

View File

@ -1,6 +1,8 @@
package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

View File

@ -8,7 +8,9 @@ scala_library(
"finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@ -18,6 +20,7 @@ scala_library(
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
],
)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.finagle.tracing.Trace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.bijection.Base64String
import com.twitter.bijection.scrooge.BinaryScalaCodec

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

View File

@ -1,8 +1,10 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
import com.twitter.tweetypie.{thriftscala => tpt}
@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
(
"has_exclusive_conversation_author_id",
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)),
("is_eligible_for_connect_boost", _ => false),
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("get_initial", _.getOrElse(GetInitialFeature, false)),
("get_newer", _.getOrElse(GetNewerFeature, false)),
("get_middle", _.getOrElse(GetMiddleFeature, false)),
("get_older", _.getOrElse(GetOlderFeature, false)),
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
("polling", _.getOrElse(PollingFeature, false)),
("tls_size_20_plus", _ => false),
("near_empty", _ => false),
("ranked_request", _ => false),
("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
(
"less_than_10_mins_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
(
"is_signup_request",
candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("empty_request", _ => false),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
(
"served_size_between_50_and_100",
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
(
"account_age_less_than_30_minutes",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"account_age_less_than_1_day",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
(
"account_age_less_than_7_days",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
("root_user_is_in_first_degree", _ => false),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
(
"part_of_utt",
_.getOrElse(EarlybirdFeature, None)
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
(
"has_home_latest_request_past_week",
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("viewer_is_employee", _ => false),
("viewer_is_timelines_employee", _ => false),
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
(
"has_ancestor_authored_by_viewer",
candidate =>
@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
(
"deep_reply",
candidate =>
@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
"tweet_age_less_than_15_seconds",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 15.seconds)),
("is_followed_topic_tweet", _ => false),
("is_recommended_topic_tweet", _ => false),
("is_topic_tweet", _ => false),
("preferred_language_matches_tweet_language", _ => false),
(
"less_than_1_hour_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
(
"device_language_matches_tweet_language",
candidate =>
candidate.getOrElse(TweetLanguageFeature, None) ==
candidate.getOrElse(DeviceLanguageFeature, None)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
("in_network", _.getOrElse(FromInNetworkSourceFeature, true)),
("viewer_follows_original_author", _ => false),
("has_account_follow_prompt", _ => false),
("has_relevance_prompt", _ => false),
("has_topic_annotation_haug_prompt", _ => false),
("has_topic_annotation_random_precision_prompt", _ => false),
("has_topic_annotation_prompt", _ => false),
("in_network", _.getOrElse(InNetworkFeature, true)),
(
"has_political_annotation",
_.getOrElse(EarlybirdFeature, None).exists(
@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
_.getOrElse(EarlybirdFeature, None)
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
("is_viewer_not_invited_to_reply", _ => false),
("is_viewer_invited_to_reply", _ => false),
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
(
"is_followed_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == BasicTopicContextFunctionalityType)),
(
"is_recommended_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == RecommendationTopicContextFunctionalityType)),
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
(
@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
(
"has_gte_100k_favs",
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
("above_neighbor_is_topic_tweet", _ => false),
("is_topic_tweet_with_neighbor_below", _ => false),
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
(
@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
(
"has_toxicity_score_above_threshold",
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
(
"text_only",
candidate =>
@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
("3_or_more_consecutive_not_in_network", _ => false),
("2_or_more_consecutive_not_in_network", _ => false),
("5_out_of_7_not_in_network", _ => false),
("7_out_of_7_not_in_network", _ => false),
("5_out_of_5_not_in_network", _ => false),
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
("has_liked_by_social_context", _ => false),
("has_followed_by_social_context", _ => false),
("has_topic_social_context", _ => false),
("timeline_entry_has_banner", _ => false),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
(
"conversation_module_has_2_displayed_tweets",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
("served_in_recap_tweet_candidate_module_injection", _ => false),
("served_in_threaded_conversation_module", _ => false)
"has_liked_by_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
.exists(candidateFeatures
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
(
"has_followed_by_social_context",
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
(
"has_topic_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(TopicIdSocialContextFeature, None)
.isDefined &&
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
(
"video_bt_10_60_sec",
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
duration > 10000 && duration <= 60000)),
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
(
"tweet_age_lte_30_minutes",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 30.minutes)),
(
"tweet_age_lte_1_hour",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 1.hour)),
(
"tweet_age_lte_6_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 6.hours)),
(
"tweet_age_lte_12_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 12.hours)),
(
"tweet_age_gte_24_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow >= 24.hours)),
)
val PredicateMap = CandidatePredicates.toMap

View File

@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelineservice.suggests.{thriftscala => st}
object ListClientEventDetailsBuilder
case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
override def apply(
@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
conversationDetails = None,
timelinesDetails = Some(
TimelinesDetails(
injectionType = Some(st.SuggestType.OrganicListTweet.name),
injectionType = Some(suggestType.name),
controllerData = None,
sourceData = None)),
articleDetails = None,

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -4,9 +4,16 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
"src/thrift/com/twitter/timelines/service:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
],
)

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tls}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -0,0 +1,18 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
class FeedbackStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
private val externalStringRegistry = externalStringRegistryProvider.get()
val seeLessOftenFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOften")
val seeLessOftenConfirmationFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.request.FollowingProduct
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => t}
import com.twitter.timelines.util.FeedbackMetadataSerializer
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@ -14,10 +14,13 @@ import javax.inject.Singleton
@Singleton
case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder: LikedBySocialContextBuilder,
listsSocialContextBuilder: ListsSocialContextBuilder,
followedBySocialContextBuilder: FollowedBySocialContextBuilder,
topicSocialContextBuilder: TopicSocialContextBuilder,
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder)
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
def apply(
@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder(query, candidate, features)
.orElse(followedBySocialContextBuilder(query, candidate, features))
.orElse(topicSocialContextBuilder(query, candidate, features))
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
.orElse(listsSocialContextBuilder(query, candidate, features))
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
case Some(_) =>
val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
// Only hydrate the social context into the root tweet in a conversation module

View File

@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
@Singleton
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry],
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
externalStringRegistry = externalStringRegistryProvider.get(),
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)

View File

@ -0,0 +1,52 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => tl}
import com.twitter.timelines.util.FeedbackRequestSerializer
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.timelineservice.thriftscala.FeedbackType
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
private val FeedbackMetadata = tl.FeedbackMetadata(
injectionType = Some(SuggestType.WhoToSubscribe),
engagementType = None,
entityIds = Seq.empty,
ttlMs = None
)
private val FeedbackRequest =
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
private val EncodedFeedbackRequest =
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
}
@Singleton
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest =
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)
override def apply(
query: PipelineQuery,
candidate: UserCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo] =
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

View File

@ -0,0 +1,50 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* "Your Lists" will be rendered for the context and a url link for your lists.
*/
@Singleton
case class ListsSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
candidateFeatures.get(SuggestTypeFeature) match {
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
Some(
GeneralContext(
contextType = ListGeneralContextType,
text = stringCenter.prepare(listString),
url = userName.map(name => ""),
contextImageUrls = None,
landingUrl = None
))
case _ => None
}
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tlst}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -0,0 +1,43 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
case class PopularInYourAreaSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
Some(
GeneralContext(
contextType = LocationGeneralContextType,
text = stringCenter.prepare(popularInYourAreaString),
url = None,
contextImageUrls = None,
landingUrl = None
))
} else None
}
}

View File

@ -1,50 +1,48 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* Renders a fixed 'You Might Like' string above all OON Tweets.
*/
@Singleton
case class YouMightLikeSocialContextBuilder @Inject() (
case class PopularVideoSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val youMightLikeString = externalStrings.socialContextYouMightLikeString
private val popularVideoString = externalStrings.socialContextPopularVideoString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true)
val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false)
if (!isInNetwork && !isRetweet) {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
Some(
GeneralContext(
contextType = SparkleGeneralContextType,
text = stringCenter.prepare(youMightLikeString),
text = stringCenter.prepare(popularVideoString),
url = None,
contextImageUrls = None,
landingUrl = None
landingUrl = Some(
Url(
urlType = DeepLink,
url = ""
)
)
))
} else {
None
}
} else None
}
}

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Singleton

View File

@ -4,95 +4,56 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider",
"finatra/inject/inject-core/src/main/scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
"representation-scorer/server/src/main/thrift:thrift-scala",
"servo/repo/src/main/scala",
"snowflake/src/main/scala/com/twitter/snowflake/id",
"src/java/com/twitter/ml/api/constant",
"src/java/com/twitter/search/common/util/lang",
"src/scala/com/twitter/ml/api/util",
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
"src/scala/com/twitter/timelines/prediction/common/util",
"src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/features/recap",
"src/scala/com/twitter/timelines/prediction/features/time_features",
"src/scala/com/twitter/timelines/prediction/adapters/request_context",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:embedding-java",
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-java",
"src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/spam/rtf:safety-result-scala",
"src/thrift/com/twitter/timelineranker:thrift-scala",
"src/thrift/com/twitter/timelines/author_features:thrift-java",
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
"src/thrift/com/twitter/timelines/impression:thrift-scala",
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"src/thrift/com/twitter/user_session_store:thrift-java",
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
"stitch/stitch-core",
"stitch/stitch-gizmoduck",
"stitch/stitch-socialgraph",
"stitch/stitch-timelineservice",
"stitch/stitch-tweetypie",
"strato/config/columns/topic-signals/tsp",
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
"timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly",
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"topic-social-proof/server/src/main/thrift:thrift-scala",
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
"twitter-config/yaml",
"user_session_store/src/main/scala/com/twitter/user_session_store",
"util/util-core",
],

View File

@ -1,12 +1,10 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
@ -17,16 +15,12 @@ import javax.inject.Singleton
@Singleton
case class FeedbackHistoryQueryFeatureHydrator @Inject() (
feedbackHistoryClient: FeedbackHistoryManhattanClient)
extends QueryFeatureHydrator[PipelineQuery]
with Conditionally[PipelineQuery] {
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableFeedbackFatigueParam)
override def hydrate(
query: PipelineQuery
): Stitch[FeatureMap] =

View File

@ -1,41 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class FollowedTopicsQueryFeatureHydrator @Inject() (
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
followedTopicsCandidateSource(request)
.map { topics =>
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
}.handle {
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
}
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
)
}

View File

@ -1,58 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.gizmoduck.{thriftscala => gt}
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.stitch.gizmoduck.Gizmoduck
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
val blueVerifiedStitch = authorIdOption
.map { authorId =>
gizmoduck
.getUserById(
userId = authorId,
queryFields = queryFields
)
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
}.getOrElse(Stitch.False)
blueVerifiedStitch.map { isBlueVerified =>
FeatureMapBuilder()
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
.build()
}
}
}

View File

@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t}
import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import javax.inject.Inject
import javax.inject.Singleton
@ -19,36 +21,39 @@ import javax.inject.Singleton
@Singleton
case class ImpressionBloomFilterQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
bloomFilter: ImpressionBloomFilter)
extends QueryFeatureHydrator[Query] {
bloomFilterClient: ManhattanStoreClient[
blm.ImpressionBloomFilterKey,
blm.ImpressionBloomFilterSeq
]) extends QueryFeatureHydrator[Query] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"ImpressionBloomFilter")
private val ImpressionBloomFilterTTL = 7.day
private val ImpressionBloomFilterFalsePositiveRate = 0.002
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
private val SurfaceArea = t.SurfaceArea.HomeTimeline
private val SurfaceArea = blm.SurfaceArea.HomeTimeline
override def hydrate(query: Query): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq =>
val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else {
bloomFilter.addElements(
userId = userId,
surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get,
bloomFilterEntrySeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = ImpressionBloomFilterFalsePositiveRate
)
}
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
}
bloomFilterClient
.get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
.map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
.map { bloomFilterSeq =>
val updatedBloomFilterSeq =
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
else {
ImpressionBloomFilter.addSeenTweetIds(
surfaceArea = SurfaceArea,
tweetIds = query.seenTweetIds.get,
bloomFilterSeq = bloomFilterSeq,
timeToLive = ImpressionBloomFilterTTL,
falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
)
}
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
}
}
override val alerts = Seq(

View File

@ -0,0 +1,41 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object InNetworkFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
val featureMaps = candidates.map { candidate =>
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
isSelfTweet || followedUserIds.contains(authorId)
}
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
}
Stitch.value(featureMaps)
}
}

View File

@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
@ -27,17 +29,27 @@ import javax.inject.Singleton
@Singleton
case class PersistenceStoreQueryFeatureHydrator @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3])
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
private val WhoToFollowExcludedUserIdsLimit = 1000
private val ServedTweetIdsDuration = 1.hour
private val ServedTweetIdsDuration = 10.minutes
private val ServedTweetIdsLimit = 100
private val ServedTweetPreviewIdsDuration = 10.hours
private val ServedTweetPreviewIdsLimit = 10
override val features: Set[Feature[_, _]] =
Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature)
Set(
ServedTweetIdsFeature,
ServedTweetPreviewIdsFeature,
PersistenceEntriesFeature,
WhoToFollowExcludedUserIdsFeature)
private val supportedClients = Seq(
ClientPlatform.IPhone,
@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
.flatMap(
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
servedTweetIdsSizeStat.add(servedTweetIds.size)
val servedTweetPreviewIds = timelineResponses
.filter(_.clientPlatform == clientPlatform)
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
.sortBy(-_.servedTime.inMilliseconds)
.flatMap(_.entries
.filter(_.entityIdType == EntityIdType.TweetPreview)
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
FeatureMapBuilder()
.add(ServedTweetIdsFeature, servedTweetIds)
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
.add(PersistenceEntriesFeature, timelineResponses)
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
.build()

View File

@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch
import com.twitter.stitch.timelineservice.TimelineService
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.map(favoritedBy => favoritedBy -> candidate.candidate.id)
}
@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
candidates.map { candidate =>
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers)
.getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
FeatureMapBuilder()

View File

@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
val realGraphScoresFeatures = realGraphFollowedUsers
.getOrElse(Seq.empty)
.sortBy(-_.score)
.map(candidate => candidate.userId -> candidate.score)
.map(candidate => candidate.userId -> scaleScore(candidate.score))
.take(RealGraphCandidateCount)
.toMap
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
}
}
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
private def scaleScore(score: Double): Double =
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
}

View File

@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.search.common.util.lang.ThriftLanguageUtil
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
import java.util.UUID
import javax.inject.Inject
import javax.inject.Singleton
@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
PullToRefreshFeature,
RequestJoinIdFeature,
ServedRequestIdFeature,
TimestampFeature,
TimestampGMTDowFeature,
TimestampGMTHourFeature,
ViewerIdFeature
)
@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
override def hydrate(query: Query): Stitch[FeatureMap] = {
val requestContext = query.deviceContext.flatMap(_.requestContextValue)
val servedRequestId = UUID.randomUUID.getMostSignificantBits
val timestamp = query.queryTime.inMilliseconds
val featureMap = FeatureMapBuilder()
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
.add(ServedRequestIdFeature, Some(servedRequestId))
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
.add(TimestampFeature, timestamp)
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
.add(HasDarkRequestFeature, hasDarkRequest)
.add(ViewerIdFeature, query.getRequiredUserId)
.add(
ViewerIdFeature,
query.getOptionalUserId
.orElse(query.getGuestId).getOrElse(
throw PipelineFailure(BadRequest, "Missing viewer id")))
.build()
Stitch.value(featureMap)

View File

@ -1,46 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
socialGraphStitchClient: SocialGraphStitchClient)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SGSFollowedUsers")
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
private val SocialGraphLimit = 14999
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
val request = sg.IdsRequest(
relationships = Seq(
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
),
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
)
socialGraphStitchClient
.ids(request).map(_.ids)
.map { followedUsers =>
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
}
}
}

View File

@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val allSocialContextUserIds =
candidates.flatMap { candidate =>
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

View File

@ -1,67 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SocialGraphService")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
val idsRequest = createIdsRequest(
userId = viewerId,
relationshipTypes = Set(sg.RelationshipType.Following),
targetIds = Some(distinctNonSelfAuthorIds)
)
socialGraphStitchClient
.ids(request = idsRequest, requestContext = None)
.map { idResult =>
authorIds.map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
val inNetworkAuthorIds = idResult.ids.toSet
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
}
}
}
private def createIdsRequest(
userId: Long,
relationshipTypes: Set[sg.RelationshipType],
targetIds: Option[Seq[Long]] = None
): sg.IdsRequest = sg.IdsRequest(
relationshipTypes.map { relationshipType =>
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
}.toSeq,
Some(sg.PageRequest(selectAll = Some(true)))
)
}

View File

@ -1,162 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.contentrecommender.{thriftscala => cr}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
import com.twitter.topiclisting.TopicListingViewerContext
import com.twitter.tsp.{thriftscala => tsp}
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
object TSPInferredTopicDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TSPInferredTopicFeatureHydrator @Inject() (
topicSocialProofClientColumn: TopicSocialProofClientColumn,
statsReceiver: StatsReceiver,
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
override val features: Set[Feature[_, _]] =
Set(
TSPInferredTopicFeature,
TSPInferredTopicDataRecordFeature,
TopicIdSocialContextFeature,
TopicContextFunctionalityTypeFeature)
private val topK = 3
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
sid.CandidateTweetSourceId.Simcluster,
sid.CandidateTweetSourceId.CroonTweet
)
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
private val DefaultFeatureMap = FeatureMapBuilder()
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
.add(TopicIdSocialContextFeature, None)
.add(TopicContextFunctionalityTypeFeature, None)
.build()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val tags = candidates.collect {
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
candidate.candidate.id -> candidate.features
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
}.toMap
val topicSocialProofRequest =
tsp.TopicSocialProofRequest(
userId = query.getRequiredUserId,
tweetIds = candidates.map(_.candidate.id).toSet,
displayLocation = cr.DisplayLocation.HomeTimeline,
topicListingSetting = tsp.TopicListingSetting.Followable,
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
bypassModes = None,
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
tags = if (tags.isEmpty) None else Some(tags)
)
topicSocialProofClientColumn.fetcher
.fetch(topicSocialProofRequest)
.map(_.v)
.map {
case Some(response) =>
candidates.map { candidate =>
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
if (topicWithScores.nonEmpty) {
keyFoundCounter.incr()
val (socialProofId, socialProofFunctionalityType) =
if (candidate.features
.getOrElse(CandidateSourceIdFeature, None)
.exists(sourcesToSetSocialProof.contains)) {
getSocialProof(topicWithScores)
} else {
(None, None)
}
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
val inferredTopicDataRecord =
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
FeatureMapBuilder()
.add(TSPInferredTopicFeature, inferredTopicFeatures)
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
.add(TopicIdSocialContextFeature, socialProofId)
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
.build()
} else {
keyLossCounter.incr()
DefaultFeatureMap
}
}
case _ =>
requestFailCounter.incr()
candidates.map { _ =>
DefaultFeatureMap
}
}
}
private def getSocialProof(
topicWithScores: Seq[tsp.TopicWithScore]
): (Option[Long], Option[TopicContextFunctionalityType]) = {
val followingTopicId = topicWithScores
.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
topicId
}
if (followingTopicId.nonEmpty) {
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
}
val implicitFollowingId = topicWithScores.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
topicId
}
if (implicitFollowingId.nonEmpty) {
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
}
(None, None)
}
private def convertTopicWithScores(
topicWithScores: Seq[tsp.TopicWithScore],
): Map[Long, Double] = {
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
}
}

View File

@ -1,251 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.FDsl._
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.features.{thriftscala => sc}
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
import com.twitter.util.Duration
import scala.collection.Searching._
object TimeFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
Stitch.value {
val richDataRecord = new RichDataRecord()
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
FeatureMapBuilder()
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
.build()
}
}
private def setTimeFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
query: PipelineQuery,
): Unit = {
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
}
private[feature_hydrator] def getTimeFeatures(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
): Option[TimeFeatures] = {
for {
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
tweetId <- Some(candidate.id)
viewerId <- query.getOptionalUserId
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
}
timeSinceSourceTweetCreation =
existingFeatures
.getOrElse(SourceTweetIdFeature, None)
.flatMap { sourceTweetId =>
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
requestTimestampMs - sourceTweetCreationTimeMs
}
}
.getOrElse(timeSinceTweetCreation)
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
} yield {
val timeFeatures = TimeFeatures(
timeSinceTweetCreation = timeSinceTweetCreation,
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
)
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
existingFeatures.getOrElse(EarlybirdFeature, None),
timeFeatures,
timeSinceSourceTweetCreation
).getOrElse(timeFeatures)
val nonPollingTimestampsMs =
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
timeFeaturesWithLastEngagement,
requestTimestampMs,
tweetCreationTimeMs,
nonPollingTimestampsMs
)
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
}
}
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
else None
}
private def addLastEngagementTimeFeatures(
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
timeFeatures: TimeFeatures,
timeSinceSourceTweetCreation: Long
): Option[TimeFeatures] = {
tweetFeaturesOpt.map { tweetFeatures =>
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
timeFeatures.copy(
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
lastFavSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
lastRetweetSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
lastReplySinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
lastQuoteSinceCreationHrs,
timeSinceSourceTweetCreation
)
)
}
}
private def addNonPollingTimeFeatures(
timeFeatures: TimeFeatures,
requestTimestampMs: Long,
creationTimeMs: Long,
nonPollingTimestampsMs: Option[Seq[Long]]
): Option[TimeFeatures] = {
for {
nonPollingTimestampsMs <- nonPollingTimestampsMs
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
} yield {
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
1.0,
timeFeatures.timeSinceTweetCreation
)
/*
* Non-polling timestamps are stored in chronological order.
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
*/
val nonPollingRequestsSinceTweetCreation =
if (nonPollingTimestampsMs.nonEmpty) {
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
} else {
0
}
/*
* Calculate the average time between non-polling requests; include
* request time in this calculation as latest timestamp.
*/
val timeBetweenNonPollingRequestsAvg =
(requestTimestampMs - earliestNonPollingTimestampMs) / math
.max(1.0, nonPollingTimestampsMs.size)
val timeFeaturesWithNonPolling = timeFeatures.copy(
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
tweetAgeRatio = Some(tweetAgeRatio)
)
timeFeaturesWithNonPolling
}
}
private[this] def getTimeSinceLastEngagementHrs(
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
timeSinceTweetCreation: Long
): Option[Double] = {
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
}
}
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
val record = richDataRecord.getRecord
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
.setFeatureValueFromOption(
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
features.timeSinceViewerAccountCreationSecs)
.setFeatureValue(
USER_ID_IS_SNOWFLAKE_ID,
features.timeSinceViewerAccountCreationSecs.isDefined
)
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
.setFeatureValueFromOption(
LAST_RETWEET_SINCE_CREATION_HRS,
features.lastRetweetSinceCreationHrs
)
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
/*
* set features whose values are optional as some users do not have non-polling timestamps
*/
features.timeBetweenNonPollingRequestsAvg.foreach(
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
)
features.timeSinceLastNonPollingRequest.foreach(
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
)
features.nonPollingRequestsSinceTweetCreation.foreach(
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
)
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
}
}

View File

@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends QueryFeatureHydrator[Query] {
private val TweetImpressionTTL = 1.day
private val TweetImpressionCap = 3000
private val TweetImpressionTTL = 2.days
private val TweetImpressionCap = 5000
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")

View File

@ -1,6 +1,9 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.util.tweetypie.RequestFields
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
import com.twitter.stitch.Stitch
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
import com.twitter.tweetypie.{thriftscala => tp}
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
class TweetypieFeatureHydrator @Inject() (
tweetypieStitchClient: TweetypieStitchClient,
statsReceiver: StatsReceiver)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature,
ExclusiveConversationAuthorIdFeature,
InReplyToTweetIdFeature,
IsHydratedFeature,
IsNsfw,
@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
SourceTweetIdFeature,
SourceUserIdFeature,
TweetTextFeature,
TweetLanguageFeature,
VisibilityReason
)
@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
): Stitch[FeatureMap] = {
val safetyLevel = query.product match {
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
case ForYouProduct => rtf.SafetyLevel.TimelineHome
case ForYouProduct =>
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
}
@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
includeQuotedTweet = true,
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
safetyLevel = Some(safetyLevel),
forUserId = Some(query.getRequiredUserId)
forUserId = query.getOptionalUserId
)
val exclusiveAuthorIdOpt =
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
val coreData = found.tweet.coreData
@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
val tweetText = coreData.map(_.text)
val tweetLanguage = found.tweet.language.map(_.language)
val tweetAuthorId = coreData.map(_.userId)
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
FeatureMapBuilder()
.add(AuthorIdFeature, tweetAuthorId)
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, inReplyToTweetId)
.add(IsHydratedFeature, true)
.add(IsNsfw, Some(isNsfw))
@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
.add(QuotedUserIdFeature, quotedTweetUserId)
.add(SourceTweetIdFeature, retweetedTweetId)
.add(SourceUserIdFeature, retweetedTweetUserId)
.add(TweetLanguageFeature, tweetLanguage)
.add(TweetTextFeature, tweetText)
.add(VisibilityReason, found.suppressReason)
.build()
// If no tweet result found, return default and pre-existing features
case _ =>
DefaultFeatureMap +
(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) +
(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) +
(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) +
(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) +
(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) +
(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) +
(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
DefaultFeatureMap ++ FeatureMapBuilder()
.add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
.add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
.add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
.add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
.add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
.build()
}
}
}

View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
import com.twitter.ml.api.DataRecordMerger
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.CompactDataRecordConverter
import com.twitter.ml.api.util.FDsl._
import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
private val originalAuthorAggregatesFeatures =
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
private val authorFeatures = originalAuthorAggregatesFeatures ++
Seq(
UserHealthFeatures.AuthorState,
UserHealthFeatures.NumAuthorFollowers,
UserHealthFeatures.NumAuthorConnectDays,
UserHealthFeatures.NumAuthorConnect)
private val featureContext = new FeatureContext(authorFeatures: _*)
override def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
private val compactDataRecordConverter = new CompactDataRecordConverter()
private val drMerger = new DataRecordMerger()
override def setFeatures(
authorFeaturesOpt: Option[af.AuthorFeatures],
richDataRecord: RichDataRecord
): Unit = {
authorFeaturesOpt.foreach { authorFeatures =>
val dataRecord = richDataRecord.getRecord
dataRecord.setFeatureValue(
UserHealthFeatures.AuthorState,
authorFeatures.user_health.user_state.getValue.toLong)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorFollowers,
authorFeatures.user_health.num_followers.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnectDays,
authorFeatures.user_health.num_connect_days.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnect,
authorFeatures.user_health.num_connect.toDouble)
val originalAuthorAggregatesDataRecord =
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
}
}
}

View File

@ -1,22 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("Phase2EdgeAggregate")
override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
Set(
UserEngagerAggregateFeature,
UserEngagerGoodClickAggregateFeature,
UserInferredTopicAggregateFeature,
UserTopicAggregateFeature,
UserMediaUnderstandingAnnotationAggregateFeature
)
}

View File

@ -4,6 +4,7 @@ scala_library(
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
@ -16,9 +17,11 @@ scala_library(
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"stitch/stitch-core",
"stitch/stitch-socialgraph",
"stitch/stitch-tweetypie",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"util/util-slf4j-api/src/main/scala",
],
exports = [
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",

View File

@ -72,7 +72,7 @@ object FeedbackFatigueFilter
originalAuthorId.exists(authorsToFilter.contains) ||
(likers.nonEmpty && eligibleLikers.isEmpty) ||
(followers.nonEmpty && eligibleFollowers.isEmpty) ||
(followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
(candidate.features.getOrElse(IsRetweetFeature, false) &&
authorId.exists(retweetersToFilter.contains))
}

View File

@ -0,0 +1,70 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
/**
* Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
*
* If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
* subscription tweets, so we explicitly filter those tweets out.
*/
@Singleton
case class InvalidSubscriptionTweetFilter @Inject() (
socialGraphClient: SocialGraph,
statsReceiver: StatsReceiver)
extends Filter[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = Stitch
.traverse(candidates) { candidate =>
val exclusiveAuthorId =
candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
if (exclusiveAuthorId.isDefined) {
val request = sg.ExistsRequest(
source = query.getRequiredUserId,
target = exclusiveAuthorId.get,
relationships =
Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
)
socialGraphClient.exists(request).map(_.exists).map { valid =>
if (!valid) invalidCounter.incr() else validCounter.incr()
valid
}
} else Stitch.value(true)
}.map { validResults =>
val (kept, removed) = candidates
.map(_.candidate)
.zip(validResults)
.partition { case (candidate, valid) => valid }
val keptCandidates = kept.map { case (candidate, _) => candidate }
val removedCandidates = removed.map { case (candidate, _) => candidate }
FilterResult(kept = keptCandidates, removed = removedCandidates)
}
}

View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Predicate which will be applied to each candidate. True indicates that the candidate will be
* @tparam Candidate - the type of the candidate
*/
trait ShouldKeepCandidate {
def apply(features: FeatureMap): Boolean
}
object PredicateFeatureFilter {
/**
* Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
* we recommend including the name of the shouldKeepCandidate parameter.
*
* @param identifier A FilterIdentifier for the new filter
* @param shouldKeepCandidate A predicate function. Candidates will be kept when
* this function returns True.
*/
def fromPredicate[Candidate <: UniversalNoun[Any]](
identifier: FilterIdentifier,
shouldKeepCandidate: ShouldKeepCandidate
): Filter[PipelineQuery, Candidate] = {
val i = identifier
new Filter[PipelineQuery, Candidate] {
override val identifier: FilterIdentifier = i
/**
* Filter the list of candidates
*
* @return a FilterResult including both the list of kept candidate and the list of removed candidates
*/
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]] = {
val allowedIds = candidates
.filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
candidate => allowedIds.contains(candidate.id)
}
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
}
}
}
}

View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] {
object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor")
override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
// Set containing best OON tweet for each authorId
val bestCandidatesForAuthorId = candidates
.filter(!_.features.getOrElse(InNetworkFeature, true))
.groupBy(_.features.getOrElse(AuthorIdFeature, None))
.values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
.toSet
val servedTweetPreviewIds =
query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
val (removed, kept) = candidates.partition { candidate =>
!candidate.features.getOrElse(InNetworkFeature, true) &&
!bestCandidatesForAuthorId.contains(candidate)
servedTweetPreviewIds.contains(candidate.candidate.id)
}
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))

View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Reply")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
!candidate.features.getOrElse(IsRetweetFeature, false)
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

View File

@ -14,7 +14,6 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"stitch/stitch-socialgraph",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
],

View File

@ -1,14 +1,14 @@
package com.twitter.home_mixer.functional_component.gate
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.stitch.Stitch
import com.twitter.util.Duration
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.util.Duration
object DismissFatigueGate {
// how long a dismiss action from user needs to be respected

View File

@ -1,18 +0,0 @@
package com.twitter.home_mixer.functional_component.gate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import scala.reflect.runtime.universe._
case class NonEmptySeqFeatureGate[T: TypeTag](
feature: Feature[PipelineQuery, Seq[T]])
extends Gate[PipelineQuery] {
override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
Stitch.value(query.features.exists(_.get(feature).nonEmpty))
}

View File

@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
// The time window for which a tweet remains editable after creation.
private val EditTimeWindow = 30.minutes
private val EditTimeWindow = 60.minutes
override def transform(query: PipelineQuery): Seq[Long] = {
val applicableCandidates = getApplicableCandidates(query)
@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
// Any tweets in it could have become stale since being served.
val previousTimelineLoadTime = applicableCandidates.head.servedTime
// The time window for editing a tweet is 30 minutes,
// so we ignore responses older than (PTL Time - 30 mins).
// The time window for editing a tweet is 60 minutes,
// so we ignore responses older than (PTL Time - 60 mins).
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)

View File

@ -6,7 +6,6 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",

Some files were not shown because too many files have changed in this diff Show More