Mirrors/the-algorithm
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2024-06-02 17:28:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: Mirrors:2f024b4829c1604f11026005aeb5d917a7f1ea4a
Branches Tags
Mirrors:main
...
compare: Mirrors:0989664e98a1c846f8e5f68a07c6ca7d9824c8b3
Branches Tags
Mirrors:main

6 Commits
2f024b4829 ... 0989664e98

Author SHA1 Message Date
Joanie Gannon
0989664e98
Merge b3909b73d5 into 72eda9a24f 2023-07-17 21:42:18 -05:00
twitter-team
72eda9a24f [opensource] Update home mixer with latest changes 2023-07-13 16:33:04 +05:30
Joanie Gannon
b3909b73d5 Updated typos in comments found in 42 more files 2023-03-31 22:16:43 -04:00
Joanie Gannon
9ccb459970 adding more 2023-03-31 20:22:10 -04:00
Joanie Gannon
4ba718b412 I found some more 2023-03-31 20:07:28 -04:00
Joanie Gannon
9953b499bf fixing typo in comment 2023-03-31 19:57:52 -04:00
438 changed files with 9386 additions and 3902 deletions
Show Stats Expand all files Collapse all files

4
ann/src/main/java/com/twitter/ann/hnsw/DistancedItemQueue.java
View File

@ -18,7 +18,7 @@ public class DistancedItemQueue<U, T> implements Iterable<DistancedItem<T>> {
private final PriorityQueue<DistancedItem<T>> queue; private final PriorityQueue<DistancedItem<T>> queue;
private final boolean minQueue; private final boolean minQueue;
/** /**
* Creates ontainer for items with their distances. * Creates container for items with their distances.
* *
* @param origin Origin (reference) point * @param origin Origin (reference) point
* @param initial Initial list of elements to add in the structure * @param initial Initial list of elements to add in the structure
@ -94,7 +94,7 @@ public class DistancedItemQueue<U, T> implements Iterable<DistancedItem<T>> {
} }
/** /**
* Dequeue all the elements from queueu with ordering mantained * Dequeue all the elements from queue with ordering maintained
* *
* @return remove all the elements in the order of the queue i.e min/max queue. * @return remove all the elements in the order of the queue i.e min/max queue.
*/ */

4
ann/src/main/java/com/twitter/ann/hnsw/HnswIndex.java
View File

@ -379,7 +379,7 @@ public class HnswIndex<T, Q> {
* This will reduce the recall. * This will reduce the recall.
* <p> * <p>
* For a full explanation of locking see this document: http://go/hnsw-locking * For a full explanation of locking see this document: http://go/hnsw-locking
* The method returns the closest nearest neighbor (can be used as an enter point) * The method returns the closest nearest neighbours (can be used as an enter point)
*/ */
private T mutuallyConnectNewElement( private T mutuallyConnectNewElement(
final T item, final T item,
@ -532,7 +532,7 @@ public class HnswIndex<T, Q> {
* @param numOfNeighbours Number of neighbours to search for. * @param numOfNeighbours Number of neighbours to search for.
* @param ef This param controls the accuracy of the search. * @param ef This param controls the accuracy of the search.
* Bigger the ef better the accuracy on the expense of latency. * Bigger the ef better the accuracy on the expense of latency.
* Keep it atleast number of neighbours to find. * Keep it at least number of neighbours to find.
* @return Neighbours * @return Neighbours
*/ */
public List<DistancedItem<T>> searchKnn(final Q query, final int numOfNeighbours, final int ef) { public List<DistancedItem<T>> searchKnn(final Q query, final int numOfNeighbours, final int ef) {

2
ann/src/main/scala/com/twitter/ann/brute_force/BruteForceIndex.scala
View File

@ -105,7 +105,7 @@ object SerializableBruteForceIndex {
} }
/** /**
* This is a class that wrapps a BruteForceIndex and provides a method for serialization. * This is a class that wraps a BruteForceIndex and provides a method for serialization.
* *
* @param bruteForceIndex all queries and updates are sent to this index. * @param bruteForceIndex all queries and updates are sent to this index.
* @param embeddingInjection injection that can convert embeddings to thrift embeddings. * @param embeddingInjection injection that can convert embeddings to thrift embeddings.

2
ann/src/main/scala/com/twitter/ann/dataflow/offline/ANNIndexBuilderBeamJob.scala
View File

@ -229,7 +229,7 @@ object ANNIndexBuilderBeamJob extends ScioBeamJob[ANNOptions] {
// Generate Index // Generate Index
processedCollection.saveAsCustomOutput( processedCollection.saveAsCustomOutput(
"Serialise to Disk", "Serialize to Disk",
OutputSink( OutputSink(
out, out,
opts.getAlgo.equals("faiss"), opts.getAlgo.equals("faiss"),

4
ann/src/main/scala/com/twitter/ann/hnsw/TypedHnswIndex.scala
View File

@ -18,7 +18,7 @@ object TypedHnswIndex {
* construction, but better index quality. At some point, increasing * construction, but better index quality. At some point, increasing
* ef_construction does not improve the quality of the index. One way to * ef_construction does not improve the quality of the index. One way to
* check if the selection of ef_construction was ok is to measure a recall * check if the selection of ef_construction was ok is to measure a recall
* for M nearest neighbor search when ef = ef_constuction: if the recall is * for M nearest neighbour search when ef = ef_construction: if the recall is
* lower than 0.9, than there is room for improvement. * lower than 0.9, than there is room for improvement.
* @param maxM The number of bi-directional links created for every new element during construction. * @param maxM The number of bi-directional links created for every new element during construction.
* Reasonable range for M is 2-100. Higher M work better on datasets with high * Reasonable range for M is 2-100. Higher M work better on datasets with high
@ -64,7 +64,7 @@ object TypedHnswIndex {
* construction, but better index quality. At some point, increasing * construction, but better index quality. At some point, increasing
* ef_construction does not improve the quality of the index. One way to * ef_construction does not improve the quality of the index. One way to
* check if the selection of ef_construction was ok is to measure a recall * check if the selection of ef_construction was ok is to measure a recall
* for M nearest neighbor search when ef = ef_constuction: if the recall is * for M nearest neighbour search when ef = ef_construction: if the recall is
* lower than 0.9, than there is room for improvement. * lower than 0.9, than there is room for improvement.
* @param maxM The number of bi-directional links created for every new element during construction. * @param maxM The number of bi-directional links created for every new element during construction.
* Reasonable range for M is 2-100. Higher M work better on datasets with high * Reasonable range for M is 2-100. Higher M work better on datasets with high

2
ann/src/main/scala/com/twitter/ann/scalding/offline/KnnTruthSetGenerator.scala
View File

@ -12,7 +12,7 @@ import com.twitter.scalding.UniqueID
import com.twitter.scalding_internal.job.TwitterExecutionApp import com.twitter.scalding_internal.job.TwitterExecutionApp
/** /**
* This job reads index embedding data, query embeddings data, and split into index set, query set and true nearest neigbor set * This job reads index embedding data, query embeddings data, and split into index set, query set and true nearest neighbour set
* from query to index. * from query to index.
*/ */
object KnnTruthSetGenerator extends TwitterExecutionApp { object KnnTruthSetGenerator extends TwitterExecutionApp {

2
ann/src/main/scala/com/twitter/ann/scalding/offline/indexbuilder/README.rst
View File

@ -95,7 +95,7 @@ General arguments (specified as **--profile.{options}**):
- **num_dimensions** Dimension of embedding in the input data. An exception will be thrown if any entry does not have a number of dimensions equal to this number. - **num_dimensions** Dimension of embedding in the input data. An exception will be thrown if any entry does not have a number of dimensions equal to this number.
- **metric** Distance metric (InnerProduct/Cosine/L2) - **metric** Distance metric (InnerProduct/Cosine/L2)
- **concurrency_level** Specifies how many parallel inserts happen to the index. This should probably be set to the number of cores on the machine. - **concurrency_level** Specifies how many parallel inserts happen to the index. This should probably be set to the number of cores on the machine.
- **algo** The kind of index you want to ouput. The supported options right now are: - **algo** The kind of index you want to output. The supported options right now are:
1. **hnsw** (Metric supported: Cosine, L2, InnerProduct) 1. **hnsw** (Metric supported: Cosine, L2, InnerProduct)

2
ann/src/main/scala/com/twitter/ann/service/loadtest/LoadTestRecorder.scala
View File

@ -179,7 +179,7 @@ class InMemoryLoadTestQueryRecorder[T](
latencyHistogram.add(queryLatency.inMicroseconds) latencyHistogram.add(queryLatency.inMicroseconds)
counter.incrementAndGet() counter.incrementAndGet()
// Requests are assumed to have started around the time time of the first time record was called // Requests are assumed to have started around the time time of the first time record was called
// plus the time it took for that query to hhave completed. // plus the time it took for that query to have completed.
val (elapsedSinceFirstCall, firstQueryLatency) = elapsedTimeFun.get() val (elapsedSinceFirstCall, firstQueryLatency) = elapsedTimeFun.get()
val durationSoFar = elapsedSinceFirstCall() + firstQueryLatency val durationSoFar = elapsedSinceFirstCall() + firstQueryLatency
elapsedTime.set(durationSoFar) elapsedTime.set(durationSoFar)

2
ann/src/main/scala/com/twitter/ann/service/query_server/common/BaseQueryIndexServer.scala
View File

@ -24,7 +24,7 @@ abstract class BaseQueryIndexServer extends ThriftServer with Mtls {
protected val environment: Flag[String] = flag[String]("environment", "service environment") protected val environment: Flag[String] = flag[String]("environment", "service environment")
/** /**
* Override with method to provide more module to guice. * Override with method to provide more module to guide.
*/ */
protected def additionalModules: Seq[Module] protected def additionalModules: Seq[Module]

2
ann/src/main/scala/com/twitter/ann/util/IndexBuilderUtils.scala
View File

@ -15,7 +15,7 @@ object IndexBuilderUtils {
concurrencyLevel: Int concurrencyLevel: Int
): Future[Int] = { ): Future[Int] = {
val count = new AtomicInteger() val count = new AtomicInteger()
// Async stream allows us to procss at most concurrentLevel futures at a time. // Async stream allows us to process at most concurrentLevel futures at a time.
Future.Unit.before { Future.Unit.before {
val stream = AsyncStream.fromSeq(embeddings) val stream = AsyncStream.fromSeq(embeddings)
val appendStream = stream.mapConcurrent(concurrencyLevel) { annEmbedding => val appendStream = stream.mapConcurrent(concurrencyLevel) { annEmbedding =>

2
cr-mixer/server/src/main/scala/com/twitter/cr_mixer/candidate_generation/UtegTweetCandidateGenerator.scala
View File

@ -57,7 +57,7 @@ class UtegTweetCandidateGenerator @Inject() (
* supported by the any existing Candidate type, so we created TweetWithScoreAndSocialProof * supported by the any existing Candidate type, so we created TweetWithScoreAndSocialProof
* instead. * instead.
* *
* However, filters and light ranker expect Candidate-typed param to work. In order to minimise the * However, filters and light ranker expect Candidate-typed param to work. In order to minimize the
* changes to them, we are doing conversions from/to TweetWithScoreAndSocialProof to/from Candidate * changes to them, we are doing conversions from/to TweetWithScoreAndSocialProof to/from Candidate
* in this method. * in this method.
*/ */

2
cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/CandidateGenerationInfo.scala
View File

@ -59,7 +59,7 @@ case class SimilarityEngineInfo(
* *
* @param sourceInfoOpt - this is optional as many consumerBased CG does not have a source * @param sourceInfoOpt - this is optional as many consumerBased CG does not have a source
* @param similarityEngineInfo - the similarity engine used in Candidate Generation (eg., TweetBasedUnifiedSE). It can be an atomic SE or an composite SE * @param similarityEngineInfo - the similarity engine used in Candidate Generation (eg., TweetBasedUnifiedSE). It can be an atomic SE or an composite SE
* @param contributingSimilarityEngines - only composite SE will have it (e.g., SANNN, UTG). Otherwise it is an empty Seq. All contributing SEs mst be atomic * @param contributingSimilarityEngines - only composite SE will have it (e.g., SANN, UTG). Otherwise it is an empty Seq. All contributing SEs mst be atomic
*/ */
case class CandidateGenerationInfo( case class CandidateGenerationInfo(
sourceInfoOpt: Option[SourceInfo], sourceInfoOpt: Option[SourceInfo],

2
cr-mixer/server/src/main/scala/com/twitter/cr_mixer/model/ModelConfig.scala
View File

@ -45,7 +45,7 @@ object ModelConfig {
val DebuggerDemo: String = "DebuggerDemo" val DebuggerDemo: String = "DebuggerDemo"
// ColdStartLookalike - this is not really a model name, it is as a placeholder to // ColdStartLookalike - this is not really a model name, it is as a placeholder to
// indicate ColdStartLookalike candidate source, which is currently being pluged into // indicate ColdStartLookalike candidate source, which is currently being plugged into
// CustomizedRetrievalCandidateGeneration temporarily. // CustomizedRetrievalCandidateGeneration temporarily.
val ColdStartLookalikeModelName: String = "ConsumersBasedUtgColdStartLookalike20220707" val ColdStartLookalikeModelName: String = "ConsumersBasedUtgColdStartLookalike20220707"

2
cr-mixer/server/src/main/scala/com/twitter/cr_mixer/similarity_engine/ConsumerBasedWalsSimilarityEngine.scala
View File

@ -63,7 +63,7 @@ case class WalsStats(scope: String, scopedStats: StatsReceiver) {
} }
// StatsMap maintains a mapping from Model's input signature to a stats receiver // StatsMap maintains a mapping from Model's input signature to a stats receiver
// The Wals model suports multiple input signature which can run different graphs internally and // The Wals model supports multiple input signature which can run different graphs internally and
// can have a different performance profile. // can have a different performance profile.
// Invoking StatsReceiver.stat() on each request can create a new stat object and can be expensive // Invoking StatsReceiver.stat() on each request can create a new stat object and can be expensive
// in performance critical paths. // in performance critical paths.

2
follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement/RepeatedProfileVisitsSource.scala
View File

@ -105,7 +105,7 @@ class RepeatedProfileVisitsSource @Inject() (
val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold) val recommendationThreshold = params.getInt(RepeatedProfileVisitsParams.RecommendationThreshold)
val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold) val bucketingThreshold = params.getInt(RepeatedProfileVisitsParams.BucketingThreshold)
// Get the list of repeatedly visited profilts. Only keep accounts with >= bucketingThreshold visits. // Get the list of repeatedly visited profiles. Only keep accounts with >= bucketingThreshold visits.
val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] = val repeatedVisitedAccountsStitch: Stitch[Map[Long, Int]] =
getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold)) getRepeatedVisitedAccounts(params, userId).map(_.filter(kv => kv._2 >= bucketingThreshold))

2
follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker/WeightedCandidateSourceRanker.scala
View File

@ -54,7 +54,7 @@ class WeightedCandidateSourceRanker[Target <: HasParams](
// Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream. // Note 1: Using map instead mapValue here since mapValue somehow caused infinite loop when used as part of Stream.
val sortAndShuffledCandidates = input.map { val sortAndShuffledCandidates = input.map {
case (source, candidates) => case (source, candidates) =>
// Note 2: toList is required here since candidates is a view, and it will result in infinit loop when used as part of Stream. // Note 2: toList is required here since candidates is a view, and it will result in infinite loop when used as part of Stream.
// Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources // Note 3: there is no real sorting logic here, it assumes the input is already sorted by candidate sources
val sortedCandidates = candidates.toList val sortedCandidates = candidates.toList
source -> shuffleFn(sortedCandidates).iterator source -> shuffleFn(sortedCandidates).iterator

2
follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling/SamplingTransformParams.scala
View File

@ -5,7 +5,7 @@ import com.twitter.timelines.configapi.FSParam
object SamplingTransformParams { object SamplingTransformParams {
case object TopKFixed // indicates how many of the fisrt K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score case object TopKFixed // indicates how many of the first K who-to-follow recommendations are reserved for the candidates with largest K CandidateUser.score where these candidates are sorted in decreasing order of score
extends FSBoundedParam[Int]( extends FSBoundedParam[Int](
name = "post_nux_ml_flow_weighted_sampling_top_k_fixed", name = "post_nux_ml_flow_weighted_sampling_top_k_fixed",
default = 0, default = 0,

2
follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/post_nux_ml/PostNuxMlCombinedRankerBuilder.scala
View File

@ -29,7 +29,7 @@ import com.twitter.timelines.configapi.HasParams
* - truncating to the top N merged results for ranking * - truncating to the top N merged results for ranking
* - ML ranker * - ML ranker
* - Interleaving ranker for producer-side experiments * - Interleaving ranker for producer-side experiments
* - impression-based fatigueing * - impression-based fatiguing
*/ */
@Singleton @Singleton
class PostNuxMlCombinedRankerBuilder[ class PostNuxMlCombinedRankerBuilder[

2
follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging/FrsLogger.scala
View File

@ -125,7 +125,7 @@ object FrsLogger {
/** The id of the current user. When the user is logged out, this method should return None. */ /** The id of the current user. When the user is logged out, this method should return None. */
override val userId: Option[Long] = clientContext.userId override val userId: Option[Long] = clientContext.userId
/** The id of the guest, which is present in logged-in or loged-out states */ /** The id of the guest, which is present in logged-in or logged-out states */
override val guestId: Option[Long] = clientContext.guestId override val guestId: Option[Long] = clientContext.guestId
/** The personalization id (pid) of the user, used to personalize Twitter services */ /** The personalization id (pid) of the user, used to personalize Twitter services */

2
follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules/FlagsModule.scala
View File

@ -8,7 +8,7 @@ object FlagsModule extends TwitterModule {
) )
flag[Boolean]( flag[Boolean](
name = "interests_opt_out_prod_enabled", name = "interests_opt_out_prod_enabled",
help = "Whether to fetch intersts opt out data from the prod strato column or not" help = "Whether to fetch interests opt out data from the prod strato column or not"
) )
flag[Boolean]( flag[Boolean](
name = "log_results", name = "log_results",

3
home-mixer/README.md
View File

@ -74,7 +74,7 @@ Timeline tabs powered by Home Mixer.
- ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer) - ScoredTweetsRecommendationPipelineConfig (main Tweet recommendation layer)
- Fetch Tweet Candidates - Fetch Tweet Candidates
- ScoredTweetsInNetworkCandidatePipelineConfig - ScoredTweetsInNetworkCandidatePipelineConfig
- ScoredTweetsCrMixerCandidatePipelineConfig - ScoredTweetsTweetMixerCandidatePipelineConfig
- ScoredTweetsUtegCandidatePipelineConfig - ScoredTweetsUtegCandidatePipelineConfig
- ScoredTweetsFrsCandidatePipelineConfig - ScoredTweetsFrsCandidatePipelineConfig
- Feature Hydration and Scoring - Feature Hydration and Scoring
@ -99,4 +99,3 @@ Timeline tabs powered by Home Mixer.
- ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service) - ListTweetsTimelineServiceCandidatePipelineConfig (fetch tweets from timeline service)
- ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules) - ConversationServiceCandidatePipelineConfig (fetch ancestors for conversation modules)
- ListTweetsAdsCandidatePipelineConfig (fetch ads) - ListTweetsAdsCandidatePipelineConfig (fetch ads)

5
home-mixer/server/src/main/scala/com/twitter/home_mixer/BUILD.bazel
View File

@ -21,6 +21,7 @@ scala_library(
"finatra/inject/inject-utils/src/main/scala", "finatra/inject/inject-utils/src/main/scala",
"home-mixer/server/src/main/resources", "home-mixer/server/src/main/resources",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/controller", "home-mixer/server/src/main/scala/com/twitter/home_mixer/controller",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/federated",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/module", "home-mixer/server/src/main/scala/com/twitter/home_mixer/module",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product",
@ -31,6 +32,10 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala", "product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala", "src/thrift/com/twitter/timelines/render:thrift-scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
"stringcenter/client", "stringcenter/client",
"stringcenter/client/src/main/java", "stringcenter/client/src/main/java",
"stringcenter/client/src/main/scala/com/twitter/stringcenter/client", "stringcenter/client/src/main/scala/com/twitter/stringcenter/client",

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerHttpServerWarmupHandler.scala
View File

@ -2,7 +2,7 @@ package com.twitter.home_mixer
import com.twitter.finatra.http.routing.HttpWarmup import com.twitter.finatra.http.routing.HttpWarmup
import com.twitter.finatra.httpclient.RequestBuilder._ import com.twitter.finatra.httpclient.RequestBuilder._
import com.twitter.inject.Logging import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler import com.twitter.inject.utils.Handler
import com.twitter.util.Try import com.twitter.util.Try
import javax.inject.Inject import javax.inject.Inject

34
home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerServer.scala
View File

@ -12,57 +12,63 @@ import com.twitter.finatra.thrift.ThriftServer
import com.twitter.finatra.thrift.filters._ import com.twitter.finatra.thrift.filters._
import com.twitter.finatra.thrift.routing.ThriftRouter import com.twitter.finatra.thrift.routing.ThriftRouter
import com.twitter.home_mixer.controller.HomeThriftController import com.twitter.home_mixer.controller.HomeThriftController
import com.twitter.home_mixer.federated.HomeMixerColumn
import com.twitter.home_mixer.module._ import com.twitter.home_mixer.module._
import com.twitter.home_mixer.param.GlobalParamConfigModule import com.twitter.home_mixer.param.GlobalParamConfigModule
import com.twitter.home_mixer.product.HomeMixerProductModule import com.twitter.home_mixer.product.HomeMixerProductModule
import com.twitter.home_mixer.{thriftscala => st} import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule import com.twitter.product_mixer.component_library.module.AccountRecommendationsMixerModule
import com.twitter.product_mixer.component_library.module.CrMixerClientModule
import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule import com.twitter.product_mixer.component_library.module.DarkTrafficFilterModule
import com.twitter.product_mixer.component_library.module.EarlybirdModule import com.twitter.product_mixer.component_library.module.EarlybirdModule
import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule import com.twitter.product_mixer.component_library.module.ExploreRankerClientModule
import com.twitter.product_mixer.component_library.module.GizmoduckClientModule import com.twitter.product_mixer.component_library.module.GizmoduckClientModule
import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule import com.twitter.product_mixer.component_library.module.OnboardingTaskServiceModule
import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule import com.twitter.product_mixer.component_library.module.SocialGraphServiceModule
import com.twitter.product_mixer.component_library.module.TimelineMixerClientModule
import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule import com.twitter.product_mixer.component_library.module.TimelineRankerClientModule
import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule import com.twitter.product_mixer.component_library.module.TimelineScorerClientModule
import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule import com.twitter.product_mixer.component_library.module.TimelineServiceClientModule
import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule import com.twitter.product_mixer.component_library.module.TweetImpressionStoreModule
import com.twitter.product_mixer.component_library.module.TweetMixerClientModule
import com.twitter.product_mixer.component_library.module.UserSessionStoreModule import com.twitter.product_mixer.component_library.module.UserSessionStoreModule
import com.twitter.product_mixer.core.controllers.ProductMixerController import com.twitter.product_mixer.core.controllers.ProductMixerController
import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper import com.twitter.product_mixer.core.module.LoggingThrowableExceptionMapper
import com.twitter.product_mixer.core.module.ProductMixerModule import com.twitter.product_mixer.core.module.ProductMixerModule
import com.twitter.product_mixer.core.module.StratoClientModule
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.fed.server.StratoFedServer
object HomeMixerServerMain extends HomeMixerServer object HomeMixerServerMain extends HomeMixerServer
class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMtls { class HomeMixerServer
extends StratoFedServer
with ThriftServer
with Mtls
with HttpServer
with HttpMtls {
override val name = "home-mixer-server" override val name = "home-mixer-server"
override val modules: Seq[Module] = Seq( override val modules: Seq[Module] = Seq(
AccountRecommendationsMixerModule, AccountRecommendationsMixerModule,
AdvertiserBrandSafetySettingsStoreModule, AdvertiserBrandSafetySettingsStoreModule,
BlenderClientModule,
ClientSentImpressionsPublisherModule, ClientSentImpressionsPublisherModule,
ConversationServiceModule, ConversationServiceModule,
CrMixerClientModule,
EarlybirdModule, EarlybirdModule,
ExploreRankerClientModule, ExploreRankerClientModule,
FeedbackHistoryClientModule,
GizmoduckClientModule, GizmoduckClientModule,
GlobalParamConfigModule, GlobalParamConfigModule,
HomeAdsCandidateSourceModule, HomeAdsCandidateSourceModule,
HomeMixerFlagsModule, HomeMixerFlagsModule,
HomeMixerProductModule, HomeMixerProductModule,
HomeMixerResourcesModule, HomeMixerResourcesModule,
HomeNaviModelClientModule,
ImpressionBloomFilterModule, ImpressionBloomFilterModule,
InjectionHistoryClientModule, InjectionHistoryClientModule,
FeedbackHistoryClientModule,
ManhattanClientsModule, ManhattanClientsModule,
ManhattanFeatureRepositoryModule, ManhattanFeatureRepositoryModule,
ManhattanTweetImpressionStoreModule, ManhattanTweetImpressionStoreModule,
MemcachedFeatureRepositoryModule, MemcachedFeatureRepositoryModule,
NaviModelClientModule,
OnboardingTaskServiceModule, OnboardingTaskServiceModule,
OptimizedStratoClientModule, OptimizedStratoClientModule,
PeopleDiscoveryServiceModule, PeopleDiscoveryServiceModule,
@ -74,24 +80,23 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
SimClustersRecentEngagementsClientModule, SimClustersRecentEngagementsClientModule,
SocialGraphServiceModule, SocialGraphServiceModule,
StaleTweetsCacheModule, StaleTweetsCacheModule,
StratoClientModule,
ThriftFeatureRepositoryModule, ThriftFeatureRepositoryModule,
TimelineMixerClientModule,
TimelineRankerClientModule, TimelineRankerClientModule,
TimelineScorerClientModule, TimelineScorerClientModule,
TimelineServiceClientModule, TimelineServiceClientModule,
TimelinesPersistenceStoreClientModule, TimelinesPersistenceStoreClientModule,
TopicSocialProofClientModule,
TweetImpressionStoreModule, TweetImpressionStoreModule,
TweetyPieClientModule, TweetMixerClientModule,
TweetypieClientModule,
TweetypieStaticEntitiesCacheClientModule, TweetypieStaticEntitiesCacheClientModule,
UserMetadataStoreModule,
UserSessionStoreModule, UserSessionStoreModule,
new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](), new DarkTrafficFilterModule[st.HomeMixer.ReqRepServicePerEndpoint](),
new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this), new MtlsThriftWebFormsModule[st.HomeMixer.MethodPerEndpoint](this),
new ProductScopeStringCenterModule() new ProductScopeStringCenterModule()
) )
def configureThrift(router: ThriftRouter): Unit = { override def configureThrift(router: ThriftRouter): Unit = {
router router
.filter[LoggingMDCFilter] .filter[LoggingMDCFilter]
.filter[TraceIdMDCFilter] .filter[TraceIdMDCFilter]
@ -111,6 +116,11 @@ class HomeMixerServer extends ThriftServer with Mtls with HttpServer with HttpMt
this.injector, this.injector,
st.HomeMixer.ExecutePipeline)) st.HomeMixer.ExecutePipeline))
override val dest: String = "/s/home-mixer/home-mixer:strato"
override val columns: Seq[Class[_ <: StratoFed.Column]] =
Seq(classOf[HomeMixerColumn])
override protected def warmup(): Unit = { override protected def warmup(): Unit = {
handle[HomeMixerThriftServerWarmupHandler]() handle[HomeMixerThriftServerWarmupHandler]()
handle[HomeMixerHttpServerWarmupHandler]() handle[HomeMixerHttpServerWarmupHandler]()

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/HomeMixerThriftServerWarmupHandler.scala
View File

@ -3,7 +3,7 @@ package com.twitter.home_mixer
import com.twitter.finagle.thrift.ClientId import com.twitter.finagle.thrift.ClientId
import com.twitter.finatra.thrift.routing.ThriftWarmup import com.twitter.finatra.thrift.routing.ThriftWarmup
import com.twitter.home_mixer.{thriftscala => st} import com.twitter.home_mixer.{thriftscala => st}
import com.twitter.inject.Logging import com.twitter.util.logging.Logging
import com.twitter.inject.utils.Handler import com.twitter.inject.utils.Handler
import com.twitter.product_mixer.core.{thriftscala => pt} import com.twitter.product_mixer.core.{thriftscala => pt}
import com.twitter.scrooge.Request import com.twitter.scrooge.Request

10
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/BUILD.bazel
View File

@ -5,19 +5,13 @@ scala_library(
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
@ -25,10 +19,6 @@ scala_library(
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/transformer",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
], ],
exports = [ exports = [
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc",

25
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfig.scala
View File

@ -1,18 +1,23 @@
package com.twitter.home_mixer.candidate_pipeline package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter
import com.twitter.home_mixer.functional_component.filter.PredicateFeatureFilter import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata
import com.twitter.product_mixer.component_library.filter.FeatureFilter import com.twitter.product_mixer.component_library.filter.FeatureFilter
import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
@ -33,8 +38,8 @@ import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipel
class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery]( class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
conversationServiceCandidateSource: ConversationServiceCandidateSource, conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator, tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator,
namesFeatureHydrator: NamesFeatureHydrator, namesFeatureHydrator: NamesFeatureHydrator,
invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
override val gates: Seq[BaseGate[Query]], override val gates: Seq[BaseGate[Query]],
override val decorator: Option[CandidateDecorator[Query, TweetCandidate]]) override val decorator: Option[CandidateDecorator[Query, TweetCandidate]])
extends DependentCandidatePipelineConfig[ extends DependentCandidatePipelineConfig[
@ -62,10 +67,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
val tweetsWithConversationMetadata = candidates.map { candidate => val tweetsWithConversationMetadata = candidates.map { candidate =>
TweetWithConversationMetadata( TweetWithConversationMetadata(
tweetId = candidate.candidateIdLong, tweetId = candidate.candidateIdLong,
userId = None, userId = candidate.features.getOrElse(AuthorIdFeature, None),
sourceTweetId = None, sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None),
sourceUserId = None, sourceUserId = candidate.features.getOrElse(SourceUserIdFeature, None),
inReplyToTweetId = None, inReplyToTweetId = candidate.features.getOrElse(InReplyToTweetIdFeature, None),
conversationId = None, conversationId = None,
ancestors = Seq.empty ancestors = Seq.empty
) )
@ -84,7 +89,10 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
override val preFilterFeatureHydrationPhase1: Seq[ override val preFilterFeatureHydrationPhase1: Seq[
BaseCandidateFeatureHydrator[Query, TweetCandidate, _] BaseCandidateFeatureHydrator[Query, TweetCandidate, _]
] = Seq(tweetypieFeatureHydrator, socialGraphServiceFeatureHydrator) ] = Seq(
tweetypieFeatureHydrator,
InNetworkFeatureHydrator,
)
override def filters: Seq[Filter[Query, TweetCandidate]] = Seq( override def filters: Seq[Filter[Query, TweetCandidate]] = Seq(
RetweetDeduplicationFilter, RetweetDeduplicationFilter,
@ -93,6 +101,7 @@ class ConversationServiceCandidatePipelineConfig[Query <: PipelineQuery](
FilterIdentifier(QuotedTweetDroppedFilterId), FilterIdentifier(QuotedTweetDroppedFilterId),
shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) }
), ),
invalidSubscriptionTweetFilter,
InvalidConversationModuleFilter InvalidConversationModuleFilter
) )

8
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/ConversationServiceCandidatePipelineConfigBuilder.scala
View File

@ -1,9 +1,9 @@
package com.twitter.home_mixer.candidate_pipeline package com.twitter.home_mixer.candidate_pipeline
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.SocialGraphServiceFeatureHydrator
import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator
import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter
import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator
import com.twitter.product_mixer.core.functional_component.gate.BaseGate import com.twitter.product_mixer.core.functional_component.gate.BaseGate
@ -15,7 +15,7 @@ import javax.inject.Singleton
class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() ( class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery] @Inject() (
conversationServiceCandidateSource: ConversationServiceCandidateSource, conversationServiceCandidateSource: ConversationServiceCandidateSource,
tweetypieFeatureHydrator: TweetypieFeatureHydrator, tweetypieFeatureHydrator: TweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator: SocialGraphServiceFeatureHydrator, invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter,
namesFeatureHydrator: NamesFeatureHydrator) { namesFeatureHydrator: NamesFeatureHydrator) {
def build( def build(
@ -25,8 +25,8 @@ class ConversationServiceCandidatePipelineConfigBuilder[Query <: PipelineQuery]
new ConversationServiceCandidatePipelineConfig( new ConversationServiceCandidatePipelineConfig(
conversationServiceCandidateSource, conversationServiceCandidateSource,
tweetypieFeatureHydrator, tweetypieFeatureHydrator,
socialGraphServiceFeatureHydrator,
namesFeatureHydrator, namesFeatureHydrator,
invalidSubscriptionTweetFilter,
gates, gates,
decorator decorator
) )

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline/EditedTweetsCandidatePipelineConfig.scala
View File

@ -1,7 +1,7 @@
package com.twitter.home_mixer.candidate_pipeline package com.twitter.home_mixer.candidate_pipeline
import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource import com.twitter.home_mixer.functional_component.candidate_source.StaleTweetsCacheCandidateSource
import com.twitter.home_mixer.functional_component.decorator.HomeFeedbackActionInfoBuilder import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator
import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer import com.twitter.home_mixer.functional_component.query_transformer.EditedTweetsCandidatePipelineQueryTransformer
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/BUILD.bazel
View File

@ -13,8 +13,5 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/urt",
"snowflake/src/main/scala/com/twitter/snowflake/id", "snowflake/src/main/scala/com/twitter/snowflake/id",
"src/thrift/com/twitter/context:twitter-context-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"twitter-context/src/main/scala",
], ],
) )

1
home-mixer/server/src/main/scala/com/twitter/home_mixer/controller/HomeThriftController.scala
View File

@ -7,6 +7,7 @@ import com.twitter.home_mixer.service.ScoredTweetsService
import com.twitter.home_mixer.{thriftscala => t} import com.twitter.home_mixer.{thriftscala => t}
import com.twitter.product_mixer.core.controllers.DebugTwitterContext import com.twitter.product_mixer.core.controllers.DebugTwitterContext
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.service.debug_query.DebugQueryService
import com.twitter.product_mixer.core.service.urt.UrtService import com.twitter.product_mixer.core.service.urt.UrtService
import com.twitter.snowflake.id.SnowflakeId import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch

24
home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/BUILD.bazel Normal file
View File

@ -0,0 +1,24 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/thrift/src/main/thrift:thrift-scala",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
"product-mixer/core/src/main/thrift/com/twitter/product_mixer/core:thrift-scala",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/timelines/render:thrift-scala",
"stitch/stitch-repo/src/main/scala",
"strato/config/columns/auth-context:auth-context-strato-client",
"strato/config/columns/gizmoduck:gizmoduck-strato-client",
"strato/config/src/thrift/com/twitter/strato/graphql/timelines:graphql-timelines-scala",
"strato/src/main/scala/com/twitter/strato/callcontext",
"strato/src/main/scala/com/twitter/strato/fed",
"strato/src/main/scala/com/twitter/strato/fed/server",
],
)

217
home-mixer/server/src/main/scala/com/twitter/home_mixer/federated/HomeMixerColumn.scala Normal file
View File

@ -0,0 +1,217 @@
package com.twitter.home_mixer.federated
import com.twitter.gizmoduck.{thriftscala => gd}
import com.twitter.home_mixer.marshaller.request.HomeMixerRequestUnmarshaller
import com.twitter.home_mixer.model.request.HomeMixerRequest
import com.twitter.home_mixer.{thriftscala => hm}
import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineRequest
import com.twitter.product_mixer.core.pipeline.product.ProductPipelineResult
import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry
import com.twitter.product_mixer.core.{thriftscala => pm}
import com.twitter.stitch.Arrow
import com.twitter.stitch.Stitch
import com.twitter.strato.callcontext.CallContext
import com.twitter.strato.catalog.OpMetadata
import com.twitter.strato.config._
import com.twitter.strato.data._
import com.twitter.strato.fed.StratoFed
import com.twitter.strato.generated.client.auth_context.AuditIpClientColumn
import com.twitter.strato.generated.client.gizmoduck.CompositeOnUserClientColumn
import com.twitter.strato.graphql.timelines.{thriftscala => gql}
import com.twitter.strato.thrift.ScroogeConv
import com.twitter.timelines.render.{thriftscala => tr}
import com.twitter.util.Try
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class HomeMixerColumn @Inject() (
homeMixerRequestUnmarshaller: HomeMixerRequestUnmarshaller,
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn,
paramsBuilder: ParamsBuilder,
productPipelineRegistry: ProductPipelineRegistry)
extends StratoFed.Column(HomeMixerColumn.Path)
with StratoFed.Fetch.Arrow {
override val contactInfo: ContactInfo = ContactInfo(
contactEmail = "",
ldapGroup = "",
slackRoomId = ""
)
override val metadata: OpMetadata =
OpMetadata(
lifecycle = Some(Lifecycle.Production),
description =
Some(Description.PlainText("Federated Strato column for Timelines served via Home Mixer"))
)
private val bouncerAccess: Seq[Policy] = Seq(BouncerAccess())
private val finatraTestServiceIdentifiers: Seq[Policy] = Seq(
ServiceIdentifierPattern(
role = "",
service = "",
env = "",
zone = Seq(""))
)
override val policy: Policy = AnyOf(bouncerAccess ++ finatraTestServiceIdentifiers)
override type Key = gql.TimelineKey
override type View = gql.HomeTimelineView
override type Value = tr.Timeline
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[gql.TimelineKey]
override val viewConv: Conv[View] = ScroogeConv.fromStruct[gql.HomeTimelineView]
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[tr.Timeline]
private def createHomeMixerRequestArrow(
compositeOnUserClientColumn: CompositeOnUserClientColumn,
auditIpClientColumn: AuditIpClientColumn
): Arrow[(Key, View), hm.HomeMixerRequest] = {
val populateUserRolesAndIp: Arrow[(Key, View), (Option[Set[String]], Option[String])] = {
val gizmoduckView: (gd.LookupContext, Set[gd.QueryFields]) =
(gd.LookupContext(), Set(gd.QueryFields.Roles))
val populateUserRoles = Arrow
.flatMap[(Key, View), Option[Set[String]]] { _ =>
Stitch.collect {
CallContext.twitterUserId.map { userId =>
compositeOnUserClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch(userId, gizmoduckView).map(_.v)
.map {
_.flatMap(_.roles.map(_.roles.toSet)).getOrElse(Set.empty)
}
}
}
}
val populateIpAddress = Arrow
.flatMap[(Key, View), Option[String]](_ =>
auditIpClientColumn.fetcher
.callStack(HomeMixerColumn.FetchCallstack)
.fetch((), ()).map(_.v))
Arrow.join(
populateUserRoles,
populateIpAddress
)
}
Arrow.zipWithArg(populateUserRolesAndIp).map {
case ((key, view), (roles, ipAddress)) =>
val deviceContextOpt = Some(
hm.DeviceContext(
isPolling = CallContext.isPolling,
requestContext = view.requestContext,
latestControlAvailable = view.latestControlAvailable,
autoplayEnabled = view.autoplayEnabled
))
val seenTweetIds = view.seenTweetIds.filter(_.nonEmpty)
val (product, productContext) = key match {
case gql.TimelineKey.HomeTimeline(_) | gql.TimelineKey.HomeTimelineV2(_) =>
(
hm.Product.ForYou,
hm.ProductContext.ForYou(
hm.ForYou(
deviceContextOpt,
seenTweetIds,
view.dspClientContext,
view.pushToHomeTweetId
)
))
case gql.TimelineKey.HomeLatestTimeline(_) | gql.TimelineKey.HomeLatestTimelineV2(_) =>
(
hm.Product.Following,
hm.ProductContext.Following(
hm.Following(deviceContextOpt, seenTweetIds, view.dspClientContext)))
case gql.TimelineKey.CreatorSubscriptionsTimeline(_) =>
(
hm.Product.Subscribed,
hm.ProductContext.Subscribed(hm.Subscribed(deviceContextOpt, seenTweetIds)))
case _ => throw new UnsupportedOperationException(s"Unknown product: $key")
}
val clientContext = pm.ClientContext(
userId = CallContext.twitterUserId,
guestId = CallContext.guestId,
guestIdAds = CallContext.guestIdAds,
guestIdMarketing = CallContext.guestIdMarketing,
appId = CallContext.clientApplicationId,
ipAddress = ipAddress,
userAgent = CallContext.userAgent,
countryCode = CallContext.requestCountryCode,
languageCode = CallContext.requestLanguageCode,
isTwoffice = CallContext.isInternalOrTwoffice,
userRoles = roles,
deviceId = CallContext.deviceId,
mobileDeviceId = CallContext.mobileDeviceId,
mobileDeviceAdId = CallContext.adId,
limitAdTracking = CallContext.limitAdTracking
)
hm.HomeMixerRequest(
clientContext = clientContext,
product = product,
productContext = Some(productContext),
maxResults = Try(view.count.get.toInt).toOption.orElse(HomeMixerColumn.MaxCount),
cursor = view.cursor.filter(_.nonEmpty)
)
}
}
override val fetch: Arrow[(Key, View), Result[Value]] = {
val transformThriftIntoPipelineRequest: Arrow[
(Key, View),
ProductPipelineRequest[HomeMixerRequest]
] = {
Arrow
.identity[(Key, View)]
.andThen {
createHomeMixerRequestArrow(compositeOnUserClientColumn, auditIpClientColumn)
}
.map {
case thriftRequest =>
val request = homeMixerRequestUnmarshaller(thriftRequest)
val params = paramsBuilder.build(
clientContext = request.clientContext,
product = request.product,
featureOverrides =
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty),
)
ProductPipelineRequest(request, params)
}
}
val underlyingProduct: Arrow[
ProductPipelineRequest[HomeMixerRequest],
ProductPipelineResult[tr.TimelineResponse]
] = Arrow
.identity[ProductPipelineRequest[HomeMixerRequest]]
.map { pipelineRequest =>
val pipelineArrow = productPipelineRegistry
.getProductPipeline[HomeMixerRequest, tr.TimelineResponse](
pipelineRequest.request.product)
.arrow
(pipelineArrow, pipelineRequest)
}.applyArrow
transformThriftIntoPipelineRequest.andThen(underlyingProduct).map {
_.result match {
case Some(result) => found(result.timeline)
case _ => missing
}
}
}
}
object HomeMixerColumn {
val Path = "home-mixer/homeMixer.Timeline"
private val FetchCallstack = s"$Path:fetch"
private val MaxCount: Option[Int] = Some(100)
}

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source/BUILD.bazel
View File

@ -10,11 +10,8 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala",
"src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search:earlybird-scala",
"stitch/stitch-timelineservice/src/main/scala", "stitch/stitch-timelineservice/src/main/scala",
"strato/config/columns/recommendations/similarity:similarity-strato-client",
"strato/src/main/scala/com/twitter/strato/client",
], ],
exports = [ exports = [
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source",

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BUILD.bazel
View File

@ -6,13 +6,11 @@ scala_library(
dependencies = [ dependencies = [
"finagle/finagle-core/src/main", "finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope",
@ -25,8 +23,6 @@ scala_library(
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"stringcenter/client", "stringcenter/client",
"stringcenter/client/src/main/java", "stringcenter/client/src/main/java",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/translation",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe", "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
], ],
) )

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeConversationServiceCandidateDecorator.scala
View File

@ -1,6 +1,8 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator
import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder
import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder
import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator
import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator

5
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/BUILD.bazel
View File

@ -8,7 +8,9 @@ scala_library(
"finagle/finagle-core/src/main", "finagle/finagle-core/src/main",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
@ -18,6 +20,7 @@ scala_library(
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala", "src/thrift/com/twitter/timelineservice/server/suggests/logging:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/injection/scribe", "timelines/src/main/scala/com/twitter/timelines/injection/scribe",
], ],
) )

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeAdsClientEventDetailsBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeAdsClientEventDetailsBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.finagle.tracing.Trace import com.twitter.finagle.tracing.Trace
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeClientEventDetailsBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeClientEventDetailsBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.bijection.Base64String import com.twitter.bijection.Base64String
import com.twitter.bijection.scrooge.BinaryScalaCodec import com.twitter.bijection.scrooge.BinaryScalaCodec

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTimelinesScoreInfoBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTimelinesScoreInfoBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient import com.twitter.home_mixer.param.HomeGlobalParams.EnableSendScoresToClient

163
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetTypePredicates.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/HomeTweetTypePredicates.scala
View File

@ -1,8 +1,10 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.builder
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures._ import com.twitter.home_mixer.model.HomeFeatures._
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures import com.twitter.timelinemixer.injection.model.candidate.SemanticCoreFeatures
import com.twitter.tweetypie.{thriftscala => tpt} import com.twitter.tweetypie.{thriftscala => tpt}
@ -25,74 +27,76 @@ object HomeTweetTypePredicates {
( (
"has_exclusive_conversation_author_id", "has_exclusive_conversation_author_id",
_.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty), _.getOrElse(ExclusiveConversationAuthorIdFeature, None).nonEmpty),
("is_eligible_for_connect_boost", _.getOrElse(AuthorIsEligibleForConnectBoostFeature, false)), ("is_eligible_for_connect_boost", _ => false),
("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)), ("hashtag", _.getOrElse(EarlybirdFeature, None).exists(_.numHashtags > 0)),
("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)), ("has_scheduled_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isScheduled)),
("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)), ("has_recorded_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isRecorded)),
("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)), ("is_read_from_cache", _.getOrElse(IsReadFromCacheFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("get_initial", _.getOrElse(GetInitialFeature, false)), ("get_initial", _.getOrElse(GetInitialFeature, false)),
("get_newer", _.getOrElse(GetNewerFeature, false)), ("get_newer", _.getOrElse(GetNewerFeature, false)),
("get_middle", _.getOrElse(GetMiddleFeature, false)), ("get_middle", _.getOrElse(GetMiddleFeature, false)),
("get_older", _.getOrElse(GetOlderFeature, false)), ("get_older", _.getOrElse(GetOlderFeature, false)),
("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)), ("pull_to_refresh", _.getOrElse(PullToRefreshFeature, false)),
("polling", _.getOrElse(PollingFeature, false)), ("polling", _.getOrElse(PollingFeature, false)),
("tls_size_20_plus", _ => false), ("near_empty", _.getOrElse(ServedSizeFeature, None).exists(_ < 3)),
("near_empty", _ => false), ("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("ranked_request", _ => false),
("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)), ("mutual_follow", _.getOrElse(EarlybirdFeature, None).exists(_.fromMutualFollow)),
(
"less_than_10_mins_since_lnpt",
_.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 10.minutes)),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)), ("has_ticketed_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasTickets)),
("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)), ("in_utis_top5", _.getOrElse(PositionFeature, None).exists(_ < 5)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
( (
"is_signup_request", "conversation_module_has_2_displayed_tweets",
candidate => candidate.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), _.getOrElse(ConversationModule2DisplayedTweetsFeature, false)),
("empty_request", _ => false), ("empty_request", _.getOrElse(ServedSizeFeature, None).exists(_ == 0)),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)), ("served_size_less_than_50", _.getOrElse(ServedSizeFeature, None).exists(_ < 50)),
( (
"served_size_between_50_and_100", "served_size_between_50_and_100",
_.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)), _.getOrElse(ServedSizeFeature, None).exists(size => size >= 50 && size < 100)),
("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)), ("authored_by_contextual_user", _.getOrElse(AuthoredByContextualUserFeature, false)),
(
"is_self_thread_tweet",
_.getOrElse(ConversationFeature, None).exists(_.isSelfThreadTweet.contains(true))),
("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty), ("has_ancestors", _.getOrElse(AncestorsFeature, Seq.empty).nonEmpty),
("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)), ("full_scoring_succeeded", _.getOrElse(FullScoringSucceededFeature, false)),
("served_size_less_than_20", _.getOrElse(ServedSizeFeature, None).exists(_ < 20)),
("served_size_less_than_10", _.getOrElse(ServedSizeFeature, None).exists(_ < 10)),
("served_size_less_than_5", _.getOrElse(ServedSizeFeature, None).exists(_ < 5)),
( (
"account_age_less_than_30_minutes", "account_age_less_than_30_minutes",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)), _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 30.minutes)),
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
( (
"account_age_less_than_1_day", "account_age_less_than_1_day",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)), _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 1.day)),
( (
"account_age_less_than_7_days", "account_age_less_than_7_days",
_.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)), _.getOrElse(AccountAgeFeature, None).exists(_.untilNow < 7.days)),
(
"directed_at_user_is_in_first_degree",
_.getOrElse(EarlybirdFeature, None).exists(_.directedAtUserIdIsInFirstDegree.contains(true))),
("root_user_is_in_first_degree", _ => false),
(
"has_semantic_core_annotation",
_.getOrElse(EarlybirdFeature, None).exists(_.semanticCoreAnnotations.nonEmpty)),
("is_request_context_foreground", _.getOrElse(IsForegroundRequestFeature, false)),
( (
"part_of_utt", "part_of_utt",
_.getOrElse(EarlybirdFeature, None) _.getOrElse(EarlybirdFeature, None)
.exists(_.semanticCoreAnnotations.exists(_.exists(annotation => .exists(_.semanticCoreAnnotations.exists(_.exists(annotation =>
annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))), annotation.domainId == SemanticCoreFeatures.UnifiedTwitterTaxonomy)))),
(
"has_home_latest_request_past_week",
_.getOrElse(FollowingLastNonPollingTimeFeature, None).exists(_.untilNow < 7.days)),
("is_utis_pos0", _.getOrElse(PositionFeature, None).exists(_ == 0)),
("is_utis_pos1", _.getOrElse(PositionFeature, None).exists(_ == 1)),
("is_utis_pos2", _.getOrElse(PositionFeature, None).exists(_ == 2)),
("is_utis_pos3", _.getOrElse(PositionFeature, None).exists(_ == 3)),
("is_utis_pos4", _.getOrElse(PositionFeature, None).exists(_ == 4)),
("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)), ("is_random_tweet", _.getOrElse(IsRandomTweetFeature, false)),
("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)), ("has_random_tweet_in_response", _.getOrElse(HasRandomTweetFeature, false)),
("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)), ("is_random_tweet_above_in_utis", _.getOrElse(IsRandomTweetAboveFeature, false)),
("is_request_context_launch", _.getOrElse(IsLaunchRequestFeature, false)),
("viewer_is_employee", _ => false),
("viewer_is_timelines_employee", _ => false),
("viewer_follows_any_topics", _.getOrElse(UserFollowedTopicsCountFeature, None).exists(_ > 0)),
( (
"has_ancestor_authored_by_viewer", "has_ancestor_authored_by_viewer",
candidate => candidate =>
@ -100,11 +104,6 @@ object HomeTweetTypePredicates {
.getOrElse(AncestorsFeature, Seq.empty).exists(ancestor => .getOrElse(AncestorsFeature, Seq.empty).exists(ancestor =>
candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)), candidate.getOrElse(ViewerIdFeature, 0L) == ancestor.userId)),
("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)), ("ancestor", _.getOrElse(IsAncestorCandidateFeature, false)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
( (
"deep_reply", "deep_reply",
candidate => candidate =>
@ -119,23 +118,22 @@ object HomeTweetTypePredicates {
"tweet_age_less_than_15_seconds", "tweet_age_less_than_15_seconds",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None) _.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 15.seconds)), .exists(_.untilNow <= 15.seconds)),
("is_followed_topic_tweet", _ => false), (
("is_recommended_topic_tweet", _ => false), "less_than_1_hour_since_lnpt",
("is_topic_tweet", _ => false), _.getOrElse(LastNonPollingTimeFeature, None).exists(_.untilNow < 1.hour)),
("preferred_language_matches_tweet_language", _ => false), ("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))),
( (
"device_language_matches_tweet_language", "device_language_matches_tweet_language",
candidate => candidate =>
candidate.getOrElse(TweetLanguageFeature, None) == candidate.getOrElse(TweetLanguageFeature, None) ==
candidate.getOrElse(DeviceLanguageFeature, None)), candidate.getOrElse(DeviceLanguageFeature, None)),
(
"root_ancestor",
candidate =>
candidate.getOrElse(IsAncestorCandidateFeature, false) && candidate
.getOrElse(InReplyToTweetIdFeature, None).isEmpty),
("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))), ("question", _.getOrElse(EarlybirdFeature, None).exists(_.hasQuestion.contains(true))),
("in_network", _.getOrElse(FromInNetworkSourceFeature, true)), ("in_network", _.getOrElse(InNetworkFeature, true)),
("viewer_follows_original_author", _ => false),
("has_account_follow_prompt", _ => false),
("has_relevance_prompt", _ => false),
("has_topic_annotation_haug_prompt", _ => false),
("has_topic_annotation_random_precision_prompt", _ => false),
("has_topic_annotation_prompt", _ => false),
( (
"has_political_annotation", "has_political_annotation",
_.getOrElse(EarlybirdFeature, None).exists( _.getOrElse(EarlybirdFeature, None).exists(
@ -153,9 +151,14 @@ object HomeTweetTypePredicates {
_.getOrElse(EarlybirdFeature, None) _.getOrElse(EarlybirdFeature, None)
.exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))), .exists(_.conversationControl.exists(_.isInstanceOf[tpt.ConversationControl.Community]))),
("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)), ("has_zero_score", _.getOrElse(ScoreFeature, None).exists(_ == 0.0)),
("is_viewer_not_invited_to_reply", _ => false), (
("is_viewer_invited_to_reply", _ => false), "is_followed_topic_tweet",
("has_gte_10_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 10))), _.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == BasicTopicContextFunctionalityType)),
(
"is_recommended_topic_tweet",
_.getOrElse(TopicContextFunctionalityTypeFeature, None)
.exists(_ == RecommendationTopicContextFunctionalityType)),
("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))), ("has_gte_100_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100))),
("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))), ("has_gte_1k_favs", _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 1000))),
( (
@ -164,8 +167,6 @@ object HomeTweetTypePredicates {
( (
"has_gte_100k_favs", "has_gte_100k_favs",
_.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))), _.getOrElse(EarlybirdFeature, None).exists(_.favCountV2.exists(_ >= 100000))),
("above_neighbor_is_topic_tweet", _ => false),
("is_topic_tweet_with_neighbor_below", _ => false),
("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)), ("has_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.hasSpace)),
("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)), ("has_live_audio_space", _.getOrElse(AudioSpaceMetaDataFeature, None).exists(_.isLive)),
( (
@ -187,6 +188,7 @@ object HomeTweetTypePredicates {
( (
"has_toxicity_score_above_threshold", "has_toxicity_score_above_threshold",
_.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))), _.getOrElse(EarlybirdFeature, None).exists(_.toxicityScore.exists(_ > 0.91))),
("is_topic_tweet", _.getOrElse(TopicIdSocialContextFeature, None).isDefined),
( (
"text_only", "text_only",
candidate => candidate =>
@ -204,23 +206,50 @@ object HomeTweetTypePredicates {
("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)), ("has_3_images", _.getOrElse(NumImagesFeature, None).exists(_ == 3)),
("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)), ("has_4_images", _.getOrElse(NumImagesFeature, None).exists(_ == 4)),
("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)), ("has_card", _.getOrElse(EarlybirdFeature, None).exists(_.hasCard)),
("3_or_more_consecutive_not_in_network", _ => false),
("2_or_more_consecutive_not_in_network", _ => false),
("5_out_of_7_not_in_network", _ => false),
("7_out_of_7_not_in_network", _ => false),
("5_out_of_5_not_in_network", _ => false),
("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)), ("user_follow_count_gte_50", _.getOrElse(UserFollowingCountFeature, None).exists(_ > 50)),
("has_liked_by_social_context", _ => false),
("has_followed_by_social_context", _ => false),
("has_topic_social_context", _ => false),
("timeline_entry_has_banner", _ => false),
("served_in_conversation_module", _.getOrElse(ServedInConversationModuleFeature, false)),
( (
"conversation_module_has_2_displayed_tweets", "has_liked_by_social_context",
_.getOrElse(ConversationModule2DisplayedTweetsFeature, false)), candidateFeatures =>
("conversation_module_has_gap", _.getOrElse(ConversationModuleHasGapFeature, false)), candidateFeatures
("served_in_recap_tweet_candidate_module_injection", _ => false), .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
("served_in_threaded_conversation_module", _ => false) .exists(candidateFeatures
.getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty).toSet.contains)),
(
"has_followed_by_social_context",
_.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty),
(
"has_topic_social_context",
candidateFeatures =>
candidateFeatures
.getOrElse(TopicIdSocialContextFeature, None)
.isDefined &&
candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined),
("video_lte_10_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ <= 10000)),
(
"video_bt_10_60_sec",
_.getOrElse(VideoDurationMsFeature, None).exists(duration =>
duration > 10000 && duration <= 60000)),
("video_gt_60_sec", _.getOrElse(VideoDurationMsFeature, None).exists(_ > 60000)),
(
"tweet_age_lte_30_minutes",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 30.minutes)),
(
"tweet_age_lte_1_hour",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 1.hour)),
(
"tweet_age_lte_6_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 6.hours)),
(
"tweet_age_lte_12_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow <= 12.hours)),
(
"tweet_age_gte_24_hours",
_.getOrElse(OriginalTweetCreationTimeFromSnowflakeFeature, None)
.exists(_.untilNow >= 24.hours)),
) )
val PredicateMap = CandidatePredicates.toMap val PredicateMap = CandidatePredicates.toMap

4
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder/ListClientEventDetailsBuilder.scala
View File

@ -8,7 +8,7 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ti
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelineservice.suggests.{thriftscala => st} import com.twitter.timelineservice.suggests.{thriftscala => st}
object ListClientEventDetailsBuilder case class ListClientEventDetailsBuilder(suggestType: st.SuggestType)
extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] { extends BaseClientEventDetailsBuilder[PipelineQuery, UniversalNoun[Any]] {
override def apply( override def apply(
@ -20,7 +20,7 @@ object ListClientEventDetailsBuilder
conversationDetails = None, conversationDetails = None,
timelinesDetails = Some( timelinesDetails = Some(
TimelinesDetails( TimelinesDetails(
injectionType = Some(st.SuggestType.OrganicListTweet.name), injectionType = Some(suggestType.name),
controllerData = None, controllerData = None,
sourceData = None)), sourceData = None)),
articleDetails = None, articleDetails = None,

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/AuthorChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/AuthorChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -9,7 +9,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

7
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BUILD.bazel
View File

@ -4,9 +4,16 @@ scala_library(
strict_deps = True, strict_deps = True,
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt/builder",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt",
"src/thrift/com/twitter/timelines/service:thrift-scala", "src/thrift/com/twitter/timelines/service:thrift-scala",
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala", "src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate",
], ],
) )

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/BlockUserChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/BlockUserChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -13,7 +13,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorBlockUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/DontLikeFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/DontLikeFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
@ -17,7 +17,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tls} import com.twitter.timelineservice.{thriftscala => tls}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/EngagerSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/EngagerSocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ExtendedReplySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ExtendedReplySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

18
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackStrings.scala Normal file
View File

@ -0,0 +1,18 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
class FeedbackStrings @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) {
private val externalStringRegistry = externalStringRegistryProvider.get()
val seeLessOftenFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOften")
val seeLessOftenConfirmationFeedbackString =
externalStringRegistry.createProdString("Feedback.seeLessOftenConfirmation")
}

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FeedbackUtil.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FeedbackUtil.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.ChildFeedbackAction

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/FollowedBySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/FollowedBySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidFollowedByUserIdsFeature

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeFeedbackActionInfoBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeFeedbackActionInfoBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProduct
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Fe
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelines.service.{thriftscala => t}
import com.twitter.timelines.util.FeedbackMetadataSerializer import com.twitter.timelines.util.FeedbackMetadataSerializer
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

10
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/HomeTweetSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeTweetSocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature
@ -14,10 +14,13 @@ import javax.inject.Singleton
@Singleton @Singleton
case class HomeTweetSocialContextBuilder @Inject() ( case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder: LikedBySocialContextBuilder, likedBySocialContextBuilder: LikedBySocialContextBuilder,
listsSocialContextBuilder: ListsSocialContextBuilder,
followedBySocialContextBuilder: FollowedBySocialContextBuilder, followedBySocialContextBuilder: FollowedBySocialContextBuilder,
topicSocialContextBuilder: TopicSocialContextBuilder, topicSocialContextBuilder: TopicSocialContextBuilder,
extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder, extendedReplySocialContextBuilder: ExtendedReplySocialContextBuilder,
receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder) receivedReplySocialContextBuilder: ReceivedReplySocialContextBuilder,
popularVideoSocialContextBuilder: PopularVideoSocialContextBuilder,
popularInYourAreaSocialContextBuilder: PopularInYourAreaSocialContextBuilder)
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
def apply( def apply(
@ -31,6 +34,9 @@ case class HomeTweetSocialContextBuilder @Inject() (
likedBySocialContextBuilder(query, candidate, features) likedBySocialContextBuilder(query, candidate, features)
.orElse(followedBySocialContextBuilder(query, candidate, features)) .orElse(followedBySocialContextBuilder(query, candidate, features))
.orElse(topicSocialContextBuilder(query, candidate, features)) .orElse(topicSocialContextBuilder(query, candidate, features))
.orElse(popularVideoSocialContextBuilder(query, candidate, features))
.orElse(listsSocialContextBuilder(query, candidate, features))
.orElse(popularInYourAreaSocialContextBuilder(query, candidate, features))
case Some(_) => case Some(_) =>
val conversationId = features.getOrElse(ConversationModuleIdFeature, None) val conversationId = features.getOrElse(ConversationModuleIdFeature, None)
// Only hydrate the social context into the root tweet in a conversation module // Only hydrate the social context into the root tweet in a conversation module

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToFollowFeedbackActionInfoBuilder.scala
View File

@ -5,7 +5,6 @@ import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.ExternalStringRegistry
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Provider import javax.inject.Provider
@ -32,12 +31,13 @@ object HomeWhoToFollowFeedbackActionInfoBuilder {
@Singleton @Singleton
case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() ( case class HomeWhoToFollowFeedbackActionInfoBuilder @Inject() (
@ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry], feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter]) @ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] { extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder( private val whoToFollowFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
externalStringRegistry = externalStringRegistryProvider.get(), seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(), stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest) encodedFeedbackRequest = Some(HomeWhoToFollowFeedbackActionInfoBuilder.EncodedFeedbackRequest)
) )

52
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/HomeWhoToSubscribeFeedbackActionInfoBuilder.scala Normal file
View File

@ -0,0 +1,52 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.WhoToFollowFeedbackActionInfoBuilder
import com.twitter.product_mixer.component_library.model.candidate.UserCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.metadata.BaseFeedbackActionInfoBuilder
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.FeedbackActionInfo
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelines.service.{thriftscala => tl}
import com.twitter.timelines.util.FeedbackRequestSerializer
import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.timelineservice.thriftscala.FeedbackType
object HomeWhoToSubscribeFeedbackActionInfoBuilder {
private val FeedbackMetadata = tl.FeedbackMetadata(
injectionType = Some(SuggestType.WhoToSubscribe),
engagementType = None,
entityIds = Seq.empty,
ttlMs = None
)
private val FeedbackRequest =
tl.DefaultFeedbackRequest2(FeedbackType.SeeFewer, FeedbackMetadata)
private val EncodedFeedbackRequest =
FeedbackRequestSerializer.serialize(tl.FeedbackRequest.DefaultFeedbackRequest2(FeedbackRequest))
}
@Singleton
case class HomeWhoToSubscribeFeedbackActionInfoBuilder @Inject() (
feedbackStrings: FeedbackStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseFeedbackActionInfoBuilder[PipelineQuery, UserCandidate] {
private val whoToSubscribeFeedbackActionInfoBuilder = WhoToFollowFeedbackActionInfoBuilder(
seeLessOftenFeedbackString = feedbackStrings.seeLessOftenFeedbackString,
seeLessOftenConfirmationFeedbackString = feedbackStrings.seeLessOftenConfirmationFeedbackString,
stringCenter = stringCenterProvider.get(),
encodedFeedbackRequest =
Some(HomeWhoToSubscribeFeedbackActionInfoBuilder.EncodedFeedbackRequest)
)
override def apply(
query: PipelineQuery,
candidate: UserCandidate,
candidateFeatures: FeatureMap
): Option[FeedbackActionInfo] =
whoToSubscribeFeedbackActionInfoBuilder.apply(query, candidate, candidateFeatures)
}

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/LikedBySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/LikedBySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.SGSValidLikedByUserIdsFeature

50
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ListsSocialContextBuilder.scala Normal file
View File

@ -0,0 +1,50 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => t}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
/**
* "Your Lists" will be rendered for the context and a url link for your lists.
*/
@Singleton
case class ListsSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val listString = externalStrings.ownedSubscribedListsModuleHeaderString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
candidateFeatures.get(SuggestTypeFeature) match {
case Some(suggestType) if suggestType == t.SuggestType.RankedListTweet =>
val userName = query.features.flatMap(_.getOrElse(UserScreenNameFeature, None))
Some(
GeneralContext(
contextType = ListGeneralContextType,
text = stringCenter.prepare(listString),
url = userName.map(name => ""),
contextImageUrls = None,
landingUrl = None
))
case _ => None
}
}
}

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/MuteUserChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/MuteUserChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -12,7 +12,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleMuteUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotInterestedTopicFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotInterestedTopicFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.PerspectiveFilteredLikedByUserIdsFeature
@ -15,7 +15,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorMarkNotInterestedTopic
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/NotRelevantChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/NotRelevantChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
@ -12,7 +12,6 @@ import com.twitter.timelines.common.{thriftscala => tlc}
import com.twitter.timelineservice.model.FeedbackInfo import com.twitter.timelineservice.model.FeedbackInfo
import com.twitter.timelineservice.model.FeedbackMetadata import com.twitter.timelineservice.model.FeedbackMetadata
import com.twitter.timelineservice.{thriftscala => tlst} import com.twitter.timelineservice.{thriftscala => tlst}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

43
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularInYourAreaSocialContextBuilder.scala Normal file
View File

@ -0,0 +1,43 @@
package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject
import javax.inject.Provider
import javax.inject.Singleton
@Singleton
case class PopularInYourAreaSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get()
private val popularInYourAreaString = externalStrings.socialContextPopularInYourAreaString
def apply(
query: PipelineQuery,
candidate: TweetCandidate,
candidateFeatures: FeatureMap
): Option[SocialContext] = {
val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
if (suggestTypeOpt.contains(st.SuggestType.RecommendedTrendTweet)) {
Some(
GeneralContext(
contextType = LocationGeneralContextType,
text = stringCenter.prepare(popularInYourAreaString),
url = None,
contextImageUrls = None,
landingUrl = None
))
} else None
}
}

32
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/YouMightLikeSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/PopularVideoSocialContextBuilder.scala
View File

@ -1,50 +1,48 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder import com.twitter.product_mixer.core.functional_component.decorator.urt.builder.social_context.BaseSocialContextBuilder
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.SocialContext
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata._
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelineservice.suggests.{thriftscala => st}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Provider import javax.inject.Provider
import javax.inject.Singleton import javax.inject.Singleton
/**
* Renders a fixed 'You Might Like' string above all OON Tweets.
*/
@Singleton @Singleton
case class YouMightLikeSocialContextBuilder @Inject() ( case class PopularVideoSocialContextBuilder @Inject() (
externalStrings: HomeMixerExternalStrings, externalStrings: HomeMixerExternalStrings,
@ProductScoped stringCenterProvider: Provider[StringCenter]) @ProductScoped stringCenterProvider: Provider[StringCenter])
extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] { extends BaseSocialContextBuilder[PipelineQuery, TweetCandidate] {
private val stringCenter = stringCenterProvider.get() private val stringCenter = stringCenterProvider.get()
private val youMightLikeString = externalStrings.socialContextYouMightLikeString private val popularVideoString = externalStrings.socialContextPopularVideoString
def apply( def apply(
query: PipelineQuery, query: PipelineQuery,
candidate: TweetCandidate, candidate: TweetCandidate,
candidateFeatures: FeatureMap candidateFeatures: FeatureMap
): Option[SocialContext] = { ): Option[SocialContext] = {
val isInNetwork = candidateFeatures.getOrElse(InNetworkFeature, true) val suggestTypeOpt = candidateFeatures.getOrElse(SuggestTypeFeature, None)
val isRetweet = candidateFeatures.getOrElse(IsRetweetFeature, false) if (suggestTypeOpt.contains(st.SuggestType.MediaTweet)) {
if (!isInNetwork && !isRetweet) {
Some( Some(
GeneralContext( GeneralContext(
contextType = SparkleGeneralContextType, contextType = SparkleGeneralContextType,
text = stringCenter.prepare(youMightLikeString), text = stringCenter.prepare(popularVideoString),
url = None, url = None,
contextImageUrls = None, contextImageUrls = None,
landingUrl = None landingUrl = Some(
Url(
urlType = DeepLink,
url = ""
)
)
)) ))
} else { } else None
None
}
} }
} }

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReceivedReplySocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReceivedReplySocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/ReportTweetChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/ReportTweetChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
@ -8,7 +8,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorReportTweet
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/RetweeterChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/RetweeterChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
@ -10,7 +10,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ch
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import com.twitter.timelines.service.{thriftscala => t} import com.twitter.timelines.service.{thriftscala => t}
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/TopicSocialContextBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/TopicSocialContextBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/UnfollowUserChildFeedbackActionBuilder.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder/UnfollowUserChildFeedbackActionBuilder.scala
View File

@ -1,4 +1,4 @@
package com.twitter.home_mixer.functional_component.decorator package com.twitter.home_mixer.functional_component.decorator.urt.builder
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
@ -11,7 +11,6 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.Ri
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RichFeedbackBehaviorToggleFollowUser
import com.twitter.product_mixer.core.product.guice.scope.ProductScoped import com.twitter.product_mixer.core.product.guice.scope.ProductScoped
import com.twitter.stringcenter.client.StringCenter import com.twitter.stringcenter.client.StringCenter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton

45
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/BUILD.bazel
View File

@ -4,95 +4,56 @@ scala_library(
strict_deps = True, strict_deps = True,
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider", "configapi/configapi-decider",
"finatra/inject/inject-core/src/main/scala", "finatra/inject/inject-core/src/main/scala",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/content",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/inferred_topic",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/twhin_embeddings",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/service", "home-mixer/server/src/main/scala/com/twitter/home_mixer/service",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content",
"joinkey/src/main/scala/com/twitter/joinkey/context", "joinkey/src/main/scala/com/twitter/joinkey/context",
"joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala", "joinkey/src/main/thrift/com/twitter/joinkey/context:joinkey-context-scala",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timelines_impression_store",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/topics",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_visibility_reason",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph",
"product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util",
"representation-scorer/server/src/main/scala/com/twitter/representationscorer/common",
"representation-scorer/server/src/main/thrift:thrift-scala",
"servo/repo/src/main/scala",
"snowflake/src/main/scala/com/twitter/snowflake/id", "snowflake/src/main/scala/com/twitter/snowflake/id",
"src/java/com/twitter/ml/api/constant",
"src/java/com/twitter/search/common/util/lang", "src/java/com/twitter/search/common/util/lang",
"src/scala/com/twitter/ml/api/util", "src/scala/com/twitter/timelines/prediction/adapters/request_context",
"src/scala/com/twitter/timelines/prediction/adapters/real_graph",
"src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/adapters/twistly",
"src/scala/com/twitter/timelines/prediction/adapters/two_hop_features",
"src/scala/com/twitter/timelines/prediction/common/util",
"src/scala/com/twitter/timelines/prediction/features/common",
"src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph",
"src/scala/com/twitter/timelines/prediction/features/recap",
"src/scala/com/twitter/timelines/prediction/features/time_features",
"src/thrift/com/twitter/gizmoduck:thrift-scala", "src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/ml/api:data-java",
"src/thrift/com/twitter/ml/api:embedding-java",
"src/thrift/com/twitter/onboarding/relevance/features:features-java",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala", "src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-java", "src/thrift/com/twitter/search/common:constants-java",
"src/thrift/com/twitter/socialgraph:thrift-scala", "src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/spam/rtf:safety-result-scala", "src/thrift/com/twitter/spam/rtf:safety-result-scala",
"src/thrift/com/twitter/timelineranker:thrift-scala", "src/thrift/com/twitter/timelineranker:thrift-scala",
"src/thrift/com/twitter/timelines/author_features:thrift-java",
"src/thrift/com/twitter/timelines/conversation_features:conversation_features-scala",
"src/thrift/com/twitter/timelines/impression:thrift-scala", "src/thrift/com/twitter/timelines/impression:thrift-scala",
"src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala", "src/thrift/com/twitter/timelines/impression_bloom_filter:thrift-scala",
"src/thrift/com/twitter/timelines/real_graph:real_graph-scala", "src/thrift/com/twitter/timelines/real_graph:real_graph-scala",
"src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala",
"src/thrift/com/twitter/topic_recos:topic_recos-thrift-java",
"src/thrift/com/twitter/tweetypie:service-scala", "src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala", "src/thrift/com/twitter/tweetypie:tweet-scala",
"src/thrift/com/twitter/user_session_store:thrift-java", "src/thrift/com/twitter/user_session_store:thrift-java",
"src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala", "src/thrift/com/twitter/wtf/candidate:wtf-candidate-scala",
"src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java",
"stitch/stitch-core", "stitch/stitch-core",
"stitch/stitch-gizmoduck", "stitch/stitch-gizmoduck",
"stitch/stitch-socialgraph", "stitch/stitch-socialgraph",
"stitch/stitch-timelineservice", "stitch/stitch-timelineservice",
"stitch/stitch-tweetypie", "stitch/stitch-tweetypie",
"strato/config/columns/topic-signals/tsp",
"strato/config/columns/topic-signals/tsp:tsp-strato-client",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence", "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/persistence",
"timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", "timelines/src/main/scala/com/twitter/timelines/clients/manhattan/store",
"timelines/src/main/scala/com/twitter/timelines/clients/user_tweet_entity_graph",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter", "timelines/src/main/scala/com/twitter/timelines/impressionstore/impressionbloomfilter",
"timelines/src/main/scala/com/twitter/timelines/impressionstore/store", "timelines/src/main/scala/com/twitter/timelines/impressionstore/store",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"topic-social-proof/server/src/main/thrift:thrift-scala",
"topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting",
"tweetconvosvc/thrift/src/main/thrift:thrift-scala",
"twitter-config/yaml",
"user_session_store/src/main/scala/com/twitter/user_session_store", "user_session_store/src/main/scala/com/twitter/user_session_store",
"util/util-core", "util/util-core",
], ],

8
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FeedbackHistoryQueryFeatureHydrator.scala
View File

@ -1,12 +1,10 @@
package com.twitter.home_mixer.functional_component.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature import com.twitter.home_mixer.model.HomeFeatures.FeedbackHistoryFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableFeedbackFatigueParam
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
@ -17,16 +15,12 @@ import javax.inject.Singleton
@Singleton @Singleton
case class FeedbackHistoryQueryFeatureHydrator @Inject() ( case class FeedbackHistoryQueryFeatureHydrator @Inject() (
feedbackHistoryClient: FeedbackHistoryManhattanClient) feedbackHistoryClient: FeedbackHistoryManhattanClient)
extends QueryFeatureHydrator[PipelineQuery] extends QueryFeatureHydrator[PipelineQuery] {
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FeedbackHistory")
override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature) override val features: Set[Feature[_, _]] = Set(FeedbackHistoryFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableFeedbackFatigueParam)
override def hydrate( override def hydrate(
query: PipelineQuery query: PipelineQuery
): Stitch[FeatureMap] = ): Stitch[FeatureMap] =

41
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/FollowedTopicsQueryFeatureHydrator.scala
View File

@ -1,41 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.UserFollowedTopicsCountFeature
import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.component_library.candidate_source.topics.FollowedTopicsCandidateSource
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
case class FollowedTopicsQueryFeatureHydrator @Inject() (
followedTopicsCandidateSource: FollowedTopicsCandidateSource)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FollowedTopics")
override val features: Set[Feature[_, _]] = Set(UserFollowedTopicsCountFeature)
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val request: StratoKeyView[Long, Unit] = StratoKeyView(query.getRequiredUserId, Unit)
followedTopicsCandidateSource(request)
.map { topics =>
FeatureMapBuilder().add(UserFollowedTopicsCountFeature, Some(topics.size)).build()
}.handle {
case _ => FeatureMapBuilder().add(UserFollowedTopicsCountFeature, None).build()
}
}
override val alerts = Seq(
HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9),
HomeMixerAlertConfig.BusinessHours.defaultLatencyAlert(1500.millis)
)
}

58
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/GizmoduckAuthorSafetyFeatureHydrator.scala
View File

@ -1,58 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.gizmoduck.{thriftscala => gt}
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.param.HomeGlobalParams.EnableGizmoduckAuthorSafetyFeatureHydratorParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.Conditionally
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.stitch.gizmoduck.Gizmoduck
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class GizmoduckAuthorSafetyFeatureHydrator @Inject() (gizmoduck: Gizmoduck)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Conditionally[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("GizmoduckAuthorSafety")
override val features: Set[Feature[_, _]] = Set(AuthorIsBlueVerifiedFeature)
override def onlyIf(query: PipelineQuery): Boolean =
query.params(EnableGizmoduckAuthorSafetyFeatureHydratorParam)
private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
val authorIdOption = existingFeatures.getOrElse(AuthorIdFeature, None)
val blueVerifiedStitch = authorIdOption
.map { authorId =>
gizmoduck
.getUserById(
userId = authorId,
queryFields = queryFields
)
.map { _.safety.flatMap(_.isBlueVerified).getOrElse(false) }
}.getOrElse(Stitch.False)
blueVerifiedStitch.map { isBlueVerified =>
FeatureMapBuilder()
.add(AuthorIsBlueVerifiedFeature, isBlueVerified)
.build()
}
}
}

45
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/ImpressionBloomFilterQueryFeatureHydrator.scala
View File

@ -3,6 +3,7 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature import com.twitter.home_mixer.model.HomeFeatures.ImpressionBloomFilterFeature
import com.twitter.home_mixer.model.request.HasSeenTweetIds import com.twitter.home_mixer.model.request.HasSeenTweetIds
import com.twitter.home_mixer.param.HomeGlobalParams.ImpressionBloomFilterFalsePositiveRateParam
import com.twitter.home_mixer.service.HomeMixerAlertConfig import com.twitter.home_mixer.service.HomeMixerAlertConfig
import com.twitter.product_mixer.core.feature.Feature import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
@ -11,7 +12,8 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Quer
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.impressionbloomfilter.{thriftscala => t} import com.twitter.timelines.clients.manhattan.store.ManhattanStoreClient
import com.twitter.timelines.impressionbloomfilter.{thriftscala => blm}
import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter import com.twitter.timelines.impressionstore.impressionbloomfilter.ImpressionBloomFilter
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -19,36 +21,39 @@ import javax.inject.Singleton
@Singleton @Singleton
case class ImpressionBloomFilterQueryFeatureHydrator[ case class ImpressionBloomFilterQueryFeatureHydrator[
Query <: PipelineQuery with HasSeenTweetIds] @Inject() ( Query <: PipelineQuery with HasSeenTweetIds] @Inject() (
bloomFilter: ImpressionBloomFilter) bloomFilterClient: ManhattanStoreClient[
extends QueryFeatureHydrator[Query] { blm.ImpressionBloomFilterKey,
blm.ImpressionBloomFilterSeq
]) extends QueryFeatureHydrator[Query] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier( override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier(
"ImpressionBloomFilter") "ImpressionBloomFilter")
private val ImpressionBloomFilterTTL = 7.day private val ImpressionBloomFilterTTL = 7.day
private val ImpressionBloomFilterFalsePositiveRate = 0.002
override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature) override val features: Set[Feature[_, _]] = Set(ImpressionBloomFilterFeature)
private val SurfaceArea = t.SurfaceArea.HomeTimeline private val SurfaceArea = blm.SurfaceArea.HomeTimeline
override def hydrate(query: Query): Stitch[FeatureMap] = { override def hydrate(query: Query): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId val userId = query.getRequiredUserId
bloomFilter.getBloomFilterSeq(userId, SurfaceArea).map { bloomFilterSeq => bloomFilterClient
val updatedBloomFilterSeq = .get(blm.ImpressionBloomFilterKey(userId, SurfaceArea))
if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq .map(_.getOrElse(blm.ImpressionBloomFilterSeq(Seq.empty)))
else { .map { bloomFilterSeq =>
bloomFilter.addElements( val updatedBloomFilterSeq =
userId = userId, if (query.seenTweetIds.forall(_.isEmpty)) bloomFilterSeq
surfaceArea = SurfaceArea, else {
tweetIds = query.seenTweetIds.get, ImpressionBloomFilter.addSeenTweetIds(
bloomFilterEntrySeq = bloomFilterSeq, surfaceArea = SurfaceArea,
timeToLive = ImpressionBloomFilterTTL, tweetIds = query.seenTweetIds.get,
falsePositiveRate = ImpressionBloomFilterFalsePositiveRate bloomFilterSeq = bloomFilterSeq,
) timeToLive = ImpressionBloomFilterTTL,
} falsePositiveRate = query.params(ImpressionBloomFilterFalsePositiveRateParam)
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build() )
} }
FeatureMapBuilder().add(ImpressionBloomFilterFeature, updatedBloomFilterSeq).build()
}
} }
override val alerts = Seq( override val alerts = Seq(

41
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/InNetworkFeatureHydrator.scala Normal file
View File

@ -0,0 +1,41 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object InNetworkFeatureHydrator
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("InNetwork")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
val followedUserIds = query.features.get.get(SGSFollowedUsersFeature).toSet
val featureMaps = candidates.map { candidate =>
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val isInNetworkOpt = candidate.features.getOrElse(AuthorIdFeature, None).map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
isSelfTweet || followedUserIds.contains(authorId)
}
FeatureMapBuilder().add(InNetworkFeature, isInNetworkOpt.getOrElse(true)).build()
}
Stitch.value(featureMaps)
}
}

29
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PersistenceStoreQueryFeatureHydrator.scala
View File

@ -2,8 +2,10 @@ package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._ import com.twitter.conversions.DurationOps._
import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent import com.twitter.common_internal.analytics.twitter_client_user_agent_parser.UserAgent
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature
import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ForYouProduct
@ -27,17 +29,27 @@ import javax.inject.Singleton
@Singleton @Singleton
case class PersistenceStoreQueryFeatureHydrator @Inject() ( case class PersistenceStoreQueryFeatureHydrator @Inject() (
timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3]) timelineResponseBatchesClient: TimelineResponseBatchesClient[TimelineResponseV3],
statsReceiver: StatsReceiver)
extends QueryFeatureHydrator[PipelineQuery] { extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("PersistenceStore")
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val servedTweetIdsSizeStat = scopedStatsReceiver.stat("ServedTweetIdsSize")
private val WhoToFollowExcludedUserIdsLimit = 1000 private val WhoToFollowExcludedUserIdsLimit = 1000
private val ServedTweetIdsDuration = 1.hour private val ServedTweetIdsDuration = 10.minutes
private val ServedTweetIdsLimit = 100 private val ServedTweetIdsLimit = 100
private val ServedTweetPreviewIdsDuration = 10.hours
private val ServedTweetPreviewIdsLimit = 10
override val features: Set[Feature[_, _]] = override val features: Set[Feature[_, _]] =
Set(ServedTweetIdsFeature, PersistenceEntriesFeature, WhoToFollowExcludedUserIdsFeature) Set(
ServedTweetIdsFeature,
ServedTweetPreviewIdsFeature,
PersistenceEntriesFeature,
WhoToFollowExcludedUserIdsFeature)
private val supportedClients = Seq( private val supportedClients = Seq(
ClientPlatform.IPhone, ClientPlatform.IPhone,
@ -80,8 +92,19 @@ case class PersistenceStoreQueryFeatureHydrator @Inject() (
.flatMap( .flatMap(
_.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit)) _.entries.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetIdsLimit))
servedTweetIdsSizeStat.add(servedTweetIds.size)
val servedTweetPreviewIds = timelineResponses
.filter(_.clientPlatform == clientPlatform)
.filter(_.servedTime >= Time.now - ServedTweetPreviewIdsDuration)
.sortBy(-_.servedTime.inMilliseconds)
.flatMap(_.entries
.filter(_.entityIdType == EntityIdType.TweetPreview)
.flatMap(_.tweetIds(includeSourceTweets = true)).take(ServedTweetPreviewIdsLimit))
FeatureMapBuilder() FeatureMapBuilder()
.add(ServedTweetIdsFeature, servedTweetIds) .add(ServedTweetIdsFeature, servedTweetIds)
.add(ServedTweetPreviewIdsFeature, servedTweetPreviewIds)
.add(PersistenceEntriesFeature, timelineResponses) .add(PersistenceEntriesFeature, timelineResponses)
.add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds) .add(WhoToFollowExcludedUserIdsFeature, whoToFollowUserIds)
.build() .build()

7
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/PerspectiveFilteredSocialContextFeatureHydrator.scala
View File

@ -10,6 +10,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.stitch.timelineservice.TimelineService import com.twitter.stitch.timelineservice.TimelineService
import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives import com.twitter.stitch.timelineservice.TimelineService.GetPerspectives
@ -37,10 +38,10 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = { ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val engagingUserIdtoTweetId = candidates.flatMap { candidate => val engagingUserIdtoTweetId = candidates.flatMap { candidate =>
candidate.features candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers) .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.map(favoritedBy => favoritedBy -> candidate.candidate.id) .map(favoritedBy => favoritedBy -> candidate.candidate.id)
} }
@ -59,7 +60,7 @@ class PerspectiveFilteredSocialContextFeatureHydrator @Inject() (timelineService
candidates.map { candidate => candidates.map { candidate =>
val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features val perspectiveFilteredFavoritedByUserIds: Seq[Long] = candidate.features
.get(FavoritedByUserIdsFeature).take(MaxCountUsers) .getOrElse(FavoritedByUserIdsFeature, Seq.empty).take(MaxCountUsers)
.filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) } .filter { userId => validUserIdTweetIds.contains((userId, candidate.candidate.id)) }
FeatureMapBuilder() FeatureMapBuilder()

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RealGraphInNetworkScoresQueryFeatureHydrator.scala
View File

@ -32,11 +32,15 @@ case class RealGraphInNetworkScoresQueryFeatureHydrator @Inject() (
val realGraphScoresFeatures = realGraphFollowedUsers val realGraphScoresFeatures = realGraphFollowedUsers
.getOrElse(Seq.empty) .getOrElse(Seq.empty)
.sortBy(-_.score) .sortBy(-_.score)
.map(candidate => candidate.userId -> candidate.score) .map(candidate => candidate.userId -> scaleScore(candidate.score))
.take(RealGraphCandidateCount) .take(RealGraphCandidateCount)
.toMap .toMap
FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build() FeatureMapBuilder().add(RealGraphInNetworkScoresFeature, realGraphScoresFeatures).build()
} }
} }
// Rescale Real Graph v2 scores from [0,1] to the v1 scores distribution [1,2.97]
private def scaleScore(score: Double): Double =
if (score >= 0.0 && score <= 1.0) score * 1.97 + 1.0 else score
} }

17
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/RequestQueryFeatureHydrator.scala
View File

@ -17,9 +17,13 @@ import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.G
import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor
import com.twitter.product_mixer.core.pipeline.HasPipelineCursor import com.twitter.product_mixer.core.pipeline.HasPipelineCursor
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
import com.twitter.search.common.util.lang.ThriftLanguageUtil import com.twitter.search.common.util.lang.ThriftLanguageUtil
import com.twitter.snowflake.id.SnowflakeId import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.dowFromTimestamp
import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter.hourFromTimestamp
import java.util.UUID import java.util.UUID
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -45,6 +49,9 @@ class RequestQueryFeatureHydrator[
PullToRefreshFeature, PullToRefreshFeature,
RequestJoinIdFeature, RequestJoinIdFeature,
ServedRequestIdFeature, ServedRequestIdFeature,
TimestampFeature,
TimestampGMTDowFeature,
TimestampGMTHourFeature,
ViewerIdFeature ViewerIdFeature
) )
@ -67,6 +74,7 @@ class RequestQueryFeatureHydrator[
override def hydrate(query: Query): Stitch[FeatureMap] = { override def hydrate(query: Query): Stitch[FeatureMap] = {
val requestContext = query.deviceContext.flatMap(_.requestContextValue) val requestContext = query.deviceContext.flatMap(_.requestContextValue)
val servedRequestId = UUID.randomUUID.getMostSignificantBits val servedRequestId = UUID.randomUUID.getMostSignificantBits
val timestamp = query.queryTime.inMilliseconds
val featureMap = FeatureMapBuilder() val featureMap = FeatureMapBuilder()
.add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt)) .add(AccountAgeFeature, query.getOptionalUserId.flatMap(SnowflakeId.timeFromIdOpt))
@ -97,8 +105,15 @@ class RequestQueryFeatureHydrator[
.add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh)) .add(PullToRefreshFeature, requestContext.contains(RequestContext.PullToRefresh))
.add(ServedRequestIdFeature, Some(servedRequestId)) .add(ServedRequestIdFeature, Some(servedRequestId))
.add(RequestJoinIdFeature, getRequestJoinId(servedRequestId)) .add(RequestJoinIdFeature, getRequestJoinId(servedRequestId))
.add(TimestampFeature, timestamp)
.add(TimestampGMTDowFeature, dowFromTimestamp(timestamp))
.add(TimestampGMTHourFeature, hourFromTimestamp(timestamp))
.add(HasDarkRequestFeature, hasDarkRequest) .add(HasDarkRequestFeature, hasDarkRequest)
.add(ViewerIdFeature, query.getRequiredUserId) .add(
ViewerIdFeature,
query.getOptionalUserId
.orElse(query.getGuestId).getOrElse(
throw PipelineFailure(BadRequest, "Missing viewer id")))
.build() .build()
Stitch.value(featureMap) Stitch.value(featureMap)

46
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSFollowedUsersQueryFeatureHydrator.scala
View File

@ -1,46 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
object SGSFollowedUsersFeature extends Feature[PipelineQuery, Seq[Long]]
@Singleton
case class SGSFollowedUsersQueryFeatureHydrator @Inject() (
socialGraphStitchClient: SocialGraphStitchClient)
extends QueryFeatureHydrator[PipelineQuery] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SGSFollowedUsers")
override val features: Set[Feature[_, _]] = Set(SGSFollowedUsersFeature)
private val SocialGraphLimit = 14999
override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = {
val userId = query.getRequiredUserId
val request = sg.IdsRequest(
relationships = Seq(
sg.SrcRelationship(userId, sg.RelationshipType.Following, hasRelationship = true),
sg.SrcRelationship(userId, sg.RelationshipType.Muting, hasRelationship = false)
),
pageRequest = Some(sg.PageRequest(count = Some(SocialGraphLimit)))
)
socialGraphStitchClient
.ids(request).map(_.ids)
.map { followedUsers =>
FeatureMapBuilder().add(SGSFollowedUsersFeature, followedUsers).build()
}
}
}

4
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SGSValidSocialContextFeatureHydrator.scala
View File

@ -12,6 +12,7 @@ import com.twitter.product_mixer.core.functional_component.feature_hydrator.Bulk
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.product_mixer.core.util.OffloadFuturePools
import com.twitter.socialgraph.{thriftscala => sg} import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph import com.twitter.stitch.socialgraph.SocialGraph
@ -41,8 +42,7 @@ class SGSValidSocialContextFeatureHydrator @Inject() (
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = { ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadStitch {
val allSocialContextUserIds = val allSocialContextUserIds =
candidates.flatMap { candidate => candidates.flatMap { candidate =>
candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++ candidate.features.getOrElse(FavoritedByUserIdsFeature, Nil).take(MaxCountUsers) ++

67
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/SocialGraphServiceFeatureHydrator.scala
View File

@ -1,67 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.{SocialGraph => SocialGraphStitchClient}
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class SocialGraphServiceFeatureHydrator @Inject() (socialGraphStitchClient: SocialGraphStitchClient)
extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("SocialGraphService")
override val features: Set[Feature[_, _]] = Set(InNetworkFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val viewerId = query.getRequiredUserId
// We use authorId and not sourceAuthorId here so that retweets are defined as in network
val authorIds = candidates.map(_.features.getOrElse(AuthorIdFeature, None).getOrElse(0L))
val distinctNonSelfAuthorIds = authorIds.filter(_ != viewerId).distinct
val idsRequest = createIdsRequest(
userId = viewerId,
relationshipTypes = Set(sg.RelationshipType.Following),
targetIds = Some(distinctNonSelfAuthorIds)
)
socialGraphStitchClient
.ids(request = idsRequest, requestContext = None)
.map { idResult =>
authorIds.map { authorId =>
// Users cannot follow themselves but this is in network by definition
val isSelfTweet = authorId == viewerId
val inNetworkAuthorIds = idResult.ids.toSet
val isInNetwork = isSelfTweet || inNetworkAuthorIds.contains(authorId) || authorId == 0L
FeatureMapBuilder().add(InNetworkFeature, isInNetwork).build()
}
}
}
private def createIdsRequest(
userId: Long,
relationshipTypes: Set[sg.RelationshipType],
targetIds: Option[Seq[Long]] = None
): sg.IdsRequest = sg.IdsRequest(
relationshipTypes.map { relationshipType =>
sg.SrcRelationship(userId, relationshipType, targets = targetIds)
}.toSeq,
Some(sg.PageRequest(selectAll = Some(true)))
)
}

162
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TSPInferredTopicFeatureHydrator.scala
View File

@ -1,162 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.contentrecommender.{thriftscala => cr}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.functional_component.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter
import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature
import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature
import com.twitter.ml.api.DataRecord
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType
import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import com.twitter.strato.generated.client.topic_signals.tsp.TopicSocialProofClientColumn
import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid}
import com.twitter.topiclisting.TopicListingViewerContext
import com.twitter.tsp.{thriftscala => tsp}
import javax.inject.Inject
import javax.inject.Singleton
import scala.collection.JavaConverters._
object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]]
object TSPInferredTopicDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
@Singleton
class TSPInferredTopicFeatureHydrator @Inject() (
topicSocialProofClientColumn: TopicSocialProofClientColumn,
statsReceiver: StatsReceiver,
) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic")
override val features: Set[Feature[_, _]] =
Set(
TSPInferredTopicFeature,
TSPInferredTopicDataRecordFeature,
TopicIdSocialContextFeature,
TopicContextFunctionalityTypeFeature)
private val topK = 3
private val sourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = Set(
sid.CandidateTweetSourceId.Simcluster,
sid.CandidateTweetSourceId.CroonTweet
)
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
private val keyFoundCounter = scopedStatsReceiver.counter("key/found")
private val keyLossCounter = scopedStatsReceiver.counter("key/loss")
private val requestFailCounter = scopedStatsReceiver.counter("request/fail")
private val DefaultFeatureMap = FeatureMapBuilder()
.add(TSPInferredTopicFeature, Map.empty[Long, Double])
.add(TSPInferredTopicDataRecordFeature, new DataRecord())
.add(TopicIdSocialContextFeature, None)
.add(TopicContextFunctionalityTypeFeature, None)
.build()
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
val tags = candidates.collect {
case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn =>
candidate.candidate.id -> candidate.features
.getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag])
}.toMap
val topicSocialProofRequest =
tsp.TopicSocialProofRequest(
userId = query.getRequiredUserId,
tweetIds = candidates.map(_.candidate.id).toSet,
displayLocation = cr.DisplayLocation.HomeTimeline,
topicListingSetting = tsp.TopicListingSetting.Followable,
context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift,
bypassModes = None,
// Only CRMixer source has this data. Convert the CRMixer metric tag to tsp metric tag.
tags = if (tags.isEmpty) None else Some(tags)
)
topicSocialProofClientColumn.fetcher
.fetch(topicSocialProofRequest)
.map(_.v)
.map {
case Some(response) =>
candidates.map { candidate =>
val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty)
if (topicWithScores.nonEmpty) {
keyFoundCounter.incr()
val (socialProofId, socialProofFunctionalityType) =
if (candidate.features
.getOrElse(CandidateSourceIdFeature, None)
.exists(sourcesToSetSocialProof.contains)) {
getSocialProof(topicWithScores)
} else {
(None, None)
}
val inferredTopicFeatures = convertTopicWithScores(topicWithScores)
val inferredTopicDataRecord =
InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head
FeatureMapBuilder()
.add(TSPInferredTopicFeature, inferredTopicFeatures)
.add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord)
.add(TopicIdSocialContextFeature, socialProofId)
.add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType)
.build()
} else {
keyLossCounter.incr()
DefaultFeatureMap
}
}
case _ =>
requestFailCounter.incr()
candidates.map { _ =>
DefaultFeatureMap
}
}
}
private def getSocialProof(
topicWithScores: Seq[tsp.TopicWithScore]
): (Option[Long], Option[TopicContextFunctionalityType]) = {
val followingTopicId = topicWithScores
.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) =>
topicId
}
if (followingTopicId.nonEmpty) {
return (followingTopicId, Some(BasicTopicContextFunctionalityType))
}
val implicitFollowingId = topicWithScores.collectFirst {
case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) =>
topicId
}
if (implicitFollowingId.nonEmpty) {
return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType))
}
(None, None)
}
private def convertTopicWithScores(
topicWithScores: Seq[tsp.TopicWithScore],
): Map[Long, Double] = {
topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap
}
}

251
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TimeFeaturesHydrator.scala
View File

@ -1,251 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.conversions.DurationOps._
import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature
import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.ml.api.DataRecord
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.FDsl._
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure
import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.search.common.features.{thriftscala => sc}
import com.twitter.snowflake.id.SnowflakeId
import com.twitter.stitch.Stitch
import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval
import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._
import com.twitter.timelines.prediction.features.time_features.TimeFeatures
import com.twitter.util.Duration
import scala.collection.Searching._
object TimeFeaturesDataRecordFeature
extends DataRecordInAFeature[TweetCandidate]
with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] {
override def defaultValue: DataRecord = new DataRecord()
}
object TimeFeaturesHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TimeFeatures")
override val features: Set[Feature[_, _]] = Set(TimeFeaturesDataRecordFeature)
override def apply(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap
): Stitch[FeatureMap] = {
Stitch.value {
val richDataRecord = new RichDataRecord()
setTimeFeatures(richDataRecord, candidate, existingFeatures, query)
FeatureMapBuilder()
.add(TimeFeaturesDataRecordFeature, richDataRecord.getRecord)
.build()
}
}
private def setTimeFeatures(
richDataRecord: RichDataRecord,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
query: PipelineQuery,
): Unit = {
val timeFeaturesOpt = getTimeFeatures(query, candidate, existingFeatures)
timeFeaturesOpt.foreach(timeFeatures => setFeatures(timeFeatures, richDataRecord))
}
private[feature_hydrator] def getTimeFeatures(
query: PipelineQuery,
candidate: TweetCandidate,
existingFeatures: FeatureMap,
): Option[TimeFeatures] = {
for {
requestTimestampMs <- Some(query.queryTime.inMilliseconds)
tweetId <- Some(candidate.id)
viewerId <- query.getOptionalUserId
tweetCreationTimeMs <- timeFromTweetOrUserId(tweetId)
timeSinceTweetCreation = requestTimestampMs - tweetCreationTimeMs
accountAgeDurationOpt = timeFromTweetOrUserId(viewerId).map { viewerAccountCreationTimeMs =>
Duration.fromMilliseconds(requestTimestampMs - viewerAccountCreationTimeMs)
}
timeSinceSourceTweetCreation =
existingFeatures
.getOrElse(SourceTweetIdFeature, None)
.flatMap { sourceTweetId =>
timeFromTweetOrUserId(sourceTweetId).map { sourceTweetCreationTimeMs =>
requestTimestampMs - sourceTweetCreationTimeMs
}
}
.getOrElse(timeSinceTweetCreation)
if (timeSinceTweetCreation > 0 && timeSinceSourceTweetCreation > 0)
} yield {
val timeFeatures = TimeFeatures(
timeSinceTweetCreation = timeSinceTweetCreation,
timeSinceSourceTweetCreation = timeSinceSourceTweetCreation,
timeSinceViewerAccountCreationSecs = accountAgeDurationOpt.map(_.inSeconds),
isDay30NewUser = accountAgeDurationOpt.map(_ < 30.days).getOrElse(false),
isMonth12NewUser = accountAgeDurationOpt.map(_ < 365.days).getOrElse(false),
accountAgeInterval = accountAgeDurationOpt.flatMap(AccountAgeInterval.fromDuration),
isTweetRecycled = false // only set in RecyclableTweetCandidateFilter, but it's not used
)
val timeFeaturesWithLastEngagement = addLastEngagementTimeFeatures(
existingFeatures.getOrElse(EarlybirdFeature, None),
timeFeatures,
timeSinceSourceTweetCreation
).getOrElse(timeFeatures)
val nonPollingTimestampsMs =
query.features.map(_.getOrElse(NonPollingTimesFeature, Seq.empty))
val timeFeaturesWithNonPollingOpt = addNonPollingTimeFeatures(
timeFeaturesWithLastEngagement,
requestTimestampMs,
tweetCreationTimeMs,
nonPollingTimestampsMs
)
timeFeaturesWithNonPollingOpt.getOrElse(timeFeaturesWithLastEngagement)
}
}
private def timeFromTweetOrUserId(tweetOrUserId: Long): Option[Long] = {
if (SnowflakeId.isSnowflakeId(tweetOrUserId))
Some(SnowflakeId(tweetOrUserId).time.inMilliseconds)
else None
}
private def addLastEngagementTimeFeatures(
tweetFeaturesOpt: Option[sc.ThriftTweetFeatures],
timeFeatures: TimeFeatures,
timeSinceSourceTweetCreation: Long
): Option[TimeFeatures] = {
tweetFeaturesOpt.map { tweetFeatures =>
val lastFavSinceCreationHrs = tweetFeatures.lastFavSinceCreationHrs.map(_.toDouble)
val lastRetweetSinceCreationHrs = tweetFeatures.lastRetweetSinceCreationHrs.map(_.toDouble)
val lastReplySinceCreationHrs = tweetFeatures.lastReplySinceCreationHrs.map(_.toDouble)
val lastQuoteSinceCreationHrs = tweetFeatures.lastQuoteSinceCreationHrs.map(_.toDouble)
timeFeatures.copy(
lastFavSinceCreationHrs = lastFavSinceCreationHrs,
lastRetweetSinceCreationHrs = lastRetweetSinceCreationHrs,
lastReplySinceCreationHrs = lastReplySinceCreationHrs,
lastQuoteSinceCreationHrs = lastQuoteSinceCreationHrs,
timeSinceLastFavoriteHrs = getTimeSinceLastEngagementHrs(
lastFavSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastRetweetHrs = getTimeSinceLastEngagementHrs(
lastRetweetSinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastReplyHrs = getTimeSinceLastEngagementHrs(
lastReplySinceCreationHrs,
timeSinceSourceTweetCreation
),
timeSinceLastQuoteHrs = getTimeSinceLastEngagementHrs(
lastQuoteSinceCreationHrs,
timeSinceSourceTweetCreation
)
)
}
}
private def addNonPollingTimeFeatures(
timeFeatures: TimeFeatures,
requestTimestampMs: Long,
creationTimeMs: Long,
nonPollingTimestampsMs: Option[Seq[Long]]
): Option[TimeFeatures] = {
for {
nonPollingTimestampsMs <- nonPollingTimestampsMs
lastNonPollingTimestampMs <- nonPollingTimestampsMs.headOption
earliestNonPollingTimestampMs <- nonPollingTimestampsMs.lastOption
} yield {
val timeSinceLastNonPollingRequest = requestTimestampMs - lastNonPollingTimestampMs
val tweetAgeRatio = timeSinceLastNonPollingRequest / math.max(
1.0,
timeFeatures.timeSinceTweetCreation
)
/*
* Non-polling timestamps are stored in chronological order.
* The latest timestamps occur first, therefore we need to explicitly search in reverse order.
*/
val nonPollingRequestsSinceTweetCreation =
if (nonPollingTimestampsMs.nonEmpty) {
nonPollingTimestampsMs.search(creationTimeMs)(Ordering[Long].reverse).insertionPoint
} else {
0
}
/*
* Calculate the average time between non-polling requests; include
* request time in this calculation as latest timestamp.
*/
val timeBetweenNonPollingRequestsAvg =
(requestTimestampMs - earliestNonPollingTimestampMs) / math
.max(1.0, nonPollingTimestampsMs.size)
val timeFeaturesWithNonPolling = timeFeatures.copy(
timeBetweenNonPollingRequestsAvg = Some(timeBetweenNonPollingRequestsAvg),
timeSinceLastNonPollingRequest = Some(timeSinceLastNonPollingRequest),
nonPollingRequestsSinceTweetCreation = Some(nonPollingRequestsSinceTweetCreation),
tweetAgeRatio = Some(tweetAgeRatio)
)
timeFeaturesWithNonPolling
}
}
private[this] def getTimeSinceLastEngagementHrs(
lastEngagementTimeSinceCreationHrsOpt: Option[Double],
timeSinceTweetCreation: Long
): Option[Double] = {
lastEngagementTimeSinceCreationHrsOpt.map { lastEngagementTimeSinceCreationHrs =>
val timeSinceTweetCreationHrs = (timeSinceTweetCreation / (60 * 60 * 1000)).toInt
timeSinceTweetCreationHrs - lastEngagementTimeSinceCreationHrs
}
}
private def setFeatures(features: TimeFeatures, richDataRecord: RichDataRecord): Unit = {
val record = richDataRecord.getRecord
.setFeatureValue(IS_TWEET_RECYCLED, features.isTweetRecycled)
.setFeatureValue(TIME_SINCE_TWEET_CREATION, features.timeSinceTweetCreation)
.setFeatureValueFromOption(
TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS,
features.timeSinceViewerAccountCreationSecs)
.setFeatureValue(
USER_ID_IS_SNOWFLAKE_ID,
features.timeSinceViewerAccountCreationSecs.isDefined
)
.setFeatureValueFromOption(ACCOUNT_AGE_INTERVAL, features.accountAgeInterval.map(_.id.toLong))
.setFeatureValue(IS_30_DAY_NEW_USER, features.isDay30NewUser)
.setFeatureValue(IS_12_MONTH_NEW_USER, features.isMonth12NewUser)
.setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, features.lastFavSinceCreationHrs)
.setFeatureValueFromOption(
LAST_RETWEET_SINCE_CREATION_HRS,
features.lastRetweetSinceCreationHrs
)
.setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, features.lastReplySinceCreationHrs)
.setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, features.lastQuoteSinceCreationHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, features.timeSinceLastFavoriteHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, features.timeSinceLastRetweetHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, features.timeSinceLastReplyHrs)
.setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, features.timeSinceLastQuoteHrs)
/*
* set features whose values are optional as some users do not have non-polling timestamps
*/
features.timeBetweenNonPollingRequestsAvg.foreach(
record.setFeatureValue(TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, _)
)
features.timeSinceLastNonPollingRequest.foreach(
record.setFeatureValue(TIME_SINCE_LAST_NON_POLLING_REQUEST, _)
)
features.nonPollingRequestsSinceTweetCreation.foreach(
record.setFeatureValue(NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, _)
)
features.tweetAgeRatio.foreach(record.setFeatureValue(TWEET_AGE_RATIO, _))
}
}

4
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetImpressionsQueryFeatureHydrator.scala
View File

@ -24,8 +24,8 @@ case class TweetImpressionsQueryFeatureHydrator[
manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient) manhattanTweetImpressionStoreClient: ManhattanTweetImpressionStoreClient)
extends QueryFeatureHydrator[Query] { extends QueryFeatureHydrator[Query] {
private val TweetImpressionTTL = 1.day private val TweetImpressionTTL = 2.days
private val TweetImpressionCap = 3000 private val TweetImpressionCap = 5000
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetImpressions")

47
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/TweetypieFeatureHydrator.scala
View File

@ -1,6 +1,9 @@
package com.twitter.home_mixer.functional_component.feature_hydrator package com.twitter.home_mixer.functional_component.feature_hydrator
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature
import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature
@ -10,11 +13,13 @@ import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature
import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature
import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature
import com.twitter.home_mixer.model.request.FollowingProduct import com.twitter.home_mixer.model.request.FollowingProduct
import com.twitter.home_mixer.model.request.ForYouProduct import com.twitter.home_mixer.model.request.ForYouProduct
import com.twitter.home_mixer.model.request.ListTweetsProduct import com.twitter.home_mixer.model.request.ListTweetsProduct
import com.twitter.home_mixer.model.request.ScoredTweetsProduct import com.twitter.home_mixer.model.request.ScoredTweetsProduct
import com.twitter.home_mixer.model.request.SubscribedProduct
import com.twitter.home_mixer.util.tweetypie.RequestFields import com.twitter.home_mixer.util.tweetypie.RequestFields
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_is_nsfw.IsNsfw
import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason import com.twitter.product_mixer.component_library.feature_hydrator.candidate.tweet_visibility_reason.VisibilityReason
@ -29,17 +34,22 @@ import com.twitter.spam.rtf.{thriftscala => rtf}
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient}
import com.twitter.tweetypie.{thriftscala => tp} import com.twitter.tweetypie.{thriftscala => tp}
import com.twitter.util.logging.Logging
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@Singleton @Singleton
class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitchClient) class TweetypieFeatureHydrator @Inject() (
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { tweetypieStitchClient: TweetypieStitchClient,
statsReceiver: StatsReceiver)
extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie") override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Tweetypie")
override val features: Set[Feature[_, _]] = Set( override val features: Set[Feature[_, _]] = Set(
AuthorIdFeature, AuthorIdFeature,
ExclusiveConversationAuthorIdFeature,
InReplyToTweetIdFeature, InReplyToTweetIdFeature,
IsHydratedFeature, IsHydratedFeature,
IsNsfw, IsNsfw,
@ -51,6 +61,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
SourceTweetIdFeature, SourceTweetIdFeature,
SourceUserIdFeature, SourceUserIdFeature,
TweetTextFeature, TweetTextFeature,
TweetLanguageFeature,
VisibilityReason VisibilityReason
) )
@ -70,9 +81,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
): Stitch[FeatureMap] = { ): Stitch[FeatureMap] = {
val safetyLevel = query.product match { val safetyLevel = query.product match {
case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest case FollowingProduct => rtf.SafetyLevel.TimelineHomeLatest
case ForYouProduct => rtf.SafetyLevel.TimelineHome case ForYouProduct =>
val inNetwork = existingFeatures.getOrElse(InNetworkFeature, true)
if (inNetwork) rtf.SafetyLevel.TimelineHome else rtf.SafetyLevel.TimelineHomeRecommendations
case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome case ScoredTweetsProduct => rtf.SafetyLevel.TimelineHome
case ListTweetsProduct => rtf.SafetyLevel.TimelineLists case ListTweetsProduct => rtf.SafetyLevel.TimelineLists
case SubscribedProduct => rtf.SafetyLevel.TimelineHomeSubscribed
case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown") case unknown => throw new UnsupportedOperationException(s"Unknown product: $unknown")
} }
@ -82,9 +96,12 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
includeQuotedTweet = true, includeQuotedTweet = true,
visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible,
safetyLevel = Some(safetyLevel), safetyLevel = Some(safetyLevel),
forUserId = Some(query.getRequiredUserId) forUserId = query.getOptionalUserId
) )
val exclusiveAuthorIdOpt =
existingFeatures.getOrElse(ExclusiveConversationAuthorIdFeature, None)
tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map { tweetypieStitchClient.getTweetFields(tweetId = candidate.id, options = tweetFieldsOptions).map {
case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) => case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), quoteOpt, _) =>
val coreData = found.tweet.coreData val coreData = found.tweet.coreData
@ -106,6 +123,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser)) found.retweetedTweet.exists(_.coreData.exists(data => data.nsfwAdmin || data.nsfwUser))
val tweetText = coreData.map(_.text) val tweetText = coreData.map(_.text)
val tweetLanguage = found.tweet.language.map(_.language)
val tweetAuthorId = coreData.map(_.userId) val tweetAuthorId = coreData.map(_.userId)
val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)) val inReplyToTweetId = coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId))
@ -127,6 +145,7 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
FeatureMapBuilder() FeatureMapBuilder()
.add(AuthorIdFeature, tweetAuthorId) .add(AuthorIdFeature, tweetAuthorId)
.add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
.add(InReplyToTweetIdFeature, inReplyToTweetId) .add(InReplyToTweetIdFeature, inReplyToTweetId)
.add(IsHydratedFeature, true) .add(IsHydratedFeature, true)
.add(IsNsfw, Some(isNsfw)) .add(IsNsfw, Some(isNsfw))
@ -137,20 +156,24 @@ class TweetypieFeatureHydrator @Inject() (tweetypieStitchClient: TweetypieStitch
.add(QuotedUserIdFeature, quotedTweetUserId) .add(QuotedUserIdFeature, quotedTweetUserId)
.add(SourceTweetIdFeature, retweetedTweetId) .add(SourceTweetIdFeature, retweetedTweetId)
.add(SourceUserIdFeature, retweetedTweetUserId) .add(SourceUserIdFeature, retweetedTweetUserId)
.add(TweetLanguageFeature, tweetLanguage)
.add(TweetTextFeature, tweetText) .add(TweetTextFeature, tweetText)
.add(VisibilityReason, found.suppressReason) .add(VisibilityReason, found.suppressReason)
.build() .build()
// If no tweet result found, return default and pre-existing features // If no tweet result found, return default and pre-existing features
case _ => case _ =>
DefaultFeatureMap + DefaultFeatureMap ++ FeatureMapBuilder()
(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None)) + .add(AuthorIdFeature, existingFeatures.getOrElse(AuthorIdFeature, None))
(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None)) + .add(ExclusiveConversationAuthorIdFeature, exclusiveAuthorIdOpt)
(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false)) + .add(InReplyToTweetIdFeature, existingFeatures.getOrElse(InReplyToTweetIdFeature, None))
(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None)) + .add(IsRetweetFeature, existingFeatures.getOrElse(IsRetweetFeature, false))
(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None)) + .add(QuotedTweetIdFeature, existingFeatures.getOrElse(QuotedTweetIdFeature, None))
(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None)) + .add(QuotedUserIdFeature, existingFeatures.getOrElse(QuotedUserIdFeature, None))
(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None)) .add(SourceTweetIdFeature, existingFeatures.getOrElse(SourceTweetIdFeature, None))
.add(SourceUserIdFeature, existingFeatures.getOrElse(SourceUserIdFeature, None))
.add(TweetLanguageFeature, existingFeatures.getOrElse(TweetLanguageFeature, None))
.build()
} }
} }
} }

59
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala
View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.adapters.author_features
import com.twitter.ml.api.DataRecordMerger
import com.twitter.ml.api.Feature
import com.twitter.ml.api.FeatureContext
import com.twitter.ml.api.RichDataRecord
import com.twitter.ml.api.util.CompactDataRecordConverter
import com.twitter.ml.api.util.FDsl._
import com.twitter.timelines.author_features.v1.{thriftjava => af}
import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase
import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig
import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures
object AuthorFeaturesAdapter extends TimelinesMutatingAdapterBase[Option[af.AuthorFeatures]] {
private val originalAuthorAggregatesFeatures =
TimelinesAggregationConfig.originalAuthorReciprocalEngagementAggregates
.buildTypedAggregateGroups().flatMap(_.allOutputFeatures)
private val authorFeatures = originalAuthorAggregatesFeatures ++
Seq(
UserHealthFeatures.AuthorState,
UserHealthFeatures.NumAuthorFollowers,
UserHealthFeatures.NumAuthorConnectDays,
UserHealthFeatures.NumAuthorConnect)
private val featureContext = new FeatureContext(authorFeatures: _*)
override def getFeatureContext: FeatureContext = featureContext
override val commonFeatures: Set[Feature[_]] = Set.empty
private val compactDataRecordConverter = new CompactDataRecordConverter()
private val drMerger = new DataRecordMerger()
override def setFeatures(
authorFeaturesOpt: Option[af.AuthorFeatures],
richDataRecord: RichDataRecord
): Unit = {
authorFeaturesOpt.foreach { authorFeatures =>
val dataRecord = richDataRecord.getRecord
dataRecord.setFeatureValue(
UserHealthFeatures.AuthorState,
authorFeatures.user_health.user_state.getValue.toLong)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorFollowers,
authorFeatures.user_health.num_followers.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnectDays,
authorFeatures.user_health.num_connect_days.toDouble)
dataRecord.setFeatureValue(
UserHealthFeatures.NumAuthorConnect,
authorFeatures.user_health.num_connect.toDouble)
val originalAuthorAggregatesDataRecord =
compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates)
drMerger.merge(dataRecord, originalAuthorAggregatesDataRecord)
}
}
}

22
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala
View File

@ -1,22 +0,0 @@
package com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates
import com.twitter.home_mixer.functional_component.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._
import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator {
override val identifier: FeatureHydratorIdentifier =
FeatureHydratorIdentifier("Phase2EdgeAggregate")
override val aggregateFeatures: Set[BaseEdgeAggregateFeature] =
Set(
UserEngagerAggregateFeature,
UserEngagerGoodClickAggregateFeature,
UserInferredTopicAggregateFeature,
UserTopicAggregateFeature,
UserMediaUnderstandingAnnotationAggregateFeature
)
}

3
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/BUILD.bazel
View File

@ -4,6 +4,7 @@ scala_library(
strict_deps = True, strict_deps = True,
tags = ["bazel-compatible"], tags = ["bazel-compatible"],
dependencies = [ dependencies = [
"home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param",
@ -16,9 +17,11 @@ scala_library(
"src/thrift/com/twitter/tweetypie:service-scala", "src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala", "src/thrift/com/twitter/tweetypie:tweet-scala",
"stitch/stitch-core", "stitch/stitch-core",
"stitch/stitch-socialgraph",
"stitch/stitch-tweetypie", "stitch/stitch-tweetypie",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"util/util-slf4j-api/src/main/scala",
], ],
exports = [ exports = [
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",

2
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/FeedbackFatigueFilter.scala
View File

@ -72,7 +72,7 @@ object FeedbackFatigueFilter
originalAuthorId.exists(authorsToFilter.contains) || originalAuthorId.exists(authorsToFilter.contains) ||
(likers.nonEmpty && eligibleLikers.isEmpty) || (likers.nonEmpty && eligibleLikers.isEmpty) ||
(followers.nonEmpty && eligibleFollowers.isEmpty) || (followers.nonEmpty && eligibleFollowers.isEmpty && likers.isEmpty) ||
(candidate.features.getOrElse(IsRetweetFeature, false) && (candidate.features.getOrElse(IsRetweetFeature, false) &&
authorId.exists(retweetersToFilter.contains)) authorId.exists(retweetersToFilter.contains))
} }

70
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/InvalidSubscriptionTweetFilter.scala Normal file
View File

@ -0,0 +1,70 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finagle.tracing.Trace
import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.socialgraph.{thriftscala => sg}
import com.twitter.stitch.Stitch
import com.twitter.stitch.socialgraph.SocialGraph
import com.twitter.util.logging.Logging
import javax.inject.Inject
import javax.inject.Singleton
/**
* Exclude invalid subscription tweets - cases where the viewer is not subscribed to the author
*
* If SGS hydration fails, `SGSInvalidSubscriptionTweetFeature` will be set to None for
* subscription tweets, so we explicitly filter those tweets out.
*/
@Singleton
case class InvalidSubscriptionTweetFilter @Inject() (
socialGraphClient: SocialGraph,
statsReceiver: StatsReceiver)
extends Filter[PipelineQuery, TweetCandidate]
with Logging {
override val identifier: FilterIdentifier = FilterIdentifier("InvalidSubscriptionTweet")
private val scopedStatsReceiver = statsReceiver.scope(identifier.toString)
private val validCounter = scopedStatsReceiver.counter("validExclusiveTweet")
private val invalidCounter = scopedStatsReceiver.counter("invalidExclusiveTweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = Stitch
.traverse(candidates) { candidate =>
val exclusiveAuthorId =
candidate.features.getOrElse(ExclusiveConversationAuthorIdFeature, None)
if (exclusiveAuthorId.isDefined) {
val request = sg.ExistsRequest(
source = query.getRequiredUserId,
target = exclusiveAuthorId.get,
relationships =
Seq(sg.Relationship(sg.RelationshipType.TierOneSuperFollowing, hasRelationship = true)),
)
socialGraphClient.exists(request).map(_.exists).map { valid =>
if (!valid) invalidCounter.incr() else validCounter.incr()
valid
}
} else Stitch.value(true)
}.map { validResults =>
val (kept, removed) = candidates
.map(_.candidate)
.zip(validResults)
.partition { case (candidate, valid) => valid }
val keptCandidates = kept.map { case (candidate, _) => candidate }
val removedCandidates = removed.map { case (candidate, _) => candidate }
FilterResult(kept = keptCandidates, removed = removedCandidates)
}
}

59
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PredicateFeatureFilter.scala
View File

@ -1,59 +0,0 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.UniversalNoun
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Predicate which will be applied to each candidate. True indicates that the candidate will be
* @tparam Candidate - the type of the candidate
*/
trait ShouldKeepCandidate {
def apply(features: FeatureMap): Boolean
}
object PredicateFeatureFilter {
/**
* Builds a simple Filter out of a predicate function from the candidate to a boolean. For clarity,
* we recommend including the name of the shouldKeepCandidate parameter.
*
* @param identifier A FilterIdentifier for the new filter
* @param shouldKeepCandidate A predicate function. Candidates will be kept when
* this function returns True.
*/
def fromPredicate[Candidate <: UniversalNoun[Any]](
identifier: FilterIdentifier,
shouldKeepCandidate: ShouldKeepCandidate
): Filter[PipelineQuery, Candidate] = {
val i = identifier
new Filter[PipelineQuery, Candidate] {
override val identifier: FilterIdentifier = i
/**
* Filter the list of candidates
*
* @return a FilterResult including both the list of kept candidate and the list of removed candidates
*/
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[Candidate]]
): Stitch[FilterResult[Candidate]] = {
val allowedIds = candidates
.filter(candidate => shouldKeepCandidate(candidate.features)).map(_.candidate.id).toSet
val (keptCandidates, removedCandidates) = candidates.map(_.candidate).partition {
candidate => allowedIds.contains(candidate.id)
}
Stitch.value(FilterResult(kept = keptCandidates, removed = removedCandidates))
}
}
}
}

20
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/KeepBestOutOfNetworkTweetPerAuthorFilter.scala → home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/PreviouslyServedTweetPreviewsFilter.scala
View File

@ -1,8 +1,6 @@
package com.twitter.home_mixer.functional_component.filter package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature import com.twitter.home_mixer.model.HomeFeatures.ServedTweetPreviewIdsFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult import com.twitter.product_mixer.core.functional_component.filter.FilterResult
@ -11,24 +9,20 @@ import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
object KeepBestOutOfNetworkTweetPerAuthorFilter extends Filter[PipelineQuery, TweetCandidate] { object PreviouslyServedTweetPreviewsFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("KeepBestOutOfNetworkTweetPerAuthor") override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedTweetPreviews")
override def apply( override def apply(
query: PipelineQuery, query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]] candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = { ): Stitch[FilterResult[TweetCandidate]] = {
// Set containing best OON tweet for each authorId
val bestCandidatesForAuthorId = candidates val servedTweetPreviewIds =
.filter(!_.features.getOrElse(InNetworkFeature, true)) query.features.map(_.getOrElse(ServedTweetPreviewIdsFeature, Seq.empty)).toSeq.flatten.toSet
.groupBy(_.features.getOrElse(AuthorIdFeature, None))
.values.map(_.maxBy(_.features.getOrElse(ScoreFeature, None)))
.toSet
val (removed, kept) = candidates.partition { candidate => val (removed, kept) = candidates.partition { candidate =>
!candidate.features.getOrElse(InNetworkFeature, true) && servedTweetPreviewIds.contains(candidate.candidate.id)
!bestCandidatesForAuthorId.contains(candidate)
} }
Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate))) Stitch.value(FilterResult(kept = kept.map(_.candidate), removed = removed.map(_.candidate)))

32
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/ReplyFilter.scala Normal file
View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object ReplyFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Reply")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
candidate.features.getOrElse(InReplyToTweetIdFeature, None).isEmpty
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

32
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter/RetweetFilter.scala Normal file
View File

@ -0,0 +1,32 @@
package com.twitter.home_mixer.functional_component.filter
import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.functional_component.filter.Filter
import com.twitter.product_mixer.core.functional_component.filter.FilterResult
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
object RetweetFilter extends Filter[PipelineQuery, TweetCandidate] {
override val identifier: FilterIdentifier = FilterIdentifier("Retweet")
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[FilterResult[TweetCandidate]] = {
val (kept, removed) = candidates
.partition { candidate =>
!candidate.features.getOrElse(IsRetweetFeature, false)
}
val filterResult = FilterResult(
kept = kept.map(_.candidate),
removed = removed.map(_.candidate)
)
Stitch.value(filterResult)
}
}

1
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/BUILD.bazel
View File

@ -14,7 +14,6 @@ scala_library(
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate",
"src/thrift/com/twitter/gizmoduck:thrift-scala", "src/thrift/com/twitter/gizmoduck:thrift-scala",
"stitch/stitch-socialgraph", "stitch/stitch-socialgraph",
"timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/manhattan",
"timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence", "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/store/persistence",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model", "timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
], ],

8
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/DismissFatigueGate.scala
View File

@ -1,14 +1,14 @@
package com.twitter.home_mixer.functional_component.gate package com.twitter.home_mixer.functional_component.gate
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.stitch.Stitch import com.twitter.stitch.Stitch
import com.twitter.util.Duration import com.twitter.timelinemixer.clients.manhattan.DismissInfo
import com.twitter.conversions.DurationOps._
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.timelineservice.suggests.thriftscala.SuggestType import com.twitter.timelineservice.suggests.thriftscala.SuggestType
import com.twitter.util.Duration
object DismissFatigueGate { object DismissFatigueGate {
// how long a dismiss action from user needs to be respected // how long a dismiss action from user needs to be respected

18
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate/NonEmptySeqFeatureGate.scala
View File

@ -1,18 +0,0 @@
package com.twitter.home_mixer.functional_component.gate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.functional_component.gate.Gate
import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
import scala.reflect.runtime.universe._
case class NonEmptySeqFeatureGate[T: TypeTag](
feature: Feature[PipelineQuery, Seq[T]])
extends Gate[PipelineQuery] {
override val identifier: GateIdentifier = GateIdentifier(s"NonEmptySeq$feature")
override def shouldContinue(query: PipelineQuery): Stitch[Boolean] =
Stitch.value(query.features.exists(_.get(feature).nonEmpty))
}

6
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/query_transformer/EditedTweetsCandidatePipelineQueryTransformer.scala
View File

@ -19,7 +19,7 @@ object EditedTweetsCandidatePipelineQueryTransformer
override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets") override val identifier: TransformerIdentifier = TransformerIdentifier("EditedTweets")
// The time window for which a tweet remains editable after creation. // The time window for which a tweet remains editable after creation.
private val EditTimeWindow = 30.minutes private val EditTimeWindow = 60.minutes
override def transform(query: PipelineQuery): Seq[Long] = { override def transform(query: PipelineQuery): Seq[Long] = {
val applicableCandidates = getApplicableCandidates(query) val applicableCandidates = getApplicableCandidates(query)
@ -29,8 +29,8 @@ object EditedTweetsCandidatePipelineQueryTransformer
// Any tweets in it could have become stale since being served. // Any tweets in it could have become stale since being served.
val previousTimelineLoadTime = applicableCandidates.head.servedTime val previousTimelineLoadTime = applicableCandidates.head.servedTime
// The time window for editing a tweet is 30 minutes, // The time window for editing a tweet is 60 minutes,
// so we ignore responses older than (PTL Time - 30 mins). // so we ignore responses older than (PTL Time - 60 mins).
val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates val inWindowCandidates: Seq[PersistenceStoreEntry] = applicableCandidates
.takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow) .takeWhile(_.servedTime.until(previousTimelineLoadTime) < EditTimeWindow)

1
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/BUILD.bazel
View File

@ -6,7 +6,6 @@ scala_library(
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/param", "home-mixer/server/src/main/scala/com/twitter/home_mixer/param",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model",
"home-mixer/server/src/main/scala/com/twitter/home_mixer/util", "home-mixer/server/src/main/scala/com/twitter/home_mixer/util",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline",
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product",

76
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/FeedbackFatigueScorer.scala
View File

@ -35,8 +35,8 @@ object FeedbackFatigueScorer
override def onlyIf(query: PipelineQuery): Boolean = override def onlyIf(query: PipelineQuery): Boolean =
query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty) query.features.exists(_.getOrElse(FeedbackHistoryFeature, Seq.empty).nonEmpty)
private val DurationForFiltering = 14.days val DurationForFiltering = 14.days
private val DurationForDiscounting = 140.days val DurationForDiscounting = 140.days
private val ScoreMultiplierLowerBound = 0.2 private val ScoreMultiplierLowerBound = 0.2
private val ScoreMultiplierUpperBound = 1.0 private val ScoreMultiplierUpperBound = 1.0
private val ScoreMultiplierIncrementsCount = 4 private val ScoreMultiplierIncrementsCount = 4
@ -76,42 +76,56 @@ object FeedbackFatigueScorer
feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty)) feedbackEntriesByEngagementType.getOrElse(tls.FeedbackEngagementType.Retweet, Seq.empty))
val featureMaps = candidates.map { candidate => val featureMaps = candidates.map { candidate =>
val multiplier = getScoreMultiplier(
candidate,
authorsToDiscount,
likersToDiscount,
followersToDiscount,
retweetersToDiscount
)
val score = candidate.features.getOrElse(ScoreFeature, None) val score = candidate.features.getOrElse(ScoreFeature, None)
val originalAuthorId =
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
val likerMultipliers = likers.flatMap(likersToDiscount.get)
val likerMultiplier =
if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size)
likerMultipliers.max
else 1.0
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
val followerMultipliers = followers.flatMap(followersToDiscount.get)
val followerMultiplier =
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size)
followerMultipliers.max
else 1.0
val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)
val retweeterMultiplier =
if (candidate.features.getOrElse(IsRetweetFeature, false))
retweetersToDiscount.getOrElse(authorId, 1.0)
else 1.0
val multiplier =
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build() FeatureMapBuilder().add(ScoreFeature, score.map(_ * multiplier)).build()
} }
Stitch.value(featureMaps) Stitch.value(featureMaps)
} }
private def getUserDiscounts( def getScoreMultiplier(
candidate: CandidateWithFeatures[TweetCandidate],
authorsToDiscount: Map[Long, Double],
likersToDiscount: Map[Long, Double],
followersToDiscount: Map[Long, Double],
retweetersToDiscount: Map[Long, Double],
): Double = {
val originalAuthorId =
CandidatesUtil.getOriginalAuthorId(candidate.features).getOrElse(0L)
val originalAuthorMultiplier = authorsToDiscount.getOrElse(originalAuthorId, 1.0)
val likers = candidate.features.getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty)
val likerMultipliers = likers.flatMap(likersToDiscount.get)
val likerMultiplier =
if (likerMultipliers.nonEmpty && likers.size == likerMultipliers.size)
likerMultipliers.max
else 1.0
val followers = candidate.features.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty)
val followerMultipliers = followers.flatMap(followersToDiscount.get)
val followerMultiplier =
if (followerMultipliers.nonEmpty && followers.size == followerMultipliers.size &&
likers.isEmpty)
followerMultipliers.max
else 1.0
val authorId = candidate.features.getOrElse(AuthorIdFeature, None).getOrElse(0L)
val retweeterMultiplier =
if (candidate.features.getOrElse(IsRetweetFeature, false))
retweetersToDiscount.getOrElse(authorId, 1.0)
else 1.0
originalAuthorMultiplier * likerMultiplier * followerMultiplier * retweeterMultiplier
}
def getUserDiscounts(
queryTime: Time, queryTime: Time,
feedbackEntries: Seq[FeedbackEntry], feedbackEntries: Seq[FeedbackEntry],
): Map[Long, Double] = { ): Map[Long, Double] = {

61
home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer/VerifiedAuthorScalingScorer.scala
View File

@ -1,61 +0,0 @@
package com.twitter.home_mixer.functional_component.scorer
import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature
import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature
import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorInNetworkMultiplierParam
import com.twitter.home_mixer.param.HomeGlobalParams.BlueVerifiedAuthorOutOfNetworkMultiplierParam
import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate
import com.twitter.product_mixer.core.feature.Feature
import com.twitter.product_mixer.core.feature.featuremap.FeatureMap
import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder
import com.twitter.product_mixer.core.functional_component.scorer.Scorer
import com.twitter.product_mixer.core.model.common.CandidateWithFeatures
import com.twitter.product_mixer.core.model.common.identifier.ScorerIdentifier
import com.twitter.product_mixer.core.pipeline.PipelineQuery
import com.twitter.stitch.Stitch
/**
* Scales scores of tweets whose author is Blue Verified by the provided scale factor
*/
object VerifiedAuthorScalingScorer extends Scorer[PipelineQuery, TweetCandidate] {
override val identifier: ScorerIdentifier = ScorerIdentifier("VerifiedAuthorScaling")
override val features: Set[Feature[_, _]] = Set(ScoreFeature)
override def apply(
query: PipelineQuery,
candidates: Seq[CandidateWithFeatures[TweetCandidate]]
): Stitch[Seq[FeatureMap]] = {
Stitch.value {
candidates.map { candidate =>
val score = candidate.features.getOrElse(ScoreFeature, None)
val updatedScore = getUpdatedScore(score, candidate, query)
FeatureMapBuilder().add(ScoreFeature, updatedScore).build()
}
}
}
/**
* We should only be applying this multiplier if the author of the candidate is Blue Verified.
* We also treat In-Network vs Out-of-Network differently.
*/
private def getUpdatedScore(
score: Option[Double],
candidate: CandidateWithFeatures[TweetCandidate],
query: PipelineQuery
): Option[Double] = {
val isAuthorBlueVerified = candidate.features.getOrElse(AuthorIsBlueVerifiedFeature, false)
if (isAuthorBlueVerified) {
val isCandidateInNetwork = candidate.features.getOrElse(InNetworkFeature, false)
val scaleFactor =
if (isCandidateInNetwork) query.params(BlueVerifiedAuthorInNetworkMultiplierParam)
else query.params(BlueVerifiedAuthorOutOfNetworkMultiplierParam)
score.map(_ * scaleFactor)
} else score
}
}

Some files were not shown because too many files have changed in this diff Show More