diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.docx new file mode 100644 index 000000000..44b97638f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala deleted file mode 100644 index 13283e4b3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/module/TweetypieStaticEntitiesCacheClientModule.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.module - -import com.google.inject.name.Named -import com.google.inject.Provides -import com.twitter.conversions.DurationOps.RichDuration -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.shared_library.memcached_client.MemcachedClientBuilder -import com.twitter.servo.cache.FinagleMemcache -import com.twitter.servo.cache.KeyTransformer -import com.twitter.servo.cache.KeyValueTransformingTtlCache -import com.twitter.servo.cache.ObservableTtlCache -import com.twitter.servo.cache.Serializer -import com.twitter.servo.cache.ThriftSerializer -import com.twitter.servo.cache.TtlCache -import com.twitter.tweetypie.{thriftscala => tp} -import javax.inject.Singleton -import org.apache.thrift.protocol.TCompactProtocol - -object TweetypieStaticEntitiesCacheClientModule extends TwitterModule { - - private val ScopeName = "TweetypieStaticEntitiesMemcache" - private val ProdDest = "/srv#/prod/local/cache/timelinescorer_tweet_core_data:twemcaches" - - private val tweetsSerializer: Serializer[tp.Tweet] = { - new ThriftSerializer[tp.Tweet](tp.Tweet, new TCompactProtocol.Factory()) - } - private val keyTransformer: KeyTransformer[Long] = { tweetId => tweetId.toString } - - @Provides - @Singleton - @Named(TweetypieStaticEntitiesCache) - def providesTweetypieStaticEntitiesCache( - statsReceiver: StatsReceiver, - serviceIdentifier: ServiceIdentifier - ): TtlCache[Long, tp.Tweet] = { - val memCacheClient = MemcachedClientBuilder.buildMemcachedClient( - destName = ProdDest, - numTries = 1, - numConnections = 1, - requestTimeout = 50.milliseconds, - globalTimeout = 100.milliseconds, - connectTimeout = 100.milliseconds, - acquisitionTimeout = 100.milliseconds, - serviceIdentifier = serviceIdentifier, - statsReceiver = statsReceiver - ) - mkCache(new FinagleMemcache(memCacheClient), statsReceiver) - } - - private def mkCache( - finagleMemcache: FinagleMemcache, - statsReceiver: StatsReceiver - ): TtlCache[Long, tp.Tweet] = { - val baseCache: KeyValueTransformingTtlCache[Long, String, tp.Tweet, Array[Byte]] = - new KeyValueTransformingTtlCache( - underlyingCache = finagleMemcache, - transformer = tweetsSerializer, - underlyingKey = keyTransformer - ) - ObservableTtlCache( - underlyingCache = baseCache, - statsReceiver = statsReceiver.scope(ScopeName), - windowSize = 1000, - name = ScopeName - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel deleted file mode 100644 index 25e9a2e31..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.docx new file mode 100644 index 000000000..fe43f4009 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.docx new file mode 100644 index 000000000..2604ecdb4 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.scala deleted file mode 100644 index 304e7bdc8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/GlobalParamConfigModule.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.home_mixer.param - -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig - -object GlobalParamConfigModule extends TwitterModule { - override def configure(): Unit = { - bind[GlobalParamConfig].to[HomeGlobalParamConfig] - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.docx new file mode 100644 index 000000000..adf27af32 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala deleted file mode 100644 index 6fbd28fce..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParamConfig.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.param - -import com.twitter.home_mixer.param.HomeGlobalParams._ -import com.twitter.product_mixer.core.functional_component.configapi.registry.GlobalParamConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Register Params that do not relate to a specific product. See GlobalParamConfig -> ParamConfig - * for hooks to register Params based on type. - */ -@Singleton -class HomeGlobalParamConfig @Inject() () extends GlobalParamConfig { - - override val booleanFSOverrides = Seq( - AdsDisableInjectionBasedOnUserRoleParam, - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - EnableImpressionBloomFilter, - EnableNahFeedbackInfoParam, - EnableNewTweetsPillAvatarsParam, - EnableScribeServedCandidatesParam, - EnableSendScoresToClient, - EnableSocialContextParam, - ) - - override val boundedIntFSOverrides = Seq( - MaxNumberReplaceInstructionsParam, - TimelinesPersistenceStoreMaxEntriesPerClient, - ) - - override val boundedDoubleFSOverrides = Seq( - ImpressionBloomFilterFalsePositiveRateParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.docx new file mode 100644 index 000000000..27a94f2df Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala deleted file mode 100644 index f19817bc9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeGlobalParams.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.param - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -/** - * Instantiate Params that do not relate to a specific product. - * - * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.supportedClientFSName]] - */ -object HomeGlobalParams { - - /** - * This param is used to disable ads injection for timelines served by home-mixer. - * It is currently used to maintain user-role based no-ads lists for automation accounts, - * and should NOT be used for other purposes. - */ - object AdsDisableInjectionBasedOnUserRoleParam - extends FSParam("home_mixer_ads_disable_injection_based_on_user_role", false) - - object EnableSendScoresToClient - extends FSParam[Boolean]( - name = "home_mixer_enable_send_scores_to_client", - default = false - ) - - object EnableNahFeedbackInfoParam - extends FSParam[Boolean]( - name = "home_mixer_enable_nah_feedback_info", - default = false - ) - - object MaxNumberReplaceInstructionsParam - extends FSBoundedParam[Int]( - name = "home_mixer_max_number_replace_instructions", - default = 100, - min = 0, - max = 200 - ) - - object TimelinesPersistenceStoreMaxEntriesPerClient - extends FSBoundedParam[Int]( - name = "home_mixer_timelines_persistence_store_max_entries_per_client", - default = 1800, - min = 500, - max = 5000 - ) - - object EnableNewTweetsPillAvatarsParam - extends FSParam[Boolean]( - name = "home_mixer_enable_new_tweets_pill_avatars", - default = true - ) - - object EnableSocialContextParam - extends FSParam[Boolean]( - name = "home_mixer_enable_social_context", - default = true - ) - - object EnableAdvertiserBrandSafetySettingsFeatureHydratorParam - extends FSParam[Boolean]( - name = "home_mixer_enable_advertiser_brand_safety_settings_feature_hydrator", - default = true - ) - - object EnableImpressionBloomFilter - extends FSParam[Boolean]( - name = "home_mixer_enable_impression_bloom_filter", - default = false - ) - - object ImpressionBloomFilterFalsePositiveRateParam - extends FSBoundedParam[Double]( - name = "home_mixer_impression_bloom_filter_false_positive_rate", - default = 0.005, - min = 0.001, - max = 0.01 - ) - - object EnableScribeServedCandidatesParam - extends FSParam[Boolean]( - name = "home_mixer_served_tweets_enable_scribing", - default = true - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.docx new file mode 100644 index 000000000..b7f68eea8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala deleted file mode 100644 index dc5f5513f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerFlagName.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.twitter.home_mixer.param - -object HomeMixerFlagName { - final val ScribeClientEventsFlag = "scribe.client_events" - final val ScribeServedCandidatesFlag = "scribe.served_candidates" - final val ScribeScoredCandidatesFlag = "scribe.scored_candidates" - final val ScribeServedCommonFeaturesAndCandidateFeaturesFlag = - "scribe.served_common_features_and_candidate_features" - final val DataRecordMetadataStoreConfigsYmlFlag = "data.record.metadata.store.configs.yml" - final val DarkTrafficFilterDeciderKey = "thrift.dark.traffic.filter.decider_key" - final val TargetFetchLatency = "target.fetch.latency" - final val TargetScoringLatency = "target.scoring.latency" -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.docx new file mode 100644 index 000000000..55804e528 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala deleted file mode 100644 index 5ea87f5ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/HomeMixerInjectionNames.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.param - -object HomeMixerInjectionNames { - final val AuthorFeatureRepository = "AuthorFeatureRepository" - final val CandidateFeaturesScribeEventPublisher = "CandidateFeaturesScribeEventPublisher" - final val CommonFeaturesScribeEventPublisher = "CommonFeaturesScribeEventPublisher" - final val EarlybirdRepository = "EarlybirdRepository" - final val EngagementsReceivedByAuthorCache = "EngagementsReceivedByAuthorCache" - final val GraphTwoHopRepository = "GraphTwoHopRepository" - final val HomeAuthorFeaturesCacheClient = "HomeAuthorFeaturesCacheClient" - final val InterestsThriftServiceClient = "InterestsThriftServiceClient" - final val BatchedStratoClientWithModerateTimeout = "BatchedStratoClientWithModerateTimeout" - final val ManhattanApolloClient = "ManhattanApolloClient" - final val ManhattanAthenaClient = "ManhattanAthenaClient" - final val ManhattanOmegaClient = "ManhattanOmegaClient" - final val ManhattanStarbuckClient = "ManhattanStarbuckClient" - final val MetricCenterUserCountingFeatureRepository = "MetricCenterUserCountingFeatureRepository" - final val MinimumFeaturesScribeEventPublisher = "MinimumFeaturesScribeEventPublisher" - final val RealGraphInNetworkScores = "RealGraphInNetworkScores" - final val RealGraphManhattanEndpoint = "RealGraphFeaturesManhattanEndpoint" - final val RealGraphFeatureRepository = "RealGraphFeatureRepository" - final val RealTimeInteractionGraphUserVertexCache = "RealTimeInteractionGraphUserVertexCache" - final val RealTimeInteractionGraphUserVertexClient = "RealTimeInteractionGraphUserVertexClient" - final val StaleTweetsCache = "StaleTweetsCache" - final val TimelineAggregateMetadataRepository = "TimelineAggregateMetadataRepository" - final val TimelineAggregatePartARepository = "TimelineAggregatePartARepository" - final val TimelineAggregatePartBRepository = "TimelineAggregatePartBRepository" - final val TimelinesRealTimeAggregateClient = "TimelinesRealTimeAggregateClient" - final val TopicCountryEngagementCache = "TopicCountryEngagementCache" - final val TopicEngagementCache = "TopicEngagementCache" - final val TweetCountryEngagementCache = "TweetCountryEngagementCache" - final val TweetEngagementCache = "TweetEngagementCache" - final val TweetypieContentRepository = "TweetypieContentRepository" - final val TweetypieStaticEntitiesCache = "TweetypieStaticEntitiesCache" - final val TwhinAuthorFollowFeatureCacheClient = "TwhinAuthorFollowFeatureCacheClient" - final val TwhinAuthorFollowFeatureRepository = "TwhinAuthorFollowFeatureRepository" - final val TwhinUserEngagementFeatureRepository = "TwhinUserEngagementFeatureRepository" - final val TwhinUserFollowFeatureRepository = "TwhinUserFollowFeatureRepository" - final val TwitterListEngagementCache = "TwitterListEngagementCache" - final val UserAuthorEngagementCache = "UserAuthorEngagementCache" - final val UserEngagementCache = "UserEngagementCache" - final val UserFollowedTopicIdsRepository = "UserFollowedTopicIdsRepository" - final val UserLanguagesRepository = "UserLanguagesRepository" - final val UserTopicEngagementForNewUserCache = "UserTopicEngagementForNewUserCache" - final val UtegSocialProofRepository = "UtegSocialProofRepository" -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel deleted file mode 100644 index 5974c186f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "servo/decider", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.docx new file mode 100644 index 000000000..7ad5baeea Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.docx new file mode 100644 index 000000000..e6fdced50 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala deleted file mode 100644 index 91f7646d9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider/DeciderKey.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.param.decider - -import com.twitter.servo.decider.DeciderKeyEnum - -/** - * These values must correspond to the deciders configured in the - * home-mixer/server/src/main/resources/config/decider.yml file - * - * @see [[com.twitter.product_mixer.core.product.ProductParamConfig.enabledDeciderKey]] - */ -object DeciderKey extends DeciderKeyEnum { - // Products - val EnableForYouProduct = Value("enable_for_you_product") - - val EnableFollowingProduct = Value("enable_following_product") - - val EnableScoredTweetsProduct = Value("enable_scored_tweets_product") - - val EnableListTweetsProduct = Value("enable_list_tweets_product") - - val EnableListRecommendedUsersProduct = Value("enable_list_recommended_users_product") - - val EnableSubscribedProduct = Value("enable_subscribed_product") - - // Candidate Pipelines - val EnableForYouScoredTweetsCandidatePipeline = - Value("enable_for_you_scored_tweets_candidate_pipeline") - - val EnableScoredTweetsTweetMixerCandidatePipeline = - Value("enable_scored_tweets_tweet_mixer_candidate_pipeline") - - val EnableScoredTweetsInNetworkCandidatePipeline = - Value("enable_scored_tweets_in_network_candidate_pipeline") - - val EnableScoredTweetsUtegCandidatePipeline = - Value("enable_scored_tweets_uteg_candidate_pipeline") - - val EnableScoredTweetsFrsCandidatePipeline = - Value("enable_scored_tweets_frs_candidate_pipeline") - - val EnableScoredTweetsListsCandidatePipeline = - Value("enable_scored_tweets_lists_candidate_pipeline") - - val EnableScoredTweetsPopularVideosCandidatePipeline = - Value("enable_scored_tweets_popular_videos_candidate_pipeline") - - val EnableScoredTweetsBackfillCandidatePipeline = - Value("enable_scored_tweets_backfill_candidate_pipeline") - - val EnableSimClustersSimilarityFeatureHydration = - Value("enable_simclusters_similarity_feature_hydration") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel deleted file mode 100644 index e4fa669d2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/subscribed/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.docx new file mode 100644 index 000000000..7781516a7 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.docx new file mode 100644 index 000000000..5ed2e206a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.scala deleted file mode 100644 index cff4da4fb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeMixerProductModule.scala +++ /dev/null @@ -1,11 +0,0 @@ -package com.twitter.home_mixer.product - -import com.twitter.inject.TwitterModule -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig - -object HomeMixerProductModule extends TwitterModule { - - override def configure(): Unit = { - bind[ProductPipelineRegistryConfig].to[HomeProductPipelineRegistryConfig] - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.docx new file mode 100644 index 000000000..99c6ec057 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala deleted file mode 100644 index 5db4bd7f6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/HomeProductPipelineRegistryConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.home_mixer.product - -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.SubscribedProduct -import com.twitter.home_mixer.product.following.FollowingProductPipelineConfig -import com.twitter.home_mixer.product.for_you.ForYouProductPipelineConfig -import com.twitter.home_mixer.product.list_recommended_users.ListRecommendedUsersProductPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.ScoredTweetsProductPipelineConfig -import com.twitter.home_mixer.product.list_tweets.ListTweetsProductPipelineConfig -import com.twitter.home_mixer.product.subscribed.SubscribedProductPipelineConfig -import com.twitter.inject.Injector -import com.twitter.product_mixer.core.product.guice.ProductScope -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistryConfig -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class HomeProductPipelineRegistryConfig @Inject() ( - injector: Injector, - productScope: ProductScope) - extends ProductPipelineRegistryConfig { - - private val followingProductPipelineConfig = productScope.let(FollowingProduct) { - injector.instance[FollowingProductPipelineConfig] - } - - private val forYouProductPipelineConfig = productScope.let(ForYouProduct) { - injector.instance[ForYouProductPipelineConfig] - } - - private val scoredTweetsProductPipelineConfig = productScope.let(ScoredTweetsProduct) { - injector.instance[ScoredTweetsProductPipelineConfig] - } - - private val listTweetsProductPipelineConfig = productScope.let(ListTweetsProduct) { - injector.instance[ListTweetsProductPipelineConfig] - } - - private val listRecommendedUsersProductPipelineConfig = - productScope.let(ListRecommendedUsersProduct) { - injector.instance[ListRecommendedUsersProductPipelineConfig] - } - - private val subscribedProductPipelineConfig = productScope.let(SubscribedProduct) { - injector.instance[SubscribedProductPipelineConfig] - } - - override val productPipelineConfigs = Seq( - followingProductPipelineConfig, - forYouProductPipelineConfig, - scoredTweetsProductPipelineConfig, - listTweetsProductPipelineConfig, - listRecommendedUsersProductPipelineConfig, - subscribedProductPipelineConfig, - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel deleted file mode 100644 index 4771fe655..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.bazel +++ /dev/null @@ -1,95 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/account_recommendations_mixer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/hermit", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweetconvosvc", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/presentation/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/java/com/twitter/search/common/schema/base", - "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/common/util/lang", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/suggests/controller_data:controller_data-scala", - "src/thrift/com/twitter/timelinemixer:thrift-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "src/thrift/com/twitter/timelinescorer:thrift-scala", - "src/thrift/com/twitter/timelinescorer/common/scoredtweetcandidate:thrift-scala", - "src/thrift/com/twitter/timelinescorer/server/internal:thrift-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "stitch/stitch-gizmoduck", - "stitch/stitch-tweetypie", - "stringcenter/client", - "stringcenter/client/src/main/java", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/clients/relevance_search", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], - exports = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.docx new file mode 100644 index 000000000..ea569d7be Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.docx new file mode 100644 index 000000000..7787d9f34 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala deleted file mode 100644 index d4898eb31..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingAdsCandidatePipelineBuilder.scala +++ /dev/null @@ -1,108 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableAdsCandidatePipelineParam -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFastAds -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.PipelineScopedOrganicItems -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingAdsCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - FollowingQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("FollowingAds") - - private val suggestType = st.SuggestType.Promoted - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - )) - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) - - def build( - organicCandidatePipelines: CandidateScope - ): AdsDependentCandidatePipelineConfig[FollowingQuery] = - adsCandidatePipelineConfigBuilder.build[FollowingQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = query => - if (query.params.getBoolean(EnableFastAds)) ads.DisplayLocation.TimelineHomeReverseChron - else ads.DisplayLocation.TimelineHome, - getOrganicItems = PipelineScopedOrganicItems( - pipelines = organicCandidatePipelines, - textFeature = TweetTextFeature, - languageFeature = TweetLanguageFeature - ), - countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), - supportedClientParam = Some(EnableAdsCandidatePipelineParam), - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate, - NonEmptyCandidatesGate(organicCandidatePipelines) - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - alerts = alerts, - urtRequest = Some(true), - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.docx new file mode 100644 index 000000000..322759471 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala deleted file mode 100644 index addb298c2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdCandidatePipelineConfig.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.home_mixer.candidate_pipeline.FollowingEarlybirdResponseFeatureTransformer -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingEarlybirdCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - followingEarlybirdQueryTransformer: FollowingEarlybirdQueryTransformer) - extends CandidatePipelineConfig[ - FollowingQuery, - t.EarlybirdRequest, - t.ThriftSearchResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("FollowingEarlybird") - - override val candidateSource: BaseCandidateSource[t.EarlybirdRequest, t.ThriftSearchResult] = - earlybirdCandidateSource - - override val gates: Seq[Gate[FollowingQuery]] = Seq( - NonEmptySeqFeatureGate(SGSFollowedUsersFeature) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - FollowingQuery, - t.EarlybirdRequest - ] = followingEarlybirdQueryTransformer - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.ThriftSearchResult] - ] = Seq(FollowingEarlybirdResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.ThriftSearchResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.docx new file mode 100644 index 000000000..425da6c9d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala deleted file mode 100644 index c388cfa92..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdQueryTransformer.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.tracing.Trace -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.BottomCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.search.queryparser.query.Conjunction -import com.twitter.search.queryparser.query.search.SearchOperator -import javax.inject.Inject -import javax.inject.Singleton -import scala.jdk.CollectionConverters.asJavaIterableConverter - -@Singleton -case class FollowingEarlybirdQueryTransformer @Inject() (clientId: ClientId) - extends CandidatePipelineQueryTransformer[FollowingQuery, t.EarlybirdRequest] { - - override def transform(query: FollowingQuery): t.EarlybirdRequest = { - val followedUserIds = - query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet - val realGraphInNetworkFollowedUserIds = - query.features.map(_.get(RealGraphInNetworkScoresFeature)).getOrElse(Map.empty).keySet - val userId = query.getRequiredUserId - val combinedUserIds = userId +: followedUserIds.toSeq - - val baseFollowedUsersSearchOperator = new SearchOperator.Builder() - .setType(SearchOperator.Type.FEATURE_VALUE_IN_ACCEPT_LIST_OR_UNSET) - .addOperand(EarlybirdFieldConstant.DIRECTED_AT_USER_ID_CSF.getFieldName) - - val followedUsersQuery = - baseFollowedUsersSearchOperator.addOperands(combinedUserIds.map(_.toString).asJava).build() - - val searchQuery = query.pipelineCursor - .map { cursor => - val sinceIdQuery = - (id: Long) => new SearchOperator(SearchOperator.Type.SINCE_ID, id.toString) - val maxIdQuery = // max ID is inclusive, so subtract 1 - (id: Long) => new SearchOperator(SearchOperator.Type.MAX_ID, (id - 1).toString) - - (cursor.cursorType, cursor.id, cursor.gapBoundaryId) match { - case (Some(TopCursor), Some(sinceId), _) => - new Conjunction(sinceIdQuery(sinceId), followedUsersQuery) - case (Some(BottomCursor), Some(maxId), _) => - new Conjunction(maxIdQuery(maxId), followedUsersQuery) - case (Some(GapCursor), Some(maxId), Some(sinceId)) => - new Conjunction(sinceIdQuery(sinceId), maxIdQuery(maxId), followedUsersQuery) - case (Some(GapCursor), _, _) => - throw PipelineFailure(MalformedCursor, "Invalid cursor " + cursor.toString) - case _ => followedUsersQuery - } - }.getOrElse(followedUsersQuery) - - val metadataOptions = t.ThriftSearchResultMetadataOptions( - getInReplyToStatusId = true, - getReferencedTweetAuthorId = true, - getFromUserId = true - ) - - t.EarlybirdRequest( - searchQuery = t.ThriftSearchQuery( - serializedQuery = Some(searchQuery.serialize), - fromUserIDFilter64 = Some(combinedUserIds), - numResults = query.requestedMaxResults.getOrElse(query.params(ServerMaxResultsParam)), - rankingMode = t.ThriftSearchRankingMode.Recency, - resultMetadataOptions = Some(metadataOptions), - searcherId = query.getOptionalUserId, - ), - getOlderResults = Some(true), // needed for archive access to older tweets - clientRequestID = Some(s"${Trace.id.traceId}"), - followedUserIds = Some(combinedUserIds), - numResultsToReturnAtRoot = Some(query.params(ServerMaxResultsParam)), - clientId = Some(clientId.name), - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.docx new file mode 100644 index 000000000..90b37d5ff Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.scala deleted file mode 100644 index 0169e6dd7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingEarlybirdResponseFeatureTransformer.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.candidate_pipeline - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.search.earlybird.{thriftscala => t} - -object FollowingEarlybirdResponseFeatureTransformer - extends CandidateFeatureTransformer[t.ThriftSearchResult] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("FollowingEarlybirdResponse") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - InReplyToTweetIdFeature, - IsRetweetFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - ) - - override def transform(candidate: t.ThriftSearchResult): FeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, candidate.tweetypieTweet.flatMap(_.coreData.map(_.userId))) - .add( - InReplyToTweetIdFeature, - candidate.tweetypieTweet.flatMap(_.coreData.flatMap(_.reply.flatMap(_.inReplyToStatusId)))) - .add(IsRetweetFeature, candidate.metadata.exists(_.isRetweet.contains(true))) - .add(SourceTweetIdFeature, candidate.sourceTweetypieTweet.map(_.id)) - .add(SourceUserIdFeature, candidate.sourceTweetypieTweet.flatMap(_.coreData.map(_.userId))) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.docx new file mode 100644 index 000000000..495984764 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala deleted file mode 100644 index efda03595..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingMixerPipelineConfig.scala +++ /dev/null @@ -1,299 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder -import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig -import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration -import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.GapIncludeInstruction -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableFlipInjectionModuleCandidatePipelineParam -import com.twitter.home_mixer.product.following.param.FollowingParam.FlipInlineInjectionModulePosition -import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowPositionParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults -import com.twitter.product_mixer.component_library.selector.SelectConditionally -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class FollowingMixerPipelineConfig @Inject() ( - followingEarlybirdCandidatePipelineConfig: FollowingEarlybirdCandidatePipelineConfig, - conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ - FollowingQuery - ], - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - followingAdsCandidatePipelineBuilder: FollowingAdsCandidatePipelineBuilder, - followingWhoToFollowCandidatePipelineConfigBuilder: FollowingWhoToFollowCandidatePipelineConfigBuilder, - flipPromptDependentCandidatePipelineConfigBuilder: FlipPromptDependentCandidatePipelineConfigBuilder, - editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[FollowingQuery], - dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, - gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, - realGraphInNetworkSourceQueryHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[FollowingQuery], - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ - FollowingQuery - ], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - FollowingQuery - ], - memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, - lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, - adsInjector: AdsInjector, - updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[FollowingQuery, Timeline], - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, - updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeTimelineServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[FollowingQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("Following") - - private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep - private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[FollowingQuery]] = Seq( - requestQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - realGraphInNetworkSourceQueryHydrator, - AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, persistenceStoreQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), - AsyncParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - resultSelectorsStep, - impressionBloomFilterQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) - ) - - private val earlybirdCandidatePipelineScope = - SpecificPipeline(followingEarlybirdCandidatePipelineConfig.identifier) - - private val conversationServiceCandidatePipelineConfig = - conversationServiceCandidatePipelineConfigBuilder.build( - Seq(NonEmptyCandidatesGate(earlybirdCandidatePipelineScope)), - HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) - ) - - private val followingAdsCandidatePipelineConfig = - followingAdsCandidatePipelineBuilder.build(earlybirdCandidatePipelineScope) - - private val followingWhoToFollowCandidatePipelineConfig = - followingWhoToFollowCandidatePipelineConfigBuilder.build(earlybirdCandidatePipelineScope) - - private val flipPromptCandidatePipelineConfig = - flipPromptDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( - supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[FollowingQuery, _, _, _]] = - Seq(followingEarlybirdCandidatePipelineConfig) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[FollowingQuery, _, _, _] - ] = Seq( - conversationServiceCandidatePipelineConfig, - followingAdsCandidatePipelineConfig, - followingWhoToFollowCandidatePipelineConfig, - flipPromptCandidatePipelineConfig, - editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig - ) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - followingAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - followingWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - ) - - override val resultSelectors: Seq[Selector[FollowingQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier - ), - DropMaxCandidates( - candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = MaxNumberReplaceInstructionsParam - ), - DropMaxCandidates( - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropModuleTooFewModuleItemResults( - candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) - ), - DropMaxModuleItemCandidates( - candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) - ), - InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), - InsertFixedPositionResults( - candidatePipeline = followingWhoToFollowCandidatePipelineConfig.identifier, - positionParam = WhoToFollowPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = flipPromptCandidatePipelineConfig.identifier, - positionParam = FlipInlineInjectionModulePosition - ), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = followingAdsCandidatePipelineConfig.identifier - ), - // This selector must come after the tweets are inserted into the results - UpdateNewTweetsPillDecoration( - pipelineScope = SpecificPipelines( - conversationServiceCandidatePipelineConfig.identifier, - newTweetsPillCandidatePipelineConfig.identifier - ), - stringCenter = stringCenterProvider.get(), - seeNewTweetsString = externalStrings.seeNewTweetsString, - tweetedString = externalStrings.tweetedString - ), - InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), - SelectConditionally( - selector = - InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), - includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) - ), - UpdateHomeClientEventDetails( - candidatePipelines = Set(conversationServiceCandidatePipelineConfig.identifier) - ), - ) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = - Seq(conversationServiceCandidatePipelineConfig.identifier), - adsCandidatePipelineIdentifier = Some(followingAdsCandidatePipelineConfig.identifier), - whoToFollowCandidatePipelineIdentifier = - Some(followingWhoToFollowCandidatePipelineConfig.identifier), - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[FollowingQuery, Timeline]] = Seq( - homeScribeClientEventSideEffect, - homeTimelineServedCandidatesSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, - truncateTimelinesPersistenceStoreSideEffect, - updateLastNonPollingTimeSideEffect, - updateTimelinesPersistenceStoreSideEffect, - ) - - override val domainMarshaller: DomainMarshaller[FollowingQuery, Timeline] = { - val instructionBuilders = Seq( - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), - ShowAlertInstructionBuilder(), - ShowCoverInstructionBuilder(), - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = - OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) - val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig(page = Some("following"), section = None, entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.docx new file mode 100644 index 000000000..f492526fa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala deleted file mode 100644 index 28c7cd6a0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingProductPipelineConfig.scala +++ /dev/null @@ -1,131 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.home_mixer.model.request.FollowingProductContext -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.following.param.FollowingParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.util.SortIndexBuilder -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Time -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingProductPipelineConfig @Inject() ( - followingMixerPipelineConfig: FollowingMixerPipelineConfig, - followingParamConfig: FollowingParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, FollowingQuery, urt.TimelineResponse] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("Following") - - override val product: Product = FollowingProduct - override val paramConfig: ProductParamConfig = followingParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): FollowingQuery = { - val context = request.productContext match { - case Some(context: FollowingProductContext) => context - case _ => throw PipelineFailure(BadRequest, "FollowingProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - /** - * Unlike other clients, newly created tweets on Android have the sort index set to the current - * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline - * if subsequent timeline responses use the sort index from the previous response instead of - * the current time. - */ - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) - .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - .map { - case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) - if id.isEmpty || gapBoundaryId.isEmpty => - throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") - case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => - val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) - case cursor => cursor - } - } - - FollowingQuery( - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - deviceContext = context.deviceContext, - seenTweetIds = context.seenTweetIds, - dspClientContext = context.dspClientContext - ) - } - - override val pipelines: Seq[PipelineConfig] = Seq(followingMixerPipelineConfig) - - override def pipelineSelector( - query: FollowingQuery - ): ComponentIdentifier = followingMixerPipelineConfig.identifier - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(1100.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(1200.millis, 15, 30) - ), - ThroughputAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(18000), - criticalPredicate = TriggerIfAbove(20000) - ), - EmptyResponseRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(65), - criticalPredicate = TriggerIfAbove(80) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.docx new file mode 100644 index 000000000..421b997b5 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala deleted file mode 100644 index cd5b1795d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/FollowingWhoToFollowCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.product.following - -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToFollowFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature -import com.twitter.home_mixer.product.following.model.FollowingQuery -import com.twitter.home_mixer.product.following.param.FollowingParam.EnableWhoToFollowCandidatePipelineParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayLocationParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowDisplayTypeIdParam -import com.twitter.home_mixer.product.following.param.FollowingParam.WhoToFollowMinInjectionIntervalParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingWhoToFollowCandidatePipelineConfigBuilder @Inject() ( - whoToFollowArmDependentCandidatePipelineConfigBuilder: WhoToFollowArmDependentCandidatePipelineConfigBuilder, - homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { - - def build( - requiredNonEmptyPipelines: CandidateScope - ): WhoToFollowArmDependentCandidatePipelineConfig[FollowingQuery] = { - val gates: Seq[BaseGate[PipelineQuery]] = Seq( - TimelinesPersistenceStoreLastInjectionGate( - WhoToFollowMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.WhoToFollow - ), - DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature), - NonEmptyCandidatesGate(requiredNonEmptyPipelines) - ) - - whoToFollowArmDependentCandidatePipelineConfigBuilder.build[FollowingQuery]( - identifier = WhoToFollowArmCandidatePipelineConfig.identifier, - supportedClientParam = Some(EnableWhoToFollowCandidatePipelineParam), - alerts = alerts, - gates = gates, - moduleDisplayTypeBuilder = - ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), - feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), - displayLocationParam = StaticParam(WhoToFollowDisplayLocationParam.default), - excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), - profileUserIdFeature = None - ) - } - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel deleted file mode 100644 index 70d5e5af9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "stringcenter/client", - "stringcenter/client/src/main/java", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.docx new file mode 100644 index 000000000..ef68a52d1 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.docx new file mode 100644 index 000000000..b0fb590a0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala deleted file mode 100644 index c45c7cf68..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/FollowingQuery.scala +++ /dev/null @@ -1,51 +0,0 @@ -package com.twitter.home_mixer.product.following.model - -import com.twitter.adserver.thriftscala.HomeTimelineType -import com.twitter.adserver.thriftscala.TimelineRequestParams -import com.twitter.home_mixer.model.HomeAdsQuery -import com.twitter.dspbidder.commons.{thriftscala => dsp} -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.home_mixer.model.request.FollowingProduct -import com.twitter.onboarding.task.service.{thriftscala => ots} -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.HasFlipInjectionParams -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class FollowingQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val deviceContext: Option[DeviceContext], - override val seenTweetIds: Option[Seq[Long]], - override val dspClientContext: Option[dsp.DspClientContext]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasDeviceContext - with HasSeenTweetIds - with HasFlipInjectionParams - with HomeAdsQuery { - override val product: Product = FollowingProduct - - override def withFeatureMap(features: FeatureMap): FollowingQuery = - copy(features = Some(features)) - - override val timelineRequestParams: Option[TimelineRequestParams] = - Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.HomeLatest))) - - // Fields below are used for FLIP Injection in Onboarding Task Service (OTS) - override val displayLocation: ots.DisplayLocation = ots.DisplayLocation.HomeLatestTimeline - override val rankingDisablerWithLatestControlsAvailable: Option[Boolean] = None - override val isEmptyState: Option[Boolean] = None - override val isFirstRequestAfterSignup: Option[Boolean] = None - override val isEndOfTimeline: Option[Boolean] = None - override val timelineId: Option[Long] = None -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.docx new file mode 100644 index 000000000..c427ce0c8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala deleted file mode 100644 index 9c6faafa7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model/HomeMixerExternalStrings.scala +++ /dev/null @@ -1,87 +0,0 @@ -package com.twitter.home_mixer.product.following.model - -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.ExternalStringRegistry -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class HomeMixerExternalStrings @Inject() ( - @ProductScoped externalStringRegistryProvider: Provider[ExternalStringRegistry]) { - val seeNewTweetsString = - externalStringRegistryProvider.get().createProdString("SeeNewTweets") - val tweetedString = - externalStringRegistryProvider.get().createProdString("Tweeted") - val muteUserString = - externalStringRegistryProvider.get().createProdString("Feedback.muteUser") - val blockUserString = externalStringRegistryProvider.get().createProdString("Feedback.blockUser") - val unfollowUserString = - externalStringRegistryProvider.get().createProdString("Feedback.unfollowUser") - val unfollowUserConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.unfollowUserConfirmation") - val reportTweetString = - externalStringRegistryProvider.get().createProdString("Feedback.reportTweet") - val dontLikeString = externalStringRegistryProvider.get().createProdString("Feedback.dontLike") - val dontLikeConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.dontLikeConfirmation") - val showFewerTweetsString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerTweets") - val showFewerTweetsConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerTweetsConfirmation") - val showFewerRetweetsString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerRetweets") - val showFewerRetweetsConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.showFewerRetweetsConfirmation") - val notRelevantString = - externalStringRegistryProvider.get().createProdString("Feedback.notRelevant") - val notRelevantConfirmationString = - externalStringRegistryProvider.get().createProdString("Feedback.notRelevantConfirmation") - - val socialContextOneUserLikedString = - externalStringRegistryProvider.get().createProdString("SocialContext.oneUserLiked") - val socialContextTwoUsersLikedString = - externalStringRegistryProvider.get().createProdString("SocialContext.twoUsersLiked") - val socialContextMoreUsersLikedString = - externalStringRegistryProvider.get().createProdString("SocialContext.moreUsersLiked") - val socialContextLikedByTimelineTitle = - externalStringRegistryProvider.get().createProdString("SocialContext.likedByTimelineTitle") - - val socialContextOneUserFollowsString = - externalStringRegistryProvider.get().createProdString("SocialContext.oneUserFollows") - val socialContextTwoUsersFollowString = - externalStringRegistryProvider.get().createProdString("SocialContext.twoUsersFollow") - val socialContextMoreUsersFollowString = - externalStringRegistryProvider.get().createProdString("SocialContext.moreUsersFollow") - val socialContextFollowedByTimelineTitle = - externalStringRegistryProvider.get().createProdString("SocialContext.followedByTimelineTitle") - - val socialContextYouMightLikeString = - externalStringRegistryProvider.get().createProdString("SocialContext.youMightLike") - - val socialContextExtendedReply = - externalStringRegistryProvider.get().createProdString("SocialContext.extendedReply") - val socialContextReceivedReply = - externalStringRegistryProvider.get().createProdString("SocialContext.receivedReply") - - val socialContextPopularVideoString = - externalStringRegistryProvider.get().createProdString("SocialContext.popularVideo") - - val socialContextPopularInYourAreaString = - externalStringRegistryProvider.get().createProdString("PopgeoTweet.socialProof") - - val listToFollowModuleHeaderString = - externalStringRegistryProvider.get().createProdString("ListToFollowModule.header") - val listToFollowModuleFooterString = - externalStringRegistryProvider.get().createProdString("ListToFollowModule.footer") - val pinnedListsModuleHeaderString = - externalStringRegistryProvider.get().createProdString("PinnedListModule.header") - val pinnedListsModuleEmptyStateMessageString = - externalStringRegistryProvider.get().createProdString("PinnedListModule.emptyStateMessage") - - val ownedSubscribedListsModuleHeaderString = - externalStringRegistryProvider.get().createProdString("OwnedSubscribedListModule.header") - val ownedSubscribedListsModuleEmptyStateMessageString = - externalStringRegistryProvider - .get().createProdString("OwnedSubscribedListModule.emptyStateMessage") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel deleted file mode 100644 index a56e3a1fd..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.docx new file mode 100644 index 000000000..77ae95745 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.docx new file mode 100644 index 000000000..497d87abf Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala deleted file mode 100644 index e43990507..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParam.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.home_mixer.product.following.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.util.Duration - -object FollowingParam { - val SupportedClientFSName = "following_supported_client" - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "following_server_max_results", - default = 100, - min = 1, - max = 500 - ) - - object EnableWhoToFollowCandidatePipelineParam - extends FSParam[Boolean]( - name = "following_enable_who_to_follow", - default = true - ) - - object EnableAdsCandidatePipelineParam - extends FSParam[Boolean]( - name = "following_enable_ads", - default = true - ) - - object EnableFlipInjectionModuleCandidatePipelineParam - extends FSParam[Boolean]( - name = "following_enable_flip_inline_injection_module", - default = true - ) - - object FlipInlineInjectionModulePosition - extends FSBoundedParam[Int]( - name = "following_flip_inline_injection_module_position", - default = 0, - min = 0, - max = 1000 - ) - - object WhoToFollowPositionParam - extends FSBoundedParam[Int]( - name = "following_who_to_follow_position", - default = 5, - min = 0, - max = 99 - ) - - object WhoToFollowMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "following_who_to_follow_min_injection_interval_in_minutes", - default = 1800.minutes, - min = 0.minutes, - max = 6000.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object WhoToFollowDisplayTypeIdParam - extends FSEnumParam[WhoToFollowModuleDisplayType.type]( - name = "following_enable_who_to_follow_display_type_id", - default = WhoToFollowModuleDisplayType.Vertical, - enum = WhoToFollowModuleDisplayType - ) - - object WhoToFollowDisplayLocationParam - extends FSParam[String]( - name = "following_who_to_follow_display_location", - default = "timeline_reverse_chron" - ) - - object EnableFastAds - extends FSParam[Boolean]( - name = "following_enable_fast_ads", - default = true - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.docx new file mode 100644 index 000000000..5a18639c6 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala deleted file mode 100644 index df3b54801..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/param/FollowingParamConfig.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.home_mixer.product.following.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.following.param.FollowingParam._ -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class FollowingParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableFollowingProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val booleanFSOverrides = - Seq( - EnableFlipInjectionModuleCandidatePipelineParam, - EnableWhoToFollowCandidatePipelineParam, - EnableAdsCandidatePipelineParam, - EnableFastAds, - ) - - override val boundedIntFSOverrides = Seq( - FlipInlineInjectionModulePosition, - WhoToFollowPositionParam, - ServerMaxResultsParam - ) - - override val stringFSOverrides = Seq(WhoToFollowDisplayLocationParam) - - override val boundedDurationFSOverrides = Seq(WhoToFollowMinInjectionIntervalParam) - - override val enumFSOverrides = Seq(WhoToFollowDisplayTypeIdParam) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel deleted file mode 100644 index 5f597c78b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.bazel +++ /dev/null @@ -1,101 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finagle/finagle-memcached/src/main/scala", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/following/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_scorer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_is_nsfw", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_follow_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/who_to_subscribe_module", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/scorer/tweet_tlx", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/scoring", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - "src/java/com/twitter/search/common/util/lang", - "src/scala/com/twitter/suggests/controller_data", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/search/common:constants-java", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "src/thrift/com/twitter/timelineservice:thrift-scala", - "stitch/stitch-tweetypie", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - ], - exports = [ - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.docx new file mode 100644 index 000000000..5ed0a2212 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.docx new file mode 100644 index 000000000..447d5b0f7 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala deleted file mode 100644 index 99ed0f584..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsCandidatePipelineBuilder.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.AdsNumOrganicItemsParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouAdsCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - ForYouQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouAds") - - private val suggestType = st.SuggestType.Promoted - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - )) - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) - - def build( - organicCandidatePipelines: Option[CandidateScope] = None - ): AdsCandidatePipelineConfig[ForYouQuery] = - adsCandidatePipelineConfigBuilder.build[ForYouQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHome), - estimateNumOrganicItems = _.params(AdsNumOrganicItemsParam).toShort, - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - alerts = alerts, - urtRequest = Some(true), - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.docx new file mode 100644 index 000000000..8361a032b Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala deleted file mode 100644 index ef65d7062..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouAdsDependentCandidatePipelineBuilder.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.model.HomeFeatures.TweetLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountTruncatedItemCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.TruncatedPipelineScopedOrganicItems -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouAdsDependentCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - ForYouQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouAdsDependent") - - private val suggestType = st.SuggestType.Promoted - - private val MaxOrganicTweets = 35 - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - )) - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) - - def build( - organicCandidatePipelines: CandidateScope - ): AdsDependentCandidatePipelineConfig[ForYouQuery] = - adsCandidatePipelineConfigBuilder.build[ForYouQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHome), - getOrganicItems = TruncatedPipelineScopedOrganicItems( - pipelines = organicCandidatePipelines, - textFeature = TweetTextFeature, - languageFeature = TweetLanguageFeature, - ordering = CandidatesUtil.scoreOrdering, - maxCount = MaxOrganicTweets - ), - countNumOrganicItems = - CountTruncatedItemCandidatesFromPipelines(organicCandidatePipelines, MaxOrganicTweets), - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate, - NonEmptyCandidatesGate(organicCandidatePipelines) - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - alerts = alerts, - urtRequest = Some(true) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.docx new file mode 100644 index 000000000..380b79c73 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala deleted file mode 100644 index 494d3a584..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouConversationServiceCandidatePipelineConfig.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.candidate_pipeline.ConversationServiceResponseFeatureTransformer -import com.twitter.home_mixer.functional_component.decorator.HomeConversationServiceCandidateDecorator -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSource -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.ConversationServiceCandidateSourceRequest -import com.twitter.product_mixer.component_library.candidate_source.tweetconvosvc.TweetWithConversationMetadata -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.gate.NoCandidatesGate -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.BaseGate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.functional_component.transformer.DependentCandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches Tweet ancestors from Conversation Service Candidate Source - */ -@Singleton -class ForYouConversationServiceCandidatePipelineConfig @Inject() ( - forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, - forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, - conversationServiceCandidateSource: ConversationServiceCandidateSource, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) - extends DependentCandidatePipelineConfig[ - ForYouQuery, - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouConversationService") - - override val gates: Seq[BaseGate[ForYouQuery]] = Seq( - NoCandidatesGate( - SpecificPipelines( - forYouTimelineScorerCandidatePipelineConfig.identifier, - forYouScoredTweetsCandidatePipelineConfig.identifier - ) - ), - NonEmptySeqFeatureGate(TimelineServiceTweetsFeature) - ) - - override val candidateSource: BaseCandidateSource[ - ConversationServiceCandidateSourceRequest, - TweetWithConversationMetadata - ] = conversationServiceCandidateSource - - override val queryTransformer: DependentCandidatePipelineQueryTransformer[ - ForYouQuery, - ConversationServiceCandidateSourceRequest - ] = { (query, candidates) => - val timelineServiceTweets = query.features - .map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).getOrElse(Seq.empty) - - val tweetsWithConversationMetadata = timelineServiceTweets.map { id => - TweetWithConversationMetadata( - tweetId = id, - userId = None, - sourceTweetId = None, - sourceUserId = None, - inReplyToTweetId = None, - conversationId = None, - ancestors = Seq.empty - ) - } - ConversationServiceCandidateSourceRequest(tweetsWithConversationMetadata) - } - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[TweetWithConversationMetadata] - ] = Seq(ConversationServiceResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - TweetWithConversationMetadata, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq( - InNetworkFeatureHydrator, - tweetypieFeatureHydrator - ) - - override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( - PreviouslyServedTweetsFilter, - RetweetDeduplicationFilter, - FeatureFilter.fromFeature(FilterIdentifier("TweetypieHydrated"), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier("QuotedTweetDropped"), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(namesFeatureHydrator) - - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = - HomeConversationServiceCandidateDecorator(homeFeedbackActionInfoBuilder) - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.docx new file mode 100644 index 000000000..7e9382fb5 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala deleted file mode 100644 index e1cf2161c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouProductPipelineConfig.scala +++ /dev/null @@ -1,142 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.ForYouProductContext -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnablePushToHomeMixerPipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsMixerPipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.for_you.param.ForYouParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.util.SortIndexBuilder -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Time -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouProductPipelineConfig @Inject() ( - forYouTimelineScorerMixerPipelineConfig: ForYouTimelineScorerMixerPipelineConfig, - forYouScoredTweetsMixerPipelineConfig: ForYouScoredTweetsMixerPipelineConfig, - forYouPushToHomeMixerPipelineConfig: ForYouPushToHomeMixerPipelineConfig, - forYouParamConfig: ForYouParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, ForYouQuery, urt.TimelineResponse] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ForYou") - - override val product: Product = ForYouProduct - - override val paramConfig: ProductParamConfig = forYouParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ForYouQuery = { - val context = request.productContext match { - case Some(context: ForYouProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ForYouProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - /** - * Unlike other clients, newly created tweets on Android have the sort index set to the current - * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline - * if subsequent timeline responses use the sort index from the previous response instead of - * the current time. - */ - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) - .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - .map { - case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => - val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) - case cursor => cursor - } - } - - ForYouQuery( - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - deviceContext = context.deviceContext, - seenTweetIds = context.seenTweetIds, - dspClientContext = context.dspClientContext, - pushToHomeTweetId = context.pushToHomeTweetId - ) - } - - override val pipelines: Seq[PipelineConfig] = Seq( - forYouTimelineScorerMixerPipelineConfig, - forYouScoredTweetsMixerPipelineConfig, - forYouPushToHomeMixerPipelineConfig - ) - - override def pipelineSelector( - query: ForYouQuery - ): ComponentIdentifier = { - if (query.pushToHomeTweetId.isDefined && query.params(EnablePushToHomeMixerPipelineParam)) - forYouPushToHomeMixerPipelineConfig.identifier - else if (query.params(EnableScoredTweetsMixerPipelineParam)) - forYouScoredTweetsMixerPipelineConfig.identifier - else forYouTimelineScorerMixerPipelineConfig.identifier - } - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(2300.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(2800.millis, 15, 30) - ), - ThroughputAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(70000), - criticalPredicate = TriggerIfAbove(80000) - ), - EmptyResponseRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(2), - criticalPredicate = TriggerIfAbove(3) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.docx new file mode 100644 index 000000000..69a356b6a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala deleted file mode 100644 index ac529f582..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeMixerPipelineConfig.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ParamGatedIncludeInstruction -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouPushToHomeMixerPipelineConfig @Inject() ( - forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, - urtTransportMarshaller: UrtTransportMarshaller) - extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouPushToHome") - - override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = - Seq(forYouPushToHomeTweetCandidatePipelineConfig) - - override val resultSelectors: Seq[Selector[ForYouQuery]] = - Seq(InsertAppendResults(forYouPushToHomeTweetCandidatePipelineConfig.identifier)) - - override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { - val instructionBuilders = Seq( - ClearCacheInstructionBuilder( - ParamGatedIncludeInstruction(ForYouParam.EnableClearCacheOnPushToHome)), - AddEntriesInstructionBuilder()) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { case item: TweetItem => item.id } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("for_you_push_to_home"), - section = None, - entityToken = None) - ) - ) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.docx new file mode 100644 index 000000000..6b8e66648 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala deleted file mode 100644 index 49df4fdf8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouPushToHomeTweetCandidatePipelineConfig.scala +++ /dev/null @@ -1,81 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.functional_component.gate.PushToHomeRequestGate -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineservice.suggests.{thriftscala => st} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouPushToHomeTweetCandidatePipelineConfig @Inject() () - extends CandidatePipelineConfig[ - ForYouQuery, - ForYouQuery, - TweetCandidate, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouPushToHomeTweet") - - override val gates: Seq[Gate[ForYouQuery]] = Seq(PushToHomeRequestGate) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ForYouQuery, - ForYouQuery - ] = identity - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[TweetCandidate] - ] = Seq(new CandidateFeatureTransformer[TweetCandidate] { - override def features: Set[Feature[_, _]] = Set(SuggestTypeFeature) - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ForYouPushToHomeTweet") - - override def transform(input: TweetCandidate): FeatureMap = - FeatureMapBuilder().add(SuggestTypeFeature, Some(st.SuggestType.Magicrec)).build() - }) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - TweetCandidate, - TweetCandidate - ] = identity - - override val candidateSource: CandidateSource[ - ForYouQuery, - TweetCandidate - ] = PassthroughCandidateSource( - CandidateSourceIdentifier("PushToHomeTweet"), - { query => query.pushToHomeTweetId.toSeq.map(TweetCandidate(_)) } - ) - - override val decorator: Option[ - CandidateDecorator[ForYouQuery, TweetCandidate] - ] = { - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = HomeClientEventInfoBuilder() - ) - Some(UrtItemCandidateDecorator(tweetItemBuilder)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.docx new file mode 100644 index 000000000..fbf3f66cc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala deleted file mode 100644 index d7716e190..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsCandidatePipelineConfig.scala +++ /dev/null @@ -1,155 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.gate.SupportedLanguagesGate -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata -import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetsProductCandidateSource -import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator -import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableScoredTweetsCandidatePipelineParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouScoredTweetsCandidatePipelineConfig @Inject() ( - scoredTweetsProductCandidateSource: ScoredTweetsProductCandidateSource, - focalTweetFeatureHydrator: FocalTweetFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) - extends CandidatePipelineConfig[ - ForYouQuery, - ForYouQuery, - ScoredTweetWithConversationMetadata, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouScoredTweets") - - private val TweetypieHydratedFilterId = "TweetypieHydrated" - private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" - private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" - private val ConversationModuleNamespace = EntryNamespace("home-conversation") - - override val gates: Seq[Gate[ForYouQuery]] = Seq(SupportedLanguagesGate) - - override val candidateSource: CandidateSource[ForYouQuery, ScoredTweetWithConversationMetadata] = - scoredTweetsProductCandidateSource - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(EnableScoredTweetsCandidatePipelineParam) - - override val queryTransformer: CandidatePipelineQueryTransformer[ForYouQuery, ForYouQuery] = - identity - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] - ] = Seq(ForYouScoredTweetsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ScoredTweetWithConversationMetadata, - TweetCandidate - ] = { sourceResults => TweetCandidate(sourceResults.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(InNetworkFeatureHydrator, namesFeatureHydrator, tweetypieFeatureHydrator) - - override val filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(QuotedTweetDroppedFilterId), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(OutOfNetworkNSFWFilterId), - shouldKeepCandidate = { features => - features.getOrElse(InNetworkFeature, false) || - !features.getOrElse(IsNsfwFeature, false) - } - ), - SocialContextFilter, - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(focalTweetFeatureHydrator) - - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = { - val clientEventInfoBuilder = HomeClientEventInfoBuilder() - - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder, - socialContextBuilder = Some(homeTweetSocialContextBuilder), - timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - moduleIdGeneration = ManualModuleId(0L), - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = tweetDecorator, - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert(10, 20) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.docx new file mode 100644 index 000000000..f6f38777d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala deleted file mode 100644 index 2137c5267..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsMixerPipelineConfig.scala +++ /dev/null @@ -1,376 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig -import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates -import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId -import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration -import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.ClearCacheIncludeInstruction -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults -import com.twitter.product_mixer.component_library.selector.SelectConditionally -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class ForYouScoredTweetsMixerPipelineConfig @Inject() ( - forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, - forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, - forYouScoredTweetsCandidatePipelineConfig: ForYouScoredTweetsCandidatePipelineConfig, - forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, - forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, - flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, - editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], - forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, - dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, - gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], - timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - adsInjector: AdsInjector, - servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, - servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, - updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, - servedStatsSideEffect: ServedStatsSideEffect, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouScoredTweets") - - private val MaxConsecutiveOutOfNetworkCandidates = 2 - - private val PushToHomeTweetPosition = 0 - - private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( - requestQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator, - timelineServiceTweetsQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - ) - - private val scoredTweetsCandidatePipelineScope = - SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) - - private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder - .build(scoredTweetsCandidatePipelineScope) - - private val forYouWhoToFollowCandidatePipelineConfig = - forYouWhoToFollowCandidatePipelineConfigBuilder.build() - - private val forYouWhoToSubscribeCandidatePipelineConfig = - forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() - - private val flipPromptCandidatePipelineConfig = - flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( - supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( - forYouScoredTweetsCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig, - forYouWhoToFollowCandidatePipelineConfig, - forYouWhoToSubscribeCandidatePipelineConfig, - forYouTweetPreviewsCandidatePipelineConfig, - flipPromptCandidatePipelineConfig - ) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[ForYouQuery, _, _, _] - ] = Seq( - forYouAdsCandidatePipelineConfig, - forYouConversationServiceCandidatePipelineConfig, - editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig - ) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - forYouScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - ) - - override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier - ), - UpdateSortCandidates( - ordering = CandidatesUtil.scoreOrdering, - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier - ), - UpdateSortModuleItemCandidates( - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, - ordering = CandidatesUtil.conversationModuleTweetsOrdering - ), - DebunchCandidates( - pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier), - mustDebunch = { - case item: ItemCandidateWithDetails => - !item.features.getOrElse(InNetworkFeature, false) - case module: ModuleCandidateWithDetails => - !module.candidates.last.features.getOrElse(InNetworkFeature, false) - }, - maxBunchSize = MaxConsecutiveOutOfNetworkCandidates - ), - UpdateConversationModuleId( - pipelineScope = SpecificPipeline(forYouScoredTweetsCandidatePipelineConfig.identifier) - ), - DropMaxCandidates( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = MaxNumberReplaceInstructionsParam - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MaxCandidatesSize) - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) - ), - // The Conversation Service pipeline will only run if the Scored Tweets pipeline returned nothing - InsertAppendResults( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier - ), - InsertAppendResults( - candidatePipeline = forYouScoredTweetsCandidatePipelineConfig.identifier - ), - InsertFixedPositionResults( - candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, - positionParam = TweetPreviewsPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - positionParam = WhoToFollowPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - positionParam = WhoToSubscribePositionParam - ), - InsertFixedPositionResults( - candidatePipeline = flipPromptCandidatePipelineConfig.identifier, - positionParam = FlipInlineInjectionModulePosition - ), - // Insert Push To Home Tweet at top of Timeline - InsertFixedPositionResults( - candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, - positionParam = StaticParam(PushToHomeTweetPosition) - ), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier - ), - // This selector must come after the tweets are inserted into the results - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowCandidatePipelineConfig.MinCandidatesSize) - ), - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) - ), - UpdateNewTweetsPillDecoration( - pipelineScope = SpecificPipelines( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouScoredTweetsCandidatePipelineConfig.identifier, - newTweetsPillCandidatePipelineConfig.identifier - ), - stringCenter = stringCenterProvider.get(), - seeNewTweetsString = externalStrings.seeNewTweetsString, - tweetedString = externalStrings.tweetedString - ), - InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), - SelectConditionally( - selector = - InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), - includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) - ), - UpdateHomeClientEventDetails( - candidatePipelines = Set( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouScoredTweetsCandidatePipelineConfig.identifier - ) - ), - ) - - private val servedCandidateKeysKafkaSideEffect = - servedCandidateKeysKafkaSideEffectBuilder.build( - Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) - - private val servedCandidateFeatureKeysKafkaSideEffect = - servedCandidateFeatureKeysKafkaSideEffectBuilder.build( - Set(forYouScoredTweetsCandidatePipelineConfig.identifier)) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = Seq( - forYouScoredTweetsCandidatePipelineConfig.identifier, - forYouConversationServiceCandidatePipelineConfig.identifier - ), - adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), - whoToFollowCandidatePipelineIdentifier = Some( - forYouWhoToFollowCandidatePipelineConfig.identifier - ), - whoToSubscribeCandidatePipelineIdentifier = - Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( - servedCandidateKeysKafkaSideEffect, - servedCandidateFeatureKeysKafkaSideEffect, - updateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect, - homeScribeClientEventSideEffect, - homeScribeServedCandidatesSideEffect, - servedStatsSideEffect - ) - - override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { - val instructionBuilders = Seq( - ClearCacheInstructionBuilder( - ClearCacheIncludeInstruction( - ClearCacheOnPtr.EnableParam, - ClearCacheOnPtr.MinEntriesParam, - ) - ), - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - // excludes alert, cover, and replace candidates - AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), - ShowAlertInstructionBuilder(), - ShowCoverInstructionBuilder(), - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("for_you_scored_tweets"), - section = None, - entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.docx new file mode 100644 index 000000000..22a6429dd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala deleted file mode 100644 index eafc09352..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouScoredTweetsResponseFeatureTransformer.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.for_you.candidate_source.ScoredTweetWithConversationMetadata -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.timelines.render.{thriftscala => tl} -import com.twitter.timelineservice.suggests.{thriftscala => tls} - -object ForYouScoredTweetsResponseFeatureTransformer - extends CandidateFeatureTransformer[ScoredTweetWithConversationMetadata] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ForYouScoredTweetsResponse") - - override val features: Set[Feature[_, _]] = Set( - AncestorsFeature, - AuthorIdFeature, - AuthorIsBlueVerifiedFeature, - AuthorIsCreatorFeature, - AuthorIsGoldVerifiedFeature, - AuthorIsGrayVerifiedFeature, - AuthorIsLegacyVerifiedFeature, - ConversationModuleFocalTweetIdFeature, - ConversationModuleIdFeature, - DirectedAtUserIdFeature, - ExclusiveConversationAuthorIdFeature, - FullScoringSucceededFeature, - FavoritedByUserIdsFeature, - FollowedByUserIdsFeature, - InNetworkFeature, - InReplyToTweetIdFeature, - InReplyToUserIdFeature, - IsAncestorCandidateFeature, - IsReadFromCacheFeature, - IsRetweetFeature, - PerspectiveFilteredLikedByUserIdsFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SGSValidFollowedByUserIdsFeature, - SGSValidLikedByUserIdsFeature, - ScoreFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - StreamToKafkaFeature, - SuggestTypeFeature, - TopicContextFunctionalityTypeFeature, - TopicIdSocialContextFeature - ) - - override def transform(input: ScoredTweetWithConversationMetadata): FeatureMap = - FeatureMapBuilder() - .add(AncestorsFeature, input.ancestors.getOrElse(Seq.empty)) - .add(AuthorIdFeature, Some(input.authorId)) - .add(AuthorIsBlueVerifiedFeature, input.authorIsBlueVerified.getOrElse(false)) - .add(AuthorIsGoldVerifiedFeature, input.authorIsGoldVerified.getOrElse(false)) - .add(AuthorIsGrayVerifiedFeature, input.authorIsGrayVerified.getOrElse(false)) - .add(AuthorIsLegacyVerifiedFeature, input.authorIsLegacyVerified.getOrElse(false)) - .add(AuthorIsCreatorFeature, input.authorIsCreator.getOrElse(false)) - .add(ConversationModuleIdFeature, input.conversationId) - .add(ConversationModuleFocalTweetIdFeature, input.conversationFocalTweetId) - .add(DirectedAtUserIdFeature, input.directedAtUserId) - .add(ExclusiveConversationAuthorIdFeature, input.exclusiveConversationAuthorId) - .add(SGSValidLikedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) - .add(SGSValidFollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) - .add(FavoritedByUserIdsFeature, input.sgsValidLikedByUserIds.getOrElse(Seq.empty)) - .add(FollowedByUserIdsFeature, input.sgsValidFollowedByUserIds.getOrElse(Seq.empty)) - .add(FullScoringSucceededFeature, true) - .add(InNetworkFeature, input.inNetwork.getOrElse(true)) - .add(InReplyToTweetIdFeature, input.inReplyToTweetId) - .add(InReplyToUserIdFeature, input.inReplyToUserId) - .add(IsAncestorCandidateFeature, input.conversationFocalTweetId.exists(_ != input.tweetId)) - .add(IsReadFromCacheFeature, input.isReadFromCache.getOrElse(false)) - .add(IsRetweetFeature, input.sourceTweetId.isDefined) - .add( - PerspectiveFilteredLikedByUserIdsFeature, - input.perspectiveFilteredLikedByUserIds.getOrElse(Seq.empty)) - .add(QuotedTweetIdFeature, input.quotedTweetId) - .add(QuotedUserIdFeature, input.quotedUserId) - .add(ScoreFeature, input.score) - .add(SourceTweetIdFeature, input.sourceTweetId) - .add(SourceUserIdFeature, input.sourceUserId) - .add(StreamToKafkaFeature, input.streamToKafka.getOrElse(false)) - .add(SuggestTypeFeature, input.suggestType.orElse(Some(tls.SuggestType.Undefined))) - .add( - TopicContextFunctionalityTypeFeature, - input.topicFunctionalityType.collect { - case tl.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType - case tl.TopicContextFunctionalityType.Recommendation => - RecommendationTopicContextFunctionalityType - case tl.TopicContextFunctionalityType.RecWithEducation => - RecWithEducationTopicContextFunctionalityType - } - ) - .add(TopicIdSocialContextFeature, input.topicId) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.docx new file mode 100644 index 000000000..32ae4c625 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala deleted file mode 100644 index bb93a4110..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerCandidatePipelineConfig.scala +++ /dev/null @@ -1,241 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeClientEventInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.HomeTimelinesScoreInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeTweetSocialContextBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.InNetworkFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.NamesFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.PerspectiveFilteredSocialContextFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.SGSValidSocialContextFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetypieFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter -import com.twitter.home_mixer.functional_component.filter.InvalidConversationModuleFilter -import com.twitter.home_mixer.functional_component.filter.InvalidSubscriptionTweetFilter -import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.functional_component.scorer.FeedbackFatigueScorer -import com.twitter.home_mixer.functional_component.scorer.OONTweetScalingScorer -import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller -import com.twitter.home_mixer.marshaller.timelines.TimelineServiceCursorMarshaller -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.IsNsfwFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetDroppedFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.product.for_you.feature_hydrator.FocalTweetFeatureHydrator -import com.twitter.home_mixer.product.for_you.filter.SocialContextFilter -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTimelineScorerCandidatePipelineParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.ScoredTweetCandidateWithFocalTweet -import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.TimelineScorerCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ManualModuleId -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.scorer.Scorer -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.model.candidate.CandidateTweetSourceId -import com.twitter.timelines.service.{thriftscala => tst} -import com.twitter.timelinescorer.{thriftscala => t} -import com.twitter.timelineservice.{thriftscala => tlst} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the Timeline Scorer Candidate Source - */ -@Singleton -class ForYouTimelineScorerCandidatePipelineConfig @Inject() ( - timelineScorerCandidateSource: TimelineScorerCandidateSource, - deviceContextMarshaller: DeviceContextMarshaller, - tweetypieFeatureHydrator: TweetypieFeatureHydrator, - sgsValidSocialContextFeatureHydrator: SGSValidSocialContextFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator: PerspectiveFilteredSocialContextFeatureHydrator, - namesFeatureHydrator: NamesFeatureHydrator, - focalTweetFeatureHydrator: FocalTweetFeatureHydrator, - invalidSubscriptionTweetFilter: InvalidSubscriptionTweetFilter, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder, - homeTweetSocialContextBuilder: HomeTweetSocialContextBuilder) - extends CandidatePipelineConfig[ - ForYouQuery, - t.ScoredTweetsRequest, - ScoredTweetCandidateWithFocalTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ForYouTimelineScorerTweets") - - private val TweetypieHydratedFilterId = "TweetypieHydrated" - private val QuotedTweetDroppedFilterId = "QuotedTweetDropped" - private val OutOfNetworkNSFWFilterId = "OutOfNetworkNSFW" - private val ConversationModuleNamespace = EntryNamespace("home-conversation") - - override val supportedClientParam: Option[FSParam[Boolean]] = - Some(EnableTimelineScorerCandidatePipelineParam) - - override val candidateSource: BaseCandidateSource[ - t.ScoredTweetsRequest, - ScoredTweetCandidateWithFocalTweet - ] = timelineScorerCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ForYouQuery, - t.ScoredTweetsRequest - ] = { query => - val deviceContext = query.deviceContext.getOrElse(DeviceContext.Empty) - - val scoredTweetsRequestContext = t.v1.ScoredTweetsRequestContext( - contextualUserId = query.clientContext.userId, - timelineId = query.clientContext.userId.map(tlst.TimelineId(tlst.TimelineType.Home, _, None)), - deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)), - seenTweetIds = query.seenTweetIds, - contextualUserContext = Some(tst.ContextualUserContext(query.clientContext.userRoles)), - timelineRequestCursor = query.pipelineCursor.flatMap(TimelineServiceCursorMarshaller(_)) - ) - - val candidateTweetSourceIds = Seq( - CandidateTweetSourceId.RecycledTweet, - CandidateTweetSourceId.OrganicTweet, - CandidateTweetSourceId.AncestorsOnlyOrganicTweet, - CandidateTweetSourceId.BackfillOrganicTweet, - CandidateTweetSourceId.CroonTweet, - CandidateTweetSourceId.RecommendedTweet, - CandidateTweetSourceId.FrsTweet, - CandidateTweetSourceId.ListTweet, - CandidateTweetSourceId.RecommendedTrendTweet, - CandidateTweetSourceId.PopularTopicTweet - ) - - val timelineServiceTweets = - query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).getOrElse(Seq.empty) - - val timelineEntries = timelineServiceTweets.map { id => - tlst.TimelineEntry.Tweet(tlst.Tweet(statusId = id, sortIndex = id)) - } - - t.ScoredTweetsRequest.V1( - t.v1.ScoredTweetsRequest( - scoredTweetsRequestContext = Some(scoredTweetsRequestContext), - candidateTweetSourceIds = - Some(candidateTweetSourceIds.flatMap(CandidateTweetSourceId.toThrift)), - maxResultsCount = query.requestedMaxResults, - organicTimeline = Some( - tlst.Timeline( - timelineId = tlst.TimelineId( - timelineType = tlst.TimelineType.Home, - id = query.getRequiredUserId, - canonicalTimelineId = None), - entries = timelineEntries, - modules = tlst.TimelineModules() - )) - ) - ) - } - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet] - ] = Seq(ForYouTimelineScorerResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ScoredTweetCandidateWithFocalTweet, - TweetCandidate - ] = { candidateWithFocalTweetId => - TweetCandidate(id = candidateWithFocalTweetId.candidate.tweetId) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq( - namesFeatureHydrator, - tweetypieFeatureHydrator, - InNetworkFeatureHydrator, - sgsValidSocialContextFeatureHydrator, - perspectiveFilteredSocialContextFeatureHydrator, - ) - - override def filters: Seq[Filter[ForYouQuery, TweetCandidate]] = Seq( - RetweetDeduplicationFilter, - FeatureFilter.fromFeature(FilterIdentifier(TweetypieHydratedFilterId), IsHydratedFeature), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(QuotedTweetDroppedFilterId), - shouldKeepCandidate = { features => !features.getOrElse(QuotedTweetDroppedFeature, false) } - ), - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(OutOfNetworkNSFWFilterId), - shouldKeepCandidate = { features => - features.getOrElse(InNetworkFeature, false) || - !features.getOrElse(IsNsfwFeature, false) - } - ), - FeedbackFatigueFilter, - RejectTweetFromViewerFilter, - SocialContextFilter, - invalidSubscriptionTweetFilter, - InvalidConversationModuleFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq(focalTweetFeatureHydrator) - - override val scorers: Seq[Scorer[ForYouQuery, TweetCandidate]] = - Seq(OONTweetScalingScorer, FeedbackFatigueScorer) - - override val decorator: Option[CandidateDecorator[ForYouQuery, TweetCandidate]] = { - val clientEventInfoBuilder = HomeClientEventInfoBuilder() - - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder, - socialContextBuilder = Some(homeTweetSocialContextBuilder), - timelinesScoreInfoBuilder = Some(HomeTimelinesScoreInfoBuilder), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder) - ) - - val tweetDecorator = UrtItemCandidateDecorator(tweetItemBuilder) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - moduleIdGeneration = ManualModuleId(0L), - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = tweetDecorator, - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } - - override val alerts: Seq[Alert] = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert(10, 20) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.docx new file mode 100644 index 000000000..63497f1c9 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala deleted file mode 100644 index 441e5796d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerMixerPipelineConfig.scala +++ /dev/null @@ -1,403 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.EditedTweetsCandidatePipelineConfig -import com.twitter.home_mixer.candidate_pipeline.NewTweetsPillCandidatePipelineConfig -import com.twitter.home_mixer.functional_component.decorator.urt.builder.AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator._ -import com.twitter.home_mixer.functional_component.selector.DebunchCandidates -import com.twitter.home_mixer.functional_component.selector.UpdateConversationModuleId -import com.twitter.home_mixer.functional_component.selector.UpdateHomeClientEventDetails -import com.twitter.home_mixer.functional_component.selector.UpdateNewTweetsPillDecoration -import com.twitter.home_mixer.functional_component.side_effect._ -import com.twitter.home_mixer.model.ClearCacheIncludeInstruction -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.param.HomeGlobalParams.MaxNumberReplaceInstructionsParam -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.product.following.model.HomeMixerExternalStrings -import com.twitter.home_mixer.product.for_you.feature_hydrator.TimelineServiceTweetsQueryFeatureHydrator -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ClearCacheOnPtr -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableFlipInjectionModuleCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.FlipInlineInjectionModulePosition -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowPositionParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribePositionParam -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateFeatureKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedCandidateKeysKafkaSideEffectBuilder -import com.twitter.home_mixer.product.for_you.side_effect.ServedStatsSideEffect -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.AsyncParamGatedQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.FlipPromptCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ClearCacheInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowCoverInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.DropModuleTooFewModuleItemResults -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.InsertFixedPositionResults -import com.twitter.product_mixer.component_library.selector.SelectConditionally -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.UpdateSortModuleItemCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipeline -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.product_mixer.core.product.guice.scope.ProductScoped -import com.twitter.stringcenter.client.StringCenter -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Provider -import javax.inject.Singleton - -@Singleton -class ForYouTimelineScorerMixerPipelineConfig @Inject() ( - forYouTimelineScorerCandidatePipelineConfig: ForYouTimelineScorerCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig: ForYouPushToHomeTweetCandidatePipelineConfig, - forYouConversationServiceCandidatePipelineConfig: ForYouConversationServiceCandidatePipelineConfig, - forYouAdsDependentCandidatePipelineBuilder: ForYouAdsDependentCandidatePipelineBuilder, - forYouWhoToFollowCandidatePipelineConfigBuilder: ForYouWhoToFollowCandidatePipelineConfigBuilder, - forYouWhoToSubscribeCandidatePipelineConfigBuilder: ForYouWhoToSubscribeCandidatePipelineConfigBuilder, - flipPromptCandidatePipelineConfigBuilder: FlipPromptCandidatePipelineConfigBuilder, - editedTweetsCandidatePipelineConfig: EditedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig: NewTweetsPillCandidatePipelineConfig[ForYouQuery], - forYouTweetPreviewsCandidatePipelineConfig: ForYouTweetPreviewsCandidatePipelineConfig, - dismissInfoQueryFeatureHydrator: DismissInfoQueryFeatureHydrator, - gizmoduckUserQueryFeatureHydrator: GizmoduckUserQueryFeatureHydrator, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ForYouQuery], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ForYouQuery], - memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator: PersistenceStoreQueryFeatureHydrator, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ForYouQuery], - timelineServiceTweetsQueryFeatureHydrator: TimelineServiceTweetsQueryFeatureHydrator, - lastNonPollingTimeQueryFeatureHydrator: LastNonPollingTimeQueryFeatureHydrator, - feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator: PreviewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - adsInjector: AdsInjector, - servedCandidateKeysKafkaSideEffectBuilder: ServedCandidateKeysKafkaSideEffectBuilder, - servedCandidateFeatureKeysKafkaSideEffectBuilder: ServedCandidateFeatureKeysKafkaSideEffectBuilder, - updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ForYouQuery, Timeline], - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, - updateTimelinesPersistenceStoreSideEffect: UpdateTimelinesPersistenceStoreSideEffect, - truncateTimelinesPersistenceStoreSideEffect: TruncateTimelinesPersistenceStoreSideEffect, - homeScribeServedCandidatesSideEffect: HomeScribeServedCandidatesSideEffect, - servedStatsSideEffect: ServedStatsSideEffect, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - externalStrings: HomeMixerExternalStrings, - @ProductScoped stringCenterProvider: Provider[StringCenter], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[ForYouQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ForYouTimelineScorer") - - private val MaxConsecutiveOutOfNetworkCandidates = 2 - - private val PushToHomeTweetPosition = 0 - - private val dependentCandidatesStep = MixerPipelineConfig.dependentCandidatePipelinesStep - private val resultSelectorsStep = MixerPipelineConfig.resultSelectorsStep - - override def fetchQueryFeatures: Seq[QueryFeatureHydrator[ForYouQuery]] = Seq( - requestQueryFeatureHydrator, - persistenceStoreQueryFeatureHydrator, - timelineServiceTweetsQueryFeatureHydrator, - feedbackHistoryQueryFeatureHydrator, - previewCreatorsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - AsyncQueryFeatureHydrator(dependentCandidatesStep, dismissInfoQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, gizmoduckUserQueryFeatureHydrator), - AsyncQueryFeatureHydrator(dependentCandidatesStep, lastNonPollingTimeQueryFeatureHydrator), - AsyncParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - resultSelectorsStep, - impressionBloomFilterQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, manhattanTweetImpressionsQueryFeatureHydrator), - AsyncQueryFeatureHydrator(resultSelectorsStep, memcacheTweetImpressionsQueryFeatureHydrator) - ) - - private val timelineScorerCandidatePipelineScope = - SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier) - - private val forYouAdsCandidatePipelineConfig = forYouAdsDependentCandidatePipelineBuilder - .build(timelineScorerCandidatePipelineScope) - - private val forYouWhoToFollowCandidatePipelineConfig = - forYouWhoToFollowCandidatePipelineConfigBuilder.build() - - private val forYouWhoToSubscribeCandidatePipelineConfig = - forYouWhoToSubscribeCandidatePipelineConfigBuilder.build() - - private val flipPromptCandidatePipelineConfig = - flipPromptCandidatePipelineConfigBuilder.build[ForYouQuery]( - supportedClientParam = Some(EnableFlipInjectionModuleCandidatePipelineParam) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[ForYouQuery, _, _, _]] = Seq( - forYouTimelineScorerCandidatePipelineConfig, - forYouPushToHomeTweetCandidatePipelineConfig, - forYouWhoToFollowCandidatePipelineConfig, - forYouWhoToSubscribeCandidatePipelineConfig, - forYouTweetPreviewsCandidatePipelineConfig, - flipPromptCandidatePipelineConfig - ) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[ForYouQuery, _, _, _] - ] = Seq( - forYouAdsCandidatePipelineConfig, - forYouConversationServiceCandidatePipelineConfig, - editedTweetsCandidatePipelineConfig, - newTweetsPillCandidatePipelineConfig - ) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - forYouTimelineScorerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToFollowCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouWhoToSubscribeCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - forYouTweetPreviewsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - flipPromptCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - editedTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - newTweetsPillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - ) - - override val resultSelectors: Seq[Selector[ForYouQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier - ), - UpdateSortCandidates( - ordering = CandidatesUtil.scoreOrdering, - candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier - ), - UpdateSortModuleItemCandidates( - candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier, - ordering = CandidatesUtil.conversationModuleTweetsOrdering - ), - DebunchCandidates( - pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier), - mustDebunch = { - case item: ItemCandidateWithDetails => - !item.features.getOrElse(InNetworkFeature, false) - case module: ModuleCandidateWithDetails => - !module.candidates.last.features.getOrElse(InNetworkFeature, false) - }, - maxBunchSize = MaxConsecutiveOutOfNetworkCandidates - ), - UpdateConversationModuleId( - pipelineScope = SpecificPipeline(forYouTimelineScorerCandidatePipelineConfig.identifier) - ), - DropMaxCandidates( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier, - maxSelectionsParam = ServerMaxResultsParam - ), - DropMaxCandidates( - candidatePipeline = editedTweetsCandidatePipelineConfig.identifier, - maxSelectionsParam = MaxNumberReplaceInstructionsParam - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MaxCandidatesSize) - ), - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MinCandidatesSize) - ), - DropMaxModuleItemCandidates( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - maxModuleItemsParam = StaticParam(WhoToSubscribeCandidatePipelineConfig.MaxCandidatesSize) - ), - // Add Conversation Service tweets to results only if the scored pipeline doesn't return any - SelectConditionally( - selector = InsertAppendResults( - candidatePipeline = forYouConversationServiceCandidatePipelineConfig.identifier), - includeSelector = (_, candidates, _) => - !candidates.exists(candidate => - forYouTimelineScorerCandidatePipelineConfig.identifier == candidate.source) - ), - InsertAppendResults(candidatePipeline = forYouTimelineScorerCandidatePipelineConfig.identifier), - InsertFixedPositionResults( - candidatePipeline = forYouTweetPreviewsCandidatePipelineConfig.identifier, - positionParam = TweetPreviewsPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - positionParam = WhoToFollowPositionParam - ), - InsertFixedPositionResults( - candidatePipeline = forYouWhoToSubscribeCandidatePipelineConfig.identifier, - positionParam = WhoToSubscribePositionParam - ), - InsertFixedPositionResults( - candidatePipeline = flipPromptCandidatePipelineConfig.identifier, - positionParam = FlipInlineInjectionModulePosition - ), - // Insert Push To Home Tweet at top of Timeline - InsertFixedPositionResults( - candidatePipeline = forYouPushToHomeTweetCandidatePipelineConfig.identifier, - positionParam = StaticParam(PushToHomeTweetPosition) - ), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = forYouAdsCandidatePipelineConfig.identifier - ), - // This selector must come after the tweets are inserted into the results - DropModuleTooFewModuleItemResults( - candidatePipeline = forYouWhoToFollowCandidatePipelineConfig.identifier, - minModuleItemsParam = StaticParam(WhoToFollowArmCandidatePipelineConfig.MinCandidatesSize) - ), - UpdateNewTweetsPillDecoration( - pipelineScope = SpecificPipelines( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouTimelineScorerCandidatePipelineConfig.identifier, - newTweetsPillCandidatePipelineConfig.identifier - ), - stringCenter = stringCenterProvider.get(), - seeNewTweetsString = externalStrings.seeNewTweetsString, - tweetedString = externalStrings.tweetedString - ), - InsertAppendResults(candidatePipeline = editedTweetsCandidatePipelineConfig.identifier), - SelectConditionally( - selector = - InsertAppendResults(candidatePipeline = newTweetsPillCandidatePipelineConfig.identifier), - includeSelector = (_, _, results) => CandidatesUtil.containsType[TweetCandidate](results) - ), - UpdateHomeClientEventDetails( - candidatePipelines = Set( - forYouConversationServiceCandidatePipelineConfig.identifier, - forYouTimelineScorerCandidatePipelineConfig.identifier - ) - ), - ) - - private val servedCandidateKeysKafkaSideEffect = - servedCandidateKeysKafkaSideEffectBuilder.build( - Set(forYouTimelineScorerCandidatePipelineConfig.identifier)) - - private val servedCandidateFeatureKeysKafkaSideEffect = - servedCandidateFeatureKeysKafkaSideEffectBuilder.build( - Set(forYouTimelineScorerCandidatePipelineConfig.identifier)) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = Seq( - forYouTimelineScorerCandidatePipelineConfig.identifier, - forYouConversationServiceCandidatePipelineConfig.identifier - ), - adsCandidatePipelineIdentifier = Some(forYouAdsCandidatePipelineConfig.identifier), - whoToFollowCandidatePipelineIdentifier = - Some(forYouWhoToFollowCandidatePipelineConfig.identifier), - whoToSubscribeCandidatePipelineIdentifier = - Some(forYouWhoToSubscribeCandidatePipelineConfig.identifier) - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[ForYouQuery, Timeline]] = Seq( - homeScribeClientEventSideEffect, - homeScribeServedCandidatesSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, - servedCandidateKeysKafkaSideEffect, - servedCandidateFeatureKeysKafkaSideEffect, - servedStatsSideEffect, - truncateTimelinesPersistenceStoreSideEffect, - updateLastNonPollingTimeSideEffect, - updateTimelinesPersistenceStoreSideEffect - ) - - override val domainMarshaller: DomainMarshaller[ForYouQuery, Timeline] = { - val instructionBuilders = Seq( - ClearCacheInstructionBuilder( - ClearCacheIncludeInstruction( - ClearCacheOnPtr.EnableParam, - ClearCacheOnPtr.MinEntriesParam - ) - ), - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - // excludes alert, cover, and replace candidates - AddEntriesWithReplaceAndShowAlertAndCoverInstructionBuilder(), - ShowAlertInstructionBuilder(), - ShowCoverInstructionBuilder(), - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { case item: TweetItem => item.id } - } - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = OrderedBottomCursorBuilder(idSelector) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("for_you_timeline_scorer"), - section = None, - entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.docx new file mode 100644 index 000000000..41a081af9 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala deleted file mode 100644 index 236ca9a26..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTimelineScorerResponseFeatureTransformer.scala +++ /dev/null @@ -1,199 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.mediaservices.commons.tweetmedia.{thriftscala => mt} -import com.twitter.product_mixer.component_library.candidate_source.timeline_scorer.ScoredTweetCandidateWithFocalTweet -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecWithEducationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.search.common.constants.thriftjava.ThriftLanguage -import com.twitter.search.common.util.lang.ThriftLanguageUtil -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.timelinemixer.injection.model.candidate.AudioSpaceMetaData -import com.twitter.timelines.conversation_features.{thriftscala => cvt} -import com.twitter.timelinescorer.common.scoredtweetcandidate.{thriftscala => stc} -import com.twitter.timelineservice.suggests.{thriftscala => tls} - -object ForYouTimelineScorerResponseFeatureTransformer - extends CandidateFeatureTransformer[ScoredTweetCandidateWithFocalTweet] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("ForYouTimelineScorerResponse") - - override val features: Set[Feature[_, _]] = Set( - AncestorsFeature, - AudioSpaceMetaDataFeature, - AuthorIdFeature, - AuthorIsBlueVerifiedFeature, - AuthorIsCreatorFeature, - AuthorIsGoldVerifiedFeature, - AuthorIsGrayVerifiedFeature, - AuthorIsLegacyVerifiedFeature, - AuthoredByContextualUserFeature, - CandidateSourceIdFeature, - ConversationFeature, - ConversationModuleFocalTweetIdFeature, - ConversationModuleIdFeature, - DirectedAtUserIdFeature, - EarlybirdFeature, - EntityTokenFeature, - ExclusiveConversationAuthorIdFeature, - FavoritedByUserIdsFeature, - FollowedByUserIdsFeature, - TopicIdSocialContextFeature, - TopicContextFunctionalityTypeFeature, - FromInNetworkSourceFeature, - FullScoringSucceededFeature, - HasDisplayedTextFeature, - InNetworkFeature, - InReplyToTweetIdFeature, - IsAncestorCandidateFeature, - IsExtendedReplyFeature, - IsRandomTweetFeature, - IsReadFromCacheFeature, - IsRetweetFeature, - IsRetweetedReplyFeature, - NonSelfFavoritedByUserIdsFeature, - NumImagesFeature, - OriginalTweetCreationTimeFromSnowflakeFeature, - PredictionRequestIdFeature, - QuotedTweetIdFeature, - ScoreFeature, - SimclustersTweetTopKClustersWithScoresFeature, - SourceTweetIdFeature, - SourceUserIdFeature, - StreamToKafkaFeature, - SuggestTypeFeature, - TweetLanguageFeature, - VideoDurationMsFeature, - ) - - // Convert language code to ISO 639-3 format - private def getLanguageISOFormatByValue(languageCodeValue: Int): String = - ThriftLanguageUtil.getLanguageCodeOf(ThriftLanguage.findByValue(languageCodeValue)) - - override def transform( - candidateWithFocalTweet: ScoredTweetCandidateWithFocalTweet - ): FeatureMap = { - val candidate: stc.v1.ScoredTweetCandidate = candidateWithFocalTweet.candidate - val focalTweetId = candidateWithFocalTweet.focalTweetIdOpt - - val originalTweetId = candidate.sourceTweetId.getOrElse(candidate.tweetId) - val tweetFeatures = candidate.tweetFeaturesMap.flatMap(_.get(originalTweetId)) - val earlybirdFeatures = tweetFeatures.flatMap(_.recapFeatures.flatMap(_.tweetFeatures)) - val directedAtUserIsInFirstDegree = - earlybirdFeatures.flatMap(_.directedAtUserIdIsInFirstDegree) - val isReply = candidate.inReplyToTweetId.nonEmpty - val isRetweet = candidate.isRetweet.getOrElse(false) - val isInNetwork = candidate.isInNetwork.getOrElse(true) - val conversationFeatures = candidate.conversationFeatures.flatMap { - case cvt.ConversationFeatures.V1(candidate) => Some(candidate) - case _ => None - } - val numImages = candidate.mediaMetaData - .map( - _.count(mediaEntity => - mediaEntity.mediaInfo.exists(_.isInstanceOf[mt.MediaInfo.ImageInfo]) || - mediaEntity.mediaInfo.isEmpty)) - val hasImage = earlybirdFeatures.exists(_.hasImage) - val hasVideo = earlybirdFeatures.exists(_.hasVideo) - val hasCard = earlybirdFeatures.exists(_.hasCard) - val hasQuote = earlybirdFeatures.exists(_.hasQuote.contains(true)) - val hasDisplayedText = earlybirdFeatures.exists(_.tweetLength.exists(length => { - val numMedia = Seq(hasVideo, (hasImage || hasCard), hasQuote).count(b => b) - val tcoLengthsPlusSpaces = 23 * numMedia + (if (numMedia > 0) numMedia - 1 else 0) - length > tcoLengthsPlusSpaces - })) - val suggestType = candidate.overrideSuggestType.orElse(Some(tls.SuggestType.Undefined)) - - val topicSocialProofMetadataOpt = candidate.entityData.flatMap(_.topicSocialProofMetadata) - val topicIdSocialContextOpt = topicSocialProofMetadataOpt.map(_.topicId) - val topicContextFunctionalityTypeOpt = - topicSocialProofMetadataOpt.map(_.topicContextFunctionalityType).collect { - case stc.v1.TopicContextFunctionalityType.Basic => BasicTopicContextFunctionalityType - case stc.v1.TopicContextFunctionalityType.Recommendation => - RecommendationTopicContextFunctionalityType - case stc.v1.TopicContextFunctionalityType.RecWithEducation => - RecWithEducationTopicContextFunctionalityType - } - - FeatureMapBuilder() - .add( - AncestorsFeature, - candidate.ancestors - .getOrElse(Seq.empty) - .map(ancestor => ta.TweetAncestor(ancestor.tweetId, ancestor.userId.getOrElse(0L)))) - .add( - AudioSpaceMetaDataFeature, - candidate.audioSpaceMetaDatalist.map(_.head).map(AudioSpaceMetaData.fromThrift)) - .add(AuthorIdFeature, Some(candidate.authorId)) - .add(AuthorIsBlueVerifiedFeature, candidate.authorIsBlueVerified.getOrElse(false)) - .add( - AuthorIsCreatorFeature, - candidate.authorIsCreator.getOrElse(false) - ) - .add(AuthorIsGoldVerifiedFeature, candidate.authorIsGoldVerified.getOrElse(false)) - .add(AuthorIsGrayVerifiedFeature, candidate.authorIsGrayVerified.getOrElse(false)) - .add(AuthorIsLegacyVerifiedFeature, candidate.authorIsLegacyVerified.getOrElse(false)) - .add( - AuthoredByContextualUserFeature, - candidate.viewerId.contains(candidate.authorId) || - candidate.viewerId.exists(candidate.sourceUserId.contains)) - .add(CandidateSourceIdFeature, candidate.candidateTweetSourceId) - .add(ConversationFeature, conversationFeatures) - .add(ConversationModuleIdFeature, candidate.conversationId) - .add(ConversationModuleFocalTweetIdFeature, focalTweetId) - .add(DirectedAtUserIdFeature, candidate.directedAtUserId) - .add(EarlybirdFeature, earlybirdFeatures) - // This is temporary, will need to be updated with the encoded string. - .add(EntityTokenFeature, Some("test_EntityTokenForYou")) - .add(ExclusiveConversationAuthorIdFeature, candidate.exclusiveConversationAuthorId) - .add(FavoritedByUserIdsFeature, candidate.favoritedByUserIds.getOrElse(Seq.empty)) - .add(FollowedByUserIdsFeature, candidate.followedByUserIds.getOrElse(Seq.empty)) - .add(TopicIdSocialContextFeature, topicIdSocialContextOpt) - .add(TopicContextFunctionalityTypeFeature, topicContextFunctionalityTypeOpt) - .add(FullScoringSucceededFeature, candidate.fullScoringSucceeded.getOrElse(false)) - .add(HasDisplayedTextFeature, hasDisplayedText) - .add(InNetworkFeature, candidate.isInNetwork.getOrElse(true)) - .add(InReplyToTweetIdFeature, candidate.inReplyToTweetId) - .add(IsAncestorCandidateFeature, candidate.isAncestorCandidate.getOrElse(false)) - .add( - IsExtendedReplyFeature, - isInNetwork && isReply && !isRetweet && directedAtUserIsInFirstDegree.contains(false)) - .add(FromInNetworkSourceFeature, candidate.isInNetwork.getOrElse(true)) - .add(IsRandomTweetFeature, candidate.isRandomTweet.getOrElse(false)) - .add(IsReadFromCacheFeature, candidate.isReadFromCache.getOrElse(false)) - .add(IsRetweetFeature, candidate.isRetweet.getOrElse(false)) - .add(IsRetweetedReplyFeature, isReply && isRetweet) - .add( - NonSelfFavoritedByUserIdsFeature, - candidate.favoritedByUserIds.getOrElse(Seq.empty).filterNot(_ == candidate.authorId)) - .add(NumImagesFeature, numImages) - .add( - OriginalTweetCreationTimeFromSnowflakeFeature, - SnowflakeId.timeFromIdOpt(originalTweetId)) - .add(PredictionRequestIdFeature, candidate.predictionRequestId) - .add(ScoreFeature, Some(candidate.score)) - .add( - SimclustersTweetTopKClustersWithScoresFeature, - candidate.simclustersTweetTopKClustersWithScores.map(_.toMap).getOrElse(Map.empty)) - .add( - StreamToKafkaFeature, - candidate.predictionRequestId.nonEmpty && candidate.fullScoringSucceeded.getOrElse(false)) - .add(SourceTweetIdFeature, candidate.sourceTweetId) - .add(SourceUserIdFeature, candidate.sourceUserId) - .add(SuggestTypeFeature, suggestType) - .add(QuotedTweetIdFeature, candidate.quotedTweetId) - .add( - TweetLanguageFeature, - earlybirdFeatures.flatMap(_.language.map(_.value)).map(getLanguageISOFormatByValue)) - .add(VideoDurationMsFeature, earlybirdFeatures.flatMap(_.videoDurationMs)) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.docx new file mode 100644 index 000000000..46bfcfa6e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala deleted file mode 100644 index c60278bfb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouTweetPreviewsCandidatePipelineConfig.scala +++ /dev/null @@ -1,143 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.filter.DropMaxCandidatesFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetPreviewsFilter -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.product.for_you.feature_hydrator.AuthorEnabledPreviewsFeatureHydrator -import com.twitter.home_mixer.product.for_you.feature_hydrator.TweetPreviewTweetypieCandidateFeatureHydrator -import com.twitter.home_mixer.product.for_you.filter.TweetPreviewTextFilter -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableTweetPreviewsCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMaxCandidatesParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.TweetPreviewsMinInjectionIntervalParam -import com.twitter.home_mixer.product.for_you.query_transformer.TweetPreviewsQueryTransformer -import com.twitter.home_mixer.product.for_you.response_transformer.TweetPreviewResponseFeatureTransformer -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature -import com.twitter.product_mixer.component_library.filter.FeatureFilter -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSourceWithExtractedFeatures -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomeTweetPreviewHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.{thriftscala => st} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouTweetPreviewsCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - authorEnabledPreviewsFeatureHydrator: AuthorEnabledPreviewsFeatureHydrator, - tweetPreviewsQueryTransformer: TweetPreviewsQueryTransformer, - tweetPreviewTweetypieCandidateFeatureHydrator: TweetPreviewTweetypieCandidateFeatureHydrator, - homeFeedbackActionInfoBuilder: HomeFeedbackActionInfoBuilder) - extends CandidatePipelineConfig[ - ForYouQuery, - eb.EarlybirdRequest, - eb.ThriftSearchResult, - TweetCandidate - ] { - - val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ForYouTweetPreviews") - - override val supportedClientParam: Option[FSParam[Boolean]] = - Some(EnableTweetPreviewsCandidatePipelineParam) - - override val gates: Seq[Gate[ForYouQuery]] = { - Seq( - TimelinesPersistenceStoreLastInjectionGate( - TweetPreviewsMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.TweetPreview - ), - NonEmptySeqFeatureGate(PreviewCreatorsFeature) - ) - } - - override val queryTransformer: CandidatePipelineQueryTransformer[ - PipelineQuery, - eb.EarlybirdRequest - ] = tweetPreviewsQueryTransformer - - override val candidateSource: CandidateSourceWithExtractedFeatures[ - eb.EarlybirdRequest, - eb.ThriftSearchResult - ] = earlybirdCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[eb.ThriftSearchResult] - ] = Seq(TweetPreviewResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - eb.ThriftSearchResult, - TweetCandidate - ] = { tweet => TweetCandidate(tweet.id) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ForYouQuery, TweetCandidate, _] - ] = Seq( - authorEnabledPreviewsFeatureHydrator, - tweetPreviewTweetypieCandidateFeatureHydrator, - ) - - override val filters: Seq[ - Filter[ForYouQuery, TweetCandidate] - ] = Seq( - PreviouslyServedTweetPreviewsFilter, - FeatureFilter - .fromFeature(FilterIdentifier("TweetPreviewVisibilityFiltering"), IsHydratedFeature), - FeatureFilter - .fromFeature(FilterIdentifier("AuthorEnabledPreviews"), AuthorEnabledPreviewsFeature), - TweetPreviewTextFilter, - DropMaxCandidatesFilter(TweetPreviewsMaxCandidatesParam) - ) - - override val decorator: Option[CandidateDecorator[PipelineQuery, TweetCandidate]] = { - val component = InjectionScribeUtil.scribeComponent(st.SuggestType.TweetPreview).get - val clientEventInfoBuilder = ClientEventInfoBuilder[PipelineQuery, TweetCandidate](component) - - val tweetItemBuilder = TweetCandidateUrtItemBuilder[PipelineQuery, TweetCandidate]( - clientEventInfoBuilder = clientEventInfoBuilder, - contextualTweetRefBuilder = Some( - ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomeTweetPreviewHydrationSafetyLevel), - outerTweetContext = None - ) - ) - ), - feedbackActionInfoBuilder = Some(homeFeedbackActionInfoBuilder), - ) - - Some(UrtItemCandidateDecorator(tweetItemBuilder)) - } - - override val alerts: Seq[Alert] = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(95)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.docx new file mode 100644 index 000000000..d0131b222 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala deleted file mode 100644 index 856f1fa6b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToFollowCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToFollowFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.model.HomeFeatures.WhoToFollowExcludedUserIdsFeature -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToFollowCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayLocationParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowDisplayTypeIdParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToFollowMinInjectionIntervalParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_follow_module.WhoToFollowArmCandidatePipelineConfigBuilder -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouWhoToFollowCandidatePipelineConfigBuilder @Inject() ( - whoToFollowArmCandidatePipelineConfigBuilder: WhoToFollowArmCandidatePipelineConfigBuilder, - homeWhoToFollowFeedbackActionInfoBuilder: HomeWhoToFollowFeedbackActionInfoBuilder) { - - def build(): WhoToFollowArmCandidatePipelineConfig[ForYouQuery] = { - val gates: Seq[Gate[ForYouQuery]] = Seq( - TimelinesPersistenceStoreLastInjectionGate( - WhoToFollowMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.WhoToFollow - ), - DismissFatigueGate(SuggestType.WhoToFollow, DismissInfoFeature) - ) - - whoToFollowArmCandidatePipelineConfigBuilder.build[ForYouQuery]( - identifier = WhoToFollowArmCandidatePipelineConfig.identifier, - supportedClientParam = Some(EnableWhoToFollowCandidatePipelineParam), - alerts = alerts, - gates = gates, - moduleDisplayTypeBuilder = - ParamWhoToFollowModuleDisplayTypeBuilder(WhoToFollowDisplayTypeIdParam), - feedbackActionInfoBuilder = Some(homeWhoToFollowFeedbackActionInfoBuilder), - excludedUserIdsFeature = Some(WhoToFollowExcludedUserIdsFeature), - displayLocationParam = WhoToFollowDisplayLocationParam - ) - } - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.docx new file mode 100644 index 000000000..58c1a4907 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala deleted file mode 100644 index bd4437b7c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/ForYouWhoToSubscribeCandidatePipelineConfigBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.for_you - -import com.twitter.home_mixer.functional_component.decorator.urt.builder.HomeWhoToSubscribeFeedbackActionInfoBuilder -import com.twitter.home_mixer.functional_component.gate.DismissFatigueGate -import com.twitter.home_mixer.functional_component.gate.TimelinesPersistenceStoreLastInjectionGate -import com.twitter.home_mixer.model.HomeFeatures.DismissInfoFeature -import com.twitter.home_mixer.model.HomeFeatures.PersistenceEntriesFeature -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableWhoToSubscribeCandidatePipelineParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeDisplayTypeIdParam -import com.twitter.home_mixer.product.for_you.param.ForYouParam.WhoToSubscribeMinInjectionIntervalParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.ParamWhoToFollowModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.who_to_subscribe_module.WhoToSubscribeCandidatePipelineConfigBuilder -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.timelineservice.model.rich.EntityIdType -import com.twitter.timelineservice.suggests.thriftscala.SuggestType -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouWhoToSubscribeCandidatePipelineConfigBuilder @Inject() ( - whoToSubscribeCandidatePipelineConfigBuilder: WhoToSubscribeCandidatePipelineConfigBuilder, - homeWhoToSubscribeFeedbackActionInfoBuilder: HomeWhoToSubscribeFeedbackActionInfoBuilder) { - - def build(): WhoToSubscribeCandidatePipelineConfig[ForYouQuery] = { - val gates: Seq[Gate[ForYouQuery]] = Seq( - TimelinesPersistenceStoreLastInjectionGate( - WhoToSubscribeMinInjectionIntervalParam, - PersistenceEntriesFeature, - EntityIdType.WhoToSubscribe - ), - DismissFatigueGate(SuggestType.WhoToSubscribe, DismissInfoFeature) - ) - - whoToSubscribeCandidatePipelineConfigBuilder.build[ForYouQuery]( - identifier = WhoToSubscribeCandidatePipelineConfig.identifier, - supportedClientParam = Some(EnableWhoToSubscribeCandidatePipelineParam), - alerts = alerts, - gates = gates, - moduleDisplayTypeBuilder = - ParamWhoToFollowModuleDisplayTypeBuilder(WhoToSubscribeDisplayTypeIdParam), - feedbackActionInfoBuilder = Some(homeWhoToSubscribeFeedbackActionInfoBuilder) - ) - } - - private val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(70), - HomeMixerAlertConfig.BusinessHours.defaultEmptyResponseRateAlert() - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel deleted file mode 100644 index 540823800..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/product_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "src/thrift/com/twitter/search:earlybird-scala", - "stitch/stitch-timelineservice/src/main/scala", - ], - exports = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.docx new file mode 100644 index 000000000..dff948354 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.docx new file mode 100644 index 000000000..33eaeb4ca Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala deleted file mode 100644 index d1daeb93e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/candidate_source/ScoredTweetsProductCandidateSource.scala +++ /dev/null @@ -1,174 +0,0 @@ -package com.twitter.home_mixer.product.for_you.candidate_source - -import com.google.inject.Provider -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProductContext -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.candidate_source.product_pipeline.ProductPipelineCandidateSource -import com.twitter.product_mixer.core.functional_component.configapi.ParamsBuilder -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.product.registry.ProductPipelineRegistry -import com.twitter.timelines.render.{thriftscala => tl} -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * [[ScoredTweetWithConversationMetadata]] - **/ -case class ScoredTweetWithConversationMetadata( - tweetId: Long, - authorId: Long, - score: Option[Double] = None, - suggestType: Option[st.SuggestType] = None, - sourceTweetId: Option[Long] = None, - sourceUserId: Option[Long] = None, - quotedTweetId: Option[Long] = None, - quotedUserId: Option[Long] = None, - inReplyToTweetId: Option[Long] = None, - inReplyToUserId: Option[Long] = None, - directedAtUserId: Option[Long] = None, - inNetwork: Option[Boolean] = None, - sgsValidLikedByUserIds: Option[Seq[Long]] = None, - sgsValidFollowedByUserIds: Option[Seq[Long]] = None, - ancestors: Option[Seq[ta.TweetAncestor]] = None, - topicId: Option[Long] = None, - topicFunctionalityType: Option[tl.TopicContextFunctionalityType] = None, - conversationId: Option[Long] = None, - conversationFocalTweetId: Option[Long] = None, - isReadFromCache: Option[Boolean] = None, - streamToKafka: Option[Boolean] = None, - exclusiveConversationAuthorId: Option[Long] = None, - authorIsBlueVerified: Option[Boolean] = None, - authorIsGoldVerified: Option[Boolean] = None, - authorIsGrayVerified: Option[Boolean] = None, - authorIsLegacyVerified: Option[Boolean] = None, - authorIsCreator: Option[Boolean] = None, - perspectiveFilteredLikedByUserIds: Option[Seq[Long]] = None) - -@Singleton -class ScoredTweetsProductCandidateSource @Inject() ( - override val productPipelineRegistry: Provider[ProductPipelineRegistry], - override val paramsBuilder: Provider[ParamsBuilder]) - extends ProductPipelineCandidateSource[ - ForYouQuery, - HomeMixerRequest, - t.ScoredTweetsResponse, - ScoredTweetWithConversationMetadata - ] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("ScoredTweetsProduct") - - private val MaxModuleSize = 3 - private val MaxAncestorsInConversation = 2 - - override def pipelineRequestTransformer(productPipelineQuery: ForYouQuery): HomeMixerRequest = { - HomeMixerRequest( - clientContext = productPipelineQuery.clientContext, - product = ScoredTweetsProduct, - productContext = Some( - ScoredTweetsProductContext( - productPipelineQuery.deviceContext, - productPipelineQuery.seenTweetIds, - productPipelineQuery.features.map(_.getOrElse(ServedTweetIdsFeature, Seq.empty)), - productPipelineQuery.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)) - )), - serializedRequestCursor = - productPipelineQuery.pipelineCursor.map(UrtCursorSerializer.serializeCursor), - maxResults = productPipelineQuery.requestedMaxResults, - debugParams = None, - homeRequestParam = false - ) - } - - override def productPipelineResultTransformer( - productPipelineResult: t.ScoredTweetsResponse - ): Seq[ScoredTweetWithConversationMetadata] = { - val scoredTweets = productPipelineResult.scoredTweets.flatMap { focalTweet => - val parentTweets = focalTweet.ancestors.getOrElse(Seq.empty).sortBy(-_.tweetId) - val (intermediates, root) = parentTweets.splitAt(parentTweets.size - 1) - val truncatedIntermediates = - intermediates.take(MaxModuleSize - MaxAncestorsInConversation).reverse - val rootScoredTweet: Seq[ScoredTweetWithConversationMetadata] = root.map { ancestor => - ScoredTweetWithConversationMetadata( - tweetId = ancestor.tweetId, - authorId = ancestor.userId, - suggestType = focalTweet.suggestType, - conversationId = Some(ancestor.tweetId), - conversationFocalTweetId = Some(focalTweet.tweetId), - exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId - ) - } - val conversationId = rootScoredTweet.headOption.map(_.tweetId) - - val tweetsToParents = - if (parentTweets.nonEmpty) parentTweets.zip(parentTweets.tail).toMap - else Map.empty[ta.TweetAncestor, ta.TweetAncestor] - - val intermediateScoredTweets = truncatedIntermediates.map { ancestor => - ScoredTweetWithConversationMetadata( - tweetId = ancestor.tweetId, - authorId = ancestor.userId, - suggestType = focalTweet.suggestType, - inReplyToTweetId = tweetsToParents.get(ancestor).map(_.tweetId), - conversationId = conversationId, - conversationFocalTweetId = Some(focalTweet.tweetId), - exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId - ) - } - val parentScoredTweets = rootScoredTweet ++ intermediateScoredTweets - - val conversationFocalTweetId = - if (parentScoredTweets.nonEmpty) Some(focalTweet.tweetId) else None - - val focalScoredTweet = ScoredTweetWithConversationMetadata( - tweetId = focalTweet.tweetId, - authorId = focalTweet.authorId, - score = focalTweet.score, - suggestType = focalTweet.suggestType, - sourceTweetId = focalTweet.sourceTweetId, - sourceUserId = focalTweet.sourceUserId, - quotedTweetId = focalTweet.quotedTweetId, - quotedUserId = focalTweet.quotedUserId, - inReplyToTweetId = parentScoredTweets.lastOption.map(_.tweetId), - inReplyToUserId = focalTweet.inReplyToUserId, - directedAtUserId = focalTweet.directedAtUserId, - inNetwork = focalTweet.inNetwork, - sgsValidLikedByUserIds = focalTweet.sgsValidLikedByUserIds, - sgsValidFollowedByUserIds = focalTweet.sgsValidFollowedByUserIds, - topicId = focalTweet.topicId, - topicFunctionalityType = focalTweet.topicFunctionalityType, - ancestors = focalTweet.ancestors, - conversationId = conversationId, - conversationFocalTweetId = conversationFocalTweetId, - isReadFromCache = focalTweet.isReadFromCache, - streamToKafka = focalTweet.streamToKafka, - exclusiveConversationAuthorId = focalTweet.exclusiveConversationAuthorId, - authorIsBlueVerified = focalTweet.authorMetadata.map(_.blueVerified), - authorIsGoldVerified = focalTweet.authorMetadata.map(_.goldVerified), - authorIsGrayVerified = focalTweet.authorMetadata.map(_.grayVerified), - authorIsLegacyVerified = focalTweet.authorMetadata.map(_.legacyVerified), - authorIsCreator = focalTweet.authorMetadata.map(_.creator), - perspectiveFilteredLikedByUserIds = focalTweet.perspectiveFilteredLikedByUserIds - ) - - parentScoredTweets :+ focalScoredTweet - } - - val dedupedTweets = scoredTweets.groupBy(_.tweetId).map { - case (_, duplicateAncestors) => duplicateAncestors.maxBy(_.score.getOrElse(0.0)) - } - - // Sort by tweet id to prevent issues with future assumptions of the root being the first - // tweet and the focal being the last tweet in a module. The tweets as a whole do not need - // to be sorted overall, only the relative order within modules must be kept. - dedupedTweets.toSeq.sortBy(_.tweetId) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.docx new file mode 100644 index 000000000..5fcaae04a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala deleted file mode 100644 index f93a5041f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/AuthorEnabledPreviewsFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorEnabledPreviewsFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetCreatorPreferencesOnUserClientColumn - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Hydrates the `AuthorEnabledPreviews` feature for tweets authored by creators by querying the - * `GetCreatorPreferences` Strato column. This feature corresponds to the `previews_enabled` field of that column. - * Given a tweet from a creator, this feature indicates whether that creator has enabled previews - * on their profile. - */ -@Singleton -class AuthorEnabledPreviewsFeatureHydrator @Inject() ( - getCreatorPreferencesOnUserClientColumn: GetCreatorPreferencesOnUserClientColumn) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("AuthorEnabledPreviews") - - override val features: Set[Feature[_, _]] = Set(AuthorEnabledPreviewsFeature) - - private val fetcher = getCreatorPreferencesOnUserClientColumn.fetcher - - private val DefaultAuthorEnabledPreviewsValue = true - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val candidateAuthors = candidates - .map(_.features.getOrElse(AuthorIdFeature, None)) - .toSet - .flatten - - // Build a map of creator -> authorEnabledPreviews, then use it to populate candidate features - val authorIdToFeatureStitch = Stitch.collect { - candidateAuthors - .map { author => - val isAuthorEnabledPreviews = fetcher.fetch(author).map { - _.v.map(_.previewsEnabled).getOrElse(DefaultAuthorEnabledPreviewsValue) - } - (author, isAuthorEnabledPreviews) - }.toMap - } - - authorIdToFeatureStitch.map { authorIdToFeatureMap => - candidates.map { - _.features.getOrElse(AuthorIdFeature, None) match { - case Some(authorId) => FeatureMapBuilder() - .add(AuthorEnabledPreviewsFeature, authorIdToFeatureMap(authorId)) - .build() - case _ => FeatureMapBuilder() - .add(AuthorEnabledPreviewsFeature, DefaultAuthorEnabledPreviewsValue) - .build() - } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel deleted file mode 100644 index 87a260b7e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "audience-rewards/thrift/src/main/thrift/common:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/tweet_tweetypie", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-core", - "stitch/stitch-socialgraph", - "stitch/stitch-timelineservice", - "strato/config/columns/audiencerewards/audienceRewardsService:audienceRewardsService-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - "timelinemixer/common/src/main/scala/com/twitter/timelinemixer/clients/feedback", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "util/util-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.docx new file mode 100644 index 000000000..ea474c77a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.docx new file mode 100644 index 000000000..2cdab46de Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala deleted file mode 100644 index 0270a81ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/FocalTweetFeatureHydrator.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetInNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetRealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.FocalTweetScreenNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.RealNamesFeature -import com.twitter.home_mixer.model.HomeFeatures.ScreenNamesFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Social context for convo modules is hydrated on the root Tweet but needs info about the focal - * Tweet (e.g. author) to render the banner. This hydrator copies focal Tweet data into the root. - */ -@Singleton -class FocalTweetFeatureHydrator @Inject() () - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FocalTweet") - - override val features: Set[Feature[_, _]] = Set( - FocalTweetAuthorIdFeature, - FocalTweetInNetworkFeature, - FocalTweetRealNamesFeature, - FocalTweetScreenNamesFeature - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(FocalTweetAuthorIdFeature, None) - .add(FocalTweetInNetworkFeature, None) - .add(FocalTweetRealNamesFeature, None) - .add(FocalTweetScreenNamesFeature, None) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - // Build a map of all the focal tweets to their corresponding features - val focalTweetIdToFeatureMap = candidates.flatMap { candidate => - val focalTweetId = candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None) - if (focalTweetId.contains(candidate.candidate.id)) { - Some(candidate.candidate.id -> candidate.features) - } else None - }.toMap - - val updatedFeatureMap = candidates.map { candidate => - val focalTweetId = candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None) - val conversationId = candidate.features.getOrElse(ConversationModuleIdFeature, None) - - // Check if the candidate is a root tweet and ensure its focal tweet's features are available - if (conversationId.contains(candidate.candidate.id) - && focalTweetId.exists(focalTweetIdToFeatureMap.contains)) { - val featureMap = focalTweetIdToFeatureMap.get(focalTweetId.get).get - FeatureMapBuilder() - .add(FocalTweetAuthorIdFeature, featureMap.getOrElse(AuthorIdFeature, None)) - .add(FocalTweetInNetworkFeature, Some(featureMap.getOrElse(InNetworkFeature, true))) - .add( - FocalTweetRealNamesFeature, - Some(featureMap.getOrElse(RealNamesFeature, Map.empty[Long, String]))) - .add( - FocalTweetScreenNamesFeature, - Some(featureMap.getOrElse(ScreenNamesFeature, Map.empty[Long, String]))) - .build() - } else DefaultFeatureMap - } - - Stitch.value(updatedFeatureMap) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.docx new file mode 100644 index 000000000..dd555e84e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala deleted file mode 100644 index e0ae2207e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TimelineServiceTweetsQueryFeatureHydrator.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.marshaller.timelines.DeviceContextMarshaller -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class TimelineServiceTweetsQueryFeatureHydrator @Inject() ( - timelineService: TimelineService, - deviceContextMarshaller: DeviceContextMarshaller) - extends QueryFeatureHydrator[PipelineQuery with HasDeviceContext] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TimelineServiceTweets") - - override val features: Set[Feature[_, _]] = Set(TimelineServiceTweetsFeature) - - private val MaxTimelineServiceTweets = 200 - - override def hydrate(query: PipelineQuery with HasDeviceContext): Stitch[FeatureMap] = { - val deviceContext = query.deviceContext.getOrElse(DeviceContext.Empty) - - val timelineQueryOptions = t.TimelineQueryOptions( - contextualUserId = query.clientContext.userId, - deviceContext = Some(deviceContextMarshaller(deviceContext, query.clientContext)) - ) - - val timelineServiceQuery = t.TimelineQuery( - timelineType = t.TimelineType.Home, - timelineId = query.getRequiredUserId, - maxCount = MaxTimelineServiceTweets.toShort, - cursor2 = None, - options = Some(timelineQueryOptions), - timelineId2 = query.clientContext.userId.map(t.TimelineId(t.TimelineType.Home, _, None)), - ) - - timelineService.getTimeline(timelineServiceQuery).map { timeline => - val tweets = timeline.entries.collect { - case t.TimelineEntry.Tweet(tweet) => tweet.statusId - } - - FeatureMapBuilder().add(TimelineServiceTweetsFeature, tweets).build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.docx new file mode 100644 index 000000000..4187d4554 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala deleted file mode 100644 index 07f3ae0e9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/feature_hydrator/TweetPreviewTweetypieCandidateFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.for_you.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsHydratedFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.product_mixer.component_library.model.candidate.BaseTweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.spam.rtf.{thriftscala => rtf} -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.{thriftscala => TP} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetPreviewTweetypieCandidateFeatureHydrator @Inject() ( - tweetypieStitchClient: TweetypieStitchClient) - extends CandidateFeatureHydrator[PipelineQuery, BaseTweetCandidate] { - - private val CoreTweetFields: Set[TP.TweetInclude] = Set[TP.TweetInclude]( - TP.TweetInclude.TweetFieldId(TP.Tweet.IdField.id), - TP.TweetInclude.TweetFieldId(TP.Tweet.CoreDataField.id) - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(TweetTextFeature, None) - .add(IsHydratedFeature, false) - .add(AuthorIdFeature, None) - .build() - - override val features: Set[Feature[_, _]] = - Set(TweetTextFeature, IsHydratedFeature, AuthorIdFeature) - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TweetPreviewTweetypie") - - override def apply( - query: PipelineQuery, - candidate: BaseTweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - tweetypieStitchClient - .getTweetFields( - tweetId = candidate.id, - options = TP.GetTweetFieldsOptions( - tweetIncludes = CoreTweetFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - visibilityPolicy = TP.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(rtf.SafetyLevel.TimelineHomeTweetPreview), - forUserId = query.getOptionalUserId - ) - ).map { - case TP.GetTweetFieldsResult(_, TP.TweetFieldsResultState.Found(found), quoteOpt, _) => - val tweetText = found.tweet.coreData.map(_.text) - FeatureMapBuilder() - .add(TweetTextFeature, tweetText) - .add(IsHydratedFeature, true) - .add(AuthorIdFeature, found.tweet.coreData.map(_.userId)) - .build() - // If no tweet result found, return default features - case _ => - DefaultFeatureMap - } - - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel deleted file mode 100644 index 0a875371d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - "stitch/stitch-core", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.docx new file mode 100644 index 000000000..b6dd510de Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.docx new file mode 100644 index 000000000..b76c3022e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala deleted file mode 100644 index 5b007969b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/SocialContextFilter.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.for_you.filter - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.for_you.param.ForYouParam -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object SocialContextFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("SocialContext") - - // Tweets from candidate sources which don't need generic like/follow/topic proof - private val AllowedSources: Set[st.SuggestType] = Set( - st.SuggestType.RankedListTweet, - st.SuggestType.RecommendedTrendTweet, - st.SuggestType.MediaTweet - ) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - val enableIsVerifiedAuthorFilter = - query.params(ForYouParam.EnableVerifiedAuthorSocialContextBypassParam) - - val enableTopicSocialContextFilter = - query.params(ForYouParam.EnableTopicSocialContextFilterParam) - - val validTweetIds = candidates - .filter { candidate => - candidate.features.getOrElse(InNetworkFeature, true) || - candidate.features.getOrElse(SuggestTypeFeature, None).exists(AllowedSources.contains) || - candidate.features.getOrElse(ConversationModuleFocalTweetIdFeature, None).isDefined || - (enableIsVerifiedAuthorFilter && isVerifiedAuthor(candidate.features)) || - hasLikedBySocialContext(candidate.features) || - hasFollowedBySocialContext(candidate.features) || - (enableTopicSocialContextFilter && hasTopicSocialContext(candidate.features)) - }.map(_.candidate.id).toSet - - val (kept, removed) = - candidates.map(_.candidate).partition(candidate => validTweetIds.contains(candidate.id)) - - Stitch.value(FilterResult(kept = kept, removed = removed)) - } - - private def isVerifiedAuthor(candidateFeatures: FeatureMap): Boolean = { - candidateFeatures.getOrElse(AuthorIsBlueVerifiedFeature, false) || - candidateFeatures.getOrElse(AuthorIsGoldVerifiedFeature, false) || - candidateFeatures.getOrElse(AuthorIsGrayVerifiedFeature, false) || - candidateFeatures.getOrElse(AuthorIsLegacyVerifiedFeature, false) - } - - private def hasLikedBySocialContext(candidateFeatures: FeatureMap): Boolean = - candidateFeatures - .getOrElse(SGSValidLikedByUserIdsFeature, Seq.empty) - .exists( - candidateFeatures - .getOrElse(PerspectiveFilteredLikedByUserIdsFeature, Seq.empty) - .toSet.contains - ) - - private def hasFollowedBySocialContext(candidateFeatures: FeatureMap): Boolean = - candidateFeatures.getOrElse(SGSValidFollowedByUserIdsFeature, Seq.empty).nonEmpty - - private def hasTopicSocialContext(candidateFeatures: FeatureMap): Boolean = { - candidateFeatures.getOrElse(TopicIdSocialContextFeature, None).isDefined && - candidateFeatures.getOrElse(TopicContextFunctionalityTypeFeature, None).isDefined - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.docx new file mode 100644 index 000000000..28cffe395 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala deleted file mode 100644 index 61125fcd3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/filter/TweetPreviewTextFilter.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.product.for_you.filter - -import com.twitter.home_mixer.model.HomeFeatures.TweetTextFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object TweetPreviewTextFilter extends Filter[PipelineQuery, TweetCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("TweetPreviewText") - - private val PreviewTextLength = 50 - private val MinTweetLength = PreviewTextLength * 2 - private val MaxNewlines = 2 - private val HttpPrefix = "http://" - private val HttpsPrefix = "https://" - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[FilterResult[TweetCandidate]] = { - - val (kept, removed) = candidates - .partition { candidate => - val text = candidate.features.get(TweetTextFeature).getOrElse("") - - text.length > MinTweetLength && - text.take(PreviewTextLength).count(_ == '\n') <= MaxNewlines && - !(text.startsWith(HttpPrefix) || text.startsWith(HttpsPrefix)) - } - - val filterResult = FilterResult( - kept = kept.map(_.candidate), - removed = removed.map(_.candidate) - ) - - Stitch.value(filterResult) - } - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel deleted file mode 100644 index 9a1d186dd..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/common", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.docx new file mode 100644 index 000000000..a19c4a553 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.docx new file mode 100644 index 000000000..9298c55ad Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala deleted file mode 100644 index 4c9d81021..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/functional_component/gate/PushToHomeRequestGate.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.home_mixer.product.for_you.functional_component.gate - -import com.twitter.home_mixer.product.for_you.model.ForYouQuery -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.stitch.Stitch - -/** - * Continues when the request is a Push-To-Home notification request - */ -object PushToHomeRequestGate extends Gate[ForYouQuery] { - override val identifier: GateIdentifier = GateIdentifier("PushToHomeRequest") - - override def shouldContinue(query: ForYouQuery): Stitch[Boolean] = - Stitch.value(query.pushToHomeTweetId.isDefined) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel deleted file mode 100644 index bcf9519f5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/quality_factor", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/flexible_injection_pipeline", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.docx new file mode 100644 index 000000000..7a7481903 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.docx new file mode 100644 index 000000000..36b2bf783 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala deleted file mode 100644 index dda427350..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouQuery.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.for_you.model - -import com.twitter.adserver.thriftscala.HomeTimelineType -import com.twitter.adserver.thriftscala.TimelineRequestParams -import com.twitter.dspbidder.commons.{thriftscala => dsp} -import com.twitter.home_mixer.model.HomeAdsQuery -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.ForYouProduct -import com.twitter.home_mixer.model.request.HasDeviceContext -import com.twitter.home_mixer.model.request.HasSeenTweetIds -import com.twitter.onboarding.task.service.{thriftscala => ots} -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.pipeline.candidate.flexible_injection_pipeline.transformer.HasFlipInjectionParams -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class ForYouQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val deviceContext: Option[DeviceContext], - override val seenTweetIds: Option[Seq[Long]], - override val dspClientContext: Option[dsp.DspClientContext], - pushToHomeTweetId: Option[Long]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasDeviceContext - with HasSeenTweetIds - with HasFlipInjectionParams - with HomeAdsQuery { - override val product: Product = ForYouProduct - - override def withFeatureMap(features: FeatureMap): ForYouQuery = - copy(features = Some(features)) - - override val timelineRequestParams: Option[TimelineRequestParams] = - Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.Home))) - - // Fields below are used for FLIP Injection in Onboarding Task Service (OTS) - override val displayLocation: ots.DisplayLocation = ots.DisplayLocation.HomeTimeline - override val rankingDisablerWithLatestControlsAvailable: Option[Boolean] = None - override val isEmptyState: Option[Boolean] = None - override val isFirstRequestAfterSignup: Option[Boolean] = None - override val isEndOfTimeline: Option[Boolean] = None - override val timelineId: Option[Long] = None -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.docx new file mode 100644 index 000000000..5789aaa56 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.scala deleted file mode 100644 index 68b0d6736..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/model/ForYouTweetsResponse.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.home_mixer.product.for_you.model - -import com.twitter.product_mixer.core.model.marshalling.HasMarshalling - -case class ForYouTweetsResponse(tweetCandidates: Seq[Long]) extends HasMarshalling diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel deleted file mode 100644 index a56e3a1fd..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "util/util-core/src/main/scala/com/twitter/conversions", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.docx new file mode 100644 index 000000000..c0cd1080c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.docx new file mode 100644 index 000000000..6d94342fb Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala deleted file mode 100644 index 5d117199f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParam.scala +++ /dev/null @@ -1,215 +0,0 @@ -package com.twitter.home_mixer.product.for_you.param - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.WhoToFollowModuleDisplayType -import com.twitter.timelines.configapi.DurationConversion -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSEnumParam -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.HasDurationConversion -import com.twitter.timelines.configapi.decider.BooleanDeciderParam -import com.twitter.util.Duration - -object ForYouParam { - val SupportedClientFSName = "for_you_supported_client" - - object EnableTopicSocialContextFilterParam - extends FSParam[Boolean]( - name = "for_you_enable_topic_social_context_filter", - default = true - ) - - object EnableVerifiedAuthorSocialContextBypassParam - extends FSParam[Boolean]( - name = "for_you_enable_verified_author_social_context_bypass", - default = true - ) - - object EnableTimelineScorerCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_timeline_scorer_candidate_pipeline", - default = false - ) - - object EnableScoredTweetsCandidatePipelineParam - extends BooleanDeciderParam(DeciderKey.EnableForYouScoredTweetsCandidatePipeline) - - object EnableWhoToFollowCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_who_to_follow", - default = true - ) - - object EnableWhoToSubscribeCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_who_to_subscribe", - default = true - ) - - object EnableTweetPreviewsCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_tweet_previews_candidate_pipeline", - default = true - ) - - object EnablePushToHomeMixerPipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_push_to_home_mixer_pipeline", - default = false - ) - - object EnableScoredTweetsMixerPipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_scored_tweets_mixer_pipeline", - default = true - ) - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "for_you_server_max_results", - default = 35, - min = 1, - max = 500 - ) - - object AdsNumOrganicItemsParam - extends FSBoundedParam[Int]( - name = "for_you_ads_num_organic_items", - default = 35, - min = 1, - max = 100 - ) - - object WhoToFollowPositionParam - extends FSBoundedParam[Int]( - name = "for_you_who_to_follow_position", - default = 5, - min = 0, - max = 99 - ) - - object WhoToFollowMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "for_you_who_to_follow_min_injection_interval_in_minutes", - default = 1800.minutes, - min = 0.minutes, - max = 6000.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object WhoToFollowDisplayTypeIdParam - extends FSEnumParam[WhoToFollowModuleDisplayType.type]( - name = "for_you_enable_who_to_follow_display_type_id", - default = WhoToFollowModuleDisplayType.Vertical, - enum = WhoToFollowModuleDisplayType - ) - - object WhoToFollowDisplayLocationParam - extends FSParam[String]( - name = "for_you_who_to_follow_display_location", - default = "timeline" - ) - - object WhoToSubscribePositionParam - extends FSBoundedParam[Int]( - name = "for_you_who_to_subscribe_position", - default = 7, - min = 0, - max = 99 - ) - - object WhoToSubscribeMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "for_you_who_to_subscribe_min_injection_interval_in_minutes", - default = 1800.minutes, - min = 0.minutes, - max = 6000.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object WhoToSubscribeDisplayTypeIdParam - extends FSEnumParam[WhoToFollowModuleDisplayType.type]( - name = "for_you_enable_who_to_subscribe_display_type_id", - default = WhoToFollowModuleDisplayType.Vertical, - enum = WhoToFollowModuleDisplayType - ) - - object TweetPreviewsPositionParam - extends FSBoundedParam[Int]( - name = "for_you_tweet_previews_position", - default = 3, - min = 0, - max = 99 - ) - - object TweetPreviewsMinInjectionIntervalParam - extends FSBoundedParam[Duration]( - "for_you_tweet_previews_min_injection_interval_in_minutes", - default = 2.hours, - min = 0.minutes, - max = 600.minutes) - with HasDurationConversion { - override val durationConversion: DurationConversion = DurationConversion.FromMinutes - } - - object TweetPreviewsMaxCandidatesParam - extends FSBoundedParam[Int]( - name = "for_you_tweet_previews_max_candidates", - default = 1, - min = 0, - // NOTE: previews are injected at a fixed position, so max candidates = 1 - // to avoid bunching of previews. - max = 1 - ) - - object EnableFlipInjectionModuleCandidatePipelineParam - extends FSParam[Boolean]( - name = "for_you_enable_flip_inline_injection_module", - default = true - ) - - object FlipInlineInjectionModulePosition - extends FSBoundedParam[Int]( - name = "for_you_flip_inline_injection_module_position", - default = 0, - min = 0, - max = 1000 - ) - - object ClearCacheOnPtr { - object EnableParam - extends FSParam[Boolean]( - name = "for_you_clear_cache_ptr_enable", - default = false - ) - - case object MinEntriesParam - extends FSBoundedParam[Int]( - name = "for_you_clear_cache_ptr_min_entries", - default = 10, - min = 0, - max = 35 - ) - } - - object EnableClearCacheOnPushToHome - extends FSParam[Boolean]( - name = "for_you_enable_clear_cache_push_to_home", - default = false - ) - - object EnableServedCandidateKafkaPublishingParam - extends FSParam[Boolean]( - name = "for_you_enable_served_candidate_kafka_publishing", - default = true - ) - - object ExperimentStatsParam - extends FSParam[String]( - name = "for_you_experiment_stats", - default = "" - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.docx new file mode 100644 index 000000000..f60566547 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala deleted file mode 100644 index 001ee57ad..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param/ForYouParamConfig.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.home_mixer.product.for_you.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.for_you.param.ForYouParam._ -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ForYouParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableForYouProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val booleanDeciderOverrides = Seq( - EnableScoredTweetsCandidatePipelineParam - ) - - override val booleanFSOverrides = Seq( - ClearCacheOnPtr.EnableParam, - EnableFlipInjectionModuleCandidatePipelineParam, - EnablePushToHomeMixerPipelineParam, - EnableScoredTweetsMixerPipelineParam, - EnableServedCandidateKafkaPublishingParam, - EnableTimelineScorerCandidatePipelineParam, - EnableTopicSocialContextFilterParam, - EnableVerifiedAuthorSocialContextBypassParam, - EnableWhoToFollowCandidatePipelineParam, - EnableWhoToSubscribeCandidatePipelineParam, - EnableTweetPreviewsCandidatePipelineParam, - EnableClearCacheOnPushToHome - ) - - override val boundedIntFSOverrides = Seq( - AdsNumOrganicItemsParam, - ClearCacheOnPtr.MinEntriesParam, - FlipInlineInjectionModulePosition, - ServerMaxResultsParam, - WhoToFollowPositionParam, - WhoToSubscribePositionParam, - TweetPreviewsPositionParam, - TweetPreviewsMaxCandidatesParam - ) - - override val stringFSOverrides = Seq( - WhoToFollowDisplayLocationParam, - ExperimentStatsParam - ) - - override val boundedDurationFSOverrides = Seq( - WhoToFollowMinInjectionIntervalParam, - WhoToSubscribeMinInjectionIntervalParam, - TweetPreviewsMinInjectionIntervalParam - ) - - override val enumFSOverrides = Seq( - WhoToFollowDisplayTypeIdParam, - WhoToSubscribeDisplayTypeIdParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel deleted file mode 100644 index 17e99c46c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "src/java/com/twitter/search/common/schema/earlybird", - "src/java/com/twitter/search/queryparser/query:core-query-nodes", - "src/java/com/twitter/search/queryparser/query/search:search-query-nodes", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.docx new file mode 100644 index 000000000..dc077532a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.docx new file mode 100644 index 000000000..47b7ef922 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala deleted file mode 100644 index 41940b823..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/query_transformer/TweetPreviewsQueryTransformer.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.home_mixer.product.for_you.query_transformer - -import com.twitter.conversions.DurationOps.richDurationFromInt -import com.twitter.finagle.thrift.ClientId -import com.twitter.finagle.tracing.Trace -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.PreviewCreatorsFeature -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.ranking.{thriftscala => scr} -import com.twitter.search.common.schema.earlybird.EarlybirdFieldConstants.EarlybirdFieldConstant -import com.twitter.search.earlybird.{thriftscala => t} -import com.twitter.search.queryparser.query.Conjunction -import com.twitter.search.queryparser.query.Query -import com.twitter.search.queryparser.query.search.SearchOperator -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetPreviewsQueryTransformer @Inject() (clientId: ClientId) - extends CandidatePipelineQueryTransformer[PipelineQuery, t.EarlybirdRequest] { - - private val MaxPreviewTweets = 200 - private val EarlybirdRelevanceTensorflowModel = "timelines_rectweet_replica" - private val SinceDuration = 7.days - - val MetadataOptions = t.ThriftSearchResultMetadataOptions( - getReferencedTweetAuthorId = true, - getFromUserId = true - ) - - override def transform(query: PipelineQuery): t.EarlybirdRequest = { - val candidatePreviewCreatorIds = - query.features.map(_.get(PreviewCreatorsFeature)).getOrElse(Seq.empty) - - val searchQuery = new Conjunction( - // Include subscriber only (aka exclusive) Tweets - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.EXCLUSIVE_FILTER_TERM) - .build(), - // Include only original Tweets - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.NATIVE_RETWEETS_FILTER_TERM) - .setOccur(Query.Occur.MUST_NOT) - .build(), - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.REPLIES_FILTER_TERM) - .setOccur(Query.Occur.MUST_NOT) - .build(), - new SearchOperator.Builder() - .setType(SearchOperator.Type.FILTER) - .addOperand(EarlybirdFieldConstant.QUOTE_FILTER_TERM) - .setOccur(Query.Occur.MUST_NOT) - .build(), - new SearchOperator(SearchOperator.Type.SINCE_TIME, SinceDuration.ago.inSeconds.toString) - ) - - t.EarlybirdRequest( - searchQuery = t.ThriftSearchQuery( - serializedQuery = Some(searchQuery.serialize), - fromUserIDFilter64 = Some(candidatePreviewCreatorIds), - numResults = MaxPreviewTweets, - rankingMode = t.ThriftSearchRankingMode.Relevance, - relevanceOptions = Some( - t.ThriftSearchRelevanceOptions( - filterDups = true, - keepDupWithHigherScore = true, - proximityScoring = true, - maxConsecutiveSameUser = Some(5), - rankingParams = Some( - scr.ThriftRankingParams( - `type` = Some(scr.ThriftScoringFunctionType.TensorflowBased), - selectedTensorflowModel = Some(EarlybirdRelevanceTensorflowModel), - minScore = -1.0e100, - applyBoosts = false, - ) - ), - ), - ), - resultMetadataOptions = Some(MetadataOptions), - searcherId = query.getOptionalUserId, - ), - getOlderResults = Some(true), // needed for archive access to older tweets - clientRequestID = Some(s"${Trace.id.traceId}"), - followedUserIds = Some(candidatePreviewCreatorIds.toSeq), - numResultsToReturnAtRoot = Some(MaxPreviewTweets), - clientId = Some(clientId.name), - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel deleted file mode 100644 index cf629b665..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "src/java/com/twitter/search/common/schema/earlybird", - "src/thrift/com/twitter/search:earlybird-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.docx new file mode 100644 index 000000000..c5e5c38ae Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.docx new file mode 100644 index 000000000..2458a5367 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala deleted file mode 100644 index 8b783db38..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/response_transformer/TweetPreviewResponseFeatureTransformer.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.home_mixer.product.for_you.response_transformer - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsTweetPreviewFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.timelineservice.suggests.{thriftscala => st} -import com.twitter.search.earlybird.{thriftscala => eb} - -object TweetPreviewResponseFeatureTransformer - extends CandidateFeatureTransformer[eb.ThriftSearchResult] { - - override val identifier: TransformerIdentifier = - TransformerIdentifier("TweetPreviewResponse") - - override val features: Set[Feature[_, _]] = - Set(AuthorIdFeature, IsTweetPreviewFeature, SuggestTypeFeature) - - def transform( - input: eb.ThriftSearchResult - ): FeatureMap = { - FeatureMapBuilder() - .add(IsTweetPreviewFeature, true) - .add(SuggestTypeFeature, Some(st.SuggestType.TweetPreview)) - .add(AuthorIdFeature, input.metadata.map(_.fromUserId)) - .build() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel deleted file mode 100644 index c320c8838..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "timelineservice/common:model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.docx new file mode 100644 index 000000000..b8281c019 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/scorer/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel deleted file mode 100644 index 90faf3d45..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "kafka/finagle-kafka/finatra-kafka/src/main/scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/timelines/served_candidates_logging:served_candidates_logging-scala", - "src/thrift/com/twitter/timelines/suggests/common:poly_data_record-java", - "timelines/ml:kafka", - "timelines/ml/cont_train/common/client/src/main/scala/com/twitter/timelines/ml/cont_train/common/client/kafka", - "timelines/ml/cont_train/common/domain/src/main/scala/com/twitter/timelines/ml/cont_train/common/domain/non_scalding", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.docx new file mode 100644 index 000000000..8d8bc46e4 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.docx new file mode 100644 index 000000000..b820040ae Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala deleted file mode 100644 index bfa7dd52e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffect.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature -import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ServedCandidateFeatureKeysAdapter -import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.ServedCandidateFeatureKeysFields -import com.twitter.timelines.ml.kafka.serde.CandidateFeatureKeySerde -import com.twitter.timelines.ml.kafka.serde.TBaseSerde -import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} -import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.timelineservice.suggests.{thriftscala => tls} -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.serialization.Serializer -import scala.collection.JavaConverters._ - -/** - * Pipeline side-effect that publishes candidate keys to a Kafka topic. - */ -class ServedCandidateFeatureKeysKafkaSideEffect( - topic: String, - sourceIdentifiers: Set[identifier.CandidatePipelineIdentifier]) - extends KafkaPublishingSideEffect[ - sc.CandidateFeatureKey, - pldr.PolyDataRecord, - PipelineQuery, - Timeline - ] - with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] { - - import ServedCandidateKafkaSideEffect._ - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedCandidateFeatureKeys") - - override def onlyIf( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) - - override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" - - override val keySerde: Serializer[sc.CandidateFeatureKey] = - CandidateFeatureKeySerde().serializer() - - override val valueSerde: Serializer[pldr.PolyDataRecord] = - TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer - - override val clientId: String = "home_mixer_served_candidate_feature_keys_producer" - - override def buildRecords( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[ProducerRecord[sc.CandidateFeatureKey, pldr.PolyDataRecord]] = { - val servedRequestIdOpt = - query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) - - extractCandidates(query, selectedCandidates, sourceIdentifiers).map { candidate => - val isReadFromCache = candidate.features.getOrElse(IsReadFromCacheFeature, false) - val servedId = candidate.features.get(ServedIdFeature).get - - val key = sc.CandidateFeatureKey( - tweetId = candidate.candidateIdLong, - viewerId = query.getRequiredUserId, - servedId = servedId) - - val record = - ServedCandidateFeatureKeysAdapter - .adaptToDataRecords( - ServedCandidateFeatureKeysFields( - candidateTweetSourceId = candidate.features - .getOrElse(CandidateSourceIdFeature, None).map(_.value.toLong).getOrElse(2L), - predictionRequestId = - candidate.features.getOrElse(PredictionRequestIdFeature, None).get, - servedRequestIdOpt = if (isReadFromCache) servedRequestIdOpt else None, - servedId = servedId, - injectionModuleName = candidate.getClass.getSimpleName, - viewerFollowsOriginalAuthor = - Some(candidate.features.getOrElse(InNetworkFeature, true)), - suggestType = candidate.features - .getOrElse(SuggestTypeFeature, None).getOrElse(tls.SuggestType.RankedOrganicTweet), - finalPositionIndex = Some(candidate.sourcePosition), - isReadFromCache = isReadFromCache - )).asScala.head - - new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record)) - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.docx new file mode 100644 index 000000000..2aa60500d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala deleted file mode 100644 index 5e4164276..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateFeatureKeysKafkaSideEffectBuilder.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ServedCandidateFeatureKeysKafkaSideEffectBuilder @Inject() ( - injectedServiceIdentifier: ServiceIdentifier) { - def build( - sourceIdentifiers: Set[CandidatePipelineIdentifier] - ): ServedCandidateFeatureKeysKafkaSideEffect = { - val topic = injectedServiceIdentifier.environment.toLowerCase match { - case "prod" => "tq_ct_served_candidate_feature_keys" - case _ => "tq_ct_served_candidate_feature_keys_staging" - } - new ServedCandidateFeatureKeysKafkaSideEffect(topic, sourceIdentifiers) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.docx new file mode 100644 index 000000000..6edc6b62a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala deleted file mode 100644 index 7da751764..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKafkaSideEffect.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature -import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.StreamToKafkaFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ModuleCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.PipelineQuery - -object ServedCandidateKafkaSideEffect { - - def extractCandidates( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - sourceIdentifiers: Set[CandidatePipelineIdentifier] - ): Seq[ItemCandidateWithDetails] = { - val servedRequestIdOpt = - query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) - - selectedCandidates.iterator - .filter(candidate => sourceIdentifiers.contains(candidate.source)) - .flatMap { - case item: ItemCandidateWithDetails => Seq(item) - case module: ModuleCandidateWithDetails => module.candidates - } - .filter(candidate => candidate.features.getOrElse(StreamToKafkaFeature, false)) - .map { candidate => - val servedId = - if (candidate.features.getOrElse(IsReadFromCacheFeature, false) && - servedRequestIdOpt.nonEmpty) - servedRequestIdOpt - else - candidate.features.getOrElse(PredictionRequestIdFeature, None) - - candidate.copy(features = candidate.features + (ServedIdFeature, servedId)) - }.toSeq - // deduplicate by (tweetId, userId, servedId) - .groupBy { candidate => - ( - candidate.candidateIdLong, - query.getRequiredUserId, - candidate.features.getOrElse(ServedIdFeature, None)) - }.values.map(_.head).toSeq - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.docx new file mode 100644 index 000000000..f3e5e161c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala deleted file mode 100644 index aa1247fbb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffect.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.home_mixer.model.HomeFeatures.IsReadFromCacheFeature -import com.twitter.home_mixer.model.HomeFeatures.PredictionRequestIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ServedRequestIdFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.EnableServedCandidateKafkaPublishingParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.product_mixer.component_library.side_effect.KafkaPublishingSideEffect -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.ml.cont_train.common.domain.non_scalding.DataRecordLoggingRelatedFeatures.tlmServedKeysFeatureContext -import com.twitter.timelines.ml.kafka.serde.ServedCandidateKeySerde -import com.twitter.timelines.ml.kafka.serde.TBaseSerde -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.served_candidates_logging.{thriftscala => sc} -import com.twitter.timelines.suggests.common.poly_data_record.{thriftjava => pldr} -import com.twitter.util.Time -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.serialization.Serializer - -/** - * Pipeline side-effect that publishes candidate keys to a Kafka topic. - */ -class ServedCandidateKeysKafkaSideEffect( - topic: String, - sourceIdentifiers: Set[CandidatePipelineIdentifier]) - extends KafkaPublishingSideEffect[ - sc.ServedCandidateKey, - pldr.PolyDataRecord, - PipelineQuery, - Timeline - ] - with PipelineResultSideEffect.Conditionally[PipelineQuery, Timeline] { - - import ServedCandidateKafkaSideEffect._ - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedCandidateKeys") - - override def onlyIf( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Boolean = query.params.getBoolean(EnableServedCandidateKafkaPublishingParam) - - override val bootstrapServer: String = "/s/kafka/timeline:kafka-tls" - - override val keySerde: Serializer[sc.ServedCandidateKey] = ServedCandidateKeySerde.serializer() - - override val valueSerde: Serializer[pldr.PolyDataRecord] = - TBaseSerde.Thrift[pldr.PolyDataRecord]().serializer - - override val clientId: String = "home_mixer_served_candidate_keys_producer" - - override def buildRecords( - query: PipelineQuery, - selectedCandidates: Seq[CandidateWithDetails], - remainingCandidates: Seq[CandidateWithDetails], - droppedCandidates: Seq[CandidateWithDetails], - response: Timeline - ): Seq[ProducerRecord[sc.ServedCandidateKey, pldr.PolyDataRecord]] = { - val servedTimestamp = Time.now.inMilliseconds - val servedRequestIdOpt = - query.features.getOrElse(FeatureMap.empty).getOrElse(ServedRequestIdFeature, None) - - extractCandidates(query, selectedCandidates, sourceIdentifiers).collect { - // Only publish non-cached tweets to the ServedCandidateKey topic - case candidate if !candidate.features.getOrElse(IsReadFromCacheFeature, false) => - val key = sc.ServedCandidateKey( - tweetId = candidate.candidateIdLong, - viewerId = query.getRequiredUserId, - servedId = -1L - ) - - val record = SRichDataRecord(new DataRecord, tlmServedKeysFeatureContext) - record.setFeatureValueFromOption( - TimelinesSharedFeatures.PREDICTION_REQUEST_ID, - candidate.features.getOrElse(PredictionRequestIdFeature, None) - ) - record - .setFeatureValueFromOption(TimelinesSharedFeatures.SERVED_REQUEST_ID, servedRequestIdOpt) - record.setFeatureValueFromOption( - TimelinesSharedFeatures.SERVED_ID, - candidate.features.getOrElse(ServedIdFeature, None) - ) - record.setFeatureValueFromOption( - TimelinesSharedFeatures.INJECTION_TYPE, - record.getFeatureValueOpt(TimelinesSharedFeatures.INJECTION_TYPE)) - record.setFeatureValue( - TimelinesSharedFeatures.SERVED_TIMESTAMP, - servedTimestamp - ) - record.record.dropUnknownFeatures() - - new ProducerRecord(topic, key, pldr.PolyDataRecord.dataRecord(record.getRecord)) - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(98.5) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.docx new file mode 100644 index 000000000..8aa036872 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala deleted file mode 100644 index 31ee389b9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedCandidateKeysKafkaSideEffectBuilder.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.finagle.mtls.authentication.ServiceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ServedCandidateKeysKafkaSideEffectBuilder @Inject() ( - injectedServiceIdentifier: ServiceIdentifier) { - def build( - sourceIdentifiers: Set[CandidatePipelineIdentifier] - ): ServedCandidateKeysKafkaSideEffect = { - val topic = injectedServiceIdentifier.environment.toLowerCase match { - case "prod" => "tq_ct_served_candidate_keys" - case _ => "tq_ct_served_candidate_keys_staging" - } - new ServedCandidateKeysKafkaSideEffect(topic, sourceIdentifiers) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.docx new file mode 100644 index 000000000..dced8abb1 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala deleted file mode 100644 index be7c2a533..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/for_you/side_effect/ServedStatsSideEffect.scala +++ /dev/null @@ -1,90 +0,0 @@ -package com.twitter.home_mixer.product.for_you.side_effect - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SuggestTypeFeature -import com.twitter.home_mixer.product.for_you.param.ForYouParam.ExperimentStatsParam -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.SideEffectIdentifier -import com.twitter.product_mixer.core.model.common.presentation.CandidateWithDetails -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ServedStatsSideEffect @Inject() (statsReceiver: StatsReceiver) - extends PipelineResultSideEffect[PipelineQuery, Timeline] { - - override val identifier: SideEffectIdentifier = SideEffectIdentifier("ServedStats") - - private val baseStatsReceiver = statsReceiver.scope(identifier.toString) - private val suggestTypeStatsReceiver = baseStatsReceiver.scope("SuggestType") - private val responseSizeStatsReceiver = baseStatsReceiver.scope("ResponseSize") - private val contentBalanceStatsReceiver = baseStatsReceiver.scope("ContentBalance") - - private val inNetworkStatsReceiver = contentBalanceStatsReceiver.scope("InNetwork") - private val outOfNetworkStatsReceiver = contentBalanceStatsReceiver.scope("OutOfNetwork") - private val replyStatsReceiver = contentBalanceStatsReceiver.scope("Reply") - private val originalStatsReceiver = contentBalanceStatsReceiver.scope("Original") - - private val emptyStatsReceiver = responseSizeStatsReceiver.scope("Empty") - private val lessThan5StatsReceiver = responseSizeStatsReceiver.scope("LessThan5") - private val lessThan10StatsReceiver = responseSizeStatsReceiver.scope("LessThan10") - - override def apply( - inputs: PipelineResultSideEffect.Inputs[PipelineQuery, Timeline] - ): Stitch[Unit] = { - val tweetCandidates = CandidatesUtil - .getItemCandidates(inputs.selectedCandidates).filter(_.isCandidateType[TweetCandidate]()) - - val expBucket = inputs.query.params(ExperimentStatsParam) - - recordSuggestTypeStats(tweetCandidates, expBucket) - recordContentBalanceStats(tweetCandidates, expBucket) - recordResponseSizeStats(tweetCandidates, expBucket) - Stitch.Unit - } - - def recordSuggestTypeStats( - candidates: Seq[ItemCandidateWithDetails], - expBucket: String - ): Unit = { - candidates.groupBy(getSuggestType).foreach { - case (suggestType, suggestTypeCandidates) => - suggestTypeStatsReceiver - .scope(expBucket).counter(suggestType).incr(suggestTypeCandidates.size) - } - } - - def recordContentBalanceStats( - candidates: Seq[ItemCandidateWithDetails], - expBucket: String - ): Unit = { - val (in, oon) = candidates.partition(_.features.getOrElse(InNetworkFeature, true)) - inNetworkStatsReceiver.counter(expBucket).incr(in.size) - outOfNetworkStatsReceiver.counter(expBucket).incr(oon.size) - - val (reply, original) = - candidates.partition(_.features.getOrElse(InReplyToTweetIdFeature, None).isDefined) - replyStatsReceiver.counter(expBucket).incr(reply.size) - originalStatsReceiver.counter(expBucket).incr(original.size) - } - - def recordResponseSizeStats( - candidates: Seq[ItemCandidateWithDetails], - expBucket: String - ): Unit = { - if (candidates.size == 0) emptyStatsReceiver.counter(expBucket).incr() - if (candidates.size < 5) lessThan5StatsReceiver.counter(expBucket).incr() - if (candidates.size < 10) lessThan10StatsReceiver.counter(expBucket).incr() - } - - private def getSuggestType(candidate: CandidateWithDetails): String = - candidate.features.getOrElse(SuggestTypeFeature, None).map(_.name).getOrElse("None") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel deleted file mode 100644 index 3a52b15dc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.bazel +++ /dev/null @@ -1,53 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/search:blender-scala", - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], - exports = [ - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.docx new file mode 100644 index 000000000..20d2d47e1 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.docx new file mode 100644 index 000000000..cbe8559aa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala deleted file mode 100644 index e0919ae9d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineConfig.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.home_mixer.product.list_recommended_users.candidate_source.BlenderUsersCandidateSource -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.gate.EmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.blender.thriftscala.ThriftBlenderRequest -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class BlenderUsersCandidatePipelineConfig @Inject() ( - blenderUsersCandidateSource: BlenderUsersCandidateSource, - isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, - isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) - extends CandidatePipelineConfig[ - ListRecommendedUsersQuery, - ThriftBlenderRequest, - Long, - UserCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("BlenderUsers") - - override val gates: Seq[Gate[ListRecommendedUsersQuery]] = - Seq(EmptySeqFeatureGate(RecentListMembersFeature)) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ListRecommendedUsersQuery, - ThriftBlenderRequest - ] = BlenderUsersCandidatePipelineQueryTransformer - - override val candidateSource: BaseCandidateSource[ThriftBlenderRequest, Long] = - blenderUsersCandidateSource - - override val resultTransformer: CandidatePipelineResultsTransformer[ - Long, - UserCandidate - ] = { candidate => UserCandidate(id = candidate) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] - ] = Seq( - isGizmoduckValidUserFeatureHydrator, - isSGSValidUserFeatureHydrator - ) - - override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { - val clientEventInfoBuilder = ClientEventInfoBuilder("user") - val userItemBuilder = UserCandidateUrtItemBuilder(clientEventInfoBuilder) - Some(UrtItemCandidateDecorator(userItemBuilder)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.docx new file mode 100644 index 000000000..ae78b1a5f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala deleted file mode 100644 index 03844fd38..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/BlenderUsersCandidatePipelineQueryTransformer.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier -import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultType -import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchRequest -import com.twitter.search.blender.thriftscala.ThriftBlenderRequest -import com.twitter.search.blender.thriftscala.ThriftBlenderTweetypieOptions -import com.twitter.search.blender.thriftscala.ThriftBlenderWorkflowID -import com.twitter.search.common.constants.thriftscala.ThriftQuerySource -import com.twitter.spam.rtf.thriftscala.SafetyLevel - -object BlenderUsersCandidatePipelineQueryTransformer - extends CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, ThriftBlenderRequest] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("BlenderUsers") - - /** - * This is a user-defined descriptor used by Blender to track the source of traffic, and it - * is different from a client id, which is set during Finagle client construction. - */ - private val ClientAppName = "timelinemixer.list_recommended_users" - - override def transform(query: ListRecommendedUsersQuery): ThriftBlenderRequest = { - - ThriftBlenderRequest( - workflowID = Some(ThriftBlenderWorkflowID.AdaptiveSearch), - userID = Some(query.getRequiredUserId), // perspectival - uiLang = query.clientContext.languageCode, // perspectival - clientAppName = Some(ClientAppName), - adaptiveSearchRequest = Some( - AdaptiveSearchRequest( - rawQuery = query.listName, - numResults = 40, - getPromotedContent = false, - resultFilter = Some(ResultType.User), - ) - ), - querySource = Some(ThriftQuerySource.TypedQuery), - getCorrections = true, - tweetypieOptions = Some( - ThriftBlenderTweetypieOptions( - safetyLevel = Some(SafetyLevel.Recommendations) - ) - ) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.docx new file mode 100644 index 000000000..552f8a695 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala deleted file mode 100644 index 756c58d72..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersCandidatePipelineConfig.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.home_mixer.product.list_recommended_users.candidate_source.SimilarityBasedUsersCandidateSource -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsGizmoduckValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsListMemberFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.IsSGSValidUserFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature -import com.twitter.home_mixer.product.list_recommended_users.filter.DropMaxCandidatesByAggregatedScoreFilter -import com.twitter.home_mixer.product.list_recommended_users.filter.PreviouslyServedUsersFilter -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.user.UserCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.decorator.CandidateDecorator -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListMemberBasedUsersCandidatePipelineConfig @Inject() ( - similarityBasedUsersCandidateSource: SimilarityBasedUsersCandidateSource, - isGizmoduckValidUserFeatureHydrator: IsGizmoduckValidUserFeatureHydrator, - isListMemberFeatureHydrator: IsListMemberFeatureHydrator, - isSGSValidUserFeatureHydrator: IsSGSValidUserFeatureHydrator) - extends CandidatePipelineConfig[ - ListRecommendedUsersQuery, - Seq[Long], - t.Candidate, - UserCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ListMemberBasedUsers") - - override val gates: Seq[Gate[ListRecommendedUsersQuery]] = - Seq(NonEmptySeqFeatureGate(RecentListMembersFeature)) - - override val queryTransformer: CandidatePipelineQueryTransformer[ListRecommendedUsersQuery, Seq[ - Long - ]] = { query => - query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)).getOrElse(Seq.empty) - } - - override val candidateSource: BaseCandidateSource[Seq[Long], t.Candidate] = - similarityBasedUsersCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.Candidate] - ] = Seq(ListMemberBasedUsersResponseFeatureTransfromer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.Candidate, - UserCandidate - ] = { candidate => - UserCandidate(id = candidate.userId) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] - ] = Seq(isListMemberFeatureHydrator) - - override val filters: Seq[Filter[ListRecommendedUsersQuery, UserCandidate]] = - Seq( - PreviouslyServedUsersFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier("IsListMember"), - shouldKeepCandidate = { features => !features.getOrElse(IsListMemberFeature, false) } - ), - DropMaxCandidatesByAggregatedScoreFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[ListRecommendedUsersQuery, UserCandidate, _] - ] = Seq( - isGizmoduckValidUserFeatureHydrator, - isSGSValidUserFeatureHydrator - ) - - override val decorator: Option[CandidateDecorator[ListRecommendedUsersQuery, UserCandidate]] = { - val clientEventInfoBuilder = ClientEventInfoBuilder("user") - val userItemBuilder = UserCandidateUrtItemBuilder(clientEventInfoBuilder) - Some(UrtItemCandidateDecorator(userItemBuilder)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.docx new file mode 100644 index 000000000..9035f29d9 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala deleted file mode 100644 index 7153d1f06..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListMemberBasedUsersResponseFeatureTransfromer.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.model.common.identifier.TransformerIdentifier - -object ListMemberBasedUsersResponseFeatureTransfromer - extends CandidateFeatureTransformer[t.Candidate] { - - override val identifier: TransformerIdentifier = TransformerIdentifier("ListMemberBasedUsers") - - override val features: Set[Feature[_, _]] = Set(ScoreFeature) - - override def transform(candidate: t.Candidate): FeatureMap = FeatureMapBuilder() - .add(ScoreFeature, candidate.score) - .build() -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.docx new file mode 100644 index 000000000..057966420 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala deleted file mode 100644 index 8c72aefec..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersMixerPipelineConfig.scala +++ /dev/null @@ -1,111 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersQueryFeatureHydrator -import com.twitter.home_mixer.product.list_recommended_users.gate.ViewerIsListOwnerGate -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ExcludedIdsMaxLengthParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UnorderedExcludeIdsBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.DropFilteredCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.user.UserItem -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.timelines.render.{thriftscala => urt} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListRecommendedUsersMixerPipelineConfig @Inject() ( - listMemberBasedUsersCandidatePipelineConfig: ListMemberBasedUsersCandidatePipelineConfig, - blenderUsersCandidatePipelineConfig: BlenderUsersCandidatePipelineConfig, - viewerIsListOwnerGate: ViewerIsListOwnerGate, - recentListMembersQueryFeatureHydrator: RecentListMembersQueryFeatureHydrator, - urtTransportMarshaller: UrtTransportMarshaller) - extends MixerPipelineConfig[ListRecommendedUsersQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ListRecommendedUsers") - - override val gates = Seq(viewerIsListOwnerGate) - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListRecommendedUsersQuery]] = - Seq(recentListMembersQueryFeatureHydrator) - - override val candidatePipelines: Seq[ - CandidatePipelineConfig[ListRecommendedUsersQuery, _, _, _] - ] = Seq( - listMemberBasedUsersCandidatePipelineConfig, - blenderUsersCandidatePipelineConfig - ) - - private val candidatePipelineIdentifiers = Set( - listMemberBasedUsersCandidatePipelineConfig.identifier, - blenderUsersCandidatePipelineConfig.identifier - ) - - override val resultSelectors: Seq[Selector[ListRecommendedUsersQuery]] = Seq( - DropFilteredCandidates( - candidatePipelines = candidatePipelineIdentifiers, - filter = candidate => - candidate.features.getOrElse(IsSGSValidUserFeature, false) && - candidate.features.getOrElse(IsGizmoduckValidUserFeature, false) - ), - DropMaxCandidates( - candidatePipelines = candidatePipelineIdentifiers, - maxSelectionsParam = ServerMaxResultsParam), - InsertAppendResults(candidatePipelineIdentifiers) - ) - - override val domainMarshaller: DomainMarshaller[ListRecommendedUsersQuery, Timeline] = { - val instructionBuilders = Seq( - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - AddEntriesWithReplaceInstructionBuilder() - ) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig( - page = Some("list_recommended_users"), - section = None, - entityToken = None))) - ) - - val excludeIdsSelector: PartialFunction[UniversalNoun[_], Long] = { - case item: UserItem => item.id - } - - val cursorBuilder = UnorderedExcludeIdsBottomCursorBuilder( - excludedIdsMaxLengthParam = ExcludedIdsMaxLengthParam, - excludeIdsSelector = excludeIdsSelector) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(cursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.docx new file mode 100644 index 000000000..221bd0d61 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala deleted file mode 100644 index f0c2ac70e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/ListRecommendedUsersProductPipelineConfig.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.RecommendedUsersCursorUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.home_mixer.model.request.ListRecommendedUsersProductContext -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Try - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListRecommendedUsersProductPipelineConfig @Inject() ( - listRecommendedUsersMixerPipelineConfig: ListRecommendedUsersMixerPipelineConfig, - listRecommendedUsersParamConfig: ListRecommendedUsersParamConfig) - extends ProductPipelineConfig[ - HomeMixerRequest, - ListRecommendedUsersQuery, - urt.TimelineResponse - ] { - - override val identifier: ProductPipelineIdentifier = - ProductPipelineIdentifier("ListRecommendedUsers") - override val product: request.Product = ListRecommendedUsersProduct - override val paramConfig: ProductParamConfig = listRecommendedUsersParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ListRecommendedUsersQuery = { - val context = request.productContext match { - case Some(context: ListRecommendedUsersProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ListRecommendedUsersProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeUnorderedExcludeIdsCursor(cursor)) - .getOrElse(RecommendedUsersCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - } - - ListRecommendedUsersQuery( - listId = context.listId, - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - selectedUserIds = context.selectedUserIds, - excludedUserIds = context.excludedUserIds, - listName = context.listName - ) - } - - override def pipelines: Seq[PipelineConfig] = Seq(listRecommendedUsersMixerPipelineConfig) - - override def pipelineSelector(query: ListRecommendedUsersQuery): ComponentIdentifier = - listRecommendedUsersMixerPipelineConfig.identifier - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(1000.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(1500.millis, 15, 30) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel deleted file mode 100644 index e1fa1de1c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source", - "src/thrift/com/twitter/hermit/candidate:hermit-candidate-scala", - "src/thrift/com/twitter/search:blender-scala", - "strato/config/columns/recommendations/similarity:similarity-strato-client", - "strato/src/main/scala/com/twitter/strato/client", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.docx new file mode 100644 index 000000000..2be57c3aa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.docx new file mode 100644 index 000000000..a247327fc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala deleted file mode 100644 index 3b7ac39fc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/BlenderUsersCandidateSource.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.candidate_source - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.search.adaptive.adaptive_results.thriftscala.AdaptiveSearchResultData -import com.twitter.search.adaptive.adaptive_results.thriftscala.Result -import com.twitter.search.adaptive.adaptive_results.thriftscala.ResultData -import com.twitter.search.blender.adaptive_search.thriftscala.AdaptiveSearchResponse -import com.twitter.search.blender.adaptive_search.thriftscala.Container -import com.twitter.search.blender.thriftscala.BlenderService -import com.twitter.search.blender.thriftscala.ThriftBlenderRequest -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class BlenderUsersCandidateSource @Inject() ( - blenderClient: BlenderService.MethodPerEndpoint) - extends CandidateSource[ThriftBlenderRequest, Long] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("BlenderUsers") - - override def apply(request: ThriftBlenderRequest): Stitch[Seq[Long]] = { - Stitch.callFuture( - blenderClient.serveV2(request).map { response => - val userIdsOpt = - response.adaptiveSearchResponse.map(extractUserIdsFromAdaptiveSearchResponse) - userIdsOpt.getOrElse(Seq.empty) - } - ) - } - - private def extractUserIdsFromAdaptiveSearchResponse( - response: AdaptiveSearchResponse - ): Seq[Long] = { - response match { - case AdaptiveSearchResponse(Some(Seq(Container(Some(results), _))), _, _) => - results.map(_.data).collect { - case AdaptiveSearchResultData.Result(Result(ResultData.User(user), _)) => - user.id - } - case _ => Seq.empty - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.docx new file mode 100644 index 000000000..cc386bae2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala deleted file mode 100644 index f5428d032..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/candidate_source/SimilarityBasedUsersCandidateSource.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.candidate_source - -import com.twitter.hermit.candidate.{thriftscala => t} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.strato.client.Fetcher -import com.twitter.strato.generated.client.recommendations.similarity.SimilarUsersBySimsOnUserClientColumn - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class SimilarityBasedUsersCandidateSource @Inject() ( - similarUsersBySimsOnUserClientColumn: SimilarUsersBySimsOnUserClientColumn) - extends CandidateSource[Seq[Long], t.Candidate] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("SimilarityBasedUsers") - - private val fetcher: Fetcher[Long, Unit, t.Candidates] = - similarUsersBySimsOnUserClientColumn.fetcher - - private val MaxCandidatesToKeep = 4000 - - override def apply(request: Seq[Long]): Stitch[Seq[t.Candidate]] = { - Stitch - .collect { - request.map { userId => - fetcher - .fetch(userId, Unit).map { result => - result.v.map(_.candidates).getOrElse(Seq.empty) - }.map { candidates => - val sortedCandidates = candidates.sortBy(-_.score) - sortedCandidates.take(MaxCandidatesToKeep) - } - } - }.map(_.flatten) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel deleted file mode 100644 index bdd2de735..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "src/thrift/com/twitter/gizmoduck:thrift-scala", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.docx new file mode 100644 index 000000000..9a6867146 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.docx new file mode 100644 index 000000000..4767ab54c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala deleted file mode 100644 index 80a41adf0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsGizmoduckValidUserFeatureHydrator.scala +++ /dev/null @@ -1,66 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsGizmoduckValidUserFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.spam.rtf.{thriftscala => rtf} -import com.twitter.stitch.Stitch -import com.twitter.stitch.gizmoduck.Gizmoduck -import com.twitter.util.Return - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class IsGizmoduckValidUserFeatureHydrator @Inject() (gizmoduck: Gizmoduck) - extends BulkCandidateFeatureHydrator[PipelineQuery, UserCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("IsGizmoduckValidUser") - - override val features: Set[Feature[_, _]] = Set(IsGizmoduckValidUserFeature) - - private val queryFields: Set[gt.QueryFields] = Set(gt.QueryFields.Safety) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val context = gt.LookupContext( - forUserId = query.getOptionalUserId, - includeProtected = true, - safetyLevel = Some(rtf.SafetyLevel.Recommendations) - ) - val userIds = candidates.map(_.candidate.id) - - Stitch - .collectToTry( - userIds.map(userId => gizmoduck.getUserById(userId, queryFields, context))).map { - userResults => - val idToUserSafetyMap = userResults - .collect { - case Return(user) => user - }.map(user => user.id -> user.safety).toMap - - candidates.map { candidate => - val safety = idToUserSafetyMap.getOrElse(candidate.candidate.id, None) - val isValidUser = safety.isDefined && - !safety.exists(_.deactivated) && - !safety.exists(_.suspended) && - !safety.exists(_.isProtected) && - !safety.flatMap(_.offboarded).getOrElse(false) - - FeatureMapBuilder() - .add(IsGizmoduckValidUserFeature, isValidUser) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.docx new file mode 100644 index 000000000..15a4f3e9b Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala deleted file mode 100644 index c9e529de5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsListMemberFeatureHydrator.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsListMemberFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class IsListMemberFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("IsListMember") - - override val features: Set[Feature[_, _]] = Set(IsListMemberFeature) - - override def apply( - query: PipelineQuery with HasListId, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val userIds = candidates.map(_.candidate.id) - val request = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship( - source = query.listId, - relationshipType = sg.RelationshipType.ListHasMember, - hasRelationship = true, - targets = Some(userIds))), - pageRequest = Some(sg.PageRequest(selectAll = Some(true))) - ) - - socialGraph.ids(request).map(_.ids).map { listMembers => - val listMembersSet = listMembers.toSet - candidates.map { candidate => - FeatureMapBuilder() - .add(IsListMemberFeature, listMembersSet.contains(candidate.candidate.id)) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.docx new file mode 100644 index 000000000..3e06376f8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala deleted file mode 100644 index de70d45d8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/IsSGSValidUserFeatureHydrator.scala +++ /dev/null @@ -1,65 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.IsSGSValidUserFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class IsSGSValidUserFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends BulkCandidateFeatureHydrator[PipelineQuery with HasListId, UserCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("IsSGSValidUser") - - override def features: Set[Feature[_, _]] = Set(IsSGSValidUserFeature) - - override def apply( - query: PipelineQuery with HasListId, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val sourceId = query.getRequiredUserId - val targetUserIds = candidates.map(_.candidate.id) - val request = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship( - source = sourceId, - relationshipType = sg.RelationshipType.Blocking, - hasRelationship = true, - targets = Some(targetUserIds)), - sg.SrcRelationship( - source = sourceId, - relationshipType = sg.RelationshipType.BlockedBy, - hasRelationship = true, - targets = Some(targetUserIds)), - sg.SrcRelationship( - source = sourceId, - relationshipType = sg.RelationshipType.Muting, - hasRelationship = true, - targets = Some(targetUserIds)) - ), - pageRequest = Some(sg.PageRequest(selectAll = Some(true))), - context = Some(sg.LookupContext(performUnion = Some(true))) - ) - - socialGraph.ids(request).map(_.ids).map(_.toSet).map { hasRelationshipUserIds => - candidates.map { candidate => - FeatureMapBuilder() - .add(IsSGSValidUserFeature, !hasRelationshipUserIds.contains(candidate.candidate.id)) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.docx new file mode 100644 index 000000000..1294db70c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala deleted file mode 100644 index 81a5ce504..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator/RecentListMembersQueryFeatureHydrator.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.feature_hydrator - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -case object RecentListMembersFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { - override val defaultValue: Seq[Long] = Seq.empty -} - -@Singleton -class RecentListMembersQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends QueryFeatureHydrator[PipelineQuery with HasListId] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RecentListMembers") - - override val features: Set[Feature[_, _]] = Set(RecentListMembersFeature) - - private val MaxRecentMembers = 10 - - override def hydrate(query: PipelineQuery with HasListId): Stitch[FeatureMap] = { - val request = sg.IdsRequest( - relationships = Seq(sg - .SrcRelationship(query.listId, sg.RelationshipType.ListHasMember, hasRelationship = true)), - pageRequest = Some(sg.PageRequest(selectAll = Some(true), count = Some(MaxRecentMembers))) - ) - socialGraph.ids(request).map(_.ids).map { listMembers => - FeatureMapBuilder().add(RecentListMembersFeature, listMembers).build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel deleted file mode 100644 index 72a5f251f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/filter", - ], - exports = [], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.docx new file mode 100644 index 000000000..ca3eb26a2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.docx new file mode 100644 index 000000000..bf63e0aa6 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala deleted file mode 100644 index a68c37450..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/DropMaxCandidatesByAggregatedScoreFilter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.filter - -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersFeatures.ScoreFeature -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -object DropMaxCandidatesByAggregatedScoreFilter extends Filter[PipelineQuery, UserCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("DropMaxCandidatesByAggregatedScore") - - private val MaxSimilarUserCandidates = 150 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[FilterResult[UserCandidate]] = { - val userIdToAggregatedScoreMap = candidates - .groupBy(_.candidate.id) - .map { - case (userId, candidates) => - val aggregatedScore = candidates.map(_.features.getOrElse(ScoreFeature, 0.0)).sum - (userId, aggregatedScore) - } - - val sortedCandidates = candidates.sortBy(candidate => - -userIdToAggregatedScoreMap.getOrElse(candidate.candidate.id, 0.0)) - - val (kept, removed) = sortedCandidates.map(_.candidate).splitAt(MaxSimilarUserCandidates) - - Stitch.value(FilterResult(kept, removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.docx new file mode 100644 index 000000000..633e6e1cb Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala deleted file mode 100644 index ac8c3107d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/filter/PreviouslyServedUsersFilter.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.filter - -import com.twitter.home_mixer.product.list_recommended_users.feature_hydrator.RecentListMembersFeature -import com.twitter.home_mixer.product.list_recommended_users.model.ListRecommendedUsersQuery -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.filter.FilterResult -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.stitch.Stitch - -object PreviouslyServedUsersFilter extends Filter[ListRecommendedUsersQuery, UserCandidate] { - - override val identifier: FilterIdentifier = FilterIdentifier("PreviouslyServedUsers") - - override def apply( - query: ListRecommendedUsersQuery, - candidates: Seq[CandidateWithFeatures[UserCandidate]] - ): Stitch[FilterResult[UserCandidate]] = { - - val recentListMembers = query.features.map(_.getOrElse(RecentListMembersFeature, Seq.empty)) - - val servedUserIds = query.pipelineCursor.map(_.excludedIds) - - val excludedUserIds = (recentListMembers.getOrElse(Seq.empty) ++ - query.selectedUserIds.getOrElse(Seq.empty) ++ - query.excludedUserIds.getOrElse(Seq.empty) ++ - servedUserIds.getOrElse(Seq.empty)).toSet - - val (removed, kept) = - candidates.map(_.candidate).partition(candidate => excludedUserIds.contains(candidate.id)) - - Stitch.value(FilterResult(kept = kept, removed = removed)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel deleted file mode 100644 index f8a7083b9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/gate", - "src/thrift/com/twitter/socialgraph:thrift-scala", - "stitch/stitch-socialgraph", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.docx new file mode 100644 index 000000000..de7bddfee Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.docx new file mode 100644 index 000000000..f716125a3 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala deleted file mode 100644 index e7ab3084e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/gate/ViewerIsListOwnerGate.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.gate - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.model.common.identifier.GateIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class ViewerIsListOwnerGate @Inject() (socialGraph: SocialGraph) - extends Gate[PipelineQuery with HasListId] { - - override val identifier: GateIdentifier = GateIdentifier("ViewerIsListOwner") - - private val relationship = sg.Relationship(relationshipType = sg.RelationshipType.ListOwning) - - override def shouldContinue(query: PipelineQuery with HasListId): Stitch[Boolean] = { - val request = sg.ExistsRequest( - source = query.getRequiredUserId, - target = query.listId, - relationships = Seq(relationship)) - socialGraph.exists(request).map(_.exists) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.bazel deleted file mode 100644 index 78302f75b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.docx new file mode 100644 index 000000000..a06801e75 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.docx new file mode 100644 index 000000000..1e2f60508 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala deleted file mode 100644 index baed8171b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersFeatures.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.model - -import com.twitter.product_mixer.component_library.model.candidate.UserCandidate -import com.twitter.product_mixer.core.feature.Feature - -object ListRecommendedUsersFeatures { - // Candidate features - object IsGizmoduckValidUserFeature extends Feature[UserCandidate, Boolean] - object IsListMemberFeature extends Feature[UserCandidate, Boolean] - object IsSGSValidUserFeature extends Feature[UserCandidate, Boolean] - object ScoreFeature extends Feature[UserCandidate, Double] -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.docx new file mode 100644 index 000000000..e666fb7fa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala deleted file mode 100644 index 15b16c25b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/model/ListRecommendedUsersQuery.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.model - -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.model.request.ListRecommendedUsersProduct -import com.twitter.product_mixer.component_library.model.cursor.UrtUnorderedExcludeIdsCursor -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class ListRecommendedUsersQuery( - override val listId: Long, - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtUnorderedExcludeIdsCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - selectedUserIds: Option[Seq[Long]], - excludedUserIds: Option[Seq[Long]], - listName: Option[String]) - extends PipelineQuery - with HasPipelineCursor[UrtUnorderedExcludeIdsCursor] - with HasListId { - - override val product: Product = ListRecommendedUsersProduct - - override def withFeatureMap(features: FeatureMap): ListRecommendedUsersQuery = - copy(features = Some(features)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.bazel deleted file mode 100644 index 18867ad9e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.docx new file mode 100644 index 000000000..f25d71b6b Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.docx new file mode 100644 index 000000000..f4a18e868 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.scala deleted file mode 100644 index 3822a1987..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParam.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.param - -import com.twitter.timelines.configapi.FSBoundedParam - -object ListRecommendedUsersParam { - val SupportedClientFSName = "list_recommended_users_supported_client" - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "list_recommended_users_server_max_results", - default = 10, - min = 1, - max = 500 - ) - - object ExcludedIdsMaxLengthParam - extends FSBoundedParam[Int]( - name = "list_recommended_users_excluded_ids_max_length", - default = 2000, - min = 0, - max = 5000 - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.docx new file mode 100644 index 000000000..426e45d6f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.scala deleted file mode 100644 index b162ef755..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_recommended_users/param/ListRecommendedUsersParamConfig.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.product.list_recommended_users.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ExcludedIdsMaxLengthParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_recommended_users.param.ListRecommendedUsersParam.SupportedClientFSName -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListRecommendedUsersParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableListRecommendedUsersProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val boundedIntFSOverrides = Seq( - ServerMaxResultsParam, - ExcludedIdsMaxLengthParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel deleted file mode 100644 index a7d22148a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.bazel +++ /dev/null @@ -1,61 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "ads-injection/lib/src/main/scala/com/twitter/goldfinch/api", - "finatra/inject/inject-core/src/main/scala", - "finatra/inject/inject-core/src/main/scala/com/twitter/inject", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/urt/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/marshaller/timelines", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_service", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/candidate/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/pipeline/candidate/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector/ads", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/java/com/twitter/product_mixer/core/product/guice/scope", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/common/presentation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/response/urt/item", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/mixer", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice", - "src/thrift/com/twitter/timelines/render:thrift-scala", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], - exports = [ - "src/thrift/com/twitter/timelines/render:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.docx new file mode 100644 index 000000000..ee3b8e885 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.docx new file mode 100644 index 000000000..2d1eb359c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala deleted file mode 100644 index 9e5a4e541..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsAdsCandidatePipelineBuilder.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.adserver.{thriftscala => ads} -import com.twitter.home_mixer.functional_component.decorator.builder.HomeAdsClientEventDetailsBuilder -import com.twitter.home_mixer.functional_component.gate.ExcludeSoftUserGate -import com.twitter.home_mixer.param.HomeGlobalParams -import com.twitter.home_mixer.param.HomeGlobalParams.EnableAdvertiserBrandSafetySettingsFeatureHydratorParam -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.EnableAdsCandidatePipelineParam -import com.twitter.product_mixer.component_library.candidate_source.ads.AdsProdThriftCandidateSource -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.contextual_ref.ContextualTweetRefBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.ad.AdsCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.ads.AdvertiserBrandSafetySettingsFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.candidate.param_gated.ParamGatedCandidateFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.model.candidate.ads.AdsCandidate -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfig -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.AdsDependentCandidatePipelineConfigBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.CountCandidatesFromPipelines -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.StaticAdsDisplayLocationBuilder -import com.twitter.product_mixer.component_library.pipeline.candidate.ads.ValidAdImpressionIdFilter -import com.twitter.product_mixer.core.functional_component.common.CandidateScope -import com.twitter.product_mixer.core.gate.ParamNotGate -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.rtf.safety_level.TimelineHomePromotedHydrationSafetyLevel -import com.twitter.product_mixer.core.model.marshalling.response.urt.contextual_ref.TweetHydrationContext -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsAdsCandidatePipelineBuilder @Inject() ( - adsCandidatePipelineConfigBuilder: AdsDependentCandidatePipelineConfigBuilder, - adsCandidateSource: AdsProdThriftCandidateSource, - advertiserBrandSafetySettingsFeatureHydrator: AdvertiserBrandSafetySettingsFeatureHydrator[ - ListTweetsQuery, - AdsCandidate - ]) { - - private val identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("ListTweetsAds") - - private val suggestType = st.SuggestType.Promoted - - private val clientEventInfoBuilder = ClientEventInfoBuilder( - component = InjectionScribeUtil.scribeComponent(suggestType).get, - detailsBuilder = Some(HomeAdsClientEventDetailsBuilder(Some(suggestType.name))) - ) - - private val contextualTweetRefBuilder = ContextualTweetRefBuilder( - TweetHydrationContext( - safetyLevelOverride = Some(TimelineHomePromotedHydrationSafetyLevel), - outerTweetContext = None - )) - - private val decorator = UrtItemCandidateDecorator( - AdsCandidateUrtItemBuilder( - tweetClientEventInfoBuilder = Some(clientEventInfoBuilder), - contextualTweetRefBuilder = Some(contextualTweetRefBuilder) - ) - ) - - def build( - organicCandidatePipelines: CandidateScope - ): AdsDependentCandidatePipelineConfig[ListTweetsQuery] = - adsCandidatePipelineConfigBuilder.build[ListTweetsQuery]( - adsCandidateSource = adsCandidateSource, - identifier = identifier, - adsDisplayLocationBuilder = - StaticAdsDisplayLocationBuilder(ads.DisplayLocation.TimelineHomeReverseChron), - countNumOrganicItems = CountCandidatesFromPipelines(organicCandidatePipelines), - supportedClientParam = Some(EnableAdsCandidatePipelineParam), - gates = Seq( - ParamNotGate( - name = "AdsDisableInjectionBasedOnUserRole", - param = HomeGlobalParams.AdsDisableInjectionBasedOnUserRoleParam - ), - ExcludeSoftUserGate, - NonEmptyCandidatesGate(organicCandidatePipelines) - ), - filters = Seq(ValidAdImpressionIdFilter), - postFilterFeatureHydration = Seq( - ParamGatedCandidateFeatureHydrator( - EnableAdvertiserBrandSafetySettingsFeatureHydratorParam, - advertiserBrandSafetySettingsFeatureHydrator - ) - ), - decorator = Some(decorator), - urtRequest = Some(true), - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.docx new file mode 100644 index 000000000..f62eb8cbe Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala deleted file mode 100644 index 2d7dd3b31..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsMixerPipelineConfig.scala +++ /dev/null @@ -1,162 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.clientapp.{thriftscala => ca} -import com.twitter.goldfinch.api.AdsInjectionSurfaceAreas -import com.twitter.home_mixer.candidate_pipeline.ConversationServiceCandidatePipelineConfigBuilder -import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.side_effect.HomeScribeClientEventSideEffect -import com.twitter.home_mixer.model.GapIncludeInstruction -import com.twitter.home_mixer.param.HomeMixerFlagName.ScribeClientEventsFlag -import com.twitter.home_mixer.product.list_tweets.decorator.ListConversationServiceCandidateDecorator -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.inject.annotations.Flag -import com.twitter.logpipeline.client.common.EventPublisher -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.gate.NonEmptyCandidatesGate -import com.twitter.product_mixer.component_library.premarshaller.urt.UrtDomainMarshaller -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.AddEntriesWithReplaceAndShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedBottomCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedGapCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.OrderedTopCursorBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceAllEntries -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ReplaceEntryInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.ShowAlertInstructionBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.StaticTimelineScribeConfigBuilder -import com.twitter.product_mixer.component_library.premarshaller.urt.builder.UrtMetadataBuilder -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.ads.AdsInjector -import com.twitter.product_mixer.component_library.selector.ads.InsertAdResults -import com.twitter.product_mixer.core.functional_component.common.SpecificPipelines -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.marshaller.response.urt.UrtTransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.UniversalNoun -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.MixerPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.Timeline -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineModule -import com.twitter.product_mixer.core.model.marshalling.response.urt.TimelineScribeConfig -import com.twitter.product_mixer.core.model.marshalling.response.urt.item.tweet.TweetItem -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.candidate.DependentCandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.mixer.MixerPipelineConfig -import com.twitter.timelines.render.{thriftscala => urt} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsMixerPipelineConfig @Inject() ( - listTweetsTimelineServiceCandidatePipelineConfig: ListTweetsTimelineServiceCandidatePipelineConfig, - conversationServiceCandidatePipelineConfigBuilder: ConversationServiceCandidatePipelineConfigBuilder[ - ListTweetsQuery - ], - listTweetsAdsCandidatePipelineBuilder: ListTweetsAdsCandidatePipelineBuilder, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ListTweetsQuery], - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - adsInjector: AdsInjector, - clientEventsScribeEventPublisher: EventPublisher[ca.LogEvent], - urtTransportMarshaller: UrtTransportMarshaller, - @Flag(ScribeClientEventsFlag) enableScribeClientEvents: Boolean) - extends MixerPipelineConfig[ListTweetsQuery, Timeline, urt.TimelineResponse] { - - override val identifier: MixerPipelineIdentifier = MixerPipelineIdentifier("ListTweets") - - private val conversationServiceCandidatePipelineConfig = - conversationServiceCandidatePipelineConfigBuilder.build( - Seq( - NonEmptyCandidatesGate( - SpecificPipelines(listTweetsTimelineServiceCandidatePipelineConfig.identifier)) - ), - ListConversationServiceCandidateDecorator() - ) - - private val listTweetsAdsCandidatePipelineConfig = listTweetsAdsCandidatePipelineBuilder.build( - SpecificPipelines(listTweetsTimelineServiceCandidatePipelineConfig.identifier) - ) - - override val candidatePipelines: Seq[CandidatePipelineConfig[ListTweetsQuery, _, _, _]] = - Seq(listTweetsTimelineServiceCandidatePipelineConfig) - - override val dependentCandidatePipelines: Seq[ - DependentCandidatePipelineConfig[ListTweetsQuery, _, _, _] - ] = - Seq(conversationServiceCandidatePipelineConfig, listTweetsAdsCandidatePipelineConfig) - - override val failOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = Map( - conversationServiceCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - listTweetsAdsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always) - - override val resultSelectors: Seq[Selector[ListTweetsQuery]] = Seq( - UpdateSortCandidates( - ordering = CandidatesUtil.reverseChronTweetsOrdering, - candidatePipeline = conversationServiceCandidatePipelineConfig.identifier - ), - InsertAppendResults(candidatePipeline = conversationServiceCandidatePipelineConfig.identifier), - InsertAdResults( - surfaceAreaName = AdsInjectionSurfaceAreas.HomeTimeline, - adsInjector = adsInjector.forSurfaceArea(AdsInjectionSurfaceAreas.HomeTimeline), - adsCandidatePipeline = listTweetsAdsCandidatePipelineConfig.identifier - ), - ) - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ListTweetsQuery]] = Seq( - requestQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator - ) - - private val homeScribeClientEventSideEffect = HomeScribeClientEventSideEffect( - enableScribeClientEvents = enableScribeClientEvents, - logPipelinePublisher = clientEventsScribeEventPublisher, - injectedTweetsCandidatePipelineIdentifiers = - Seq(conversationServiceCandidatePipelineConfig.identifier), - adsCandidatePipelineIdentifier = Some(listTweetsAdsCandidatePipelineConfig.identifier), - ) - - override val resultSideEffects: Seq[PipelineResultSideEffect[ListTweetsQuery, Timeline]] = - Seq(homeScribeClientEventSideEffect) - - override val domainMarshaller: DomainMarshaller[ListTweetsQuery, Timeline] = { - val instructionBuilders = Seq( - ReplaceEntryInstructionBuilder(ReplaceAllEntries), - AddEntriesWithReplaceAndShowAlertInstructionBuilder(), - ShowAlertInstructionBuilder() - ) - - val idSelector: PartialFunction[UniversalNoun[_], Long] = { - // exclude ads while determining tweet cursor values - case item: TweetItem if item.promotedMetadata.isEmpty => item.id - case module: TimelineModule - if module.items.headOption.exists(_.item.isInstanceOf[TweetItem]) => - module.items.last.item match { - case item: TweetItem => item.id - } - } - - val topCursorBuilder = OrderedTopCursorBuilder(idSelector) - val bottomCursorBuilder = - OrderedBottomCursorBuilder(idSelector, GapIncludeInstruction.inverse()) - val gapCursorBuilder = OrderedGapCursorBuilder(idSelector, GapIncludeInstruction) - - val metadataBuilder = UrtMetadataBuilder( - title = None, - scribeConfigBuilder = Some( - StaticTimelineScribeConfigBuilder( - TimelineScribeConfig(page = Some("list_tweets"), section = None, entityToken = None))) - ) - - UrtDomainMarshaller( - instructionBuilders = instructionBuilders, - metadataBuilder = Some(metadataBuilder), - cursorBuilders = Seq(topCursorBuilder, bottomCursorBuilder, gapCursorBuilder) - ) - } - - override val transportMarshaller: TransportMarshaller[Timeline, urt.TimelineResponse] = - urtTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.docx new file mode 100644 index 000000000..2264af174 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala deleted file mode 100644 index b4b5b1924..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsProductPipelineConfig.scala +++ /dev/null @@ -1,130 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.marshaller.timelines.ChronologicalCursorUnmarshaller -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.home_mixer.model.request.ListTweetsProductContext -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.service.HomeMixerAlertConfig.DefaultNotificationGroup -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.functional_component.common.alert.Alert -import com.twitter.product_mixer.core.functional_component.common.alert.EmptyResponseRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.LatencyAlert -import com.twitter.product_mixer.core.functional_component.common.alert.P99 -import com.twitter.product_mixer.core.functional_component.common.alert.SuccessRateAlert -import com.twitter.product_mixer.core.functional_component.common.alert.ThroughputAlert -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfAbove -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfBelow -import com.twitter.product_mixer.core.functional_component.common.alert.predicate.TriggerIfLatencyAbove -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.GapCursor -import com.twitter.product_mixer.core.model.marshalling.response.urt.operation.TopCursor -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.MalformedCursor -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.product_mixer.core.util.SortIndexBuilder -import com.twitter.timelines.configapi.Params -import com.twitter.timelines.render.{thriftscala => urt} -import com.twitter.timelines.util.RequestCursorSerializer -import com.twitter.util.Time -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsProductPipelineConfig @Inject() ( - listTweetsMixerPipelineConfig: ListTweetsMixerPipelineConfig, - listTweetsParamConfig: ListTweetsParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, ListTweetsQuery, urt.TimelineResponse] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ListTweets") - override val product: request.Product = ListTweetsProduct - override val paramConfig: ProductParamConfig = listTweetsParamConfig - override val denyLoggedOutUsers: Boolean = false - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ListTweetsQuery = { - val context = request.productContext match { - case Some(context: ListTweetsProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ListTweetsProductContext not found") - } - - val debugOptions = request.debugParams.flatMap(_.debugOptions) - - /** - * Unlike other clients, newly created tweets on Android have the sort index set to the current - * time instead of the top sort index + 1, so these tweets get stuck at the top of the timeline - * if subsequent timeline responses use the sort index from the previous response instead of - * the current time. - */ - val pipelineCursor = request.serializedRequestCursor.flatMap { cursor => - Try(UrtCursorSerializer.deserializeOrderedCursor(cursor)) - .getOrElse(ChronologicalCursorUnmarshaller(RequestCursorSerializer.deserialize(cursor))) - .map { - case UrtOrderedCursor(_, id, Some(GapCursor), gapBoundaryId) - if id.isEmpty || gapBoundaryId.isEmpty => - throw PipelineFailure(MalformedCursor, "Gap Cursor bounds not defined") - case topCursor @ UrtOrderedCursor(_, _, Some(TopCursor), _) => - val queryTime = debugOptions.flatMap(_.requestTimeOverride).getOrElse(Time.now) - topCursor.copy(initialSortIndex = SortIndexBuilder.timeToId(queryTime)) - case cursor => cursor - } - } - - ListTweetsQuery( - params = params, - clientContext = request.clientContext, - features = None, - pipelineCursor = pipelineCursor, - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = debugOptions, - listId = context.listId, - deviceContext = context.deviceContext, - dspClientContext = context.dspClientContext - ) - } - - override def pipelines: Seq[PipelineConfig] = Seq(listTweetsMixerPipelineConfig) - - override def pipelineSelector(query: ListTweetsQuery): ComponentIdentifier = - listTweetsMixerPipelineConfig.identifier - - override val alerts: Seq[Alert] = Seq( - SuccessRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfBelow(99.9, 20, 30), - criticalPredicate = TriggerIfBelow(99.9, 30, 30), - ), - LatencyAlert( - notificationGroup = DefaultNotificationGroup, - percentile = P99, - warnPredicate = TriggerIfLatencyAbove(300.millis, 15, 30), - criticalPredicate = TriggerIfLatencyAbove(400.millis, 15, 30) - ), - ThroughputAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(3000), - criticalPredicate = TriggerIfAbove(4000) - ), - EmptyResponseRateAlert( - notificationGroup = DefaultNotificationGroup, - warnPredicate = TriggerIfAbove(65), - criticalPredicate = TriggerIfAbove(80) - ) - ) - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.docx new file mode 100644 index 000000000..79fb5644d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala deleted file mode 100644 index 1af95e873..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/ListTweetsTimelineServiceCandidatePipelineConfig.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets - -import com.twitter.home_mixer.candidate_pipeline.TimelineServiceResponseFeatureTransformer -import com.twitter.home_mixer.marshaller.timelines.TimelineServiceCursorMarshaller -import com.twitter.home_mixer.product.list_tweets.model.ListTweetsQuery -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.component_library.candidate_source.timeline_service.TimelineServiceTweetCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsTimelineServiceCandidatePipelineConfig @Inject() ( - timelineServiceTweetCandidateSource: TimelineServiceTweetCandidateSource) - extends CandidatePipelineConfig[ListTweetsQuery, t.TimelineQuery, t.Tweet, TweetCandidate] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ListTweetsTimelineServiceTweets") - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ListTweetsQuery, - t.TimelineQuery - ] = { query => - val timelineQueryOptions = t.TimelineQueryOptions( - contextualUserId = query.clientContext.userId, - ) - - t.TimelineQuery( - timelineType = t.TimelineType.List, - timelineId = query.listId, - maxCount = query.maxResults(ServerMaxResultsParam).toShort, - cursor2 = query.pipelineCursor.flatMap(TimelineServiceCursorMarshaller(_)), - options = Some(timelineQueryOptions), - timelineId2 = Some(t.TimelineId(t.TimelineType.List, query.listId, None)) - ) - } - - override def candidateSource: BaseCandidateSource[t.TimelineQuery, t.Tweet] = - timelineServiceTweetCandidateSource - - override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] = { - sourceResult => TweetCandidate(id = sourceResult.statusId) - } - - override val featuresFromCandidateSourceTransformers: Seq[CandidateFeatureTransformer[t.Tweet]] = - Seq(TimelineServiceResponseFeatureTransformer) - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.7) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel deleted file mode 100644 index 4258ffc36..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/decorator/builder", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/decorator/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - "timelines/src/main/scala/com/twitter/timelines/injection/scribe", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.docx new file mode 100644 index 000000000..3c0afa0a1 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.docx new file mode 100644 index 000000000..ff399b2d2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala deleted file mode 100644 index a02b628e8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/ListConversationServiceCandidateDecorator.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.decorator - -import com.twitter.home_mixer.functional_component.decorator.builder.HomeConversationModuleMetadataBuilder -import com.twitter.home_mixer.functional_component.decorator.builder.ListClientEventDetailsBuilder -import com.twitter.home_mixer.model.HomeFeatures.ConversationModuleFocalTweetIdFeature -import com.twitter.product_mixer.component_library.decorator.urt.UrtItemCandidateDecorator -import com.twitter.product_mixer.component_library.decorator.urt.UrtMultipleModulesDecorator -import com.twitter.product_mixer.component_library.decorator.urt.builder.item.tweet.TweetCandidateUrtItemBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.metadata.ClientEventInfoBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.StaticModuleDisplayTypeBuilder -import com.twitter.product_mixer.component_library.decorator.urt.builder.timeline_module.TimelineModuleBuilder -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.model.marshalling.response.urt.EntryNamespace -import com.twitter.product_mixer.core.model.marshalling.response.urt.timeline_module.VerticalConversation -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.injection.scribe.InjectionScribeUtil -import com.twitter.timelineservice.suggests.{thriftscala => st} - -object ListConversationServiceCandidateDecorator { - - private val ConversationModuleNamespace = EntryNamespace("list-conversation") - - def apply(): Some[UrtMultipleModulesDecorator[PipelineQuery, TweetCandidate, Long]] = { - val suggestType = st.SuggestType.OrganicListTweet - val component = InjectionScribeUtil.scribeComponent(suggestType).get - val clientEventInfoBuilder = ClientEventInfoBuilder( - component = component, - detailsBuilder = Some(ListClientEventDetailsBuilder(st.SuggestType.OrganicListTweet)) - ) - val tweetItemBuilder = TweetCandidateUrtItemBuilder( - clientEventInfoBuilder = clientEventInfoBuilder - ) - - val moduleBuilder = TimelineModuleBuilder( - entryNamespace = ConversationModuleNamespace, - clientEventInfoBuilder = clientEventInfoBuilder, - displayTypeBuilder = StaticModuleDisplayTypeBuilder(VerticalConversation), - metadataBuilder = Some(HomeConversationModuleMetadataBuilder()) - ) - - Some( - UrtMultipleModulesDecorator( - urtItemCandidateDecorator = UrtItemCandidateDecorator(tweetItemBuilder), - moduleBuilder = moduleBuilder, - groupByKey = (_, _, candidateFeatures) => - candidateFeatures.getOrElse(ConversationModuleFocalTweetIdFeature, None) - )) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel deleted file mode 100644 index 3ed81a196..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/decorator/urt/builder", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.docx new file mode 100644 index 000000000..4d817946f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/decorator/builder/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.bazel deleted file mode 100644 index fac4ba949..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], - exports = [ - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/cursor", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/query/ads", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.docx new file mode 100644 index 000000000..2581f4a4f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.docx new file mode 100644 index 000000000..92db3e2bb Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.scala deleted file mode 100644 index ce8c73c0a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/model/ListTweetsQuery.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.model - -import com.twitter.adserver.thriftscala.HomeTimelineType -import com.twitter.adserver.thriftscala.TimelineRequestParams -import com.twitter.dspbidder.commons.{thriftscala => dsp} -import com.twitter.home_mixer.model.HomeAdsQuery -import com.twitter.home_mixer.model.request.DeviceContext -import com.twitter.home_mixer.model.request.HasListId -import com.twitter.home_mixer.model.request.ListTweetsProduct -import com.twitter.product_mixer.component_library.model.cursor.UrtOrderedCursor -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.marshalling.request._ -import com.twitter.product_mixer.core.pipeline.HasPipelineCursor -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.timelines.configapi.Params - -case class ListTweetsQuery( - override val params: Params, - override val clientContext: ClientContext, - override val pipelineCursor: Option[UrtOrderedCursor], - override val requestedMaxResults: Option[Int], - override val debugOptions: Option[DebugOptions], - override val features: Option[FeatureMap], - override val listId: Long, - override val deviceContext: Option[DeviceContext], - override val dspClientContext: Option[dsp.DspClientContext]) - extends PipelineQuery - with HasPipelineCursor[UrtOrderedCursor] - with HasListId - with HomeAdsQuery { - override val product: Product = ListTweetsProduct - - override def withFeatureMap(features: FeatureMap): ListTweetsQuery = - copy(features = Some(features)) - - override val timelineRequestParams: Option[TimelineRequestParams] = - Some(TimelineRequestParams(homeTimelineType = Some(HomeTimelineType.HomeLatest))) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.bazel deleted file mode 100644 index 18867ad9e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/javax/inject:javax.inject", - "configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param/decider", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.docx new file mode 100644 index 000000000..a777011ca Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.docx new file mode 100644 index 000000000..283e41499 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.scala deleted file mode 100644 index 827d29f10..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParam.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.param - -import com.twitter.timelines.configapi.FSBoundedParam -import com.twitter.timelines.configapi.FSParam - -object ListTweetsParam { - val SupportedClientFSName = "list_tweets_supported_client" - - object EnableAdsCandidatePipelineParam - extends FSParam[Boolean]( - name = "list_tweets_enable_ads", - default = false - ) - - object ServerMaxResultsParam - extends FSBoundedParam[Int]( - name = "list_tweets_server_max_results", - default = 100, - min = 1, - max = 500 - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.docx new file mode 100644 index 000000000..979785e56 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.scala deleted file mode 100644 index 01575db92..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/list_tweets/param/ListTweetsParamConfig.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.twitter.home_mixer.product.list_tweets.param - -import com.twitter.home_mixer.param.decider.DeciderKey -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.EnableAdsCandidatePipelineParam -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.list_tweets.param.ListTweetsParam.SupportedClientFSName -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.servo.decider.DeciderKeyName -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListTweetsParamConfig @Inject() () extends ProductParamConfig { - override val enabledDeciderKey: DeciderKeyName = DeciderKey.EnableListTweetsProduct - override val supportedClientFSName: String = SupportedClientFSName - - override val booleanFSOverrides = - Seq(EnableAdsCandidatePipelineParam) - - override val boundedIntFSOverrides = Seq( - ServerMaxResultsParam - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel deleted file mode 100644 index 4bbbec94d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.bazel +++ /dev/null @@ -1,40 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/marshaller", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/scoring_pipeline", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/selector", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/side_effect", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/async", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/impressed_tweets", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/param_gated", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/premarshaller/urt", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/selector", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/side_effect", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/product", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/recommendation", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.docx new file mode 100644 index 000000000..c7a50c98d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.docx new file mode 100644 index 000000000..0e6bad501 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala deleted file mode 100644 index 27278db0c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsProductPipelineConfig.scala +++ /dev/null @@ -1,74 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets - -import com.twitter.home_mixer.model.HomeFeatures.ServedTweetIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.model.request.HomeMixerRequest -import com.twitter.home_mixer.model.request.ScoredTweetsProduct -import com.twitter.home_mixer.model.request.ScoredTweetsProductContext -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.ServerMaxResultsParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParamConfig -import com.twitter.home_mixer.service.HomeMixerAccessPolicy.DefaultHomeMixerAccessPolicy -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.product_mixer.component_library.premarshaller.cursor.UrtCursorSerializer -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.common.access_policy.AccessPolicy -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ProductPipelineIdentifier -import com.twitter.product_mixer.core.model.marshalling.request.Product -import com.twitter.product_mixer.core.pipeline.PipelineConfig -import com.twitter.product_mixer.core.pipeline.pipeline_failure.BadRequest -import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure -import com.twitter.product_mixer.core.pipeline.product.ProductPipelineConfig -import com.twitter.product_mixer.core.product.ProductParamConfig -import com.twitter.timelines.configapi.Params -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsProductPipelineConfig @Inject() ( - scoredTweetsRecommendationPipelineConfig: ScoredTweetsRecommendationPipelineConfig, - scoredTweetsParamConfig: ScoredTweetsParamConfig) - extends ProductPipelineConfig[HomeMixerRequest, ScoredTweetsQuery, t.ScoredTweets] { - - override val identifier: ProductPipelineIdentifier = ProductPipelineIdentifier("ScoredTweets") - - override val product: Product = ScoredTweetsProduct - - override val paramConfig: ProductParamConfig = scoredTweetsParamConfig - - override def pipelineQueryTransformer( - request: HomeMixerRequest, - params: Params - ): ScoredTweetsQuery = { - val context = request.productContext match { - case Some(context: ScoredTweetsProductContext) => context - case _ => throw PipelineFailure(BadRequest, "ScoredTweetsProductContext not found") - } - - val featureMap = FeatureMapBuilder() - .add(ServedTweetIdsFeature, context.servedTweetIds.getOrElse(Seq.empty)) - .add(TimelineServiceTweetsFeature, context.backfillTweetIds.getOrElse(Seq.empty)) - .build() - - ScoredTweetsQuery( - params = params, - clientContext = request.clientContext, - pipelineCursor = - request.serializedRequestCursor.flatMap(UrtCursorSerializer.deserializeOrderedCursor), - requestedMaxResults = Some(params(ServerMaxResultsParam)), - debugOptions = request.debugParams.flatMap(_.debugOptions), - features = Some(featureMap), - deviceContext = context.deviceContext, - seenTweetIds = context.seenTweetIds, - qualityFactorStatus = None - ) - } - - override val pipelines: Seq[PipelineConfig] = Seq(scoredTweetsRecommendationPipelineConfig) - - override def pipelineSelector(query: ScoredTweetsQuery): ComponentIdentifier = - scoredTweetsRecommendationPipelineConfig.identifier - - override val debugAccessPolicies: Set[AccessPolicy] = DefaultHomeMixerAccessPolicy -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.docx new file mode 100644 index 000000000..2d2d4469a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala deleted file mode 100644 index e3d50040b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/ScoredTweetsRecommendationPipelineConfig.scala +++ /dev/null @@ -1,340 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.functional_component.feature_hydrator.FeedbackHistoryQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.ImpressionBloomFilterQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RealGraphInNetworkScoresQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.RequestQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.feature_hydrator.TweetImpressionsQueryFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.FeedbackFatigueFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslySeenTweetsFilter -import com.twitter.home_mixer.functional_component.filter.PreviouslyServedTweetsFilter -import com.twitter.home_mixer.functional_component.filter.RejectTweetFromViewerFilter -import com.twitter.home_mixer.functional_component.filter.RetweetDeduplicationFilter -import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsEventBusSideEffect -import com.twitter.home_mixer.functional_component.side_effect.PublishClientSentImpressionsManhattanSideEffect -import com.twitter.home_mixer.functional_component.side_effect.PublishImpressionBloomFilterSideEffect -import com.twitter.home_mixer.functional_component.side_effect.UpdateLastNonPollingTimeSideEffect -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InNetworkFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature -import com.twitter.home_mixer.model.HomeFeatures.ScoreFeature -import com.twitter.home_mixer.param.HomeGlobalParams.EnableImpressionBloomFilter -import com.twitter.home_mixer.param.HomeMixerFlagName.TargetFetchLatency -import com.twitter.home_mixer.param.HomeMixerFlagName.TargetScoringLatency -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsBackfillCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsFrsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsInNetworkCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsListsCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsPopularVideosCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsTweetMixerCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.ScoredTweetsUtegCandidatePipelineConfig -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.CachedScoredTweetsQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealTimeInteractionGraphUserVertexQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RequestTimeQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserEngagementQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TwhinUserFollowQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserLanguagesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.UserStateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartAAggregateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.PartBAggregateQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.UserEngagementRealTimeAggregatesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.filter.DuplicateConversationTweetsFilter -import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorFilter -import com.twitter.home_mixer.product.scored_tweets.filter.OutOfNetworkCompetitorURLFilter -import com.twitter.home_mixer.product.scored_tweets.filter.ScoredTweetsSocialContextFilter -import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseDomainMarshaller -import com.twitter.home_mixer.product.scored_tweets.marshaller.ScoredTweetsResponseTransportMarshaller -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsResponse -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxInNetworkResultsParam -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.MaxOutOfNetworkResultsParam -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsHeuristicScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.scoring_pipeline.ScoredTweetsModelScoringPipelineConfig -import com.twitter.home_mixer.product.scored_tweets.selector.KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType -import com.twitter.home_mixer.product.scored_tweets.side_effect.CachedScoredTweetsSideEffect -import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeScoredCandidatesSideEffect -import com.twitter.home_mixer.product.scored_tweets.side_effect.ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect -import com.twitter.home_mixer.{thriftscala => t} -import com.twitter.inject.annotations.Flag -import com.twitter.product_mixer.component_library.feature_hydrator.query.async.AsyncQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.impressed_tweets.ImpressedTweetsQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.param_gated.ParamGatedQueryFeatureHydrator -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersQueryFeatureHydrator -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.component_library.selector.DropDuplicateCandidates -import com.twitter.product_mixer.component_library.selector.DropFilteredMaxCandidates -import com.twitter.product_mixer.component_library.selector.DropMaxCandidates -import com.twitter.product_mixer.component_library.selector.IdAndClassDuplicationKey -import com.twitter.product_mixer.component_library.selector.InsertAppendResults -import com.twitter.product_mixer.component_library.selector.PickFirstCandidateMerger -import com.twitter.product_mixer.component_library.selector.UpdateSortCandidates -import com.twitter.product_mixer.component_library.selector.sorter.FeatureValueSorter -import com.twitter.product_mixer.core.functional_component.common.AllExceptPipelines -import com.twitter.product_mixer.core.functional_component.common.AllPipelines -import com.twitter.product_mixer.core.functional_component.configapi.StaticParam -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.marshaller.TransportMarshaller -import com.twitter.product_mixer.core.functional_component.premarshaller.DomainMarshaller -import com.twitter.product_mixer.core.functional_component.selector.Selector -import com.twitter.product_mixer.core.functional_component.side_effect.PipelineResultSideEffect -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ComponentIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.ScoringPipelineIdentifier -import com.twitter.product_mixer.core.model.common.presentation.ItemCandidateWithDetails -import com.twitter.product_mixer.core.pipeline.FailOpenPolicy -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.product_mixer.core.pipeline.recommendation.RecommendationPipelineConfig -import com.twitter.product_mixer.core.pipeline.scoring.ScoringPipelineConfig -import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault -import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig -import com.twitter.product_mixer.core.quality_factor.QualityFactorConfig -import com.twitter.util.Duration - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsRecommendationPipelineConfig @Inject() ( - scoredTweetsInNetworkCandidatePipelineConfig: ScoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig: ScoredTweetsUtegCandidatePipelineConfig, - scoredTweetsTweetMixerCandidatePipelineConfig: ScoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig: ScoredTweetsFrsCandidatePipelineConfig, - scoredTweetsListsCandidatePipelineConfig: ScoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig: ScoredTweetsPopularVideosCandidatePipelineConfig, - scoredTweetsBackfillCandidatePipelineConfig: ScoredTweetsBackfillCandidatePipelineConfig, - cachedScoredTweetsCandidatePipelineConfig: CachedScoredTweetsCandidatePipelineConfig, - requestQueryFeatureHydrator: RequestQueryFeatureHydrator[ScoredTweetsQuery], - requestTimeQueryFeatureHydrator: RequestTimeQueryFeatureHydrator, - realTimeInteractionGraphUserVertexQueryFeatureHydrator: RealTimeInteractionGraphUserVertexQueryFeatureHydrator, - userStateQueryFeatureHydrator: UserStateQueryFeatureHydrator, - userEngagementRealTimeAggregatesFeatureHydrator: UserEngagementRealTimeAggregatesFeatureHydrator, - twhinUserEngagementQueryFeatureHydrator: TwhinUserEngagementQueryFeatureHydrator, - twhinUserFollowQueryFeatureHydrator: TwhinUserFollowQueryFeatureHydrator, - cachedScoredTweetsQueryFeatureHydrator: CachedScoredTweetsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator: SGSFollowedUsersQueryFeatureHydrator, - scoredTweetsModelScoringPipelineConfig: ScoredTweetsModelScoringPipelineConfig, - impressionBloomFilterQueryFeatureHydrator: ImpressionBloomFilterQueryFeatureHydrator[ - ScoredTweetsQuery - ], - manhattanTweetImpressionsQueryFeatureHydrator: TweetImpressionsQueryFeatureHydrator[ - ScoredTweetsQuery - ], - memcacheTweetImpressionsQueryFeatureHydrator: ImpressedTweetsQueryFeatureHydrator, - listIdsQueryFeatureHydrator: ListIdsQueryFeatureHydrator, - feedbackHistoryQueryFeatureHydrator: FeedbackHistoryQueryFeatureHydrator, - publishClientSentImpressionsEventBusSideEffect: PublishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect: PublishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect: PublishImpressionBloomFilterSideEffect, - realGraphInNetworkScoresQueryFeatureHydrator: RealGraphInNetworkScoresQueryFeatureHydrator, - realGraphQueryFeatureHydrator: RealGraphQueryFeatureHydrator, - userLanguagesFeatureHydrator: UserLanguagesFeatureHydrator, - partAAggregateQueryFeatureHydrator: PartAAggregateQueryFeatureHydrator, - partBAggregateQueryFeatureHydrator: PartBAggregateQueryFeatureHydrator, - cachedScoredTweetsSideEffect: CachedScoredTweetsSideEffect, - scribeScoredCandidatesSideEffect: ScribeScoredCandidatesSideEffect, - scribeServedCommonFeaturesAndCandidateFeaturesSideEffect: ScribeServedCommonFeaturesAndCandidateFeaturesSideEffect, - updateLastNonPollingTimeSideEffect: UpdateLastNonPollingTimeSideEffect[ - ScoredTweetsQuery, - ScoredTweetsResponse - ], - @Flag(TargetFetchLatency) targetFetchLatency: Duration, - @Flag(TargetScoringLatency) targetScoringLatency: Duration) - extends RecommendationPipelineConfig[ - ScoredTweetsQuery, - TweetCandidate, - ScoredTweetsResponse, - t.ScoredTweetsResponse - ] { - - override val identifier: RecommendationPipelineIdentifier = - RecommendationPipelineIdentifier("ScoredTweets") - - private val SubscriptionReplyFilterId = "SubscriptionReply" - private val MaxBackfillTweets = 50 - - private val scoringStep = RecommendationPipelineConfig.scoringPipelinesStep - - override val fetchQueryFeatures: Seq[QueryFeatureHydrator[ScoredTweetsQuery]] = Seq( - requestQueryFeatureHydrator, - realGraphInNetworkScoresQueryFeatureHydrator, - cachedScoredTweetsQueryFeatureHydrator, - sgsFollowedUsersQueryFeatureHydrator, - ParamGatedQueryFeatureHydrator( - EnableImpressionBloomFilter, - impressionBloomFilterQueryFeatureHydrator - ), - manhattanTweetImpressionsQueryFeatureHydrator, - memcacheTweetImpressionsQueryFeatureHydrator, - listIdsQueryFeatureHydrator, - userStateQueryFeatureHydrator, - AsyncQueryFeatureHydrator(scoringStep, feedbackHistoryQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, realGraphQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, requestTimeQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, userLanguagesFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, userEngagementRealTimeAggregatesFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, realTimeInteractionGraphUserVertexQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, twhinUserFollowQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, twhinUserEngagementQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, partAAggregateQueryFeatureHydrator), - AsyncQueryFeatureHydrator(scoringStep, partBAggregateQueryFeatureHydrator), - ) - - override val candidatePipelines: Seq[ - CandidatePipelineConfig[ScoredTweetsQuery, _, _, TweetCandidate] - ] = Seq( - cachedScoredTweetsCandidatePipelineConfig, - scoredTweetsInNetworkCandidatePipelineConfig, - scoredTweetsUtegCandidatePipelineConfig, - scoredTweetsTweetMixerCandidatePipelineConfig, - scoredTweetsFrsCandidatePipelineConfig, - scoredTweetsListsCandidatePipelineConfig, - scoredTweetsPopularVideosCandidatePipelineConfig, - scoredTweetsBackfillCandidatePipelineConfig - ) - - override val postCandidatePipelinesSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - DropDuplicateCandidates( - pipelineScope = AllPipelines, - duplicationKey = IdAndClassDuplicationKey, - mergeStrategy = PickFirstCandidateMerger - ), - InsertAppendResults(AllPipelines) - ) - - override val globalFilters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - // sort these to have the "cheaper" filters run first - RejectTweetFromViewerFilter, - RetweetDeduplicationFilter, - PreviouslySeenTweetsFilter, - PreviouslyServedTweetsFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(SubscriptionReplyFilterId), - shouldKeepCandidate = { features => - features.getOrElse(InReplyToTweetIdFeature, None).isEmpty || - features.getOrElse(ExclusiveConversationAuthorIdFeature, None).isEmpty - } - ), - FeedbackFatigueFilter - ) - - override val candidatePipelineFailOpenPolicies: Map[CandidatePipelineIdentifier, FailOpenPolicy] = - Map( - cachedScoredTweetsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsInNetworkCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsUtegCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsFrsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsListsCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> FailOpenPolicy.Always, - scoredTweetsBackfillCandidatePipelineConfig.identifier -> FailOpenPolicy.Always - ) - - override val scoringPipelineFailOpenPolicies: Map[ScoringPipelineIdentifier, FailOpenPolicy] = - Map( - ScoredTweetsHeuristicScoringPipelineConfig.identifier -> FailOpenPolicy.Always - ) - - private val candidatePipelineQualityFactorConfig = LinearLatencyQualityFactorConfig( - qualityFactorBounds = BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 0.95), - initialDelay = 60.seconds, - targetLatency = targetFetchLatency, - targetLatencyPercentile = 95.0, - delta = 0.00125 - ) - - private val scoringPipelineQualityFactorConfig = - candidatePipelineQualityFactorConfig.copy(targetLatency = targetScoringLatency) - - override val qualityFactorConfigs: Map[ComponentIdentifier, QualityFactorConfig] = Map( - // candidate pipelines - scoredTweetsInNetworkCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsUtegCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsTweetMixerCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsFrsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsListsCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsPopularVideosCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - scoredTweetsBackfillCandidatePipelineConfig.identifier -> candidatePipelineQualityFactorConfig, - // scoring pipelines - scoredTweetsModelScoringPipelineConfig.identifier -> scoringPipelineQualityFactorConfig, - ) - - override val scoringPipelines: Seq[ScoringPipelineConfig[ScoredTweetsQuery, TweetCandidate]] = - Seq( - // scoring pipeline - run on non-cached candidates only since cached ones are already scored - scoredTweetsModelScoringPipelineConfig, - // re-scoring pipeline - run on all candidates since these are request specific - ScoredTweetsHeuristicScoringPipelineConfig - ) - - override val postScoringFilters = Seq( - ScoredTweetsSocialContextFilter, - OutOfNetworkCompetitorFilter, - OutOfNetworkCompetitorURLFilter, - DuplicateConversationTweetsFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier("IsSupportAccountReply"), - shouldKeepCandidate = { features => - !features.getOrElse(IsSupportAccountReplyFeature, false) - }) - ) - - override val resultSelectors: Seq[Selector[ScoredTweetsQuery]] = Seq( - KeepBestOutOfNetworkCandidatePerAuthorPerSuggestType(AllPipelines), - UpdateSortCandidates(AllPipelines, FeatureValueSorter.descending(ScoreFeature)), - DropFilteredMaxCandidates( - pipelineScope = - AllExceptPipelines(Set(scoredTweetsBackfillCandidatePipelineConfig.identifier)), - filter = { - case ItemCandidateWithDetails(_, _, features) => - features.getOrElse(InNetworkFeature, false) - case _ => false - }, - maxSelectionsParam = MaxInNetworkResultsParam - ), - DropFilteredMaxCandidates( - pipelineScope = AllPipelines, - filter = { - case ItemCandidateWithDetails(_, _, features) => - !features.getOrElse(InNetworkFeature, false) - case _ => false - }, - maxSelectionsParam = MaxOutOfNetworkResultsParam - ), - DropMaxCandidates( - candidatePipeline = scoredTweetsBackfillCandidatePipelineConfig.identifier, - maxSelectionsParam = StaticParam(MaxBackfillTweets) - ), - InsertAppendResults(AllPipelines) - ) - - override val resultSideEffects: Seq[ - PipelineResultSideEffect[ScoredTweetsQuery, ScoredTweetsResponse] - ] = Seq( - cachedScoredTweetsSideEffect, - publishClientSentImpressionsEventBusSideEffect, - publishClientSentImpressionsManhattanSideEffect, - publishImpressionBloomFilterSideEffect, - scribeScoredCandidatesSideEffect, - scribeServedCommonFeaturesAndCandidateFeaturesSideEffect, - updateLastNonPollingTimeSideEffect - ) - - override val domainMarshaller: DomainMarshaller[ - ScoredTweetsQuery, - ScoredTweetsResponse - ] = ScoredTweetsResponseDomainMarshaller - - override val transportMarshaller: TransportMarshaller[ - ScoredTweetsResponse, - t.ScoredTweetsResponse - ] = ScoredTweetsResponseTransportMarshaller -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel deleted file mode 100644 index c5bb294a1..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.bazel +++ /dev/null @@ -1,37 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "explore/explore-ranker/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/filter", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/explore_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/tweet_mixer", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/marshaller/request", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "tweet-mixer/thrift/src/main/thrift:thrift-scala", - ], - exports = [ - "src/thrift/com/twitter/timelineranker:thrift-scala", - "tweet-mixer/thrift/src/main/thrift:thrift-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.docx new file mode 100644 index 000000000..2ef4b8453 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.docx new file mode 100644 index 000000000..8e3eeb68e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala deleted file mode 100644 index 65a42cbee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/CachedScoredTweetsCandidatePipelineConfig.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.CachedScoredTweetsCandidatePipelineConfig._ -import com.twitter.home_mixer.product.scored_tweets.candidate_source.CachedScoredTweetsCandidateSource -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.response_transformer.CachedScoredTweetsResponseFeatureTransformer -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from Scored Tweets Cache. - */ -@Singleton -class CachedScoredTweetsCandidatePipelineConfig @Inject() ( - cachedScoredTweetsCandidateSource: CachedScoredTweetsCandidateSource) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - ScoredTweetsQuery, - hmt.ScoredTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = Identifier - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - ScoredTweetsQuery - ] = identity - - override val candidateSource: BaseCandidateSource[ScoredTweetsQuery, hmt.ScoredTweet] = - cachedScoredTweetsCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[hmt.ScoredTweet] - ] = Seq(CachedScoredTweetsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - hmt.ScoredTweet, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } -} - -object CachedScoredTweetsCandidatePipelineConfig { - val Identifier: CandidatePipelineIdentifier = CandidatePipelineIdentifier("CachedScoredTweets") -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.docx new file mode 100644 index 000000000..d3406c727 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala deleted file mode 100644 index c34031c78..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsBackfillCandidatePipelineConfig.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.functional_component.filter.ReplyFilter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.TimelineServiceTweetsFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.gate.MinTimeSinceLastRequestGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.EnableBackfillCandidatePipelineParam -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsBackfillResponseFeatureTransformer -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.candidate_source.PassthroughCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.FSParam -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsBackfillCandidatePipelineConfig @Inject() ( - tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - ScoredTweetsQuery, - Long, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsBackfill") - - private val HasAuthorFilterId = "HasAuthor" - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableBackfillParam) - - override val supportedClientParam: Option[FSParam[Boolean]] = - Some(EnableBackfillCandidatePipelineParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = - Seq( - MinTimeSinceLastRequestGate, - NonEmptySeqFeatureGate(TimelineServiceTweetsFeature), - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - ScoredTweetsQuery - ] = identity - - override def candidateSource: CandidateSource[ScoredTweetsQuery, Long] = - PassthroughCandidateSource( - identifier = CandidateSourceIdentifier("ScoredTweetsBackfill"), - candidateExtractor = { query => - query.features.map(_.getOrElse(TimelineServiceTweetsFeature, Seq.empty)).toSeq.flatten - } - ) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[Long] - ] = Seq(ScoredTweetsBackfillResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[Long, TweetCandidate] = { - sourceResult => TweetCandidate(id = sourceResult) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesHydrator) - - override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - ReplyFilter, - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(HasAuthorFilterId), - shouldKeepCandidate = _.getOrElse(AuthorIdFeature, None).isDefined - ) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.docx new file mode 100644 index 000000000..c11725a1f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala deleted file mode 100644 index f25a839b7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsFrsCandidatePipelineConfig.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerFrsQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsFrsResponseFeatureTransformer -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerRecapCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineranker.{thriftscala => tlr} -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that takes user recommendations from Follow Recommendation Service (FRS) - * and makes a TimelineRanker->Earlybird query for tweet candidates from those users. - * Additionally, the candidate pipeline hydrates followedByUserIds so that followed-by social proof - * can be used. - */ -@Singleton -class ScoredTweetsFrsCandidatePipelineConfig @Inject() ( - timelineRankerRecapCandidateSource: TimelineRankerRecapCandidateSource, - frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - tlr.RecapQuery, - tlr.CandidateTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsFrs") - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableFrsParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryFeatureHydration: Seq[ - BaseQueryFeatureHydrator[ScoredTweetsQuery, _] - ] = Seq(frsSeedUsersQueryFeatureHydrator) - - override val candidateSource: BaseCandidateSource[tlr.RecapQuery, tlr.CandidateTweet] = - timelineRankerRecapCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - tlr.RecapQuery - ] = TimelineRankerFrsQueryTransformer(identifier) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[tlr.CandidateTweet] - ] = Seq(ScoredTweetsFrsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - tlr.CandidateTweet, - TweetCandidate - ] = { candidate => TweetCandidate(candidate.tweet.get.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.docx new file mode 100644 index 000000000..4bfd7fe31 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala deleted file mode 100644 index 63bd1add8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsInNetworkCandidatePipelineConfig.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.IsExtendedReplyFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ReplyFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RetweetSourceTweetFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.filter.RetweetSourceTweetRemovingFilter -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerInNetworkQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsInNetworkResponseFeatureTransformer -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineranker.{thriftscala => t} -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config to fetch in-network tweets from Timeline Ranker's Recycled source - */ -@Singleton -class ScoredTweetsInNetworkCandidatePipelineConfig @Inject() ( - timelineRankerInNetworkCandidateSource: TimelineRankerInNetworkCandidateSource, - replyFeatureHydrator: ReplyFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - t.RecapQuery, - t.CandidateTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsInNetwork") - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableInNetworkParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val candidateSource: BaseCandidateSource[t.RecapQuery, t.CandidateTweet] = - timelineRankerInNetworkCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - t.RecapQuery - ] = TimelineRankerInNetworkQueryTransformer(identifier) - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] - ] = Seq(RetweetSourceTweetFeatureHydrator) - - override def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - RetweetSourceTweetRemovingFilter - ) - - override val postFilterFeatureHydration: Seq[ - BaseCandidateFeatureHydrator[PipelineQuery, TweetCandidate, _] - ] = Seq(IsExtendedReplyFeatureHydrator, replyFeatureHydrator) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.CandidateTweet] - ] = Seq(ScoredTweetsInNetworkResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.CandidateTweet, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweet.get.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.docx new file mode 100644 index 000000000..ddcdc6824 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala deleted file mode 100644 index 1161a8278..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsListsCandidatePipelineConfig.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.functional_component.filter.ReplyFilter -import com.twitter.home_mixer.functional_component.filter.RetweetFilter -import com.twitter.home_mixer.product.scored_tweets.candidate_source.ListsCandidateSource -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.ListIdsFeature -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsListsResponseFeatureTransformer -import com.twitter.product_mixer.component_library.gate.NonEmptySeqFeatureGate -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam -import com.twitter.timelineservice.{thriftscala => t} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsListsCandidatePipelineConfig @Inject() ( - listsCandidateSource: ListsCandidateSource, - tweetypieStaticEntitiesHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - Seq[t.TimelineQuery], - t.Tweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsLists") - - private val MaxTweetsToFetchPerList = 20 - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableListsParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - NonEmptySeqFeatureGate(ListIdsFeature), - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - Seq[t.TimelineQuery] - ] = { query => - val listIds = query.features.map(_.get(ListIdsFeature)).get - listIds.map { listId => - t.TimelineQuery( - timelineType = t.TimelineType.List, - timelineId = listId, - maxCount = MaxTweetsToFetchPerList.toShort, - options = Some(t.TimelineQueryOptions(query.clientContext.userId)), - timelineId2 = Some(t.TimelineId(t.TimelineType.List, listId, None)) - ) - } - } - - override def candidateSource: CandidateSource[Seq[t.TimelineQuery], t.Tweet] = - listsCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.Tweet] - ] = Seq(ScoredTweetsListsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[t.Tweet, TweetCandidate] = { - sourceResult => TweetCandidate(id = sourceResult.statusId) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesHydrator) - - override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = - Seq(ReplyFilter, RetweetFilter) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.docx new file mode 100644 index 000000000..ab93cb904 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala deleted file mode 100644 index c8b28b0ff..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsPopularVideosCandidatePipelineConfig.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.explore_ranker.{thriftscala => ert} -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsPopularVideosResponseFeatureTransformer -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.product_mixer.component_library.candidate_source.explore_ranker.ExploreRankerImmersiveRecsCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ScoredTweetsPopularVideosCandidatePipelineConfig @Inject() ( - exploreRankerCandidateSource: ExploreRankerImmersiveRecsCandidateSource, - tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - ert.ExploreRankerRequest, - ert.ExploreTweetRecommendation, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsPopularVideos") - - private val MaxTweetsToFetch = 40 - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnablePopularVideosParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - ert.ExploreRankerRequest - ] = { query => - val excludedTweetIds = query.features.map( - CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) - - ert.ExploreRankerRequest( - clientContext = ClientContextMarshaller(query.clientContext), - product = ert.Product.HomeTimelineVideoInline, - productContext = Some( - ert.ProductContext.HomeTimelineVideoInline(ert.HomeTimelineVideoInline(excludedTweetIds))), - maxResults = Some(MaxTweetsToFetch) - ) - } - - override def candidateSource: BaseCandidateSource[ - ert.ExploreRankerRequest, - ert.ExploreTweetRecommendation - ] = exploreRankerCandidateSource - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[ert.ExploreTweetRecommendation] - ] = Seq(ScoredTweetsPopularVideosResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - ert.ExploreTweetRecommendation, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesFeatureHydrator) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.docx new file mode 100644 index 000000000..8412589ba Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala deleted file mode 100644 index df16e3f11..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsTweetMixerCandidatePipelineConfig.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.tweet_mixer.{thriftscala => t} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TweetypieStaticEntitiesFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsTweetMixerResponseFeatureTransformer -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.product_mixer.component_library.candidate_source.tweet_mixer.TweetMixerCandidateSource -import com.twitter.product_mixer.component_library.filter.PredicateFeatureFilter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseCandidateFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.marshaller.request.ClientContextMarshaller -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.model.common.identifier.FilterIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelines.configapi.decider.DeciderParam - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from TweetMixer. - */ -@Singleton -class ScoredTweetsTweetMixerCandidatePipelineConfig @Inject() ( - tweetMixerCandidateSource: TweetMixerCandidateSource, - tweetypieStaticEntitiesFeatureHydrator: TweetypieStaticEntitiesFeatureHydrator) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - t.TweetMixerRequest, - t.TweetResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsTweetMixer") - - val HasAuthorFilterId = "HasAuthor" - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableTweetMixerParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam), - ) - - override val candidateSource: BaseCandidateSource[t.TweetMixerRequest, t.TweetResult] = - tweetMixerCandidateSource - - private val MaxTweetsToFetch = 400 - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - t.TweetMixerRequest - ] = { query => - val maxCount = (query.getQualityFactorCurrentValue(identifier) * MaxTweetsToFetch).toInt - - val excludedTweetIds = query.features.map( - CachedScoredTweetsHelper.tweetImpressionsAndCachedScoredTweets(_, identifier)) - - t.TweetMixerRequest( - clientContext = ClientContextMarshaller(query.clientContext), - product = t.Product.HomeRecommendedTweets, - productContext = Some( - t.ProductContext.HomeRecommendedTweetsProductContext( - t.HomeRecommendedTweetsProductContext(excludedTweetIds = excludedTweetIds.map(_.toSet)))), - maxResults = Some(maxCount) - ) - } - - override val preFilterFeatureHydrationPhase1: Seq[ - BaseCandidateFeatureHydrator[ScoredTweetsQuery, TweetCandidate, _] - ] = Seq(tweetypieStaticEntitiesFeatureHydrator) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.TweetResult] - ] = Seq(ScoredTweetsTweetMixerResponseFeatureTransformer) - - override val filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq( - PredicateFeatureFilter.fromPredicate( - FilterIdentifier(HasAuthorFilterId), - shouldKeepCandidate = _.getOrElse(AuthorIdFeature, None).isDefined - ) - ) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.TweetResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweetId) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.docx new file mode 100644 index 000000000..a0581e189 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala deleted file mode 100644 index 9178d0789..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/ScoredTweetsUtegCandidatePipelineConfig.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline - -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CandidatePipeline -import com.twitter.home_mixer.product.scored_tweets.query_transformer.TimelineRankerUtegQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.ScoredTweetsUtegResponseFeatureTransformer -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerUtegCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.timelineranker.{thriftscala => t} -import com.twitter.timelines.configapi.decider.DeciderParam -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the Timeline Ranker UTEG Candidate Source - */ -@Singleton -class ScoredTweetsUtegCandidatePipelineConfig @Inject() ( - timelineRankerUtegCandidateSource: TimelineRankerUtegCandidateSource) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - t.UtegLikedByTweetsQuery, - t.CandidateTweet, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsUteg") - - override val enabledDeciderParam: Option[DeciderParam[Boolean]] = - Some(CandidatePipeline.EnableUtegParam) - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val candidateSource: BaseCandidateSource[t.UtegLikedByTweetsQuery, t.CandidateTweet] = - timelineRankerUtegCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - t.UtegLikedByTweetsQuery - ] = TimelineRankerUtegQueryTransformer(identifier) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[t.CandidateTweet] - ] = Seq(ScoredTweetsUtegResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - t.CandidateTweet, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.tweet.get.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel deleted file mode 100644 index 618c4a081..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/gate", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/query_transformer/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/response_transformer/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/earlybird", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/filter", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/gate", - "src/thrift/com/twitter/search:earlybird-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.docx new file mode 100644 index 000000000..370e2c282 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.docx new file mode 100644 index 000000000..2276818f7 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala deleted file mode 100644 index 0d59bf3ca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdFrsCandidatePipelineConfig.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird - -import com.twitter.finagle.thrift.ClientId -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.FrsSeedUsersQueryFeatureHydrator -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdFrsQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdFrsResponseFeatureTransformer -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BaseQueryFeatureHydrator -import com.twitter.product_mixer.core.functional_component.filter.Filter -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => eb} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the earlybird FRS Candidate Source - */ -@Singleton -class ScoredTweetsEarlybirdFrsCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - frsSeedUsersQueryFeatureHydrator: FrsSeedUsersQueryFeatureHydrator, - clientId: ClientId) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - eb.EarlybirdRequest, - eb.ThriftSearchResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsEarlybirdFrs") - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val queryFeatureHydration: Seq[ - BaseQueryFeatureHydrator[ScoredTweetsQuery, _] - ] = Seq(frsSeedUsersQueryFeatureHydrator) - - override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = - earlybirdCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - eb.EarlybirdRequest - ] = EarlybirdFrsQueryTransformer(identifier, clientId = Some(clientId.name)) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[eb.ThriftSearchResult] - ] = Seq(ScoredTweetsEarlybirdFrsResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - eb.ThriftSearchResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.id) } - - override def filters: Seq[Filter[ScoredTweetsQuery, TweetCandidate]] = Seq.empty -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.docx new file mode 100644 index 000000000..4ef96c569 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala deleted file mode 100644 index acb26ce42..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_pipeline/earlybird/ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_pipeline.earlybird - -import com.twitter.finagle.thrift.ClientId -import com.twitter.home_mixer.functional_component.candidate_source.EarlybirdCandidateSource -import com.twitter.home_mixer.product.scored_tweets.gate.MinCachedTweetsGate -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.product.scored_tweets.query_transformer.earlybird.EarlybirdInNetworkQueryTransformer -import com.twitter.home_mixer.product.scored_tweets.response_transformer.earlybird.ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.functional_component.candidate_source.BaseCandidateSource -import com.twitter.product_mixer.core.functional_component.gate.Gate -import com.twitter.product_mixer.core.functional_component.transformer.CandidateFeatureTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineQueryTransformer -import com.twitter.product_mixer.core.functional_component.transformer.CandidatePipelineResultsTransformer -import com.twitter.product_mixer.core.model.common.identifier.CandidatePipelineIdentifier -import com.twitter.product_mixer.core.pipeline.candidate.CandidatePipelineConfig -import com.twitter.search.earlybird.{thriftscala => eb} -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Candidate Pipeline Config that fetches tweets from the earlybird InNetwork Candidate Source - */ -@Singleton -class ScoredTweetsEarlybirdInNetworkCandidatePipelineConfig @Inject() ( - earlybirdCandidateSource: EarlybirdCandidateSource, - clientId: ClientId) - extends CandidatePipelineConfig[ - ScoredTweetsQuery, - eb.EarlybirdRequest, - eb.ThriftSearchResult, - TweetCandidate - ] { - - override val identifier: CandidatePipelineIdentifier = - CandidatePipelineIdentifier("ScoredTweetsEarlybirdInNetwork") - - override val gates: Seq[Gate[ScoredTweetsQuery]] = Seq( - MinCachedTweetsGate(identifier, CachedScoredTweets.MinCachedTweetsParam) - ) - - override val candidateSource: BaseCandidateSource[eb.EarlybirdRequest, eb.ThriftSearchResult] = - earlybirdCandidateSource - - override val queryTransformer: CandidatePipelineQueryTransformer[ - ScoredTweetsQuery, - eb.EarlybirdRequest - ] = EarlybirdInNetworkQueryTransformer(identifier, clientId = Some(clientId.name)) - - override val featuresFromCandidateSourceTransformers: Seq[ - CandidateFeatureTransformer[eb.ThriftSearchResult] - ] = Seq(ScoredTweetsEarlybirdInNetworkResponseFeatureTransformer) - - override val resultTransformer: CandidatePipelineResultsTransformer[ - eb.ThriftSearchResult, - TweetCandidate - ] = { sourceResult => TweetCandidate(id = sourceResult.id) } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel deleted file mode 100644 index 4461d8460..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product", - "src/thrift/com/twitter/timelineservice:thrift-scala", - "stitch/stitch-timelineservice", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.docx new file mode 100644 index 000000000..73a9a92c2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.docx new file mode 100644 index 000000000..9c9d38fa0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala deleted file mode 100644 index a5f556e7c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/CachedScoredTweetsCandidateSource.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_source - -import com.twitter.home_mixer.util.CachedScoredTweetsHelper -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class CachedScoredTweetsCandidateSource @Inject() () - extends CandidateSource[PipelineQuery, hmt.ScoredTweet] { - - override val identifier: CandidateSourceIdentifier = - CandidateSourceIdentifier("CachedScoredTweets") - - override def apply(request: PipelineQuery): Stitch[Seq[hmt.ScoredTweet]] = { - Stitch.value( - request.features.map(CachedScoredTweetsHelper.unseenCachedScoredTweets).getOrElse(Seq.empty)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.docx new file mode 100644 index 000000000..93cb19cbe Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala deleted file mode 100644 index 395ac7da8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/candidate_source/ListsCandidateSource.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.candidate_source - -import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource -import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier -import com.twitter.stitch.Stitch -import com.twitter.stitch.timelineservice.TimelineService -import com.twitter.timelineservice.{thriftscala => tls} - -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class ListsCandidateSource @Inject() (timelineService: TimelineService) - extends CandidateSource[Seq[tls.TimelineQuery], tls.Tweet] { - - override val identifier: CandidateSourceIdentifier = CandidateSourceIdentifier("Lists") - - override def apply(requests: Seq[tls.TimelineQuery]): Stitch[Seq[tls.Tweet]] = { - val timelines = Stitch.traverse(requests) { request => timelineService.getTimeline(request) } - - timelines.map { - _.flatMap { - _.entries.collect { case tls.TimelineEntry.Tweet(tweet) => tweet } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.docx new file mode 100644 index 000000000..c78082694 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala deleted file mode 100644 index 9c5e8f3ab..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AncestorFeatureHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AncestorsFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.tweetconvosvc.tweet_ancestor.{thriftscala => ta} -import com.twitter.tweetconvosvc.{thriftscala => tcs} -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AncestorFeatureHydrator @Inject() ( - conversationServiceClient: tcs.ConversationService.MethodPerEndpoint) - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Ancestor") - - override val features: Set[Feature[_, _]] = Set(AncestorsFeature) - - private val DefaultFeatureMap = FeatureMapBuilder().add(AncestorsFeature, Seq.empty).build() - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { - if (existingFeatures.getOrElse(InReplyToTweetIdFeature, None).isDefined) { - val ancestorsRequest = tcs.GetAncestorsRequest(Seq(candidate.id)) - conversationServiceClient.getAncestors(ancestorsRequest).map { getAncestorsResponse => - val ancestors = getAncestorsResponse.ancestors.headOption - .collect { - case tcs.TweetAncestorsResult.TweetAncestors(ancestorsResult) - if ancestorsResult.nonEmpty => - ancestorsResult.head.ancestors ++ getTruncatedRootTweet(ancestorsResult.head) - }.getOrElse(Seq.empty) - - FeatureMapBuilder().add(AncestorsFeature, ancestors).build() - } - } else Future.value(DefaultFeatureMap) - } - - private def getTruncatedRootTweet( - ancestors: ta.TweetAncestors, - ): Option[ta.TweetAncestor] = { - ancestors.conversationRootAuthorId.collect { - case rootAuthorId - if ancestors.state == ta.ReplyState.Partial && - ancestors.ancestors.last.tweetId != ancestors.conversationId => - ta.TweetAncestor(ancestors.conversationId, rootAuthorId) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.docx new file mode 100644 index 000000000..6ecb4ac57 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala deleted file mode 100644 index c8dd6a934..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorFeatureHydrator.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.AuthorFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features.AuthorFeaturesAdapter -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.KeyValueResult -import com.twitter.stitch.Stitch -import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.util.Future -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object AuthorFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class AuthorFeatureHydrator @Inject() ( - @Named(AuthorFeatureRepository) client: KeyValueRepository[Seq[Long], Long, af.AuthorFeatures], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("AuthorFeature") - - override val features: Set[Feature[_, _]] = Set(AuthorFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, af.AuthorFeatures]] = - if (authorIds.nonEmpty) client(authorIds) - else Future.value(KeyValueResult.empty) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - - FeatureMapBuilder().add(AuthorFeature, transformedValue).build() - } - } - } - - private def postTransformer(authorFeatures: Try[Option[af.AuthorFeatures]]): Try[DataRecord] = { - authorFeatures.map { - _.map { features => AuthorFeaturesAdapter.adaptToDataRecords(features).asScala.head } - .getOrElse(new DataRecord()) - } - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - CandidatesUtil.getOriginalAuthorId(candidate.features) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.docx new file mode 100644 index 000000000..8ea939f11 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala deleted file mode 100644 index cc66983cc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/AuthorIsCreatorFeatureHydrator.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsCreatorFeature -import com.twitter.home_mixer.util.MissingKeyException -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.strato.generated.client.audiencerewards.audienceRewardsService.GetSuperFollowEligibilityOnUserClientColumn -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class AuthorIsCreatorFeatureHydrator @Inject() ( - getSuperFollowEligibilityOnUserClientColumn: GetSuperFollowEligibilityOnUserClientColumn, - statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("AuthorIsCreator") - - override val features: Set[Feature[_, _]] = - Set(AuthorIsCreatorFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - - private val MissingKeyFeatureMap = FeatureMapBuilder() - .add(AuthorIsCreatorFeature, Throw(MissingKeyException)) - .build() - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(AuthorIsCreatorFeature, false) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - OffloadFuturePools.offloadStitch { - val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)).distinct - Stitch - .collect { - authorIds.map { authorId => - getSuperFollowEligibilityOnUserClientColumn.fetcher - .fetch(authorId) - .map { authorId -> _.v } - } - }.map { authorIdsToIsCreator => - val authorIdsToIsCreatorMap = authorIdsToIsCreator.toMap - candidates.map { candidate => - candidate.features.getOrElse(AuthorIdFeature, None) match { - case Some(authorId) => - authorIdsToIsCreatorMap.get(authorId) match { - case Some(response) => - keyFoundCounter.incr() - FeatureMapBuilder() - .add(AuthorIsCreatorFeature, response.getOrElse(false)).build() - case _ => - keyFailureCounter.incr() - DefaultFeatureMap - } - case _ => MissingKeyFeatureMap - } - } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel deleted file mode 100644 index 58a566de8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.bazel +++ /dev/null @@ -1,70 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "follow-recommendations-service/thrift/src/main/thrift:thrift-scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/service", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/earlybird", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util/tweetypie/content", - "home-mixer/thrift/src/main/thrift:thrift-scala", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/recommendations", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/candidate_source/timeline_ranker", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/feature_hydrator/query/social_graph", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/candidate_source/strato", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "representation-scorer/server/src/main/scala/com/twitter/representationscorer/common", - "servo/repo/src/main/scala", - "servo/util/src/main/scala", - "snowflake/src/main/scala/com/twitter/snowflake/id", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/adapters/conversation_features", - "src/scala/com/twitter/timelines/prediction/adapters/real_graph", - "src/scala/com/twitter/timelines/prediction/adapters/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/adapters/twistly", - "src/scala/com/twitter/timelines/prediction/adapters/two_hop_features", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/common/util", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/realtime_interaction_graph", - "src/scala/com/twitter/timelines/prediction/features/time_features", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/onboarding/relevance/features:features-java", - "src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala", - "src/thrift/com/twitter/search:earlybird-scala", - "src/thrift/com/twitter/timelineranker:thrift-scala", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "stitch/stitch-gizmoduck", - "stitch/stitch-socialgraph", - "stitch/stitch-tweetypie", - "strato/config/columns/audiencerewards/audienceRewardsService:getSuperFollowEligibility-strato-client", - "strato/config/columns/ml/featureStore:featureStore-strato-client", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/topics", - "timelines/src/main/scala/com/twitter/timelines/clients/strato/twistly", - "timelines/src/main/scala/com/twitter/timelines/common/model", - "timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils", - "timelines/src/main/scala/com/twitter/timelines/model/candidate", - "timelines/src/main/scala/com/twitter/timelines/model/types", - "topic-social-proof/server/src/main/thrift:thrift-scala", - "topiclisting/topiclisting-core/src/main/scala/com/twitter/topiclisting", - "tweetconvosvc/thrift/src/main/thrift:thrift-scala", - "user_session_store/src/main/scala/com/twitter/user_session_store", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.docx new file mode 100644 index 000000000..bb3b78e34 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.docx new file mode 100644 index 000000000..f09f31f64 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala deleted file mode 100644 index 873f043be..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/CachedScoredTweetsQueryFeatureHydrator.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam.CachedScoredTweets -import com.twitter.home_mixer.{thriftscala => hmt} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.TtlCache -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Time - -import javax.inject.Inject -import javax.inject.Singleton - -/** - * Fetch scored Tweets from cache and exclude the expired ones - */ -@Singleton -case class CachedScoredTweetsQueryFeatureHydrator @Inject() ( - scoredTweetsCache: TtlCache[Long, hmt.ScoredTweetsResponse]) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("CachedScoredTweets") - - override val features: Set[Feature[_, _]] = Set(CachedScoredTweetsFeature) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - val tweetScoreTtl = query.params(CachedScoredTweets.TTLParam) - - Stitch.callFuture(scoredTweetsCache.get(Seq(userId))).map { keyValueResult => - keyValueResult(userId) match { - case Return(cachedCandidatesOpt) => - val cachedScoredTweets = cachedCandidatesOpt.map(_.scoredTweets).getOrElse(Seq.empty) - val nonExpiredTweets = cachedScoredTweets.filter { tweet => - tweet.lastScoredTimestampMs.exists(Time.fromMilliseconds(_).untilNow < tweetScoreTtl) - } - FeatureMapBuilder().add(CachedScoredTweetsFeature, nonExpiredTweets).build() - case Throw(exception) => throw exception - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.docx new file mode 100644 index 000000000..1fd801a26 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala deleted file mode 100644 index cc3c2aeeb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/EarlybirdFeatureHydrator.scala +++ /dev/null @@ -1,151 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.EarlybirdAdapter -import com.twitter.home_mixer.model.HomeFeatures.DeviceLanguageFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdSearchResultFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.TweetUrlsFeature -import com.twitter.home_mixer.model.HomeFeatures.UserScreenNameFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EarlybirdRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.home_mixer.util.earlybird.EarlybirdResponseUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.search.earlybird.{thriftscala => eb} -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object EarlybirdDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class EarlybirdFeatureHydrator @Inject() ( - @Named(EarlybirdRepository) client: KeyValueRepository[ - (Seq[Long], Long), - Long, - eb.ThriftSearchResult - ], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Earlybird") - - override val features: Set[Feature[_, _]] = Set( - EarlybirdDataRecordFeature, - EarlybirdFeature, - EarlybirdSearchResultFeature, - TweetUrlsFeature - ) - - override val statScope: String = identifier.toString - - private val scopedStatsReceiver = statsReceiver.scope(statScope) - private val originalKeyFoundCounter = scopedStatsReceiver.counter("originalKey/found") - private val originalKeyLossCounter = scopedStatsReceiver.counter("originalKey/loss") - - private val ebSearchResultNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = - candidate => candidate.features.getOrElse(EarlybirdSearchResultFeature, None).isEmpty - private val ebFeaturesNotExistPredicate: CandidateWithFeatures[TweetCandidate] => Boolean = - candidate => candidate.features.getOrElse(EarlybirdFeature, None).isEmpty - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val candidatesToHydrate = candidates.filter { candidate => - val isEmpty = - ebFeaturesNotExistPredicate(candidate) && ebSearchResultNotExistPredicate(candidate) - if (isEmpty) originalKeyLossCounter.incr() else originalKeyFoundCounter.incr() - isEmpty - } - - client((candidatesToHydrate.map(_.candidate.id), query.getRequiredUserId)) - .map(handleResponse(query, candidates, _, candidatesToHydrate)) - } - - private def handleResponse( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]], - results: KeyValueResult[Long, eb.ThriftSearchResult], - candidatesToHydrate: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[FeatureMap] = { - val queryFeatureMap = query.features.getOrElse(FeatureMap.empty) - val userLanguages = queryFeatureMap.getOrElse(UserLanguagesFeature, Seq.empty) - val uiLanguageCode = queryFeatureMap.getOrElse(DeviceLanguageFeature, None) - val screenName = queryFeatureMap.getOrElse(UserScreenNameFeature, None) - val followedUserIds = queryFeatureMap.getOrElse(SGSFollowedUsersFeature, Seq.empty).toSet - - val searchResults = candidatesToHydrate - .map { candidate => - observedGet(Some(candidate.candidate.id), results) - }.collect { - case Return(Some(value)) => value - } - - val allSearchResults = searchResults ++ - candidates.filter(!ebSearchResultNotExistPredicate(_)).flatMap { candidate => - candidate.features - .getOrElse(EarlybirdSearchResultFeature, None) - } - val idToSearchResults = allSearchResults.map(sr => sr.id -> sr).toMap - val tweetIdToEbFeatures = EarlybirdResponseUtil.getTweetThriftFeaturesByTweetId( - searcherUserId = query.getRequiredUserId, - screenName = screenName, - userLanguages = userLanguages, - uiLanguageCode = uiLanguageCode, - followedUserIds = followedUserIds, - mutuallyFollowingUserIds = Set.empty, - searchResults = allSearchResults, - sourceTweetSearchResults = Seq.empty, - ) - - candidates.map { candidate => - val transformedEbFeatures = tweetIdToEbFeatures.get(candidate.candidate.id) - val earlybirdFeatures = - if (transformedEbFeatures.nonEmpty) transformedEbFeatures - else candidate.features.getOrElse(EarlybirdFeature, None) - - val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) - val sourceTweetEbFeatures = - candidate.features.getOrElse(SourceTweetEarlybirdFeature, None) - - val originalTweetEbFeatures = - if (candidateIsRetweet && sourceTweetEbFeatures.nonEmpty) - sourceTweetEbFeatures - else earlybirdFeatures - - val earlybirdDataRecord = - EarlybirdAdapter.adaptToDataRecords(originalTweetEbFeatures).asScala.head - - FeatureMapBuilder() - .add(EarlybirdFeature, earlybirdFeatures) - .add(EarlybirdDataRecordFeature, earlybirdDataRecord) - .add(EarlybirdSearchResultFeature, idToSearchResults.get(candidate.candidate.id)) - .add(TweetUrlsFeature, earlybirdFeatures.flatMap(_.urlsList).getOrElse(Seq.empty)) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.docx new file mode 100644 index 000000000..8ae74f428 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala deleted file mode 100644 index 2508067b0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/FrsSeedUsersQueryFeatureHydrator.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.follow_recommendations.{thriftscala => frs} -import com.twitter.home_mixer.product.scored_tweets.model.ScoredTweetsQuery -import com.twitter.product_mixer.component_library.candidate_source.recommendations.UserFollowRecommendationsCandidateSource -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.candidate_source.strato.StratoKeyView -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Singleton - -object FrsSeedUserIdsFeature extends Feature[TweetCandidate, Option[Seq[Long]]] -object FrsUserToFollowedByUserIdsFeature extends Feature[TweetCandidate, Map[Long, Seq[Long]]] - -@Singleton -case class FrsSeedUsersQueryFeatureHydrator @Inject() ( - userFollowRecommendationsCandidateSource: UserFollowRecommendationsCandidateSource) - extends QueryFeatureHydrator[ScoredTweetsQuery] { - - private val maxUsersToFetch = 100 - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("FrsSeedUsers") - - override def features: Set[Feature[_, _]] = Set( - FrsSeedUserIdsFeature, - FrsUserToFollowedByUserIdsFeature - ) - - override def hydrate(query: ScoredTweetsQuery): Stitch[FeatureMap] = { - val frsRequest = frs.RecommendationRequest( - clientContext = frs.ClientContext(query.getOptionalUserId), - displayLocation = frs.DisplayLocation.HomeTimelineTweetRecs, - maxResults = Some(maxUsersToFetch) - ) - - userFollowRecommendationsCandidateSource(StratoKeyView(frsRequest, Unit)) - .map { userRecommendations: Seq[frs.UserRecommendation] => - val seedUserIds = userRecommendations.map(_.userId) - val seedUserIdsSet = seedUserIds.toSet - - val userToFollowedByUserIds: Map[Long, Seq[Long]] = userRecommendations.flatMap { - userRecommendation => - if (seedUserIdsSet.contains(userRecommendation.userId)) { - val followProof = - userRecommendation.reason.flatMap(_.accountProof).flatMap(_.followProof) - val followedByUserIds = followProof.map(_.userIds).getOrElse(Seq.empty) - Some(userRecommendation.userId -> followedByUserIds) - } else { - None - } - }.toMap - - FeatureMapBuilder() - .add(FrsSeedUserIdsFeature, Some(seedUserIds)) - .add(FrsUserToFollowedByUserIdsFeature, userToFollowedByUserIds) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.docx new file mode 100644 index 000000000..bb910f834 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala deleted file mode 100644 index 7c96ef474..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GizmoduckAuthorFeatureHydrator.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.ads.entities.db.{thriftscala => ae} -import com.twitter.gizmoduck.{thriftscala => gt} -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsBlueVerifiedFeature -import com.twitter.home_mixer.model.HomeFeatures.AuthorIsProtectedFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.IsSupportAccountReplyFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.snowflake.id.SnowflakeId -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class GizmoduckAuthorFeatureHydrator @Inject() (gizmoduck: gt.UserService.MethodPerEndpoint) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("GizmoduckAuthor") - - override val features: Set[Feature[_, _]] = - Set(AuthorIsBlueVerifiedFeature, AuthorIsProtectedFeature, IsSupportAccountReplyFeature) - - private val queryFields: Set[gt.QueryFields] = - Set(gt.QueryFields.AdvertiserAccount, gt.QueryFields.Profile, gt.QueryFields.Safety) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val authorIds = candidates.flatMap(_.features.getOrElse(AuthorIdFeature, None)) - - val response = gizmoduck.get( - userIds = authorIds.distinct, - queryFields = queryFields, - context = gt.LookupContext() - ) - - response.map { hydratedAuthors => - val userMetadataMap = hydratedAuthors - .collect { - case userResult if userResult.user.isDefined => - val user = userResult.user.get - val blueVerified = user.safety.flatMap(_.isBlueVerified).getOrElse(false) - val isProtected = user.safety.exists(_.isProtected) - (user.id, (blueVerified, isProtected)) - }.toMap.withDefaultValue((false, false)) - - candidates.map { candidate => - val authorId = candidate.features.get(AuthorIdFeature).get - val (isBlueVerified, isProtected) = userMetadataMap(authorId) - - // Some accounts run promotions on Twitter and send replies automatically. - // We assume that a reply that took more than one minute is not an auto-reply. - // If time difference doesn't exist, this means that one of the tweets was - // not snowflake and therefore much older, and therefore OK as an extended reply. - val timeDifference = candidate.features.getOrElse(InReplyToTweetIdFeature, None).map { - SnowflakeId.timeFromId(candidate.candidate.id) - SnowflakeId.timeFromId(_) - } - val isAutoReply = timeDifference.exists(_ < 1.minute) - - FeatureMapBuilder() - .add(AuthorIsBlueVerifiedFeature, isBlueVerified) - .add(AuthorIsProtectedFeature, isProtected) - .add(IsSupportAccountReplyFeature, isAutoReply) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.docx new file mode 100644 index 000000000..f9fa9c4e3 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala deleted file mode 100644 index 03d70445e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/GraphTwoHopFeatureHydrator.scala +++ /dev/null @@ -1,97 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.graph_feature_service.{thriftscala => gfs} -import com.twitter.home_mixer.model.HomeFeatures.FollowedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.GraphTwoHopRepository -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.two_hop_features.TwoHopFeaturesAdapter -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object GraphTwoHopFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class GraphTwoHopFeatureHydrator @Inject() ( - @Named(GraphTwoHopRepository) client: KeyValueRepository[(Seq[Long], Long), Long, Seq[ - gfs.IntersectionValue - ]], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("GraphTwoHop") - - override val features: Set[Feature[_, _]] = Set(GraphTwoHopFeature, FollowedByUserIdsFeature) - - override val statScope: String = identifier.toString - - private val twoHopFeaturesAdapter = new TwoHopFeaturesAdapter - - private val FollowFeatureType = gfs.FeatureType(gfs.EdgeType.Following, gfs.EdgeType.FollowedBy) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - // Apply filters to in network candidates for retweets only. - val (inNetworkCandidates, oonCandidates) = candidates.partition { candidate => - candidate.features.getOrElse(FromInNetworkSourceFeature, false) - } - - val inNetworkCandidatesToHydrate = - inNetworkCandidates.filter(_.features.getOrElse(IsRetweetFeature, false)) - - val candidatesToHydrate = (inNetworkCandidatesToHydrate ++ oonCandidates) - .flatMap(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)).distinct - - val response = client((candidatesToHydrate, query.getRequiredUserId)) - - response.map { result => - candidates.map { candidate => - val originalAuthorId = CandidatesUtil.getOriginalAuthorId(candidate.features) - - val value = observedGet(key = originalAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - val followedByUserIds = value.toOption - .flatMap(getFollowedByUserIds(_)) - .getOrElse(Seq.empty) - - FeatureMapBuilder() - .add(GraphTwoHopFeature, transformedValue) - .add(FollowedByUserIdsFeature, followedByUserIds) - .build() - } - } - } - - private def getFollowedByUserIds(input: Option[Seq[gfs.IntersectionValue]]): Option[Seq[Long]] = - input.map(_.filter(_.featureType == FollowFeatureType).flatMap(_.intersectionIds).flatten) - - private def postTransformer(input: Try[Option[Seq[gfs.IntersectionValue]]]): Try[DataRecord] = - input.map(twoHopFeaturesAdapter.adaptToDataRecords(_).asScala.head) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.docx new file mode 100644 index 000000000..e238a4fbe Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala deleted file mode 100644 index fb114e55c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/IsExtendedReplyFeatureHydrator.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsExtendedReplyFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.product_mixer.component_library.feature_hydrator.query.social_graph.SGSFollowedUsersFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch - -object IsExtendedReplyFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("IsExtendedReply") - - override def features: Set[Feature[_, _]] = Set(IsExtendedReplyFeature) - - private val TrueFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, true).build() - private val FalseFeatureMap = FeatureMapBuilder().add(IsExtendedReplyFeature, false).build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val followedUsers = - query.features.map(_.get(SGSFollowedUsersFeature)).getOrElse(Seq.empty).toSet - - candidates.map { candidate => - val features = candidate.features - val isExtendedReply = features.getOrElse(InReplyToTweetIdFeature, None).nonEmpty && - !features.getOrElse(IsRetweetFeature, false) && - features.getOrElse(InReplyToUserIdFeature, None).exists(!followedUsers.contains(_)) - - if (isExtendedReply) TrueFeatureMap else FalseFeatureMap - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.docx new file mode 100644 index 000000000..8bdc806c3 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala deleted file mode 100644 index db60d8866..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ListIdsQueryFeatureHydrator.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.socialgraph.{thriftscala => sg} -import com.twitter.stitch.Stitch -import com.twitter.stitch.socialgraph.SocialGraph -import javax.inject.Inject -import javax.inject.Singleton - -case object ListIdsFeature extends FeatureWithDefaultOnFailure[PipelineQuery, Seq[Long]] { - override val defaultValue: Seq[Long] = Seq.empty -} - -@Singleton -class ListIdsQueryFeatureHydrator @Inject() (socialGraph: SocialGraph) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ListIds") - - override val features: Set[Feature[_, _]] = Set(ListIdsFeature) - - private val MaxListsToFetch = 20 - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - - val ownedSubscribedRequest = sg.IdsRequest( - relationships = Seq( - sg.SrcRelationship(userId, sg.RelationshipType.ListIsSubscriber, hasRelationship = true), - sg.SrcRelationship(userId, sg.RelationshipType.ListOwning, hasRelationship = true) - ), - pageRequest = Some(sg.PageRequest(selectAll = Some(false), count = Some(MaxListsToFetch))), - context = Some( - sg.LookupContext( - includeInactive = false, - performUnion = Some(true), - includeAll = Some(false) - ) - ) - ) - - socialGraph.ids(ownedSubscribedRequest).map { response => - FeatureMapBuilder().add(ListIdsFeature, response.ids).build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.docx new file mode 100644 index 000000000..b152860cc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala deleted file mode 100644 index bc51aabb9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/MetricCenterUserCountingFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.MetricCenterUserCountingFeatureRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.onboarding.relevance.features.{thriftjava => rf} -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object MetricCenterUserCountingFeature - extends Feature[TweetCandidate, Option[rf.MCUserCountingFeatures]] - -@Singleton -class MetricCenterUserCountingFeatureHydrator @Inject() ( - @Named(MetricCenterUserCountingFeatureRepository) client: KeyValueRepository[Seq[ - Long - ], Long, rf.MCUserCountingFeatures], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("MetricCenterUserCounting") - - override val features: Set[Feature[_, _]] = Set(MetricCenterUserCountingFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val userIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, rf.MCUserCountingFeatures]] = - if (userIds.isEmpty) Future.value(KeyValueResult.empty) else client(userIds) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - FeatureMapBuilder().add(MetricCenterUserCountingFeature, value).build() - } - } - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - candidate.features - .getTry(AuthorIdFeature) - .toOption - .flatten - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.docx new file mode 100644 index 000000000..82ddcfbae Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala deleted file mode 100644 index a48d3742e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphQueryFeatureHydrator.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealGraphFeatureRepository -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.Repository -import com.twitter.timelines.real_graph.{thriftscala => rg} -import com.twitter.stitch.Stitch -import com.twitter.timelines.model.UserId -import com.twitter.timelines.real_graph.v1.thriftscala.RealGraphEdgeFeatures -import com.twitter.user_session_store.{thriftscala => uss} - -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object RealGraphFeatures extends Feature[PipelineQuery, Option[Map[UserId, RealGraphEdgeFeatures]]] - -@Singleton -class RealGraphQueryFeatureHydrator @Inject() ( - @Named(RealGraphFeatureRepository) repository: Repository[Long, Option[uss.UserSession]]) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphFeatures") - - override val features: Set[Feature[_, _]] = Set(RealGraphFeatures) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - Stitch.callFuture { - repository(query.getRequiredUserId).map { userSession => - val realGraphFeaturesMap = userSession.flatMap { userSession => - userSession.realGraphFeatures.collect { - case rg.RealGraphFeatures.V1(realGraphFeatures) => - val edgeFeatures = realGraphFeatures.edgeFeatures ++ realGraphFeatures.oonEdgeFeatures - edgeFeatures.map { edge => edge.destId -> edge }.toMap - } - } - - FeatureMapBuilder().add(RealGraphFeatures, realGraphFeaturesMap).build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.docx new file mode 100644 index 000000000..2ffc0db1c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala deleted file mode 100644 index 6048213aa..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerAuthorFeatureHydrator.scala +++ /dev/null @@ -1,123 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures -import com.twitter.home_mixer.util.MissingKeyException -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter -import com.twitter.timelines.prediction.adapters.real_graph.RealGraphFeaturesAdapter -import com.twitter.timelines.real_graph.v1.{thriftscala => v1} -import com.twitter.timelines.real_graph.{thriftscala => rg} -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object RealGraphViewerAuthorDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object RealGraphViewerAuthorsDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class RealGraphViewerAuthorFeatureHydrator @Inject() () - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphViewerAuthor") - - override val features: Set[Feature[_, _]] = - Set(RealGraphViewerAuthorDataRecordFeature, RealGraphViewerAuthorsDataRecordFeature) - - private val realGraphEdgeFeaturesAdapter = new RealGraphFeaturesAdapter - private val realGraphEdgeFeaturesCombineAdapter = - new RealGraphEdgeFeaturesCombineAdapter(prefix = "authors.realgraph") - - private val MissingKeyFeatureMap = FeatureMapBuilder() - .add(RealGraphViewerAuthorDataRecordFeature, Throw(MissingKeyException)) - .add(RealGraphViewerAuthorsDataRecordFeature, Throw(MissingKeyException)) - .build() - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offload { - val viewerId = query.getRequiredUserId - val realGraphFeatures = query.features - .flatMap(_.getOrElse(RealGraphFeatures, None)) - .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) - - existingFeatures.getOrElse(AuthorIdFeature, None) match { - case Some(authorId) => - val realGraphAuthorFeatures = - getRealGraphViewerAuthorFeatures(viewerId, authorId, realGraphFeatures) - val realGraphAuthorDataRecord = realGraphEdgeFeaturesAdapter - .adaptToDataRecords(realGraphAuthorFeatures).asScala.headOption.getOrElse(new DataRecord) - - val combinedRealGraphFeaturesDataRecord = for { - inReplyToAuthorId <- existingFeatures.getOrElse(InReplyToUserIdFeature, None) - } yield { - val combinedRealGraphFeatures = - getCombinedRealGraphFeatures(Seq(authorId, inReplyToAuthorId), realGraphFeatures) - realGraphEdgeFeaturesCombineAdapter - .adaptToDataRecords(Some(combinedRealGraphFeatures)).asScala.headOption - .getOrElse(new DataRecord) - } - - FeatureMapBuilder() - .add(RealGraphViewerAuthorDataRecordFeature, realGraphAuthorDataRecord) - .add( - RealGraphViewerAuthorsDataRecordFeature, - combinedRealGraphFeaturesDataRecord.getOrElse(new DataRecord)) - .build() - case _ => MissingKeyFeatureMap - } - } - - private def getRealGraphViewerAuthorFeatures( - viewerId: Long, - authorId: Long, - realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] - ): rg.UserRealGraphFeatures = { - realGraphEdgeFeaturesMap.get(authorId) match { - case Some(realGraphEdgeFeatures) => - rg.UserRealGraphFeatures( - srcId = viewerId, - features = rg.RealGraphFeatures.V1( - v1.RealGraphFeatures(edgeFeatures = Seq(realGraphEdgeFeatures)))) - case _ => - rg.UserRealGraphFeatures( - srcId = viewerId, - features = rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = Seq.empty))) - } - } -} - -object RealGraphViewerAuthorFeatureHydrator { - def getCombinedRealGraphFeatures( - userIds: Seq[Long], - realGraphEdgeFeaturesMap: Map[Long, v1.RealGraphEdgeFeatures] - ): rg.RealGraphFeatures = { - val edgeFeatures = userIds.flatMap(realGraphEdgeFeaturesMap.get) - rg.RealGraphFeatures.V1(v1.RealGraphFeatures(edgeFeatures = edgeFeatures)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.docx new file mode 100644 index 000000000..d618633e8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala deleted file mode 100644 index 47e2431cf..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealGraphViewerRelatedUsersFeatureHydrator.scala +++ /dev/null @@ -1,73 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.real_graph.RealGraphEdgeFeaturesCombineAdapter -import com.twitter.timelines.real_graph.v1.{thriftscala => v1} -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object RealGraphViewerRelatedUsersDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class RealGraphViewerRelatedUsersFeatureHydrator @Inject() () - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealGraphViewerRelatedUsers") - - override val features: Set[Feature[_, _]] = Set(RealGraphViewerRelatedUsersDataRecordFeature) - - private val RealGraphEdgeFeaturesCombineAdapter = new RealGraphEdgeFeaturesCombineAdapter - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offload { - val realGraphQueryFeatures = query.features - .flatMap(_.getOrElse(RealGraphFeatures, None)) - .getOrElse(Map.empty[Long, v1.RealGraphEdgeFeatures]) - - val allRelatedUserIds = getRelatedUserIds(existingFeatures) - val realGraphFeatures = RealGraphViewerAuthorFeatureHydrator.getCombinedRealGraphFeatures( - allRelatedUserIds, - realGraphQueryFeatures - ) - val realGraphFeaturesDataRecord = RealGraphEdgeFeaturesCombineAdapter - .adaptToDataRecords(Some(realGraphFeatures)).asScala.headOption - .getOrElse(new DataRecord) - - FeatureMapBuilder() - .add(RealGraphViewerRelatedUsersDataRecordFeature, realGraphFeaturesDataRecord) - .build() - } - - private def getRelatedUserIds(features: FeatureMap): Seq[Long] = { - (CandidatesUtil.getEngagerUserIds(features) ++ - features.getOrElse(AuthorIdFeature, None) ++ - features.getOrElse(MentionUserIdFeature, Seq.empty) ++ - features.getOrElse(SourceUserIdFeature, None) ++ - features.getOrElse(DirectedAtUserIdFeature, None)).distinct - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.docx new file mode 100644 index 000000000..f03ad0fe0 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala deleted file mode 100644 index f690ff0e0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphEdgeFeatureHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.adapters.realtime_interaction_graph.RealTimeInteractionGraphFeaturesAdapter -import com.twitter.timelines.prediction.features.realtime_interaction_graph.RealTimeInteractionGraphEdgeFeatures -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object RealTimeInteractionGraphEdgeFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class RealTimeInteractionGraphEdgeFeatureHydrator @Inject() () - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealTimeInteractionGraphEdge") - - override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphEdgeFeature) - - private val realTimeInteractionGraphFeaturesAdapter = new RealTimeInteractionGraphFeaturesAdapter - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val userVertex = - query.features.flatMap(_.getOrElse(RealTimeInteractionGraphUserVertexQueryFeature, None)) - val realTimeInteractionGraphFeaturesMap = - userVertex.map(RealTimeInteractionGraphEdgeFeatures(_, Time.now)) - - candidates.map { candidate => - val feature = candidate.features.getOrElse(AuthorIdFeature, None).flatMap { authorId => - realTimeInteractionGraphFeaturesMap.flatMap(_.get(authorId)) - } - - val dataRecordFeature = - realTimeInteractionGraphFeaturesAdapter.adaptToDataRecords(feature).asScala.head - - FeatureMapBuilder().add(RealTimeInteractionGraphEdgeFeature, dataRecordFeature).build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.docx new file mode 100644 index 000000000..9930c2626 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala deleted file mode 100644 index 60d82cc28..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RealTimeInteractionGraphUserVertexQueryFeatureHydrator.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.RealTimeInteractionGraphUserVertexCache -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.stitch.Stitch -import com.twitter.wtf.real_time_interaction_graph.{thriftscala => ig} -import javax.inject.Inject -import javax.inject.Singleton - -object RealTimeInteractionGraphUserVertexQueryFeature - extends Feature[PipelineQuery, Option[ig.UserVertex]] - -@Singleton -class RealTimeInteractionGraphUserVertexQueryFeatureHydrator @Inject() ( - @Named(RealTimeInteractionGraphUserVertexCache) client: ReadCache[Long, ig.UserVertex], - override val statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RealTimeInteractionGraphUserVertex") - - override val features: Set[Feature[_, _]] = Set(RealTimeInteractionGraphUserVertexQueryFeature) - - override val statScope: String = identifier.toString - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - - Stitch.callFuture( - client.get(Seq(userId)).map { results => - val feature = observedGet(key = Some(userId), keyValueResult = results) - FeatureMapBuilder() - .add(RealTimeInteractionGraphUserVertexQueryFeature, feature) - .build() - } - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.docx new file mode 100644 index 000000000..674ea726d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala deleted file mode 100644 index 80d857a26..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/ReplyFeatureHydrator.scala +++ /dev/null @@ -1,242 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.InReplyToContentFeatureAdapter -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird.InReplyToEarlybirdAdapter -import com.twitter.home_mixer.util.ReplyRetweetUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.conversation_features.v1.thriftscala.ConversationFeatures -import com.twitter.timelines.conversation_features.{thriftscala => cf} -import com.twitter.timelines.prediction.adapters.conversation_features.ConversationFeaturesAdapter -import com.twitter.util.Duration -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object InReplyToTweetHydratedEarlybirdFeature - extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] - -object ConversationDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object InReplyToEarlybirdDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object InReplyToTweetypieContentDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -/** - * The purpose of this hydrator is to - * 1) hydrate simple features into replies and their ancestor tweets - * 2) keep both the normal replies and ancestor source candidates, but hydrate into the candidates - * features useful for predicting the quality of the replies and source ancestor tweets. - */ -@Singleton -class ReplyFeatureHydrator @Inject() (statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("ReplyTweet") - - override val features: Set[Feature[_, _]] = Set( - ConversationDataRecordFeature, - InReplyToTweetHydratedEarlybirdFeature, - InReplyToEarlybirdDataRecordFeature, - InReplyToTweetypieContentDataRecordFeature - ) - - private val defaulDataRecord: DataRecord = new DataRecord() - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(ConversationDataRecordFeature, defaulDataRecord) - .add(InReplyToTweetHydratedEarlybirdFeature, None) - .add(InReplyToEarlybirdDataRecordFeature, defaulDataRecord) - .add(InReplyToTweetypieContentDataRecordFeature, defaulDataRecord) - .build() - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val hydratedReplyCounter = scopedStatsReceiver.counter("hydratedReply") - private val hydratedAncestorCounter = scopedStatsReceiver.counter("hydratedAncestor") - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val replyToInReplyToTweetMap = - ReplyRetweetUtil.replyTweetIdToInReplyToTweetMap(candidates) - val candidatesWithRepliesHydrated = candidates.map { candidate => - replyToInReplyToTweetMap - .get(candidate.candidate.id).map { inReplyToTweet => - hydratedReplyCounter.incr() - hydratedReplyCandidate(candidate, inReplyToTweet) - }.getOrElse((candidate, None, None)) - } - - /** - * Update ancestor tweets with descendant replies and hydrate simple features from one of - * the descendants. - */ - val ancestorTweetToDescendantRepliesMap = - ReplyRetweetUtil.ancestorTweetIdToDescendantRepliesMap(candidates) - val candidatesWithRepliesAndAncestorTweetsHydrated = candidatesWithRepliesHydrated.map { - case ( - maybeAncestorTweetCandidate, - updatedReplyConversationFeatures, - inReplyToTweetEarlyBirdFeature) => - ancestorTweetToDescendantRepliesMap - .get(maybeAncestorTweetCandidate.candidate.id) - .map { descendantReplies => - hydratedAncestorCounter.incr() - val (ancestorTweetCandidate, updatedConversationFeatures): ( - CandidateWithFeatures[TweetCandidate], - Option[ConversationFeatures] - ) = - hydrateAncestorTweetCandidate( - maybeAncestorTweetCandidate, - descendantReplies, - updatedReplyConversationFeatures) - (ancestorTweetCandidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) - } - .getOrElse( - ( - maybeAncestorTweetCandidate, - inReplyToTweetEarlyBirdFeature, - updatedReplyConversationFeatures)) - } - - candidatesWithRepliesAndAncestorTweetsHydrated.map { - case (candidate, inReplyToTweetEarlyBirdFeature, updatedConversationFeatures) => - val conversationDataRecordFeature = updatedConversationFeatures - .map(f => ConversationFeaturesAdapter.adaptToDataRecord(cf.ConversationFeatures.V1(f))) - .getOrElse(defaulDataRecord) - - val inReplyToEarlybirdDataRecord = - InReplyToEarlybirdAdapter - .adaptToDataRecords(inReplyToTweetEarlyBirdFeature).asScala.head - val inReplyToContentDataRecord = InReplyToContentFeatureAdapter - .adaptToDataRecords( - inReplyToTweetEarlyBirdFeature.map(ContentFeatures.fromThrift)).asScala.head - - FeatureMapBuilder() - .add(ConversationDataRecordFeature, conversationDataRecordFeature) - .add(InReplyToTweetHydratedEarlybirdFeature, inReplyToTweetEarlyBirdFeature) - .add(InReplyToEarlybirdDataRecordFeature, inReplyToEarlybirdDataRecord) - .add(InReplyToTweetypieContentDataRecordFeature, inReplyToContentDataRecord) - .build() - case _ => DefaultFeatureMap - } - } - - private def hydratedReplyCandidate( - replyCandidate: CandidateWithFeatures[TweetCandidate], - inReplyToTweetCandidate: CandidateWithFeatures[TweetCandidate] - ): ( - CandidateWithFeatures[TweetCandidate], - Option[ConversationFeatures], - Option[ThriftTweetFeatures] - ) = { - val tweetedAfterInReplyToTweetInSecs = - ( - originalTweetAgeFromSnowflake(inReplyToTweetCandidate), - originalTweetAgeFromSnowflake(replyCandidate)) match { - case (Some(inReplyToTweetAge), Some(replyTweetAge)) => - Some((inReplyToTweetAge - replyTweetAge).inSeconds.toLong) - case _ => None - } - - val existingConversationFeatures = Some( - replyCandidate.features - .getOrElse(ConversationFeature, None).getOrElse(ConversationFeatures())) - - val updatedConversationFeatures = existingConversationFeatures match { - case Some(v1) => - Some( - v1.copy( - tweetedAfterInReplyToTweetInSecs = tweetedAfterInReplyToTweetInSecs, - isSelfReply = Some( - replyCandidate.features.getOrElse( - AuthorIdFeature, - None) == inReplyToTweetCandidate.features.getOrElse(AuthorIdFeature, None)) - ) - ) - case _ => None - } - - // Note: if inReplyToTweet is a retweet, we need to read early bird feature from the merged - // early bird feature field from RetweetSourceTweetFeatureHydrator class. - // But if inReplyToTweet is a reply, we return its early bird feature directly - val inReplyToTweetThriftTweetFeaturesOpt = { - if (inReplyToTweetCandidate.features.getOrElse(IsRetweetFeature, false)) { - inReplyToTweetCandidate.features.getOrElse(SourceTweetEarlybirdFeature, None) - } else { - inReplyToTweetCandidate.features.getOrElse(EarlybirdFeature, None) - } - } - - (replyCandidate, updatedConversationFeatures, inReplyToTweetThriftTweetFeaturesOpt) - } - - private def hydrateAncestorTweetCandidate( - ancestorTweetCandidate: CandidateWithFeatures[TweetCandidate], - descendantReplies: Seq[CandidateWithFeatures[TweetCandidate]], - updatedReplyConversationFeatures: Option[ConversationFeatures] - ): (CandidateWithFeatures[TweetCandidate], Option[ConversationFeatures]) = { - // Ancestor could be a reply. For example, in thread: tweetA -> tweetB -> tweetC, - // tweetB is a reply and ancestor at the same time. Hence, tweetB's conversation feature - // will be updated by hydratedReplyCandidate and hydrateAncestorTweetCandidate functions. - val existingConversationFeatures = - if (updatedReplyConversationFeatures.nonEmpty) - updatedReplyConversationFeatures - else - Some( - ancestorTweetCandidate.features - .getOrElse(ConversationFeature, None).getOrElse(ConversationFeatures())) - - val updatedConversationFeatures = existingConversationFeatures match { - case Some(v1) => - Some( - v1.copy( - hasDescendantReplyCandidate = Some(true), - hasInNetworkDescendantReply = - Some(descendantReplies.exists(_.features.getOrElse(InNetworkFeature, false))) - )) - case _ => None - } - (ancestorTweetCandidate, updatedConversationFeatures) - } - - private def originalTweetAgeFromSnowflake( - candidate: CandidateWithFeatures[TweetCandidate] - ): Option[Duration] = { - SnowflakeId - .timeFromIdOpt( - candidate.features - .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id)) - .map(Time.now - _) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.docx new file mode 100644 index 000000000..e5831e676 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala deleted file mode 100644 index fd9656d8c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RequestTimeQueryFeatureHydrator.scala +++ /dev/null @@ -1,122 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.conversions.DurationOps._ -import com.twitter.home_mixer.model.HomeFeatures.FollowingLastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.LastNonPollingTimeFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.util.FDsl._ -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.features.time_features.AccountAgeInterval -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.ACCOUNT_AGE_INTERVAL -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_12_MONTH_NEW_USER -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.IS_30_DAY_NEW_USER -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_BETWEEN_NON_POLLING_REQUESTS_AVG -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_LAST_NON_POLLING_REQUEST -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures.USER_ID_IS_SNOWFLAKE_ID -import com.twitter.user_session_store.ReadRequest -import com.twitter.user_session_store.ReadWriteUserSessionStore -import com.twitter.user_session_store.UserSessionDataset -import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Singleton - -object RequestTimeDataRecordFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -case class RequestTimeQueryFeatureHydrator @Inject() ( - userSessionStore: ReadWriteUserSessionStore) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("RequestTime") - - override val features: Set[Feature[_, _]] = Set( - FollowingLastNonPollingTimeFeature, - LastNonPollingTimeFeature, - NonPollingTimesFeature, - RequestTimeDataRecordFeature - ) - - private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.NonPollingTimes) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - userSessionStore - .read(ReadRequest(query.getRequiredUserId, datasets)) - .map { userSession => - val nonPollingTimestamps = userSession.flatMap(_.nonPollingTimestamps) - - val lastNonPollingTime = nonPollingTimestamps - .flatMap(_.nonPollingTimestampsMs.headOption) - .map(Time.fromMilliseconds) - - val followingLastNonPollingTime = nonPollingTimestamps - .flatMap(_.mostRecentHomeLatestNonPollingTimestampMs) - .map(Time.fromMilliseconds) - - val nonPollingTimes = nonPollingTimestamps - .map(_.nonPollingTimestampsMs) - .getOrElse(Seq.empty) - - val requestTimeDataRecord = getRequestTimeDataRecord(query, nonPollingTimes) - - FeatureMapBuilder() - .add(FollowingLastNonPollingTimeFeature, followingLastNonPollingTime) - .add(LastNonPollingTimeFeature, lastNonPollingTime) - .add(NonPollingTimesFeature, nonPollingTimes) - .add(RequestTimeDataRecordFeature, requestTimeDataRecord) - .build() - } - } - - def getRequestTimeDataRecord(query: PipelineQuery, nonPollingTimes: Seq[Long]): DataRecord = { - val requestTimeMs = query.queryTime.inMillis - val accountAge = SnowflakeId.timeFromIdOpt(query.getRequiredUserId) - val timeSinceAccountCreation = accountAge.map(query.queryTime.since) - val timeSinceEarliestNonPollingRequest = - nonPollingTimes.lastOption.map(requestTimeMs - _) - val timeSinceLastNonPollingRequest = - nonPollingTimes.headOption.map(requestTimeMs - _) - - new DataRecord() - .setFeatureValue(USER_ID_IS_SNOWFLAKE_ID, accountAge.isDefined) - .setFeatureValue( - IS_30_DAY_NEW_USER, - timeSinceAccountCreation.map(_ < 30.days).getOrElse(false) - ) - .setFeatureValue( - IS_12_MONTH_NEW_USER, - timeSinceAccountCreation.map(_ < 365.days).getOrElse(false) - ) - .setFeatureValueFromOption( - ACCOUNT_AGE_INTERVAL, - timeSinceAccountCreation.flatMap(AccountAgeInterval.fromDuration).map(_.id.toLong) - ) - .setFeatureValueFromOption( - TIME_SINCE_VIEWER_ACCOUNT_CREATION_SECS, - timeSinceAccountCreation.map(_.inSeconds.toDouble) - ) - .setFeatureValueFromOption( - TIME_BETWEEN_NON_POLLING_REQUESTS_AVG, - timeSinceEarliestNonPollingRequest.map(_.toDouble / math.max(1.0, nonPollingTimes.size)) - ) - .setFeatureValueFromOption( - TIME_SINCE_LAST_NON_POLLING_REQUEST, - timeSinceLastNonPollingRequest.map(_.toDouble) - ) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.docx new file mode 100644 index 000000000..ba7292f1e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala deleted file mode 100644 index 33cbef884..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/RetweetSourceTweetFeatureHydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures._ -import com.twitter.product_mixer.component_library.candidate_source.timeline_ranker.TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures -import com.twitter.stitch.Stitch -import com.twitter.timelineranker.thriftscala.CandidateTweet - -object SourceTweetEarlybirdFeature extends Feature[TweetCandidate, Option[ThriftTweetFeatures]] - -/** - * Feature Hydrator that bulk hydrates source tweets' features to retweet candidates - */ -object RetweetSourceTweetFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("RetweetSourceTweet") - - override val features: Set[Feature[_, _]] = Set( - SourceTweetEarlybirdFeature, - ) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(SourceTweetEarlybirdFeature, None) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val sourceTweetsByTweetId: Option[Map[Long, CandidateTweet]] = { - query.features.map( - _.getOrElse( - TimelineRankerInNetworkSourceTweetsByTweetIdMapFeature, - Map.empty[Long, CandidateTweet])) - } - - /** - * Return DefaultFeatureMap (no-op to candidate) when it is unfeasible to hydrate the - * source tweet's feature to the current candidate: early bird does not return source - * tweets info / candidate is not a retweet / sourceTweetId is not found - */ - Stitch.value { - if (sourceTweetsByTweetId.exists(_.nonEmpty)) { - candidates.map { candidate => - val candidateIsRetweet = candidate.features.getOrElse(IsRetweetFeature, false) - val sourceTweetId = candidate.features.getOrElse(SourceTweetIdFeature, None) - if (!candidateIsRetweet || sourceTweetId.isEmpty) { - DefaultFeatureMap - } else { - val sourceTweet = sourceTweetsByTweetId.flatMap(_.get(sourceTweetId.get)) - if (sourceTweet.nonEmpty) { - val source = sourceTweet.get - FeatureMapBuilder() - .add(SourceTweetEarlybirdFeature, source.features) - .build() - } else { - DefaultFeatureMap - } - } - } - } else { - candidates.map(_ => DefaultFeatureMap) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.docx new file mode 100644 index 000000000..f7d4ef46c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala deleted file mode 100644 index f66966f00..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersEngagementSimilarityFeatureHydrator.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.product.scored_tweets.param.ScoredTweetsParam -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.strato.twistly.SimClustersRecentEngagementSimilarityClient -import com.twitter.timelines.configapi.decider.BooleanDeciderParam -import com.twitter.timelines.prediction.adapters.twistly.SimClustersRecentEngagementSimilarityFeaturesAdapter -import javax.inject.Inject -import javax.inject.Singleton - -object SimClustersEngagementSimilarityFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class SimClustersEngagementSimilarityFeatureHydrator @Inject() ( - simClustersEngagementSimilarityClient: SimClustersRecentEngagementSimilarityClient) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SimClustersEngagementSimilarity") - - override val features: Set[Feature[_, _]] = Set(SimClustersEngagementSimilarityFeature) - - private val simClustersRecentEngagementSimilarityFeaturesAdapter = - new SimClustersRecentEngagementSimilarityFeaturesAdapter - - override def onlyIf(query: PipelineQuery): Boolean = { - val param: BooleanDeciderParam = - ScoredTweetsParam.EnableSimClustersSimilarityFeatureHydrationDeciderParam - query.params.apply(param) - } - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val tweetToCandidates = candidates.map(candidate => candidate.candidate.id -> candidate).toMap - val tweetIds = tweetToCandidates.keySet.toSeq - val userId = query.getRequiredUserId - val userTweetEdges = tweetIds.map(tweetId => (userId, tweetId)) - simClustersEngagementSimilarityClient - .getSimClustersRecentEngagementSimilarityScores(userTweetEdges).map { - simClustersRecentEngagementSimilarityScoresMap => - candidates.map { candidate => - val similarityFeatureOpt = simClustersRecentEngagementSimilarityScoresMap - .get(userId -> candidate.candidate.id).flatten - val dataRecordOpt = similarityFeatureOpt.map { similarityFeature => - simClustersRecentEngagementSimilarityFeaturesAdapter - .adaptToDataRecords(similarityFeature) - .get(0) - } - FeatureMapBuilder() - .add(SimClustersEngagementSimilarityFeature, dataRecordOpt.getOrElse(new DataRecord)) - .build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.docx new file mode 100644 index 000000000..c075bc748 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala deleted file mode 100644 index c938f34c5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/SimClustersUserTweetScoresHydrator.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.dal.personal_data.{thriftjava => pd} -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordOptionalFeature -import com.twitter.product_mixer.core.feature.datarecord.DoubleDataRecordCompatible -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.strato.catalog.Fetch -import com.twitter.strato.generated.client.ml.featureStore.SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn -import javax.inject.Inject -import javax.inject.Singleton - -object SimClustersUserInterestedInTweetEmbeddingDataRecordFeature - extends DataRecordOptionalFeature[TweetCandidate, Double] - with DoubleDataRecordCompatible { - override val featureName: String = - "user-tweet.recommendations.sim_clusters_scores.user_interested_in_tweet_embedding_dot_product_20m_145k_2020" - override val personalDataTypes: Set[pd.PersonalDataType] = - Set(pd.PersonalDataType.InferredInterests) -} - -@Singleton -class SimClustersUserTweetScoresHydrator @Inject() ( - simClustersColumn: SimClustersUserInterestedInTweetEmbeddingDotProduct20M145K2020OnUserTweetEdgeClientColumn, - statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("SimClustersUserTweetScores") - - override val features: Set[Feature[_, _]] = Set( - SimClustersUserInterestedInTweetEmbeddingDataRecordFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - private val keySkipCounter = scopedStatsReceiver.counter("key/skip") - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, None) - .build() - private val MinFavToHydrate = 9 - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - Stitch.run { - Stitch.collect { - candidates.map { candidate => - val ebFeatures = candidate.features.getOrElse(EarlybirdFeature, None) - val favCount = ebFeatures.flatMap(_.favCountV2).getOrElse(0) - - if (ebFeatures.isEmpty || favCount >= MinFavToHydrate) { - simClustersColumn.fetcher - .fetch((query.getRequiredUserId, candidate.candidate.id), Unit) - .map { - case Fetch.Result(response, _) => - if (response.nonEmpty) keyFoundCounter.incr() else keyLossCounter.incr() - FeatureMapBuilder() - .add(SimClustersUserInterestedInTweetEmbeddingDataRecordFeature, response) - .build() - case _ => - keyFailureCounter.incr() - DefaultFeatureMap - } - } else { - keySkipCounter.incr() - Stitch.value(DefaultFeatureMap) - } - } - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.docx new file mode 100644 index 000000000..c76cefa2e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala deleted file mode 100644 index f6047730f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TSPInferredTopicFeatureHydrator.scala +++ /dev/null @@ -1,148 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.contentrecommender.{thriftscala => cr} -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.model.HomeFeatures.TSPMetricTagFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicContextFunctionalityTypeFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic.InferredTopicAdapter -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.BasicTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.RecommendationTopicContextFunctionalityType -import com.twitter.product_mixer.core.model.marshalling.response.urt.metadata.TopicContextFunctionalityType -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.clients.strato.topics.TopicSocialProofClient -import com.twitter.timelineservice.suggests.logging.candidate_tweet_source_id.{thriftscala => sid} -import com.twitter.topiclisting.TopicListingViewerContext -import com.twitter.tsp.{thriftscala => tsp} -import javax.inject.Inject -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TSPInferredTopicFeature extends Feature[TweetCandidate, Map[Long, Double]] -object TSPInferredTopicDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TSPInferredTopicFeatureHydrator @Inject() ( - topicSocialProofClient: TopicSocialProofClient) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TSPInferredTopic") - - override val features: Set[Feature[_, _]] = Set( - TSPInferredTopicFeature, - TSPInferredTopicDataRecordFeature, - TopicIdSocialContextFeature, - TopicContextFunctionalityTypeFeature - ) - - private val topK = 3 - - private val SourcesToSetSocialProof: Set[sid.CandidateTweetSourceId] = - Set(sid.CandidateTweetSourceId.Simcluster) - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(TSPInferredTopicFeature, Map.empty[Long, Double]) - .add(TSPInferredTopicDataRecordFeature, new DataRecord()) - .add(TopicIdSocialContextFeature, None) - .add(TopicContextFunctionalityTypeFeature, None) - .build() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val tags = candidates.collect { - case candidate if candidate.features.getTry(TSPMetricTagFeature).isReturn => - candidate.candidate.id -> candidate.features - .getOrElse(TSPMetricTagFeature, Set.empty[tsp.MetricTag]) - }.toMap - - val topicSocialProofRequest = tsp.TopicSocialProofRequest( - userId = query.getRequiredUserId, - tweetIds = candidates.map(_.candidate.id).toSet, - displayLocation = cr.DisplayLocation.HomeTimeline, - topicListingSetting = tsp.TopicListingSetting.Followable, - context = TopicListingViewerContext.fromClientContext(query.clientContext).toThrift, - bypassModes = None, - // Only TweetMixer source has this data. Convert the TweetMixer metric tag to tsp metric tag. - tags = if (tags.isEmpty) None else Some(tags) - ) - - topicSocialProofClient - .getTopicTweetSocialProofResponse(topicSocialProofRequest) - .map { - case Some(response) => - handleResponse(response, candidates) - case _ => candidates.map { _ => DefaultFeatureMap } - } - } - - private def handleResponse( - response: tsp.TopicSocialProofResponse, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[FeatureMap] = { - candidates.map { candidate => - val topicWithScores = response.socialProofs.getOrElse(candidate.candidate.id, Seq.empty) - if (topicWithScores.nonEmpty) { - val (socialProofId, socialProofFunctionalityType) = - if (candidate.features - .getOrElse(CandidateSourceIdFeature, None) - .exists(SourcesToSetSocialProof.contains)) { - getSocialProof(topicWithScores) - } else (None, None) - - val inferredTopicFeatures = - topicWithScores.sortBy(-_.score).take(topK).map(a => (a.topicId, a.score)).toMap - - val inferredTopicDataRecord = - InferredTopicAdapter.adaptToDataRecords(inferredTopicFeatures).asScala.head - - FeatureMapBuilder() - .add(TSPInferredTopicFeature, inferredTopicFeatures) - .add(TSPInferredTopicDataRecordFeature, inferredTopicDataRecord) - .add(TopicIdSocialContextFeature, socialProofId) - .add(TopicContextFunctionalityTypeFeature, socialProofFunctionalityType) - .build() - } else DefaultFeatureMap - } - } - - private def getSocialProof( - topicWithScores: Seq[tsp.TopicWithScore] - ): (Option[Long], Option[TopicContextFunctionalityType]) = { - val followingTopicId = topicWithScores.collectFirst { - case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.Following)) => topicId - } - - if (followingTopicId.nonEmpty) { - return (followingTopicId, Some(BasicTopicContextFunctionalityType)) - } - - val implicitFollowingId = topicWithScores.collectFirst { - case tsp.TopicWithScore(topicId, _, _, Some(tsp.TopicFollowType.ImplicitFollow)) => - topicId - } - - if (implicitFollowingId.nonEmpty) { - return (implicitFollowingId, Some(RecommendationTopicContextFunctionalityType)) - } - - (None, None) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.docx new file mode 100644 index 000000000..558c55db2 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala deleted file mode 100644 index 1c237bf18..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetMetaDataFeatureHydrator.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.CandidateSourceIdFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.util.DataRecordConverters._ -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import java.lang.{Long => JLong} - -object TweetMetaDataDataRecord - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object TweetMetaDataFeatureHydrator - extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetMetaData") - - override def features: Set[Feature[_, _]] = Set(TweetMetaDataDataRecord) - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = OffloadFuturePools.offload { - val richDataRecord = new RichDataRecord() - setFeatures(richDataRecord, candidate, existingFeatures) - FeatureMapBuilder().add(TweetMetaDataDataRecord, richDataRecord.getRecord).build() - } - - private def setFeatures( - richDataRecord: RichDataRecord, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Unit = { - richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, candidate.id) - - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, - CandidatesUtil.getOriginalAuthorId(existingFeatures)) - - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.CANDIDATE_TWEET_SOURCE_ID, - existingFeatures.getOrElse(CandidateSourceIdFeature, None).map(_.value.toLong)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.docx new file mode 100644 index 000000000..acea8c01c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala deleted file mode 100644 index 947016826..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetTimeFeatureHydrator.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.EarlybirdFeature -import com.twitter.home_mixer.model.HomeFeatures.NonPollingTimesFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.util.FDsl._ -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.CandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.snowflake.id.SnowflakeId -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.features.time_features.TimeDataRecordFeatures._ -import com.twitter.util.Duration -import scala.collection.Searching._ - -object TweetTimeDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -object TweetTimeFeatureHydrator extends CandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetTime") - - override val features: Set[Feature[_, _]] = Set(TweetTimeDataRecordFeature) - - override def apply( - query: PipelineQuery, - candidate: TweetCandidate, - existingFeatures: FeatureMap - ): Stitch[FeatureMap] = { - val tweetFeatures = existingFeatures.getOrElse(EarlybirdFeature, None) - val timeSinceTweetCreation = SnowflakeId.timeFromIdOpt(candidate.id).map(query.queryTime.since) - val timeSinceTweetCreationMs = timeSinceTweetCreation.map(_.inMillis) - - val timeSinceSourceTweetCreationOpt = existingFeatures - .getOrElse(SourceTweetIdFeature, None) - .flatMap { sourceTweetId => - SnowflakeId.timeFromIdOpt(sourceTweetId).map(query.queryTime.since) - }.orElse(timeSinceTweetCreation) - - val lastFavSinceCreationHrs = - tweetFeatures.flatMap(_.lastFavSinceCreationHrs).map(_.toDouble) - val lastRetweetSinceCreationHrs = - tweetFeatures.flatMap(_.lastRetweetSinceCreationHrs).map(_.toDouble) - val lastReplySinceCreationHrs = - tweetFeatures.flatMap(_.lastReplySinceCreationHrs).map(_.toDouble) - val lastQuoteSinceCreationHrs = - tweetFeatures.flatMap(_.lastQuoteSinceCreationHrs).map(_.toDouble) - val timeSinceLastFavoriteHrs = - getTimeSinceLastEngagementHrs(lastFavSinceCreationHrs, timeSinceSourceTweetCreationOpt) - val timeSinceLastRetweetHrs = - getTimeSinceLastEngagementHrs(lastRetweetSinceCreationHrs, timeSinceSourceTweetCreationOpt) - val timeSinceLastReplyHrs = - getTimeSinceLastEngagementHrs(lastReplySinceCreationHrs, timeSinceSourceTweetCreationOpt) - val timeSinceLastQuoteHrs = - getTimeSinceLastEngagementHrs(lastQuoteSinceCreationHrs, timeSinceSourceTweetCreationOpt) - - val nonPollingTimestampsMs = query.features.get.getOrElse(NonPollingTimesFeature, Seq.empty) - val timeSinceLastNonPollingRequest = - nonPollingTimestampsMs.headOption.map(query.queryTime.inMillis - _) - - val nonPollingRequestsSinceTweetCreation = - if (nonPollingTimestampsMs.nonEmpty && timeSinceTweetCreationMs.isDefined) { - nonPollingTimestampsMs - .search(timeSinceTweetCreationMs.get)(Ordering[Long].reverse) - .insertionPoint - } else 0.0 - - val tweetAgeRatio = - if (timeSinceTweetCreationMs.exists(_ > 0.0) && timeSinceLastNonPollingRequest.isDefined) { - timeSinceLastNonPollingRequest.get / timeSinceTweetCreationMs.get.toDouble - } else 0.0 - - val dataRecord = new DataRecord() - .setFeatureValue(IS_TWEET_RECYCLED, false) - .setFeatureValue(TWEET_AGE_RATIO, tweetAgeRatio) - .setFeatureValueFromOption( - TIME_SINCE_TWEET_CREATION, - timeSinceTweetCreationMs.map(_.toDouble) - ) - .setFeatureValue( - NON_POLLING_REQUESTS_SINCE_TWEET_CREATION, - nonPollingRequestsSinceTweetCreation - ) - .setFeatureValueFromOption(LAST_FAVORITE_SINCE_CREATION_HRS, lastFavSinceCreationHrs) - .setFeatureValueFromOption(LAST_RETWEET_SINCE_CREATION_HRS, lastRetweetSinceCreationHrs) - .setFeatureValueFromOption(LAST_REPLY_SINCE_CREATION_HRS, lastReplySinceCreationHrs) - .setFeatureValueFromOption(LAST_QUOTE_SINCE_CREATION_HRS, lastQuoteSinceCreationHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_FAVORITE_HRS, timeSinceLastFavoriteHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_RETWEET_HRS, timeSinceLastRetweetHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_REPLY_HRS, timeSinceLastReplyHrs) - .setFeatureValueFromOption(TIME_SINCE_LAST_QUOTE_HRS, timeSinceLastQuoteHrs) - - Stitch.value(FeatureMapBuilder().add(TweetTimeDataRecordFeature, dataRecord).build()) - } - - private def getTimeSinceLastEngagementHrs( - lastEngagementTimeSinceCreationHrsOpt: Option[Double], - timeSinceTweetCreation: Option[Duration] - ): Option[Double] = lastEngagementTimeSinceCreationHrsOpt.flatMap { lastEngagementTimeHrs => - timeSinceTweetCreation.map(_.inHours - lastEngagementTimeHrs) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.docx new file mode 100644 index 000000000..cb115fd1f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala deleted file mode 100644 index 93039e8bc..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieContentFeatureHydrator.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.escherbird.{thriftscala => esb} -import com.twitter.finagle.stats.Stat -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.MediaUnderstandingAnnotationIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieContentRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content.ContentFeatureAdapter -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.home_mixer.util.tweetypie.content.FeatureExtractionHelper -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.timelines.prediction.common.util.MediaUnderstandingAnnotations -import com.twitter.tweetypie.{thriftscala => tp} -import com.twitter.util.Future -import com.twitter.util.Return -import com.twitter.util.Throw -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TweetypieContentDataRecordFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TweetypieContentFeatureHydrator @Inject() ( - @Named(TweetypieContentRepository) client: KeyValueRepository[Seq[Long], Long, tp.Tweet], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("TweetypieContent") - - override val features: Set[Feature[_, _]] = Set( - MediaUnderstandingAnnotationIdsFeature, - TweetypieContentDataRecordFeature - ) - - override val statScope: String = identifier.toString - - private val bulkRequestLatencyStat = - statsReceiver.scope(statScope).scope("bulkRequest").stat("latency_ms") - private val postTransformerLatencyStat = - statsReceiver.scope(statScope).scope("postTransformer").stat("latency_ms") - private val bulkPostTransformerLatencyStat = - statsReceiver.scope(statScope).scope("bulkPostTransformer").stat("latency_ms") - - private val DefaultDataRecord: DataRecord = new DataRecord() - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val tweetIdsToHydrate = candidates.map(getCandidateOriginalTweetId).distinct - - val response: Future[KeyValueResult[Long, tp.Tweet]] = Stat.timeFuture(bulkRequestLatencyStat) { - if (tweetIdsToHydrate.isEmpty) Future.value(KeyValueResult.empty) - else client(tweetIdsToHydrate) - } - - response.flatMap { result => - Stat.timeFuture(bulkPostTransformerLatencyStat) { - OffloadFuturePools - .parallelize[CandidateWithFeatures[TweetCandidate], Try[(Seq[Long], DataRecord)]]( - candidates, - parTransformer(result, _), - parallelism = 32, - default = Return((Seq.empty, DefaultDataRecord)) - ).map { - _.map { - case Return(result) => - FeatureMapBuilder() - .add(MediaUnderstandingAnnotationIdsFeature, result._1) - .add(TweetypieContentDataRecordFeature, result._2) - .build() - case Throw(e) => - FeatureMapBuilder() - .add(MediaUnderstandingAnnotationIdsFeature, Throw(e)) - .add(TweetypieContentDataRecordFeature, Throw(e)) - .build() - } - } - } - } - } - - private def parTransformer( - result: KeyValueResult[Long, tp.Tweet], - candidate: CandidateWithFeatures[TweetCandidate] - ): Try[(Seq[Long], DataRecord)] = { - val originalTweetId = Some(getCandidateOriginalTweetId(candidate)) - val value = observedGet(key = originalTweetId, keyValueResult = result) - Stat.time(postTransformerLatencyStat)(postTransformer(value)) - } - - private def postTransformer( - result: Try[Option[tp.Tweet]] - ): Try[(Seq[Long], DataRecord)] = { - result.map { tweet => - val transformedValue = tweet.map(FeatureExtractionHelper.extractFeatures) - val semanticAnnotations = transformedValue - .flatMap { contentFeatures => - contentFeatures.semanticCoreAnnotations.map { - getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds - } - }.getOrElse(Seq.empty) - val dataRecord = ContentFeatureAdapter.adaptToDataRecords(transformedValue).asScala.head - (semanticAnnotations, dataRecord) - } - } - - private def getCandidateOriginalTweetId( - candidate: CandidateWithFeatures[TweetCandidate] - ): Long = { - candidate.features - .getOrElse(SourceTweetIdFeature, None).getOrElse(candidate.candidate.id) - } - - private def getNonSensitiveHighRecallMediaUnderstandingAnnotationEntityIds( - semanticCoreAnnotations: Seq[esb.TweetEntityAnnotation] - ): Seq[Long] = - semanticCoreAnnotations - .filter(MediaUnderstandingAnnotations.isEligibleNonSensitiveHighRecallMUAnnotation) - .map(_.entityId) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.docx new file mode 100644 index 000000000..20d87294e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala deleted file mode 100644 index 0ea9e8591..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TweetypieStaticEntitiesFeatureHydrator.scala +++ /dev/null @@ -1,161 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.google.inject.name.Named -import com.twitter.conversions.DurationOps.RichDuration -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.DirectedAtUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.ExclusiveConversationAuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.HasImageFeature -import com.twitter.home_mixer.model.HomeFeatures.HasVideoFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.IsRetweetFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.QuotedUserIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceUserIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TweetypieStaticEntitiesCache -import com.twitter.home_mixer.util.tweetypie.RequestFields -import com.twitter.home_mixer.util.tweetypie.content.TweetMediaFeaturesExtractor -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.TtlCache -import com.twitter.spam.rtf.{thriftscala => sp} -import com.twitter.stitch.Stitch -import com.twitter.stitch.tweetypie.{TweetyPie => TweetypieStitchClient} -import com.twitter.tweetypie.{thriftscala => tp} -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class TweetypieStaticEntitiesFeatureHydrator @Inject() ( - tweetypieStitchClient: TweetypieStitchClient, - @Named(TweetypieStaticEntitiesCache) cacheClient: TtlCache[Long, tp.Tweet]) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TweetypieStaticEntities") - - override val features: Set[Feature[_, _]] = Set( - AuthorIdFeature, - DirectedAtUserIdFeature, - ExclusiveConversationAuthorIdFeature, - HasImageFeature, - HasVideoFeature, - InReplyToTweetIdFeature, - InReplyToUserIdFeature, - IsRetweetFeature, - MentionScreenNameFeature, - MentionUserIdFeature, - QuotedTweetIdFeature, - QuotedUserIdFeature, - SourceTweetIdFeature, - SourceUserIdFeature - ) - - private val CacheTTL = 24.hours - - private val DefaultFeatureMap = FeatureMapBuilder() - .add(AuthorIdFeature, None) - .add(DirectedAtUserIdFeature, None) - .add(ExclusiveConversationAuthorIdFeature, None) - .add(HasImageFeature, false) - .add(HasVideoFeature, false) - .add(InReplyToTweetIdFeature, None) - .add(InReplyToUserIdFeature, None) - .add(IsRetweetFeature, false) - .add(MentionScreenNameFeature, Seq.empty) - .add(MentionUserIdFeature, Seq.empty) - .add(QuotedTweetIdFeature, None) - .add(QuotedUserIdFeature, None) - .add(SourceTweetIdFeature, None) - .add(SourceUserIdFeature, None) - .build() - - /** - * Steps: - * 1. query cache with all candidates - * 2. create a cached feature map - * 3. iterate candidates to hydrate features - * 3.a transform cached candidates - * 3.b hydrate non-cached candidates from Tweetypie and write to cache - */ - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = { - val tweetIds = candidates.map(_.candidate.id) - val cachedTweetsMapFu = cacheClient - .get(tweetIds) - .map(_.found) - - Stitch.callFuture(cachedTweetsMapFu).flatMap { cachedTweets => - Stitch.collect { - candidates.map { candidate => - if (cachedTweets.contains(candidate.candidate.id)) - Stitch.value(createFeatureMap(cachedTweets(candidate.candidate.id))) - else readFromTweetypie(query, candidate) - } - } - } - } - - private def createFeatureMap(tweet: tp.Tweet): FeatureMap = { - val coreData = tweet.coreData - val quotedTweet = tweet.quotedTweet - val mentions = tweet.mentions.getOrElse(Seq.empty) - val share = coreData.flatMap(_.share) - val reply = coreData.flatMap(_.reply) - - FeatureMapBuilder() - .add(AuthorIdFeature, coreData.map(_.userId)) - .add(DirectedAtUserIdFeature, coreData.flatMap(_.directedAtUser.map(_.userId))) - .add( - ExclusiveConversationAuthorIdFeature, - tweet.exclusiveTweetControl.map(_.conversationAuthorId)) - .add(HasImageFeature, TweetMediaFeaturesExtractor.hasImage(tweet)) - .add(HasVideoFeature, TweetMediaFeaturesExtractor.hasVideo(tweet)) - .add(InReplyToTweetIdFeature, reply.flatMap(_.inReplyToStatusId)) - .add(InReplyToUserIdFeature, reply.map(_.inReplyToUserId)) - .add(IsRetweetFeature, share.isDefined) - .add(MentionScreenNameFeature, mentions.map(_.screenName)) - .add(MentionUserIdFeature, mentions.flatMap(_.userId)) - .add(QuotedTweetIdFeature, quotedTweet.map(_.tweetId)) - .add(QuotedUserIdFeature, quotedTweet.map(_.userId)) - .add(SourceTweetIdFeature, share.map(_.sourceStatusId)) - .add(SourceUserIdFeature, share.map(_.sourceUserId)) - .build() - } - - private def readFromTweetypie( - query: PipelineQuery, - candidate: CandidateWithFeatures[TweetCandidate] - ): Stitch[FeatureMap] = { - tweetypieStitchClient - .getTweetFields( - tweetId = candidate.candidate.id, - options = tp.GetTweetFieldsOptions( - tweetIncludes = RequestFields.TweetStaticEntitiesFields, - includeRetweetedTweet = false, - includeQuotedTweet = false, - forUserId = query.getOptionalUserId, // Needed to get protected Tweets for certain users - visibilityPolicy = tp.TweetVisibilityPolicy.UserVisible, - safetyLevel = Some(sp.SafetyLevel.FilterNone) // VF is handled in the For You product - ) - ).map { - case tp.GetTweetFieldsResult(_, tp.TweetFieldsResultState.Found(found), _, _) => - cacheClient.set(candidate.candidate.id, found.tweet, CacheTTL) - createFeatureMap(found.tweet) - case _ => - DefaultFeatureMap + (AuthorIdFeature, candidate.features.getOrElse(AuthorIdFeature, None)) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.docx new file mode 100644 index 000000000..910b8de60 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala deleted file mode 100644 index 2e531bfb2..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinAuthorFollowFeatureHydrator.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinAuthorFollowFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinAuthorFollowEmbeddingsAdapter -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.servo.repository.KeyValueResult -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TwhinAuthorFollowFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TwhinAuthorFollowFeatureHydrator @Inject() ( - @Named(TwhinAuthorFollowFeatureRepository) - client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwhinAuthorFollow") - - override val features: Set[Feature[_, _]] = Set(TwhinAuthorFollowFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, ml.FloatTensor]] = - if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - - FeatureMapBuilder().add(TwhinAuthorFollowFeature, transformedValue).build() - } - } - } - - private def postTransformer(embedding: Try[Option[ml.FloatTensor]]): Try[DataRecord] = { - embedding.map { floatTensor => - TwhinAuthorFollowEmbeddingsAdapter.adaptToDataRecords(floatTensor).asScala.head - } - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - CandidatesUtil.getOriginalAuthorId(candidate.features) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.docx new file mode 100644 index 000000000..0cf0c99aa Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala deleted file mode 100644 index f43abc98f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserEngagementQueryFeatureHydrator.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserEngagementFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserEngagementEmbeddingsAdapter -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TwhinUserEngagementFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TwhinUserEngagementQueryFeatureHydrator @Inject() ( - @Named(TwhinUserEngagementFeatureRepository) - client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwhinUserEngagement") - - override val features: Set[Feature[_, _]] = Set(TwhinUserEngagementFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - Stitch.callFuture(client(Seq(userId))).map { results => - val embedding: Option[ml.FloatTensor] = results(userId) match { - case Return(value) => - if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() - else keyLossCounter.incr() - value - case Throw(_) => - keyFailureCounter.incr() - None - case _ => - None - } - - val dataRecord = - TwhinUserEngagementEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head - - FeatureMapBuilder() - .add(TwhinUserEngagementFeature, dataRecord) - .build() - } - } - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.docx new file mode 100644 index 000000000..eab6943bf Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala deleted file mode 100644 index 925043e1e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/TwhinUserFollowQueryFeatureHydrator.scala +++ /dev/null @@ -1,70 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TwhinUserFollowFeatureRepository -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings.TwhinUserFollowEmbeddingsAdapter -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Return -import com.twitter.util.Throw -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton -import scala.collection.JavaConverters._ - -object TwhinUserFollowFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TwhinUserFollowQueryFeatureHydrator @Inject() ( - @Named(TwhinUserFollowFeatureRepository) - client: KeyValueRepository[Seq[Long], Long, ml.FloatTensor], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TwhinUserFollow") - - override val features: Set[Feature[_, _]] = Set(TwhinUserFollowFeature) - - private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName) - private val keyFoundCounter = scopedStatsReceiver.counter("key/found") - private val keyLossCounter = scopedStatsReceiver.counter("key/loss") - private val keyFailureCounter = scopedStatsReceiver.counter("key/failure") - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val userId = query.getRequiredUserId - Stitch.callFuture(client(Seq(userId))).map { results => - val embedding: Option[ml.FloatTensor] = results(userId) match { - case Return(value) => - if (value.exists(_.floats.nonEmpty)) keyFoundCounter.incr() - else keyLossCounter.incr() - value - case Throw(_) => - keyFailureCounter.incr() - None - case _ => - None - } - - val dataRecord = TwhinUserFollowEmbeddingsAdapter.adaptToDataRecords(embedding).asScala.head - - FeatureMapBuilder() - .add(TwhinUserFollowFeature, dataRecord) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.docx new file mode 100644 index 000000000..cd9d21a05 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala deleted file mode 100644 index 90f618fd0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserFollowedTopicIdsFeatureHydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserFollowedTopicIdsRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import com.twitter.util.Future -import com.twitter.util.Try -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object UserFollowedTopicIdsFeature extends Feature[TweetCandidate, Seq[Long]] - -@Singleton -class UserFollowedTopicIdsFeatureHydrator @Inject() ( - @Named(UserFollowedTopicIdsRepository) - client: KeyValueRepository[Seq[Long], Long, Seq[Long]], - override val statsReceiver: StatsReceiver) - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("UserFollowedTopicIds") - - override val features: Set[Feature[_, _]] = Set(UserFollowedTopicIdsFeature) - - override val statScope: String = identifier.toString - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyAuthorIds = extractKeys(candidates) - val authorIds = possiblyAuthorIds.flatten - - val response: Future[KeyValueResult[Long, Seq[Long]]] = - if (authorIds.isEmpty) Future.value(KeyValueResult.empty) else client(authorIds) - - response.map { result => - possiblyAuthorIds.map { possiblyAuthorId => - val value = observedGet(key = possiblyAuthorId, keyValueResult = result) - val transformedValue = postTransformer(value) - - FeatureMapBuilder().add(UserFollowedTopicIdsFeature, transformedValue).build() - } - } - } - - private def postTransformer(input: Try[Option[Seq[Long]]]): Try[Seq[Long]] = { - input.map(_.getOrElse(Seq.empty[Long])) - } - - private def extractKeys( - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - candidate.features - .getTry(AuthorIdFeature) - .toOption - .flatten - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.docx new file mode 100644 index 000000000..146861657 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala deleted file mode 100644 index e2431c43e..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserLanguagesFeatureHydrator.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UserLanguagesRepository -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.search.common.constants.{thriftscala => scc} -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object UserLanguagesFeature extends Feature[PipelineQuery, Seq[scc.ThriftLanguage]] - -@Singleton -case class UserLanguagesFeatureHydrator @Inject() ( - @Named(UserLanguagesRepository) client: KeyValueRepository[Seq[Long], Long, Seq[ - scc.ThriftLanguage - ]], - statsReceiver: StatsReceiver) - extends QueryFeatureHydrator[PipelineQuery] - with ObservedKeyValueResultHandler { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("UserLanguages") - - override val features: Set[Feature[_, _]] = Set(UserLanguagesFeature) - - override val statScope: String = identifier.toString - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val key = query.getRequiredUserId - Stitch.callFuture(client(Seq(key))).map { result => - val feature = - observedGet(key = Some(key), keyValueResult = result).map(_.getOrElse(Seq.empty)) - FeatureMapBuilder() - .add(UserLanguagesFeature, feature) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.docx new file mode 100644 index 000000000..78fd09717 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala deleted file mode 100644 index 5a61f31fb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UserStateQueryFeatureHydrator.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.UserStateFeature -import com.twitter.home_mixer.service.HomeMixerAlertConfig -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.stitch.Stitch -import com.twitter.timelines.user_health.v1.{thriftscala => uhv1} -import com.twitter.timelines.user_health.{thriftscala => uh} -import com.twitter.user_session_store.ReadOnlyUserSessionStore -import com.twitter.user_session_store.ReadRequest -import com.twitter.user_session_store.UserSessionDataset -import com.twitter.user_session_store.UserSessionDataset.UserSessionDataset -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -case class UserStateQueryFeatureHydrator @Inject() ( - userSessionStore: ReadOnlyUserSessionStore) - extends QueryFeatureHydrator[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("UserState") - - override val features: Set[Feature[_, _]] = Set(UserStateFeature) - - private val datasets: Set[UserSessionDataset] = Set(UserSessionDataset.UserHealth) - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - userSessionStore - .read(ReadRequest(query.getRequiredUserId, datasets)) - .map { userSession => - val userState = userSession.flatMap { - _.userHealth match { - case Some(uh.UserHealth.V1(uhv1.UserHealth(userState))) => userState - case _ => None - } - } - - FeatureMapBuilder() - .add(UserStateFeature, userState) - .build() - } - } - - override val alerts = Seq( - HomeMixerAlertConfig.BusinessHours.defaultSuccessRateAlert(99.9) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.docx new file mode 100644 index 000000000..0cda0b771 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala deleted file mode 100644 index 96b9657e7..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/UtegFeatureHydrator.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator - -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByCountFeature -import com.twitter.home_mixer.model.HomeFeatures.FavoritedByUserIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.FromInNetworkSourceFeature -import com.twitter.home_mixer.model.HomeFeatures.InReplyToTweetIdFeature -import com.twitter.home_mixer.model.HomeFeatures.RealGraphInNetworkScoresFeature -import com.twitter.home_mixer.model.HomeFeatures.RepliedByCountFeature -import com.twitter.home_mixer.model.HomeFeatures.RepliedByEngagerIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.RetweetedByCountFeature -import com.twitter.home_mixer.model.HomeFeatures.RetweetedByEngagerIdsFeature -import com.twitter.home_mixer.model.HomeFeatures.SourceTweetIdFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.UtegSocialProofRepository -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.Conditionally -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.recos.recos_common.{thriftscala => rc} -import com.twitter.recos.user_tweet_entity_graph.{thriftscala => uteg} -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.servo.repository.KeyValueRepository -import com.twitter.stitch.Stitch -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -@Singleton -class UtegFeatureHydrator @Inject() ( - @Named(UtegSocialProofRepository) client: KeyValueRepository[ - (Seq[Long], (Long, Map[Long, Double])), - Long, - uteg.TweetRecommendation - ]) extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with Conditionally[PipelineQuery] { - - override val identifier: FeatureHydratorIdentifier = FeatureHydratorIdentifier("Uteg") - - override val features: Set[Feature[_, _]] = Set( - FavoritedByUserIdsFeature, - RetweetedByEngagerIdsFeature, - RepliedByEngagerIdsFeature, - FavoritedByCountFeature, - RetweetedByCountFeature, - RepliedByCountFeature - ) - - override def onlyIf(query: PipelineQuery): Boolean = query.features - .exists(_.getOrElse(RealGraphInNetworkScoresFeature, Map.empty[Long, Double]).nonEmpty) - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val seedUserWeights = query.features.map(_.get(RealGraphInNetworkScoresFeature)).get - - val sourceTweetIds = candidates.flatMap(_.features.getOrElse(SourceTweetIdFeature, None)) - val inReplyToTweetIds = candidates.flatMap(_.features.getOrElse(InReplyToTweetIdFeature, None)) - val tweetIds = candidates.map(_.candidate.id) - val tweetIdsToSend = (tweetIds ++ sourceTweetIds ++ inReplyToTweetIds).distinct - - val utegQuery = (tweetIdsToSend, (query.getRequiredUserId, seedUserWeights)) - - client(utegQuery).map(handleResponse(candidates, _)) - } - - private def handleResponse( - candidates: Seq[CandidateWithFeatures[TweetCandidate]], - results: KeyValueResult[Long, uteg.TweetRecommendation], - ): Seq[FeatureMap] = { - candidates.map { candidate => - val inNetwork = candidate.features.getOrElse(FromInNetworkSourceFeature, false) - val candidateProof = results(candidate.candidate.id).toOption.flatten - val sourceProof = candidate.features - .getOrElse(SourceTweetIdFeature, None).flatMap(results(_).toOption.flatten) - val proofs = Seq(candidateProof, sourceProof).flatten.map(_.socialProofByType) - - val favoritedBy = proofs.flatMap(_.get(rc.SocialProofType.Favorite)).flatten - val retweetedBy = proofs.flatMap(_.get(rc.SocialProofType.Retweet)).flatten - val repliedBy = proofs.flatMap(_.get(rc.SocialProofType.Reply)).flatten - - val (favoritedByCount, retweetedByCount, repliedByCount) = - if (!inNetwork) { - (favoritedBy.size.toDouble, retweetedBy.size.toDouble, repliedBy.size.toDouble) - } else { (0.0, 0.0, 0.0) } - - FeatureMapBuilder() - .add(FavoritedByUserIdsFeature, favoritedBy) - .add(RetweetedByEngagerIdsFeature, retweetedBy) - .add(RepliedByEngagerIdsFeature, repliedBy) - .add(FavoritedByCountFeature, favoritedByCount) - .add(RetweetedByCountFeature, retweetedByCount) - .add(RepliedByCountFeature, repliedByCount) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.docx new file mode 100644 index 000000000..4a17b5550 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala deleted file mode 100644 index cbaa3f18c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/AuthorFeaturesAdapter.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.author_features - -import com.twitter.home_mixer.util.DataRecordUtil -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.util.CompactDataRecordConverter -import com.twitter.ml.api.util.FDsl._ -import com.twitter.timelines.author_features.v1.{thriftjava => af} -import com.twitter.timelines.prediction.common.adapters.TimelinesAdapterBase -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig -import com.twitter.timelines.prediction.features.user_health.UserHealthFeatures -import scala.collection.JavaConverters._ - -object AuthorFeaturesAdapter extends TimelinesAdapterBase[af.AuthorFeatures] { - - private val Prefix = "original_author.timelines.original_author_aggregates." - - private val typedAggregateGroups = - TimelinesAggregationConfig.originalAuthorAggregatesV1.buildTypedAggregateGroups() - - private val aggregateFeaturesRenameMap: Map[Feature[_], Feature[_]] = - typedAggregateGroups.map(_.outputFeaturesToRenamedOutputFeatures(Prefix)).reduce(_ ++ _) - - private val prefixedOriginalAuthorAggregateFeatures = - typedAggregateGroups.flatMap(_.allOutputFeatures).map { feature => - aggregateFeaturesRenameMap.getOrElse(feature, feature) - } - - private val authorFeatures = prefixedOriginalAuthorAggregateFeatures ++ Seq( - UserHealthFeatures.AuthorState, - UserHealthFeatures.NumAuthorFollowers, - UserHealthFeatures.NumAuthorConnectDays, - UserHealthFeatures.NumAuthorConnect - ) - - private val aggregateFeatureContext: FeatureContext = - new FeatureContext(typedAggregateGroups.flatMap(_.allOutputFeatures).asJava) - - private lazy val prefixedAggregateFeatureContext: FeatureContext = - new FeatureContext(prefixedOriginalAuthorAggregateFeatures.asJava) - - override val getFeatureContext: FeatureContext = new FeatureContext(authorFeatures: _*) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - private val compactDataRecordConverter = new CompactDataRecordConverter() - - override def adaptToDataRecords( - authorFeatures: af.AuthorFeatures - ): java.util.List[DataRecord] = { - val dataRecord = - if (authorFeatures.aggregates != null) { - val originalAuthorAggregatesDataRecord = - compactDataRecordConverter.compactDataRecordToDataRecord(authorFeatures.aggregates) - - DataRecordUtil.applyRename( - originalAuthorAggregatesDataRecord, - aggregateFeatureContext, - prefixedAggregateFeatureContext, - aggregateFeaturesRenameMap) - } else new DataRecord - - if (authorFeatures.user_health != null) { - val userHealth = authorFeatures.user_health - - if (userHealth.user_state != null) { - dataRecord.setFeatureValue( - UserHealthFeatures.AuthorState, - userHealth.user_state.getValue.toLong - ) - } - - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorFollowers, - userHealth.num_followers.toDouble - ) - - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorConnectDays, - userHealth.num_connect_days.toDouble - ) - - dataRecord.setFeatureValue( - UserHealthFeatures.NumAuthorConnect, - userHealth.num_connect.toDouble - ) - } - - List(dataRecord).asJava - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel deleted file mode 100644 index 7a00a9b26..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/util", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/common/aggregates", - "src/scala/com/twitter/timelines/prediction/features/user_health", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - "timelines/data_processing/ml_util/aggregation_framework", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.docx new file mode 100644 index 000000000..349cba45c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/author_features/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel deleted file mode 100644 index 6a9393121..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.bazel +++ /dev/null @@ -1,18 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/conversation_features", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.docx new file mode 100644 index 000000000..6bf1c6070 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.docx new file mode 100644 index 000000000..eb7bae97c Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala deleted file mode 100644 index 2d73ece45..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/ContentFeatureAdapter.scala +++ /dev/null @@ -1,273 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content - -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters._ -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.common.adapters.TweetLengthType -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.prediction.features.conversation_features.ConversationFeatures -import scala.collection.JavaConverters._ - -object ContentFeatureAdapter extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - ConversationFeatures.IS_SELF_THREAD_TWEET, - ConversationFeatures.IS_LEAF_IN_SELF_THREAD, - TimelinesSharedFeatures.ASPECT_RATIO_DEN, - TimelinesSharedFeatures.ASPECT_RATIO_NUM, - TimelinesSharedFeatures.BIT_RATE, - TimelinesSharedFeatures.CLASSIFICATION_LABELS, - TimelinesSharedFeatures.COLOR_1_BLUE, - TimelinesSharedFeatures.COLOR_1_GREEN, - TimelinesSharedFeatures.COLOR_1_PERCENTAGE, - TimelinesSharedFeatures.COLOR_1_RED, - TimelinesSharedFeatures.FACE_AREAS, - TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, - TimelinesSharedFeatures.HAS_DESCRIPTION, - TimelinesSharedFeatures.HAS_QUESTION, - TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, - TimelinesSharedFeatures.HAS_TITLE, - TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, - TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, - TimelinesSharedFeatures.HEIGHT_1, - TimelinesSharedFeatures.HEIGHT_2, - TimelinesSharedFeatures.HEIGHT_3, - TimelinesSharedFeatures.HEIGHT_4, - TimelinesSharedFeatures.IS_360, - TimelinesSharedFeatures.IS_EMBEDDABLE, - TimelinesSharedFeatures.IS_MANAGED, - TimelinesSharedFeatures.IS_MONETIZABLE, - TimelinesSharedFeatures.MEDIA_PROVIDERS, - TimelinesSharedFeatures.NUM_CAPS, - TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, - TimelinesSharedFeatures.NUM_FACES, - TimelinesSharedFeatures.NUM_MEDIA_TAGS, - TimelinesSharedFeatures.NUM_NEWLINES, - TimelinesSharedFeatures.NUM_STICKERS, - TimelinesSharedFeatures.NUM_WHITESPACES, - TimelinesSharedFeatures.RESIZE_METHOD_1, - TimelinesSharedFeatures.RESIZE_METHOD_2, - TimelinesSharedFeatures.RESIZE_METHOD_3, - TimelinesSharedFeatures.RESIZE_METHOD_4, - TimelinesSharedFeatures.TWEET_LENGTH, - TimelinesSharedFeatures.TWEET_LENGTH_TYPE, - TimelinesSharedFeatures.VIDEO_DURATION, - TimelinesSharedFeatures.VIEW_COUNT, - TimelinesSharedFeatures.WIDTH_1, - TimelinesSharedFeatures.WIDTH_2, - TimelinesSharedFeatures.WIDTH_3, - TimelinesSharedFeatures.WIDTH_4, - ) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - private def getTweetLengthType(tweetLength: Int): Long = { - tweetLength match { - case x if 0 > x || 280 < x => TweetLengthType.INVALID - case x if 0 <= x && x <= 30 => TweetLengthType.VERY_SHORT - case x if 30 < x && x <= 60 => TweetLengthType.SHORT - case x if 60 < x && x <= 90 => TweetLengthType.MEDIUM - case x if 90 < x && x <= 140 => TweetLengthType.LENGTHY - case x if 140 < x && x <= 210 => TweetLengthType.VERY_LENGTHY - case x if x > 210 => TweetLengthType.MAXIMUM_LENGTH - } - } - - override def setFeatures( - contentFeatures: Option[ContentFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (contentFeatures.nonEmpty) { - val features = contentFeatures.get - // Conversation Features - richDataRecord.setFeatureValueFromOption( - ConversationFeatures.IS_SELF_THREAD_TWEET, - Some(features.selfThreadMetadata.nonEmpty) - ) - richDataRecord.setFeatureValueFromOption( - ConversationFeatures.IS_LEAF_IN_SELF_THREAD, - features.selfThreadMetadata.map(_.isLeaf) - ) - - // Media Features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.ASPECT_RATIO_DEN, - features.aspectRatioDen.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.ASPECT_RATIO_NUM, - features.aspectRatioNum.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.BIT_RATE, - features.bitRate.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_1, - features.heights.flatMap(_.lift(0)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_2, - features.heights.flatMap(_.lift(1)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_3, - features.heights.flatMap(_.lift(2)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HEIGHT_4, - features.heights.flatMap(_.lift(3)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_MEDIA_TAGS, - features.numMediaTags.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_1, - features.resizeMethods.flatMap(_.lift(0)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_2, - features.resizeMethods.flatMap(_.lift(1)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_3, - features.resizeMethods.flatMap(_.lift(2)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.RESIZE_METHOD_4, - features.resizeMethods.flatMap(_.lift(3)).map(_.toLong) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.VIDEO_DURATION, - features.videoDurationMs.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_1, - features.widths.flatMap(_.lift(0)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_2, - features.widths.flatMap(_.lift(1)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_3, - features.widths.flatMap(_.lift(2)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WIDTH_4, - features.widths.flatMap(_.lift(3)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_COLOR_PALLETTE_ITEMS, - features.numColors.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_RED, - features.dominantColorRed.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_BLUE, - features.dominantColorBlue.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_GREEN, - features.dominantColorGreen.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.COLOR_1_PERCENTAGE, - features.dominantColorPercentage - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.MEDIA_PROVIDERS, - features.mediaOriginProviders.map(_.toSet.asJava) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_360, - features.is360 - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.VIEW_COUNT, - features.viewCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_MANAGED, - features.isManaged - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_MONETIZABLE, - features.isMonetizable - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_EMBEDDABLE, - features.isEmbeddable - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_STICKERS, - features.stickerIds.map(_.length.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_FACES, - features.faceAreas.map(_.length.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FACE_AREAS, - // guard for exception from max on empty seq - features.faceAreas.map(faceAreas => - faceAreas.map(_.toDouble).reduceOption(_ max _).getOrElse(0.0)) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_SELECTED_PREVIEW_IMAGE, - features.hasSelectedPreviewImage - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_TITLE, - features.hasTitle - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_DESCRIPTION, - features.hasDescription - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_VISIT_SITE_CALL_TO_ACTION, - features.hasVisitSiteCallToAction - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_APP_INSTALL_CALL_TO_ACTION, - features.hasAppInstallCallToAction - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_WATCH_NOW_CALL_TO_ACTION, - features.hasWatchNowCallToAction - ) - // text features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_CAPS, - Some(features.numCaps.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.TWEET_LENGTH, - Some(features.length.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.TWEET_LENGTH_TYPE, - Some(getTweetLengthType(features.length.toInt)) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_WHITESPACES, - Some(features.numWhiteSpaces.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_QUESTION, - Some(features.hasQuestion) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.NUM_NEWLINES, - features.numNewlines.map(_.toDouble) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.docx new file mode 100644 index 000000000..673c25219 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala deleted file mode 100644 index fa27ddac0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/content/InReplyToContentFeatureAdapter.scala +++ /dev/null @@ -1,75 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.content - -import com.twitter.home_mixer.model.ContentFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures - -object InReplyToContentFeatureAdapter - extends TimelinesMutatingAdapterBase[Option[ContentFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - // Media Features - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, - InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, - InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, - InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, - // TextFeatures - InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, - InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, - InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, - ) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - contentFeatures: Option[ContentFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (contentFeatures.nonEmpty) { - val features = contentFeatures.get - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_DEN, - features.aspectRatioNum.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.ASPECT_RATIO_NUM, - features.aspectRatioNum.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HEIGHT_1, - features.heights.flatMap(_.lift(0)).map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HEIGHT_2, - features.heights.flatMap(_.lift(1)).map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.VIDEO_DURATION, - features.videoDurationMs.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.NUM_CAPS, - Some(features.numCaps.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.TWEET_LENGTH, - Some(features.length.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HAS_QUESTION, - Some(features.hasQuestion) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel deleted file mode 100644 index 9428c0d39..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/scala/com/twitter/timelines/prediction/features/recap", - "src/scala/com/twitter/timelines/util", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/search/common:features-scala", - "timelines/src/main/scala/com/twitter/timelines/util", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.docx new file mode 100644 index 000000000..e76f211db Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.docx new file mode 100644 index 000000000..48d966986 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala deleted file mode 100644 index 5a0207e8a..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/EarlybirdAdapter.scala +++ /dev/null @@ -1,457 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird - -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters._ -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import com.twitter.timelines.prediction.features.recap.RecapFeatures -import com.twitter.timelines.util.UrlExtractorUtil -import java.lang.{Boolean => JBoolean} -import java.lang.{Double => JDouble} -import java.util.{Map => JMap} -import scala.collection.JavaConverters._ - -object EarlybirdAdapter extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - RecapFeatures.BIDIRECTIONAL_FAV_COUNT, - RecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - RecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - RecapFeatures.BLENDER_SCORE, - RecapFeatures.CONTAINS_MEDIA, - RecapFeatures.CONVERSATIONAL_COUNT, - RecapFeatures.EMBEDS_IMPRESSION_COUNT, - RecapFeatures.EMBEDS_URL_COUNT, - RecapFeatures.FAV_COUNT, - RecapFeatures.FAV_COUNT_V2, - RecapFeatures.FROM_INACTIVE_USER, - RecapFeatures.FROM_MUTUAL_FOLLOW, - RecapFeatures.FROM_VERIFIED_ACCOUNT, - RecapFeatures.HAS_CARD, - RecapFeatures.HAS_CONSUMER_VIDEO, - RecapFeatures.HAS_HASHTAG, - RecapFeatures.HAS_IMAGE, - RecapFeatures.HAS_LINK, - RecapFeatures.HAS_MENTION, - RecapFeatures.HAS_MULTIPLE_HASHTAGS_OR_TRENDS, - RecapFeatures.HAS_MULTIPLE_MEDIA, - RecapFeatures.HAS_NATIVE_IMAGE, - RecapFeatures.HAS_NATIVE_VIDEO, - RecapFeatures.HAS_NEWS, - RecapFeatures.HAS_PERISCOPE, - RecapFeatures.HAS_PRO_VIDEO, - RecapFeatures.HAS_TREND, - RecapFeatures.HAS_VIDEO, - RecapFeatures.HAS_VINE, - RecapFeatures.HAS_VISIBLE_LINK, - RecapFeatures.IS_AUTHOR_BOT, - RecapFeatures.IS_AUTHOR_NEW, - RecapFeatures.IS_AUTHOR_NSFW, - RecapFeatures.IS_AUTHOR_PROFILE_EGG, - RecapFeatures.IS_AUTHOR_SPAM, - RecapFeatures.IS_BUSINESS_SCORE, - RecapFeatures.IS_OFFENSIVE, - RecapFeatures.IS_REPLY, - RecapFeatures.IS_RETWEET, - RecapFeatures.IS_RETWEETER_BOT, - RecapFeatures.IS_RETWEETER_NEW, - RecapFeatures.IS_RETWEETER_NSFW, - RecapFeatures.IS_RETWEETER_PROFILE_EGG, - RecapFeatures.IS_RETWEETER_SPAM, - RecapFeatures.IS_RETWEET_OF_REPLY, - RecapFeatures.IS_SENSITIVE, - RecapFeatures.LANGUAGE, - RecapFeatures.LINK_COUNT, - RecapFeatures.LINK_LANGUAGE, - RecapFeatures.MATCH_SEARCHER_LANGS, - RecapFeatures.MATCH_SEARCHER_MAIN_LANG, - RecapFeatures.MATCH_UI_LANG, - RecapFeatures.MENTIONED_SCREEN_NAMES, - RecapFeatures.MENTION_SEARCHER, - RecapFeatures.NUM_HASHTAGS, - RecapFeatures.NUM_MENTIONS, - RecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - RecapFeatures.PROBABLY_FROM_FOLLOWED_AUTHOR, - RecapFeatures.REPLY_COUNT, - RecapFeatures.REPLY_COUNT_V2, - RecapFeatures.REPLY_OTHER, - RecapFeatures.REPLY_SEARCHER, - RecapFeatures.RETWEET_COUNT, - RecapFeatures.RETWEET_COUNT_V2, - RecapFeatures.RETWEET_DIRECTED_AT_USER_IN_FIRST_DEGREE, - RecapFeatures.RETWEET_OF_MUTUAL_FOLLOW, - RecapFeatures.RETWEET_OTHER, - RecapFeatures.RETWEET_SEARCHER, - RecapFeatures.SIGNATURE, - RecapFeatures.SOURCE_AUTHOR_REP, - RecapFeatures.TEXT_SCORE, - RecapFeatures.TWEET_COUNT_FROM_USER_IN_SNAPSHOT, - RecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - RecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - RecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - RecapFeatures.URL_DOMAINS, - RecapFeatures.USER_REP, - RecapFeatures.VIDEO_VIEW_COUNT, - // shared features - TimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - TimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - TimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - TimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - TimelinesSharedFeatures.EMBEDS_IMPRESSION_COUNT_V2, - TimelinesSharedFeatures.EMBEDS_URL_COUNT_V2, - TimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - TimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - TimelinesSharedFeatures.DECAYED_REPLY_COUNT, - TimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - TimelinesSharedFeatures.FAKE_FAVORITE_COUNT, - TimelinesSharedFeatures.FAKE_RETWEET_COUNT, - TimelinesSharedFeatures.FAKE_REPLY_COUNT, - TimelinesSharedFeatures.FAKE_QUOTE_COUNT, - TimelinesSharedFeatures.QUOTE_COUNT, - TimelinesSharedFeatures.EARLYBIRD_SCORE, - // Safety features - TimelinesSharedFeatures.LABEL_ABUSIVE_FLAG, - TimelinesSharedFeatures.LABEL_ABUSIVE_HI_RCL_FLAG, - TimelinesSharedFeatures.LABEL_DUP_CONTENT_FLAG, - TimelinesSharedFeatures.LABEL_NSFW_HI_PRC_FLAG, - TimelinesSharedFeatures.LABEL_NSFW_HI_RCL_FLAG, - TimelinesSharedFeatures.LABEL_SPAM_FLAG, - TimelinesSharedFeatures.LABEL_SPAM_HI_RCL_FLAG, - // periscope features - TimelinesSharedFeatures.PERISCOPE_EXISTS, - TimelinesSharedFeatures.PERISCOPE_IS_LIVE, - TimelinesSharedFeatures.PERISCOPE_HAS_BEEN_FEATURED, - TimelinesSharedFeatures.PERISCOPE_IS_CURRENTLY_FEATURED, - TimelinesSharedFeatures.PERISCOPE_IS_FROM_QUALITY_SOURCE, - // VISIBLE_TOKEN_RATIO - TimelinesSharedFeatures.VISIBLE_TOKEN_RATIO, - TimelinesSharedFeatures.HAS_QUOTE, - TimelinesSharedFeatures.IS_COMPOSER_SOURCE_CAMERA, - // health features - TimelinesSharedFeatures.PREPORTED_TWEET_SCORE, - // media - TimelinesSharedFeatures.CLASSIFICATION_LABELS - ) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - ebFeatures: Option[sc.ThriftTweetFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (ebFeatures.nonEmpty) { - val features = ebFeatures.get - richDataRecord.setFeatureValue[JDouble]( - RecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - features.prevUserTweetEngagement.toDouble - ) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_SENSITIVE, features.isSensitiveContent) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_MULTIPLE_MEDIA, features.hasMultipleMedia) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_PROFILE_EGG, features.isAuthorProfileEgg) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_NEW, features.isAuthorNew) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.NUM_MENTIONS, features.numMentions.toDouble) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_MENTION, features.numMentions > 0) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.NUM_HASHTAGS, features.numHashtags.toDouble) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_HASHTAG, features.numHashtags > 0) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.LINK_LANGUAGE, features.linkLanguage.toDouble) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_NSFW, features.isAuthorNSFW) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_SPAM, features.isAuthorSpam) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_AUTHOR_BOT, features.isAuthorBot) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.LANGUAGE, - features.language.map(_.getValue.toLong)) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.SIGNATURE, - features.signature.map(_.toLong)) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.FROM_INACTIVE_USER, features.fromInActiveUser) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.PROBABLY_FROM_FOLLOWED_AUTHOR, - features.probablyFromFollowedAuthor) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.FROM_MUTUAL_FOLLOW, features.fromMutualFollow) - richDataRecord.setFeatureValue[JBoolean]( - RecapFeatures.FROM_VERIFIED_ACCOUNT, - features.fromVerifiedAccount) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.USER_REP, features.userRep) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.IS_BUSINESS_SCORE, features.isBusinessScore) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_CONSUMER_VIDEO, features.hasConsumerVideo) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_PRO_VIDEO, features.hasProVideo) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VINE, features.hasVine) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_PERISCOPE, features.hasPeriscope) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_NATIVE_VIDEO, features.hasNativeVideo) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_NATIVE_IMAGE, features.hasNativeImage) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_CARD, features.hasCard) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_IMAGE, features.hasImage) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_NEWS, features.hasNews) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_VIDEO, features.hasVideo) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.CONTAINS_MEDIA, features.containsMedia) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.RETWEET_SEARCHER, features.retweetSearcher) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.REPLY_SEARCHER, features.replySearcher) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.MENTION_SEARCHER, features.mentionSearcher) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.REPLY_OTHER, features.replyOther) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.RETWEET_OTHER, features.retweetOther) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_REPLY, features.isReply) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_RETWEET, features.isRetweet) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.IS_OFFENSIVE, features.isOffensive) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.MATCH_UI_LANG, features.matchesUILang) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.MATCH_SEARCHER_MAIN_LANG, - features.matchesSearcherMainLang) - richDataRecord.setFeatureValue[JBoolean]( - RecapFeatures.MATCH_SEARCHER_LANGS, - features.matchesSearcherLangs) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.BIDIRECTIONAL_FAV_COUNT, - features.bidirectionalFavCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - features.unidirectionalFavCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - features.bidirectionalReplyCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - features.unidirectionalReplyCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - features.bidirectionalRetweetCount) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - features.unidirectionalRetweetCount) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.CONVERSATIONAL_COUNT, features.conversationCount) - richDataRecord.setFeatureValue[JDouble]( - RecapFeatures.TWEET_COUNT_FROM_USER_IN_SNAPSHOT, - features.tweetCountFromUserInSnapshot - ) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.IS_RETWEETER_PROFILE_EGG, - features.isRetweeterProfileEgg) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_NEW, features.isRetweeterNew) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_BOT, features.isRetweeterBot) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_NSFW, features.isRetweeterNSFW) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEETER_SPAM, features.isRetweeterSpam) - richDataRecord - .setFeatureValue[JBoolean]( - RecapFeatures.RETWEET_OF_MUTUAL_FOLLOW, - features.retweetOfMutualFollow) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.SOURCE_AUTHOR_REP, features.sourceAuthorRep) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.IS_RETWEET_OF_REPLY, features.isRetweetOfReply) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.RETWEET_DIRECTED_AT_USER_IN_FIRST_DEGREE, - features.retweetDirectedAtUserInFirstDegree - ) - richDataRecord - .setFeatureValue[JDouble]( - RecapFeatures.EMBEDS_IMPRESSION_COUNT, - features.embedsImpressionCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.EMBEDS_URL_COUNT, features.embedsUrlCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.VIDEO_VIEW_COUNT, features.videoViewCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.REPLY_COUNT, features.replyCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](RecapFeatures.RETWEET_COUNT, features.retweetCount.toDouble) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.FAV_COUNT, features.favCount.toDouble) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.BLENDER_SCORE, features.blenderScore) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.TEXT_SCORE, features.textScore) - richDataRecord - .setFeatureValue[JBoolean](RecapFeatures.HAS_VISIBLE_LINK, features.hasVisibleLink) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_LINK, features.hasLink) - richDataRecord.setFeatureValue[JBoolean](RecapFeatures.HAS_TREND, features.hasTrend) - richDataRecord.setFeatureValue[JBoolean]( - RecapFeatures.HAS_MULTIPLE_HASHTAGS_OR_TRENDS, - features.hasMultipleHashtagsOrTrends - ) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.FAV_COUNT_V2, - features.favCountV2.map(_.toDouble)) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.RETWEET_COUNT_V2, - features.retweetCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.REPLY_COUNT_V2, - features.replyCountV2.map(_.toDouble)) - richDataRecord.setFeatureValueFromOption( - RecapFeatures.MENTIONED_SCREEN_NAMES, - features.mentionsList.map(_.toSet.asJava) - ) - val urls = features.urlsList.getOrElse(Seq.empty) - richDataRecord.setFeatureValue( - RecapFeatures.URL_DOMAINS, - urls.toSet.flatMap(UrlExtractorUtil.extractDomain).asJava) - richDataRecord.setFeatureValue[JDouble](RecapFeatures.LINK_COUNT, urls.size.toDouble) - // shared features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - features.weightedFavoriteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - features.weightedRetweetCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - features.weightedReplyCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - features.weightedQuoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.EMBEDS_IMPRESSION_COUNT_V2, - features.embedsImpressionCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.EMBEDS_URL_COUNT_V2, - features.embedsUrlCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - features.decayedFavoriteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - features.decayedRetweetCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_REPLY_COUNT, - features.decayedReplyCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - features.decayedQuoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_FAVORITE_COUNT, - features.fakeFavoriteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_RETWEET_COUNT, - features.fakeRetweetCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_REPLY_COUNT, - features.fakeReplyCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.FAKE_QUOTE_COUNT, - features.fakeQuoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.QUOTE_COUNT, - features.quoteCount.map(_.toDouble) - ) - richDataRecord.setFeatureValue[JDouble]( - TimelinesSharedFeatures.EARLYBIRD_SCORE, - features.earlybirdScore - ) - // safety features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_ABUSIVE_FLAG, - features.labelAbusiveFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_ABUSIVE_HI_RCL_FLAG, - features.labelAbusiveHiRclFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_DUP_CONTENT_FLAG, - features.labelDupContentFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_NSFW_HI_PRC_FLAG, - features.labelNsfwHiPrcFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_NSFW_HI_RCL_FLAG, - features.labelNsfwHiRclFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_SPAM_FLAG, - features.labelSpamFlag - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.LABEL_SPAM_HI_RCL_FLAG, - features.labelSpamHiRclFlag - ) - // periscope features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_EXISTS, - features.periscopeExists - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_IS_LIVE, - features.periscopeIsLive - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_HAS_BEEN_FEATURED, - features.periscopeHasBeenFeatured - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_IS_CURRENTLY_FEATURED, - features.periscopeIsCurrentlyFeatured - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PERISCOPE_IS_FROM_QUALITY_SOURCE, - features.periscopeIsFromQualitySource - ) - // misc features - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.VISIBLE_TOKEN_RATIO, - features.visibleTokenRatio.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.HAS_QUOTE, - features.hasQuote - ) - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.IS_COMPOSER_SOURCE_CAMERA, - features.isComposerSourceCamera - ) - // health scores - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.PREPORTED_TWEET_SCORE, - features.pReportedTweetScore - ) - // media - richDataRecord.setFeatureValueFromOption( - TimelinesSharedFeatures.CLASSIFICATION_LABELS, - features.mediaClassificationInfo.map(_.toMap.asJava.asInstanceOf[JMap[String, JDouble]]) - ) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.docx new file mode 100644 index 000000000..1591d7c83 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala deleted file mode 100644 index b6a7b0ad3..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/earlybird/InReplyToEarlybirdAdapter.scala +++ /dev/null @@ -1,206 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.earlybird - -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.DataRecordConverters.RichDataRecordWrapper -import com.twitter.search.common.features.{thriftscala => sc} -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.InReplyToTweetTimelinesSharedFeatures -import com.twitter.timelines.prediction.features.recap.InReplyToRecapFeatures -import java.lang.{Boolean => JBoolean} -import java.lang.{Double => JDouble} - -object InReplyToEarlybirdAdapter - extends TimelinesMutatingAdapterBase[Option[sc.ThriftTweetFeatures]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - // TextFeatures - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, - InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, - InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, - InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, - InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - InReplyToRecapFeatures.IS_SENSITIVE, - InReplyToRecapFeatures.IS_AUTHOR_NEW, - InReplyToRecapFeatures.NUM_MENTIONS, - InReplyToRecapFeatures.HAS_MENTION, - InReplyToRecapFeatures.HAS_HASHTAG, - InReplyToRecapFeatures.IS_AUTHOR_NSFW, - InReplyToRecapFeatures.IS_AUTHOR_SPAM, - InReplyToRecapFeatures.IS_AUTHOR_BOT, - InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, - InReplyToRecapFeatures.USER_REP, - InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, - InReplyToRecapFeatures.HAS_IMAGE, - InReplyToRecapFeatures.HAS_NEWS, - InReplyToRecapFeatures.HAS_VIDEO, - InReplyToRecapFeatures.HAS_VISIBLE_LINK, - InReplyToRecapFeatures.IS_OFFENSIVE, - InReplyToRecapFeatures.IS_REPLY, - InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, - InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - InReplyToRecapFeatures.CONVERSATIONAL_COUNT, - InReplyToRecapFeatures.REPLY_COUNT, - InReplyToRecapFeatures.RETWEET_COUNT, - InReplyToRecapFeatures.FAV_COUNT, - InReplyToRecapFeatures.TEXT_SCORE, - InReplyToRecapFeatures.FAV_COUNT_V2, - InReplyToRecapFeatures.RETWEET_COUNT_V2, - InReplyToRecapFeatures.REPLY_COUNT_V2) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - ebFeatures: Option[sc.ThriftTweetFeatures], - richDataRecord: RichDataRecord - ): Unit = { - if (ebFeatures.nonEmpty) { - val features = ebFeatures.get - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_FAV_COUNT, - features.weightedFavoriteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_RETWEET_COUNT, - features.weightedRetweetCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_REPLY_COUNT, - features.weightedReplyCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.WEIGHTED_QUOTE_COUNT, - features.weightedQuoteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_FAVORITE_COUNT, - features.decayedFavoriteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_RETWEET_COUNT, - features.decayedRetweetCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_REPLY_COUNT, - features.decayedReplyCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.DECAYED_QUOTE_COUNT, - features.decayedQuoteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.QUOTE_COUNT, - features.quoteCount.map(_.toDouble) - ) - - richDataRecord.setFeatureValueFromOption( - InReplyToTweetTimelinesSharedFeatures.HAS_QUOTE, - features.hasQuote - ) - - if (features.earlybirdScore > 0) - richDataRecord.setFeatureValue[JDouble]( - InReplyToTweetTimelinesSharedFeatures.EARLYBIRD_SCORE, - features.earlybirdScore - ) - - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.PREV_USER_TWEET_ENGAGEMENT, - features.prevUserTweetEngagement.toDouble - ) - - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_SENSITIVE, features.isSensitiveContent) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NEW, features.isAuthorNew) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.NUM_MENTIONS, - features.numMentions.toDouble) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_MENTION, (features.numMentions > 0)) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_HASHTAG, (features.numHashtags > 0)) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_NSFW, features.isAuthorNSFW) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_SPAM, features.isAuthorSpam) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_AUTHOR_BOT, features.isAuthorBot) - richDataRecord.setFeatureValue[JBoolean]( - InReplyToRecapFeatures.FROM_MUTUAL_FOLLOW, - features.fromMutualFollow) - richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.USER_REP, features.userRep) - richDataRecord.setFeatureValue[JBoolean]( - InReplyToRecapFeatures.FROM_VERIFIED_ACCOUNT, - features.fromVerifiedAccount) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_IMAGE, features.hasImage) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_NEWS, features.hasNews) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VIDEO, features.hasVideo) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.HAS_VISIBLE_LINK, features.hasVisibleLink) - richDataRecord - .setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_OFFENSIVE, features.isOffensive) - richDataRecord.setFeatureValue[JBoolean](InReplyToRecapFeatures.IS_REPLY, features.isReply) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.BIDIRECTIONAL_REPLY_COUNT, - features.bidirectionalReplyCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.UNIDIRECTIONAL_REPLY_COUNT, - features.unidirectionalReplyCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.BIDIRECTIONAL_RETWEET_COUNT, - features.bidirectionalRetweetCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.UNIDIRECTIONAL_RETWEET_COUNT, - features.unidirectionalRetweetCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.BIDIRECTIONAL_FAV_COUNT, - features.bidirectionalFavCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.UNIDIRECTIONAL_FAV_COUNT, - features.unidirectionalFavCount) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.CONVERSATIONAL_COUNT, - features.conversationCount) - richDataRecord - .setFeatureValue[JDouble](InReplyToRecapFeatures.REPLY_COUNT, features.replyCount.toDouble) - richDataRecord.setFeatureValue[JDouble]( - InReplyToRecapFeatures.RETWEET_COUNT, - features.retweetCount.toDouble) - richDataRecord - .setFeatureValue[JDouble](InReplyToRecapFeatures.FAV_COUNT, features.favCount.toDouble) - richDataRecord.setFeatureValue[JDouble](InReplyToRecapFeatures.TEXT_SCORE, features.textScore) - richDataRecord.setFeatureValueFromOption( - InReplyToRecapFeatures.FAV_COUNT_V2, - features.favCountV2.map(_.toDouble)) - richDataRecord.setFeatureValueFromOption( - InReplyToRecapFeatures.RETWEET_COUNT_V2, - features.retweetCountV2.map(_.toDouble) - ) - richDataRecord.setFeatureValueFromOption( - InReplyToRecapFeatures.REPLY_COUNT_V2, - features.replyCountV2.map(_.toDouble)) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel deleted file mode 100644 index acf19eabb..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/ml/api:data-java", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.docx new file mode 100644 index 000000000..a6e0c02b8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.docx new file mode 100644 index 000000000..7b3a94477 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala deleted file mode 100644 index 1439cc2ad..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/inferred_topic/InferredTopicAdapter.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.inferred_topic - -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import scala.collection.JavaConverters._ - -object InferredTopicAdapter extends TimelinesMutatingAdapterBase[Map[Long, Double]] { - - override val getFeatureContext: FeatureContext = new FeatureContext( - TimelinesSharedFeatures.INFERRED_TOPIC_IDS) - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - inferredTopicFeatures: Map[Long, Double], - richDataRecord: RichDataRecord - ): Unit = { - richDataRecord.setFeatureValue( - TimelinesSharedFeatures.INFERRED_TOPIC_IDS, - inferredTopicFeatures.keys.map(_.toString).toSet.asJava) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel deleted file mode 100644 index d1a7281c5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/features/common", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/timelines/author_features:thrift-java", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.docx new file mode 100644 index 000000000..8049b02bb Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.docx new file mode 100644 index 000000000..fe6d6b850 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala deleted file mode 100644 index 6c4b79eee..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCandidateFeaturesAdapter.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features - -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import java.lang.{Long => JLong} - -case class NonMLCandidateFeatures( - tweetId: Long, - sourceTweetId: Option[Long], - originalAuthorId: Option[Long], -) - -/** - * define non ml features adapter to create a data record which includes many non ml features - * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. - */ -object NonMLCandidateFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCandidateFeatures] { - - private val featureContext = new FeatureContext( - SharedFeatures.TWEET_ID, - // For Secondary Engagement data generation - TimelinesSharedFeatures.SOURCE_TWEET_ID, - TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, - ) - - override def getFeatureContext: FeatureContext = featureContext - - override val commonFeatures: Set[Feature[_]] = Set.empty - - override def setFeatures( - nonMLCandidateFeatures: NonMLCandidateFeatures, - richDataRecord: RichDataRecord - ): Unit = { - richDataRecord.setFeatureValue[JLong](SharedFeatures.TWEET_ID, nonMLCandidateFeatures.tweetId) - nonMLCandidateFeatures.sourceTweetId.foreach( - richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.SOURCE_TWEET_ID, _)) - nonMLCandidateFeatures.originalAuthorId.foreach( - richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.ORIGINAL_AUTHOR_ID, _)) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.docx new file mode 100644 index 000000000..b126d2ba8 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala deleted file mode 100644 index a2777538c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/non_ml_features/NonMLCommonFeaturesAdapter.scala +++ /dev/null @@ -1,48 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.non_ml_features - -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase -import com.twitter.timelines.prediction.features.common.TimelinesSharedFeatures -import java.lang.{Long => JLong} - -case class NonMLCommonFeatures( - userId: Long, - predictionRequestId: Option[Long], - servedTimestamp: Long, -) - -/** - * define non ml features adapter to create a data record which includes many non ml features - * e.g. predictionRequestId, userId, tweetId to be used as joined key in batch pipeline. - */ -object NonMLCommonFeaturesAdapter extends TimelinesMutatingAdapterBase[NonMLCommonFeatures] { - - private val featureContext = new FeatureContext( - SharedFeatures.USER_ID, - TimelinesSharedFeatures.PREDICTION_REQUEST_ID, - TimelinesSharedFeatures.SERVED_TIMESTAMP, - ) - - override def getFeatureContext: FeatureContext = featureContext - - override val commonFeatures: Set[Feature[_]] = Set( - SharedFeatures.USER_ID, - TimelinesSharedFeatures.PREDICTION_REQUEST_ID, - TimelinesSharedFeatures.SERVED_TIMESTAMP, - ) - - override def setFeatures( - nonMLCommonFeatures: NonMLCommonFeatures, - richDataRecord: RichDataRecord - ): Unit = { - richDataRecord.setFeatureValue[JLong](SharedFeatures.USER_ID, nonMLCommonFeatures.userId) - nonMLCommonFeatures.predictionRequestId.foreach( - richDataRecord.setFeatureValue[JLong](TimelinesSharedFeatures.PREDICTION_REQUEST_ID, _)) - richDataRecord.setFeatureValue[JLong]( - TimelinesSharedFeatures.SERVED_TIMESTAMP, - nonMLCommonFeatures.servedTimestamp) - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel deleted file mode 100644 index bfff86d3f..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/thrift/com/twitter/ml/api:data-java", - "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.docx new file mode 100644 index 000000000..fcd76133d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.docx new file mode 100644 index 000000000..b3eeade93 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala deleted file mode 100644 index dce0a81db..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/PassThroughAdapter.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.IRecordOneToOneAdapter - -object PassThroughAdapter extends IRecordOneToOneAdapter[Seq[DataRecord]] { - override def adaptToDataRecord(record: Seq[DataRecord]): DataRecord = - record.headOption.getOrElse(new DataRecord) - - // This is not necessary and should not be used. - override def getFeatureContext = ??? -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.docx new file mode 100644 index 000000000..c53d1ac00 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala deleted file mode 100644 index 4c54f78d8..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates/SparseAggregatesToDenseAdapter.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.conversion.CombineCountsPolicy -import com.twitter.timelines.prediction.common.adapters.TimelinesIRecordAdapter - -class SparseAggregatesToDenseAdapter(policy: CombineCountsPolicy) - extends TimelinesIRecordAdapter[Seq[DataRecord]] { - - override def setFeatures(input: Seq[DataRecord], mutableDataRecord: RichDataRecord): Unit = - policy.defaultMergeRecord(mutableDataRecord.getRecord, input.toList) - - override val getFeatureContext: FeatureContext = - new FeatureContext(policy.outputFeaturesPostMerge.toSeq: _*) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel deleted file mode 100644 index c32c29ce5..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api:api-base", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/ml/api:data-scala", - "src/thrift/com/twitter/ml/api:embedding-java", - "src/thrift/com/twitter/ml/api:embedding-scala", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.docx new file mode 100644 index 000000000..a83a9c64d Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.docx new file mode 100644 index 000000000..f89562d90 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala deleted file mode 100644 index c2830e462..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/twhin_embeddings/TwhinEmbeddingsAdapter.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.twhin_embeddings - -import com.twitter.ml.api.DataType -import com.twitter.ml.api.Feature -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.ml.api.util.ScalaToJavaDataRecordConversions -import com.twitter.ml.api.{thriftscala => ml} -import com.twitter.timelines.prediction.common.adapters.TimelinesMutatingAdapterBase - -sealed trait TwhinEmbeddingsAdapter extends TimelinesMutatingAdapterBase[Option[ml.FloatTensor]] { - def twhinEmbeddingsFeature: Feature.Tensor - - override def getFeatureContext: FeatureContext = new FeatureContext( - twhinEmbeddingsFeature - ) - - override def setFeatures( - embedding: Option[ml.FloatTensor], - richDataRecord: RichDataRecord - ): Unit = { - embedding.foreach { floatTensor => - richDataRecord.setFeatureValue( - twhinEmbeddingsFeature, - ScalaToJavaDataRecordConversions.scalaTensor2Java( - ml.GeneralTensor - .FloatTensor(floatTensor))) - } - } -} - -object TwhinEmbeddingsFeatures { - val TwhinAuthorFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "original_author.twhin.tw_hi_n.author_follow_as_float_tensor", - DataType.FLOAT - ) - - val TwhinUserEngagementEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "user.twhin.tw_hi_n.user_engagement_as_float_tensor", - DataType.FLOAT - ) - - val TwhinUserFollowEmbeddingsFeature: Feature.Tensor = new Feature.Tensor( - "user.twhin.tw_hi_n.user_follow_as_float_tensor", - DataType.FLOAT - ) -} - -object TwhinAuthorFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { - override val twhinEmbeddingsFeature: Feature.Tensor = - TwhinEmbeddingsFeatures.TwhinAuthorFollowEmbeddingsFeature - - override val commonFeatures: Set[Feature[_]] = Set.empty -} - -object TwhinUserEngagementEmbeddingsAdapter extends TwhinEmbeddingsAdapter { - override val twhinEmbeddingsFeature: Feature.Tensor = - TwhinEmbeddingsFeatures.TwhinUserEngagementEmbeddingsFeature - - override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) -} - -object TwhinUserFollowEmbeddingsAdapter extends TwhinEmbeddingsAdapter { - override val twhinEmbeddingsFeature: Feature.Tensor = - TwhinEmbeddingsFeatures.TwhinUserFollowEmbeddingsFeature - - override val commonFeatures: Set[Feature[_]] = Set(twhinEmbeddingsFeature) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.docx new file mode 100644 index 000000000..6bff3756f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala deleted file mode 100644 index 00dee1209..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeatureInfo.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.ml.api.FeatureContext -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.TypedAggregateGroup -import scala.jdk.CollectionConverters.asJavaIterableConverter - -// A helper class deriving aggregate feature info from the given configuration parameters. -class AggregateFeatureInfo( - val aggregateGroups: Set[AggregateGroup], - val aggregateType: AggregateType) { - - private val typedAggregateGroups = aggregateGroups.flatMap(_.buildTypedAggregateGroups()).toList - - val featureContext: FeatureContext = - new FeatureContext( - (typedAggregateGroups.flatMap(_.allOutputFeatures) ++ - typedAggregateGroups.flatMap(_.allOutputKeys) ++ - Seq(TypedAggregateGroup.timestampFeature)).asJava) - - val feature: BaseAggregateRootFeature = - AggregateFeatureInfo.pickFeature(aggregateType) -} - -object AggregateFeatureInfo { - val features: Set[BaseAggregateRootFeature] = - Set(PartAAggregateRootFeature, PartBAggregateRootFeature) - - def pickFeature(aggregateType: AggregateType): BaseAggregateRootFeature = { - val filtered = features.filter(_.aggregateTypes.contains(aggregateType)) - require( - filtered.size == 1, - "requested AggregateType must be backed by exactly one physical store.") - filtered.head - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.docx new file mode 100644 index 000000000..68b0c3fbd Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala deleted file mode 100644 index 5fb599240..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/AggregateFeaturesToDecodeWithMetadata.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.timelinemixer.injection.repository.uss.VersionedAggregateFeaturesDecoder -import com.twitter.ml.api.DataRecord -import com.twitter.timelines.aggregate_interactions.thriftjava.UserAggregateInteractions -import com.twitter.timelines.aggregate_interactions.v17.thriftjava.{ - UserAggregateInteractions => V17UserAggregateInteractions -} -import com.twitter.timelines.aggregate_interactions.v1.thriftjava.{ - UserAggregateInteractions => V1UserAggregateInteractions -} -import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import java.lang.{Long => JLong} -import java.util.Collections -import java.util.{Map => JMap} - -private[offline_aggregates] case class AggregateFeaturesToDecodeWithMetadata( - metadataOpt: Option[DenseFeatureMetadata], - aggregates: UserAggregateInteractions) { - def toDataRecord(dr: DenseCompactDataRecord): DataRecord = - VersionedAggregateFeaturesDecoder.fromJDenseCompact( - metadataOpt, - dr.versionId, - NullStatsReceiver, - s"V${dr.versionId}" - )(dr) - - def userAggregatesOpt: Option[DenseCompactDataRecord] = { - aggregates.getSetField match { - case UserAggregateInteractions._Fields.V17 => - Option(aggregates.getV17.user_aggregates) - case _ => - None - } - } - - def userAuthorAggregates = extract(_.user_author_aggregates) - def userEngagerAggregates = extract(_.user_engager_aggregates) - def userMentionAggregates = extract(_.user_mention_aggregates) - def userOriginalAuthorAggregates = extract(_.user_original_author_aggregates) - def userRequestDowAggregates = extract(_.user_request_dow_aggregates) - def userRequestHourAggregates = extract(_.user_request_hour_aggregates) - def rectweetUserSimclustersTweetAggregates = extract(_.rectweet_user_simclusters_tweet_aggregates) - def userTwitterListAggregates = extract(_.user_list_aggregates) - def userTopicAggregates = extract(_.user_topic_aggregates) - def userInferredTopicAggregates = extract(_.user_inferred_topic_aggregates) - def userMediaUnderstandingAnnotationAggregates = extract( - _.user_media_understanding_annotation_aggregates) - - private def extract[T]( - v17Fn: V17UserAggregateInteractions => JMap[JLong, DenseCompactDataRecord] - ): JMap[JLong, DenseCompactDataRecord] = { - aggregates.getSetField match { - case UserAggregateInteractions._Fields.V17 => - v17Fn(aggregates.getV17) - case _ => - Collections.emptyMap() - } - } -} - -object AggregateFeaturesToDecodeWithMetadata { - val empty = new AggregateFeaturesToDecodeWithMetadata( - None, - UserAggregateInteractions.v1(new V1UserAggregateInteractions())) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel deleted file mode 100644 index 10f696409..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.bazel +++ /dev/null @@ -1,40 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/functional_component/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/adapters/offline_aggregates", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/component-library/src/main/scala/com/twitter/product_mixer/component_library/model/candidate", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/featuremap/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/feature_hydrator", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "servo/repo/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/scala/com/twitter/timelines/prediction/adapters/request_context", - "src/scala/com/twitter/timelines/prediction/common/adapters:base", - "src/scala/com/twitter/timelines/prediction/common/aggregates", - "src/scala/com/twitter/timelines/util", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/timelines/suggests/common:data_record_metadata-scala", - "src/thrift/com/twitter/timelines/suggests/common:dense_data_record-scala", - "src/thrift/com/twitter/tweetypie:service-scala", - "src/thrift/com/twitter/tweetypie:tweet-scala", - "src/thrift/com/twitter/user_session_store:thrift-java", - "stitch/stitch-core", - "timelinemixer/server/src/main/scala/com/twitter/timelinemixer/injection/repository/uss:versioned-aggregate-features-decoder", - "timelines/data_processing/jobs/timeline_ranking_user_features:mini", - "timelines/data_processing/ml_util/aggregation_framework:common_types", - "timelines/data_processing/ml_util/aggregation_framework/conversion:for-timelines", - "timelineservice/common/src/main/scala/com/twitter/timelineservice/model", - "user_session_store/src/main/scala/com/twitter/user_session_store", - "util/util-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.docx new file mode 100644 index 000000000..3b5820a97 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.docx new file mode 100644 index 000000000..a7196779a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala deleted file mode 100644 index c81ed5054..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.Repository -import com.twitter.stitch.Stitch -import com.twitter.timelines.aggregate_interactions.thriftjava.UserAggregateInteractions -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import com.twitter.user_session_store.thriftjava.UserSession -import com.twitter.util.Future - -abstract class BaseAggregateQueryFeatureHydrator( - featureRepository: Repository[Long, Option[UserSession]], - metadataRepository: Repository[Int, Option[DenseFeatureMetadata]], - feature: Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]]) - extends QueryFeatureHydrator[PipelineQuery] { - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - val viewerId = query.getRequiredUserId - - Stitch.callFuture( - featureRepository(viewerId) - .flatMap { userSession: Option[UserSession] => - val featuresWithMetadata: Option[Future[AggregateFeaturesToDecodeWithMetadata]] = - userSession - .flatMap(decodeUserSession(_)) - - featuresWithMetadata - .map { fu: Future[AggregateFeaturesToDecodeWithMetadata] => fu.map(Some(_)) } - .getOrElse(Future.None) - .map { value => - FeatureMapBuilder() - .add(feature, value) - .build() - } - } - ) - } - - private def decodeUserSession( - session: UserSession - ): Option[Future[AggregateFeaturesToDecodeWithMetadata]] = { - Option(session.user_aggregate_interactions).flatMap { aggregates => - aggregates.getSetField match { - case UserAggregateInteractions._Fields.V17 => - Some( - getAggregateFeaturesWithMetadata( - aggregates.getV17.user_aggregates.versionId, - UserAggregateInteractions.v17(aggregates.getV17)) - ) - case _ => - None - } - } - } - - private def getAggregateFeaturesWithMetadata( - versionId: Int, - userAggregateInteractions: UserAggregateInteractions, - ): Future[AggregateFeaturesToDecodeWithMetadata] = { - metadataRepository(versionId) - .map(AggregateFeaturesToDecodeWithMetadata(_, userAggregateInteractions)) - } -} - -trait BaseAggregateRootFeature - extends Feature[PipelineQuery, Option[AggregateFeaturesToDecodeWithMetadata]] { - def aggregateStores: Set[StoreConfig[_]] - - lazy val aggregateTypes: Set[AggregateType] = aggregateStores.map(_.aggregateType) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.docx new file mode 100644 index 000000000..ec356c4bc Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala deleted file mode 100644 index 43cfa967b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/BaseEdgeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,93 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.IRecordOneToOneAdapter -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType.AggregateType -import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord -import java.lang.{Long => JLong} -import java.util.{Map => JMap} - -abstract case class BaseEdgeAggregateFeature( - aggregateGroups: Set[AggregateGroup], - aggregateType: AggregateType, - extractMapFn: AggregateFeaturesToDecodeWithMetadata => JMap[JLong, DenseCompactDataRecord], - adapter: IRecordOneToOneAdapter[Seq[DataRecord]], - getSecondaryKeysFn: CandidateWithFeatures[TweetCandidate] => Seq[Long]) - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord - - private val rootFeatureInfo = new AggregateFeatureInfo(aggregateGroups, aggregateType) - val featureContext: FeatureContext = rootFeatureInfo.featureContext - val rootFeature: BaseAggregateRootFeature = rootFeatureInfo.feature -} - -trait BaseEdgeAggregateFeatureHydrator - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] { - - def aggregateFeatures: Set[BaseEdgeAggregateFeature] - - override def features = aggregateFeatures.asInstanceOf[Set[Feature[_, _]]] - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offload { - val featureMapBuilders: Seq[FeatureMapBuilder] = - for (_ <- candidates) yield FeatureMapBuilder() - - aggregateFeatures.foreach { feature => - val featureValues = hydrateAggregateFeature(query, candidates, feature) - (featureMapBuilders zip featureValues).foreach { - case (featureMapBuilder, featureValue) => featureMapBuilder.add(feature, featureValue) - } - } - - featureMapBuilders.map(_.build()) - } - - private def hydrateAggregateFeature( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]], - feature: BaseEdgeAggregateFeature - ): Seq[DataRecord] = { - val rootFeature = feature.rootFeature - val extractMapFn = feature.extractMapFn - val featureContext = feature.featureContext - val secondaryIds: Seq[Seq[Long]] = candidates.map(feature.getSecondaryKeysFn) - - val featuresToDecodeWithMetadata = query.features - .flatMap(_.getOrElse(rootFeature, None)) - .getOrElse(AggregateFeaturesToDecodeWithMetadata.empty) - - // Decode the DenseCompactDataRecords into DataRecords for each required secondary id. - val decoded: Map[Long, DataRecord] = Utils.selectAndTransform( - secondaryIds.flatten.distinct, - featuresToDecodeWithMetadata.toDataRecord, - extractMapFn(featuresToDecodeWithMetadata) - ) - - // Remove unnecessary features in-place. This is safe because the underlying DataRecords - // are unique and have just been generated in the previous step. - decoded.values.foreach(Utils.filterDataRecord(_, featureContext)) - - // Put features into the FeatureMapBuilders - secondaryIds.map { ids => - val dataRecords = ids.flatMap(decoded.get) - feature.adapter.adaptToDataRecord(dataRecords) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.docx new file mode 100644 index 000000000..b8fbba991 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala deleted file mode 100644 index f769bb980..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/EdgeAggregateFeatures.scala +++ /dev/null @@ -1,118 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.PassThroughAdapter -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.adapters.offline_aggregates.SparseAggregatesToDenseAdapter -import com.twitter.home_mixer.model.HomeFeatures.AuthorIdFeature -import com.twitter.home_mixer.model.HomeFeatures.MentionScreenNameFeature -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.TSPInferredTopicFeature -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig.CombineCountPolicies - -object EdgeAggregateFeatures { - - object UserAuthorAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = TimelinesAggregationConfig.userAuthorAggregatesV2 ++ Set( - TimelinesAggregationConfig.userAuthorAggregatesV5, - TimelinesAggregationConfig.tweetSourceUserAuthorAggregatesV1, - TimelinesAggregationConfig.twitterWideUserAuthorAggregates - ), - aggregateType = AggregateType.UserAuthor, - extractMapFn = _.userAuthorAggregates, - adapter = PassThroughAdapter, - getSecondaryKeysFn = _.features.getOrElse(AuthorIdFeature, None).toSeq - ) - - object UserOriginalAuthorAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userOriginalAuthorAggregatesV1), - aggregateType = AggregateType.UserOriginalAuthor, - extractMapFn = _.userOriginalAuthorAggregates, - adapter = PassThroughAdapter, - getSecondaryKeysFn = candidate => - CandidatesUtil.getOriginalAuthorId(candidate.features).toSeq - ) - - object UserTopicAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userTopicAggregates, - TimelinesAggregationConfig.userTopicAggregatesV2, - ), - aggregateType = AggregateType.UserTopic, - extractMapFn = _.userTopicAggregates, - adapter = PassThroughAdapter, - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(TopicIdSocialContextFeature, None).toSeq - ) - - object UserMentionAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userMentionAggregates), - aggregateType = AggregateType.UserMention, - extractMapFn = _.userMentionAggregates, - adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.MentionCountsPolicy), - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(MentionScreenNameFeature, Seq.empty).map(_.hashCode.toLong) - ) - - object UserInferredTopicAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userInferredTopicAggregates, - ), - aggregateType = AggregateType.UserInferredTopic, - extractMapFn = _.userInferredTopicAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.UserInferredTopicCountsPolicy), - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq - ) - - object UserInferredTopicAggregateV2Feature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userInferredTopicAggregatesV2 - ), - aggregateType = AggregateType.UserInferredTopic, - extractMapFn = _.userInferredTopicAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.UserInferredTopicV2CountsPolicy), - getSecondaryKeysFn = candidate => - candidate.features.getOrElse(TSPInferredTopicFeature, Map.empty[Long, Double]).keys.toSeq - ) - - object UserMediaUnderstandingAnnotationAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set( - TimelinesAggregationConfig.userMediaUnderstandingAnnotationAggregates), - aggregateType = AggregateType.UserMediaUnderstandingAnnotation, - extractMapFn = _.userMediaUnderstandingAnnotationAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.UserMediaUnderstandingAnnotationCountsPolicy), - getSecondaryKeysFn = candidate => - CandidatesUtil.getMediaUnderstandingAnnotationIds(candidate.features) - ) - - object UserEngagerAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userEngagerAggregates), - aggregateType = AggregateType.UserEngager, - extractMapFn = _.userEngagerAggregates, - adapter = new SparseAggregatesToDenseAdapter(CombineCountPolicies.EngagerCountsPolicy), - getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) - ) - - object UserEngagerGoodClickAggregateFeature - extends BaseEdgeAggregateFeature( - aggregateGroups = Set(TimelinesAggregationConfig.userEngagerGoodClickAggregates), - aggregateType = AggregateType.UserEngager, - extractMapFn = _.userEngagerAggregates, - adapter = new SparseAggregatesToDenseAdapter( - CombineCountPolicies.EngagerGoodClickCountsPolicy), - getSecondaryKeysFn = candidate => CandidatesUtil.getEngagerUserIds(candidate.features) - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.docx new file mode 100644 index 000000000..c86f29294 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala deleted file mode 100644 index f18cceaca..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartAAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartARepository -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.servo.repository.Repository -import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartAStoreRegister -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import com.twitter.user_session_store.thriftjava.UserSession -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object PartAAggregateRootFeature extends BaseAggregateRootFeature { - override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartAStoreRegister.allStores -} - -@Singleton -class PartAAggregateQueryFeatureHydrator @Inject() ( - @Named(TimelineAggregatePartARepository) - repository: Repository[Long, Option[UserSession]], - @Named(TimelineAggregateMetadataRepository) - metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) - extends BaseAggregateQueryFeatureHydrator( - repository, - metadataRepository, - PartAAggregateRootFeature - ) { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("PartAAggregateQuery") - - override val features = Set(PartAAggregateRootFeature) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.docx new file mode 100644 index 000000000..e94208bf7 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala deleted file mode 100644 index fa0336165..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/PartBAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,144 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregateMetadataRepository -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TimelineAggregatePartBRepository -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.repository.Repository -import com.twitter.stitch.Stitch -import com.twitter.timelines.data_processing.jobs.timeline_ranking_user_features.TimelinesPartBStoreRegister -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateType -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.StoreConfig -import com.twitter.timelines.prediction.adapters.request_context.RequestContextAdapter -import com.twitter.timelines.prediction.common.aggregates.TimelinesAggregationConfig -import com.twitter.timelines.suggests.common.dense_data_record.thriftscala.DenseFeatureMetadata -import com.twitter.user_session_store.thriftjava.UserSession -import com.twitter.util.Time -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object PartBAggregateRootFeature extends BaseAggregateRootFeature { - override val aggregateStores: Set[StoreConfig[_]] = TimelinesPartBStoreRegister.allStores -} - -object UserAggregateFeature - extends DataRecordInAFeature[PipelineQuery] - with FeatureWithDefaultOnFailure[PipelineQuery, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class PartBAggregateQueryFeatureHydrator @Inject() ( - @Named(TimelineAggregatePartBRepository) - repository: Repository[Long, Option[UserSession]], - @Named(TimelineAggregateMetadataRepository) - metadataRepository: Repository[Int, Option[DenseFeatureMetadata]]) - extends BaseAggregateQueryFeatureHydrator( - repository, - metadataRepository, - PartBAggregateRootFeature - ) { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("PartBAggregateQuery") - - override val features: Set[Feature[_, _]] = - Set(PartBAggregateRootFeature, UserAggregateFeature) - - private val userAggregateFeatureInfo = new AggregateFeatureInfo( - aggregateGroups = Set( - TimelinesAggregationConfig.userAggregatesV2, - TimelinesAggregationConfig.userAggregatesV5Continuous, - TimelinesAggregationConfig.userAggregatesV6, - TimelinesAggregationConfig.twitterWideUserAggregates, - ), - aggregateType = AggregateType.User - ) - - private val userHourAggregateFeatureInfo = new AggregateFeatureInfo( - aggregateGroups = Set( - TimelinesAggregationConfig.userRequestHourAggregates, - ), - aggregateType = AggregateType.UserRequestHour - ) - - private val userDowAggregateFeatureInfo = new AggregateFeatureInfo( - aggregateGroups = Set( - TimelinesAggregationConfig.userRequestDowAggregates - ), - aggregateType = AggregateType.UserRequestDow - ) - - require( - userAggregateFeatureInfo.feature == PartBAggregateRootFeature, - "UserAggregates feature must be provided by the PartB data source.") - require( - userHourAggregateFeatureInfo.feature == PartBAggregateRootFeature, - "UserRequstHourAggregates feature must be provided by the PartB data source.") - require( - userDowAggregateFeatureInfo.feature == PartBAggregateRootFeature, - "UserRequestDowAggregates feature must be provided by the PartB data source.") - - override def hydrate(query: PipelineQuery): Stitch[FeatureMap] = { - // Hydrate TimelineAggregatePartBFeature and UserAggregateFeature sequentially. - super.hydrate(query).map { featureMap => - val time: Time = Time.now - val hourOfDay = RequestContextAdapter.hourFromTimestamp(time.inMilliseconds) - val dayOfWeek = RequestContextAdapter.dowFromTimestamp(time.inMilliseconds) - - val dr = featureMap - .get(PartBAggregateRootFeature).map { featuresWithMetadata => - val userAggregatesDr = - featuresWithMetadata.userAggregatesOpt - .map(featuresWithMetadata.toDataRecord) - val userRequestHourAggregatesDr = - Option(featuresWithMetadata.userRequestHourAggregates.get(hourOfDay)) - .map(featuresWithMetadata.toDataRecord) - val userRequestDowAggregatesDr = - Option(featuresWithMetadata.userRequestDowAggregates.get(dayOfWeek)) - .map(featuresWithMetadata.toDataRecord) - - dropUnknownFeatures(userAggregatesDr, userAggregateFeatureInfo.featureContext) - - dropUnknownFeatures( - userRequestHourAggregatesDr, - userHourAggregateFeatureInfo.featureContext) - - dropUnknownFeatures( - userRequestDowAggregatesDr, - userDowAggregateFeatureInfo.featureContext) - - mergeDataRecordOpts( - userAggregatesDr, - userRequestHourAggregatesDr, - userRequestDowAggregatesDr) - - }.getOrElse(new DataRecord()) - - featureMap + (UserAggregateFeature, dr) - } - } - - private val drMerger = new DataRecordMerger - private def mergeDataRecordOpts(dataRecordOpts: Option[DataRecord]*): DataRecord = - dataRecordOpts.flatten.foldLeft(new DataRecord) { (l, r) => - drMerger.merge(l, r) - l - } - - private def dropUnknownFeatures( - dataRecordOpt: Option[DataRecord], - featureContext: FeatureContext - ): Unit = - dataRecordOpt.foreach(new RichDataRecord(_, featureContext).dropUnknownFeatures()) - -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.docx new file mode 100644 index 000000000..0e05d4282 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala deleted file mode 100644 index d9df51d5c..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase1EdgeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures._ -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class Phase1EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("Phase1EdgeAggregate") - - override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = Set( - UserAuthorAggregateFeature, - UserOriginalAuthorAggregateFeature, - UserMentionAggregateFeature - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.docx new file mode 100644 index 000000000..849f658ef Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala deleted file mode 100644 index cfbb0b2c9..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Phase2EdgeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserEngagerGoodClickAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserInferredTopicAggregateV2Feature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserMediaUnderstandingAnnotationAggregateFeature -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates.EdgeAggregateFeatures.UserTopicAggregateFeature -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import javax.inject.Inject -import javax.inject.Singleton - -@Singleton -class Phase2EdgeAggregateFeatureHydrator @Inject() extends BaseEdgeAggregateFeatureHydrator { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("Phase2EdgeAggregate") - - override val aggregateFeatures: Set[BaseEdgeAggregateFeature] = - Set( - UserEngagerAggregateFeature, - UserEngagerGoodClickAggregateFeature, - UserInferredTopicAggregateFeature, - UserInferredTopicAggregateV2Feature, - UserTopicAggregateFeature, - UserMediaUnderstandingAnnotationAggregateFeature - ) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.docx new file mode 100644 index 000000000..c69a2db24 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala deleted file mode 100644 index 6ae1a0f4b..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/offline_aggregates/Utils.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.offline_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.RichDataRecord -import com.twitter.timelines.suggests.common.dense_data_record.thriftjava.DenseCompactDataRecord - -private[offline_aggregates] object Utils { - - /** - * Selects only those values in map that correspond to the keys in ids and apply the provided - * transform to the selected values. This is a convenience method for use by Timelines Aggregation - * Framework based features. - * - * @param idsToSelect The set of ids to extract values for. - * @param transform A transform to apply to the selected values. - * @param map Map[Long, DenseCompactDataRecord] - */ - def selectAndTransform( - idsToSelect: Seq[Long], - transform: DenseCompactDataRecord => DataRecord, - map: java.util.Map[java.lang.Long, DenseCompactDataRecord], - ): Map[Long, DataRecord] = { - val filtered: Seq[(Long, DataRecord)] = - for { - id <- idsToSelect if map.containsKey(id) - } yield { - id -> transform(map.get(id)) - } - filtered.toMap - } - - def filterDataRecord(dr: DataRecord, featureContext: FeatureContext): Unit = { - new RichDataRecord(dr, featureContext).dropUnknownFeatures() - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel deleted file mode 100644 index 1954dd77d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.bazel +++ /dev/null @@ -1,26 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "finatra/inject/inject-core/src/main/scala", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/model/request", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/param", - "home-mixer/server/src/main/scala/com/twitter/home_mixer/util", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/feature/datarecord", - "product-mixer/core/src/main/scala/com/twitter/product_mixer/core/util", - "servo/repo/src/main/scala", - "src/java/com/twitter/ml/api:api-base", - "src/java/com/twitter/ml/api/constant", - "src/scala/com/twitter/ml/api/util", - "src/scala/com/twitter/timelines/prediction/common/aggregates/real_time:base-config", - "src/thrift/com/twitter/ml/api:data-java", - "src/thrift/com/twitter/wtf/real_time_interaction_graph:wtf-real_time_interaction_graph-thrift-java", - "stitch/stitch-core", - "timelines/data_processing/ml_util/aggregation_framework:common_types", - "timelines/data_processing/ml_util/aggregation_framework/heron", - "util/util-core", - ], -) diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.docx new file mode 100644 index 000000000..bc0c7193e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BUILD.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.docx new file mode 100644 index 000000000..d832ec30f Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala deleted file mode 100644 index 41b565ded..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateBulkCandidateFeatureHydrator.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.BulkCandidateFeatureHydrator -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch - -trait BaseRealTimeAggregateBulkCandidateFeatureHydrator[K] - extends BulkCandidateFeatureHydrator[PipelineQuery, TweetCandidate] - with BaseRealtimeAggregateHydrator[K] { - - val outputFeature: DataRecordInAFeature[TweetCandidate] - - override def features: Set[Feature[_, _]] = Set(outputFeature) - - override lazy val statScope: String = identifier.toString - - def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[K]] - - override def apply( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Stitch[Seq[FeatureMap]] = OffloadFuturePools.offloadFuture { - val possiblyKeys = keysFromQueryAndCandidates(query, candidates) - fetchAndConstructDataRecords(possiblyKeys).map { dataRecords => - dataRecords.map { dataRecord => - FeatureMapBuilder().add(outputFeature, dataRecord).build() - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.docx new file mode 100644 index 000000000..3563345ac Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala deleted file mode 100644 index 8f5b17d64..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealTimeAggregateQueryFeatureHydrator.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.product_mixer.core.feature.Feature -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.feature.featuremap.FeatureMap -import com.twitter.product_mixer.core.feature.featuremap.FeatureMapBuilder -import com.twitter.product_mixer.core.functional_component.feature_hydrator.QueryFeatureHydrator -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.product_mixer.core.util.OffloadFuturePools -import com.twitter.stitch.Stitch - -trait BaseRealTimeAggregateQueryFeatureHydrator[K] - extends QueryFeatureHydrator[PipelineQuery] - with BaseRealtimeAggregateHydrator[K] { - - val outputFeature: DataRecordInAFeature[PipelineQuery] - - override def features: Set[Feature[_, _]] = Set(outputFeature) - - override lazy val statScope: String = identifier.toString - - def keysFromQueryAndCandidates( - query: PipelineQuery - ): Option[K] - - override def hydrate( - query: PipelineQuery - ): Stitch[FeatureMap] = OffloadFuturePools.offloadFuture { - val possiblyKeys = keysFromQueryAndCandidates(query) - fetchAndConstructDataRecords(Seq(possiblyKeys)).map { dataRecords => - FeatureMapBuilder() - .add(outputFeature, dataRecords.head) - .build() - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.docx new file mode 100644 index 000000000..44e86ef4e Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala deleted file mode 100644 index f97820be0..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/BaseRealtimeAggregateHydrator.scala +++ /dev/null @@ -1,138 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates.BaseRealtimeAggregateHydrator._ -import com.twitter.home_mixer.util.DataRecordUtil -import com.twitter.home_mixer.util.ObservedKeyValueResultHandler -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.DataRecordMerger -import com.twitter.ml.api.FeatureContext -import com.twitter.ml.api.constant.SharedFeatures -import com.twitter.ml.api.util.SRichDataRecord -import com.twitter.ml.api.{Feature => MLApiFeature} -import com.twitter.servo.cache.ReadCache -import com.twitter.servo.keyvalue.KeyValueResult -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.util.Future -import com.twitter.util.Time -import com.twitter.util.Try -import java.lang.{Double => JDouble} -import scala.collection.JavaConverters._ - -trait BaseRealtimeAggregateHydrator[K] extends ObservedKeyValueResultHandler { - - val client: ReadCache[K, DataRecord] - - val aggregateGroups: Seq[AggregateGroup] - - val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map.empty - - private lazy val typedAggregateGroupsList = aggregateGroups.map(_.buildTypedAggregateGroups()) - - private lazy val featureContexts: Seq[FeatureContext] = typedAggregateGroupsList.map { - typedAggregateGroups => - new FeatureContext( - (SharedFeatures.TIMESTAMP +: typedAggregateGroups.flatMap(_.allOutputFeatures)).asJava - ) - } - - private lazy val aggregateFeaturesRenameMap: Map[MLApiFeature[_], MLApiFeature[_]] = { - val prefixes: Seq[Option[String]] = aggregateGroups.map(aggregateGroupToPrefix.get) - - typedAggregateGroupsList - .zip(prefixes).map { - case (typedAggregateGroups, prefix) => - if (prefix.nonEmpty) - typedAggregateGroups - .map { - _.outputFeaturesToRenamedOutputFeatures(prefix.get) - }.reduce(_ ++ _) - else - Map.empty[MLApiFeature[_], MLApiFeature[_]] - }.reduce(_ ++ _) - } - - private lazy val renamedFeatureContexts: Seq[FeatureContext] = - typedAggregateGroupsList.map { typedAggregateGroups => - val renamedAllOutputFeatures = typedAggregateGroups.flatMap(_.allOutputFeatures).map { - feature => aggregateFeaturesRenameMap.getOrElse(feature, feature) - } - - new FeatureContext(renamedAllOutputFeatures.asJava) - } - - private lazy val decays: Seq[TimeDecay] = typedAggregateGroupsList.map { typedAggregateGroups => - RealTimeAggregateTimeDecay( - typedAggregateGroups.flatMap(_.continuousFeatureIdsToHalfLives).toMap) - .apply(_, _) - } - - private val drMerger = new DataRecordMerger - - private def postTransformer(dataRecord: Try[Option[DataRecord]]): Try[DataRecord] = { - dataRecord.map { - case Some(dr) => - val newDr = new DataRecord() - featureContexts.zip(renamedFeatureContexts).zip(decays).foreach { - case ((featureContext, renamedFeatureContext), decay) => - val decayedDr = applyDecay(dr, featureContext, decay) - val renamedDr = DataRecordUtil.applyRename( - dataRecord = decayedDr, - featureContext, - renamedFeatureContext, - aggregateFeaturesRenameMap) - drMerger.merge(newDr, renamedDr) - } - newDr - case _ => new DataRecord - } - } - - def fetchAndConstructDataRecords(possiblyKeys: Seq[Option[K]]): Future[Seq[Try[DataRecord]]] = { - val keys = possiblyKeys.flatten - - val response: Future[KeyValueResult[K, DataRecord]] = - if (keys.isEmpty) Future.value(KeyValueResult.empty) - else { - val batchResponses = keys - .grouped(RequestBatchSize) - .map(keyGroup => client.get(keyGroup)) - .toSeq - - Future.collect(batchResponses).map(_.reduce(_ ++ _)) - } - - response.map { result => - possiblyKeys.map { possiblyKey => - val value = observedGet(key = possiblyKey, keyValueResult = result) - postTransformer(value) - } - } - } -} - -object BaseRealtimeAggregateHydrator { - private val RequestBatchSize = 5 - - type TimeDecay = scala.Function2[com.twitter.ml.api.DataRecord, scala.Long, scala.Unit] - - private def applyDecay( - dataRecord: DataRecord, - featureContext: FeatureContext, - decay: TimeDecay - ): DataRecord = { - def time: Long = Time.now.inMillis - - val richFullDr = new SRichDataRecord(dataRecord, featureContext) - val richNewDr = new SRichDataRecord(new DataRecord, featureContext) - val featureIterator = featureContext.iterator() - featureIterator.forEachRemaining { feature => - if (richFullDr.hasFeature(feature)) { - val typedFeature = feature.asInstanceOf[MLApiFeature[JDouble]] - richNewDr.setFeatureValue(typedFeature, richFullDr.getFeatureValue(typedFeature)) - } - } - val resultDr = richNewDr.getRecord - decay(resultDr, time) - resultDr - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.docx new file mode 100644 index 000000000..4275e1b9a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 990fe1ca4..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.param.HomeMixerInjectionNames.EngagementsReceivedByAuthorCache -import com.twitter.home_mixer.util.CandidatesUtil -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object EngagementsReceivedByAuthorRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class EngagementsReceivedByAuthorRealTimeAggregateFeatureHydrator @Inject() ( - @Named(EngagementsReceivedByAuthorCache) override val client: ReadCache[Long, DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("EngagementsReceivedByAuthorRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - EngagementsReceivedByAuthorRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - authorEngagementRealTimeAggregatesProd, - authorShareEngagementsRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - authorShareEngagementsRealTimeAggregates -> "original_author.timelines.author_share_engagements_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = - candidates.map(candidate => CandidatesUtil.getOriginalAuthorId(candidate.features)) -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.docx new file mode 100644 index 000000000..6fc55d85a Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala deleted file mode 100644 index dbecd12c6..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/RealTimeAggregateTimeDecay.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.ml.api.DataRecord -import com.twitter.ml.api.constant.SharedFeatures.TIMESTAMP -import com.twitter.util.Duration - -/** - * The default TimeDecay implementation for real time aggregates. - * - * @param featureIdToHalfLife A precomputed map from aggregate feature ids to their half lives. - * @param timestampFeatureId A discrete timestamp feature id. - */ -case class RealTimeAggregateTimeDecay( - featureIdToHalfLife: Map[Long, Duration], - timestampFeatureId: Long = TIMESTAMP.getFeatureId) { - - /** - * Mutates the data record which is just a reference to the input. - * - * @param record Data record to apply decay to (is mutated). - * @param timeNow The current read time (in milliseconds) to decay counts forward to. - */ - def apply(record: DataRecord, timeNow: Long): Unit = { - if (record.isSetDiscreteFeatures) { - val discreteFeatures = record.getDiscreteFeatures - if (discreteFeatures.containsKey(timestampFeatureId)) { - if (record.isSetContinuousFeatures) { - val ctsFeatures = record.getContinuousFeatures - - val storedTimestamp: Long = discreteFeatures.get(timestampFeatureId) - val scaledDt = if (timeNow > storedTimestamp) { - (timeNow - storedTimestamp).toDouble * math.log(2) - } else 0.0 - featureIdToHalfLife.foreach { - case (featureId, halfLife) => - if (ctsFeatures.containsKey(featureId)) { - val storedValue = ctsFeatures.get(featureId) - val alpha = - if (halfLife.inMilliseconds != 0) math.exp(-scaledDt / halfLife.inMilliseconds) - else 0 - val decayedValue: Double = alpha * storedValue - record.putToContinuousFeatures(featureId, decayedValue) - } - } - } - discreteFeatures.remove(timestampFeatureId) - } - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.docx new file mode 100644 index 000000000..a2f24a494 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 13b2a0d2d..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicCountryEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.google.inject.name.Named -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicCountryEngagementCache -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Singleton - -object TopicCountryEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TopicCountryEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(TopicCountryEngagementCache) override val client: ReadCache[(Long, String), DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[(Long, String)] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TopicCountryEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - TopicCountryEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - topicCountryRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - topicCountryRealTimeAggregates -> "topic-country_code.timelines.topic_country_engagement_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[(Long, String)]] = { - candidates.map { candidate => - val maybeTopicId = candidate.features - .getTry(TopicIdSocialContextFeature) - .toOption - .flatten - - val maybeCountryCode = query.clientContext.countryCode - - for { - topicId <- maybeTopicId - countryCode <- maybeCountryCode - } yield (topicId, countryCode) - } - } -} diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.docx b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.docx new file mode 100644 index 000000000..df3575d87 Binary files /dev/null and b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.docx differ diff --git a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala b/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala deleted file mode 100644 index 12a2e1c47..000000000 --- a/home-mixer/server/src/main/scala/com/twitter/home_mixer/product/scored_tweets/feature_hydrator/real_time_aggregates/TopicEngagementRealTimeAggregateFeatureHydrator.scala +++ /dev/null @@ -1,60 +0,0 @@ -package com.twitter.home_mixer.product.scored_tweets.feature_hydrator.real_time_aggregates - -import com.twitter.finagle.stats.StatsReceiver -import com.twitter.home_mixer.model.HomeFeatures.TopicIdSocialContextFeature -import com.twitter.home_mixer.param.HomeMixerInjectionNames.TopicEngagementCache -import com.twitter.ml.api.DataRecord -import com.twitter.product_mixer.component_library.model.candidate.TweetCandidate -import com.twitter.product_mixer.core.feature.FeatureWithDefaultOnFailure -import com.twitter.product_mixer.core.feature.datarecord.DataRecordInAFeature -import com.twitter.product_mixer.core.model.common.CandidateWithFeatures -import com.twitter.product_mixer.core.model.common.identifier.FeatureHydratorIdentifier -import com.twitter.product_mixer.core.pipeline.PipelineQuery -import com.twitter.servo.cache.ReadCache -import com.twitter.timelines.data_processing.ml_util.aggregation_framework.AggregateGroup -import com.twitter.timelines.prediction.common.aggregates.real_time.TimelinesOnlineAggregationFeaturesOnlyConfig._ -import javax.inject.Inject -import javax.inject.Named -import javax.inject.Singleton - -object TopicEngagementRealTimeAggregateFeature - extends DataRecordInAFeature[TweetCandidate] - with FeatureWithDefaultOnFailure[TweetCandidate, DataRecord] { - override def defaultValue: DataRecord = new DataRecord() -} - -@Singleton -class TopicEngagementRealTimeAggregateFeatureHydrator @Inject() ( - @Named(TopicEngagementCache) override val client: ReadCache[Long, DataRecord], - override val statsReceiver: StatsReceiver) - extends BaseRealTimeAggregateBulkCandidateFeatureHydrator[Long] { - - override val identifier: FeatureHydratorIdentifier = - FeatureHydratorIdentifier("TopicEngagementRealTimeAggregate") - - override val outputFeature: DataRecordInAFeature[TweetCandidate] = - TopicEngagementRealTimeAggregateFeature - - override val aggregateGroups: Seq[AggregateGroup] = Seq( - topicEngagementRealTimeAggregatesProd, - topicEngagement24HourRealTimeAggregatesProd, - topicShareEngagementsRealTimeAggregates - ) - - override val aggregateGroupToPrefix: Map[AggregateGroup, String] = Map( - topicEngagement24HourRealTimeAggregatesProd -> "topic.timelines.topic_engagement_24_hour_real_time_aggregates.", - topicShareEngagementsRealTimeAggregates -> "topic.timelines.topic_share_engagements_real_time_aggregates." - ) - - override def keysFromQueryAndCandidates( - query: PipelineQuery, - candidates: Seq[CandidateWithFeatures[TweetCandidate]] - ): Seq[Option[Long]] = { - candidates.map { candidate => - candidate.features - .getTry(TopicIdSocialContextFeature) - .toOption - .flatten - } - } -}